diff --git "a/laya_multilingual_e8_L512_options32.mlmodelc/model.mil" "b/laya_multilingual_e8_L512_options32.mlmodelc/model.mil" new file mode 100644--- /dev/null +++ "b/laya_multilingual_e8_L512_options32.mlmodelc/model.mil" @@ -0,0 +1,2125 @@ +program(1.0) +[buildInfo = dict, tensor>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.25.2"}})] +{ + func main(tensor attention_mask, tensor input_ids, tensor marker_map, tensor question_type) { + tensor var_1093_to_fp16 = const()[name = tensor("op_1093_to_fp16"), val = tensor(0x1p+0)]; + tensor var_1092_to_fp16_dtype_0 = const()[name = tensor("op_1092_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor attention_mask_to_fp16 = cast(dtype = var_1092_to_fp16_dtype_0, x = attention_mask)[name = tensor("cast_5")]; + tensor var_1095_cast_fp16 = sub(x = var_1093_to_fp16, y = attention_mask_to_fp16)[name = tensor("op_1095_cast_fp16")]; + tensor var_1100 = const()[name = tensor("op_1100"), val = tensor([1, 1, 1, 512])]; + tensor var_1101_cast_fp16 = reshape(shape = var_1100, x = var_1095_cast_fp16)[name = tensor("op_1101_cast_fp16")]; + tensor var_1102_to_fp16 = const()[name = tensor("op_1102_to_fp16"), val = tensor(-0x1.388p+13)]; + tensor pad_cast_fp16 = mul(x = var_1101_cast_fp16, y = var_1102_to_fp16)[name = tensor("pad_cast_fp16")]; + tensor full_mask_to_fp16 = const()[name = tensor("full_mask_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64)))]; + tensor attention_mask_3_cast_fp16 = add(x = full_mask_to_fp16, y = pad_cast_fp16)[name = tensor("attention_mask_3_cast_fp16")]; + tensor band_mask_to_fp16 = const()[name = tensor("band_mask_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(524416)))]; + tensor attention_mask_cast_fp16 = add(x = band_mask_to_fp16, y = pad_cast_fp16)[name = tensor("attention_mask_cast_fp16")]; + tensor input_3_batch_dims_0 = const()[name = tensor("input_3_batch_dims_0"), val = tensor(0)]; + tensor input_3_validate_indices_0 = const()[name = tensor("input_3_validate_indices_0"), val = tensor(false)]; + tensor model_encoder_embeddings_tok_embeddings_weight_to_fp16_quantized = constexpr_affine_dequantize()[axis = tensor(0), name = tensor("model_encoder_embeddings_tok_embeddings_weight_to_fp16_quantized"), quantized_data = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(1048768))), scale = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(197912896))), zero_point = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(197656832)))]; + tensor greater_equal_0_y_0 = const()[name = tensor("greater_equal_0_y_0"), val = tensor(0)]; + tensor greater_equal_0 = greater_equal(x = input_ids, y = greater_equal_0_y_0)[name = tensor("greater_equal_0")]; + tensor slice_by_index_0 = const()[name = tensor("slice_by_index_0"), val = tensor(256000)]; + tensor add_0 = add(x = input_ids, y = slice_by_index_0)[name = tensor("add_0")]; + tensor select_0 = select(a = input_ids, b = add_0, cond = greater_equal_0)[name = tensor("select_0")]; + tensor greater_equal_0_y_0_1 = const()[name = tensor("greater_equal_0_y_0_1"), val = tensor(0)]; + tensor greater_equal_0_1 = greater_equal(x = select_0, y = greater_equal_0_y_0_1)[name = tensor("greater_equal_0_1")]; + tensor slice_by_index_0_1 = const()[name = tensor("slice_by_index_0_1"), val = tensor(256000)]; + tensor add_0_1 = add(x = select_0, y = slice_by_index_0_1)[name = tensor("add_0_1")]; + tensor select_0_1 = select(a = select_0, b = add_0_1, cond = greater_equal_0_1)[name = tensor("select_0_1")]; + tensor input_3_cast_fp16_axis_0 = const()[name = tensor("input_3_cast_fp16_axis_0"), val = tensor(0)]; + tensor input_3_cast_fp16 = gather(axis = input_3_cast_fp16_axis_0, batch_dims = input_3_batch_dims_0, indices = select_0_1, validate_indices = input_3_validate_indices_0, x = model_encoder_embeddings_tok_embeddings_weight_to_fp16_quantized)[name = tensor("input_3_cast_fp16")]; + tensor input_5_axes_0 = const()[name = tensor("input_5_axes_0"), val = tensor([-1])]; + tensor model_encoder_embeddings_norm_weight_to_fp16 = const()[name = tensor("model_encoder_embeddings_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(198424960)))]; + tensor var_1116_to_fp16 = const()[name = tensor("op_1116_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_5_cast_fp16 = layer_norm(axes = input_5_axes_0, epsilon = var_1116_to_fp16, gamma = model_encoder_embeddings_norm_weight_to_fp16, x = input_3_cast_fp16)[name = tensor("input_5_cast_fp16")]; + tensor var_1131 = const()[name = tensor("op_1131"), val = tensor(-1)]; + tensor model_encoder_layers_0_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(198426560)))]; + tensor linear_0_bias_0_to_fp16 = const()[name = tensor("linear_0_bias_0_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(201965568)))]; + tensor linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_0_attn_Wqkv_weight_to_fp16, x = input_5_cast_fp16)[name = tensor("linear_0_cast_fp16")]; + tensor var_1144 = const()[name = tensor("op_1144"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_3_cast_fp16 = reshape(shape = var_1144, x = linear_0_cast_fp16)[name = tensor("qkv_3_cast_fp16")]; + tensor var_1146_split_sizes_0 = const()[name = tensor("op_1146_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1146_axis_0 = const()[name = tensor("op_1146_axis_0"), val = tensor(-3)]; + tensor var_1146_cast_fp16_0, tensor var_1146_cast_fp16_1, tensor var_1146_cast_fp16_2 = split(axis = var_1146_axis_0, split_sizes = var_1146_split_sizes_0, x = qkv_3_cast_fp16)[name = tensor("op_1146_cast_fp16")]; + tensor squeeze_0_axes_0 = const()[name = tensor("squeeze_0_axes_0"), val = tensor([-3])]; + tensor squeeze_0_cast_fp16 = squeeze(axes = squeeze_0_axes_0, x = var_1146_cast_fp16_0)[name = tensor("squeeze_0_cast_fp16")]; + tensor squeeze_1_axes_0 = const()[name = tensor("squeeze_1_axes_0"), val = tensor([-3])]; + tensor squeeze_1_cast_fp16 = squeeze(axes = squeeze_1_axes_0, x = var_1146_cast_fp16_1)[name = tensor("squeeze_1_cast_fp16")]; + tensor squeeze_2_axes_0 = const()[name = tensor("squeeze_2_axes_0"), val = tensor([-3])]; + tensor squeeze_2_cast_fp16 = squeeze(axes = squeeze_2_axes_0, x = var_1146_cast_fp16_2)[name = tensor("squeeze_2_cast_fp16")]; + tensor q_1_perm_0 = const()[name = tensor("q_1_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_1_perm_0 = const()[name = tensor("k_1_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_1_perm_0 = const()[name = tensor("value_1_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor cos_3_to_fp16 = const()[name = tensor("cos_3_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(201970240)))]; + tensor q_1_cast_fp16 = transpose(perm = q_1_perm_0, x = squeeze_0_cast_fp16)[name = tensor("transpose_107")]; + tensor var_1156_cast_fp16 = mul(x = q_1_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1156_cast_fp16")]; + tensor x1_1_begin_0 = const()[name = tensor("x1_1_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_1_end_0 = const()[name = tensor("x1_1_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_1_end_mask_0 = const()[name = tensor("x1_1_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_1_cast_fp16 = slice_by_index(begin = x1_1_begin_0, end = x1_1_end_0, end_mask = x1_1_end_mask_0, x = q_1_cast_fp16)[name = tensor("x1_1_cast_fp16")]; + tensor x2_1_begin_0 = const()[name = tensor("x2_1_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_1_end_0 = const()[name = tensor("x2_1_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_1_end_mask_0 = const()[name = tensor("x2_1_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_1_cast_fp16 = slice_by_index(begin = x2_1_begin_0, end = x2_1_end_0, end_mask = x2_1_end_mask_0, x = q_1_cast_fp16)[name = tensor("x2_1_cast_fp16")]; + tensor const_4_promoted_to_fp16 = const()[name = tensor("const_4_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1168_cast_fp16 = mul(x = x2_1_cast_fp16, y = const_4_promoted_to_fp16)[name = tensor("op_1168_cast_fp16")]; + tensor var_1170_interleave_0 = const()[name = tensor("op_1170_interleave_0"), val = tensor(false)]; + tensor var_1170_cast_fp16 = concat(axis = var_1131, interleave = var_1170_interleave_0, values = (var_1168_cast_fp16, x1_1_cast_fp16))[name = tensor("op_1170_cast_fp16")]; + tensor sin_3_to_fp16 = const()[name = tensor("sin_3_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(202035840)))]; + tensor var_1171_cast_fp16 = mul(x = var_1170_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1171_cast_fp16")]; + tensor q_embed_1_cast_fp16 = add(x = var_1156_cast_fp16, y = var_1171_cast_fp16)[name = tensor("q_embed_1_cast_fp16")]; + tensor k_1_cast_fp16 = transpose(perm = k_1_perm_0, x = squeeze_1_cast_fp16)[name = tensor("transpose_106")]; + tensor var_1174_cast_fp16 = mul(x = k_1_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1174_cast_fp16")]; + tensor x1_3_begin_0 = const()[name = tensor("x1_3_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_3_end_0 = const()[name = tensor("x1_3_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_3_end_mask_0 = const()[name = tensor("x1_3_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_3_cast_fp16 = slice_by_index(begin = x1_3_begin_0, end = x1_3_end_0, end_mask = x1_3_end_mask_0, x = k_1_cast_fp16)[name = tensor("x1_3_cast_fp16")]; + tensor x2_3_begin_0 = const()[name = tensor("x2_3_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_3_end_0 = const()[name = tensor("x2_3_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_3_end_mask_0 = const()[name = tensor("x2_3_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_3_cast_fp16 = slice_by_index(begin = x2_3_begin_0, end = x2_3_end_0, end_mask = x2_3_end_mask_0, x = k_1_cast_fp16)[name = tensor("x2_3_cast_fp16")]; + tensor const_7_promoted_to_fp16 = const()[name = tensor("const_7_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1186_cast_fp16 = mul(x = x2_3_cast_fp16, y = const_7_promoted_to_fp16)[name = tensor("op_1186_cast_fp16")]; + tensor var_1188_interleave_0 = const()[name = tensor("op_1188_interleave_0"), val = tensor(false)]; + tensor var_1188_cast_fp16 = concat(axis = var_1131, interleave = var_1188_interleave_0, values = (var_1186_cast_fp16, x1_3_cast_fp16))[name = tensor("op_1188_cast_fp16")]; + tensor var_1189_cast_fp16 = mul(x = var_1188_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1189_cast_fp16")]; + tensor k_embed_1_cast_fp16 = add(x = var_1174_cast_fp16, y = var_1189_cast_fp16)[name = tensor("k_embed_1_cast_fp16")]; + tensor var_1194_transpose_x_1 = const()[name = tensor("op_1194_transpose_x_1"), val = tensor(false)]; + tensor var_1194_transpose_y_1 = const()[name = tensor("op_1194_transpose_y_1"), val = tensor(true)]; + tensor var_1194_cast_fp16 = matmul(transpose_x = var_1194_transpose_x_1, transpose_y = var_1194_transpose_y_1, x = q_embed_1_cast_fp16, y = k_embed_1_cast_fp16)[name = tensor("op_1194_cast_fp16")]; + tensor var_1195_to_fp16 = const()[name = tensor("op_1195_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_1_cast_fp16 = mul(x = var_1194_cast_fp16, y = var_1195_to_fp16)[name = tensor("attn_weights_1_cast_fp16")]; + tensor input_7_cast_fp16 = add(x = attn_weights_1_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_7_cast_fp16")]; + tensor var_1198_cast_fp16 = softmax(axis = var_1131, x = input_7_cast_fp16)[name = tensor("op_1198_cast_fp16")]; + tensor attn_output_1_transpose_x_0 = const()[name = tensor("attn_output_1_transpose_x_0"), val = tensor(false)]; + tensor attn_output_1_transpose_y_0 = const()[name = tensor("attn_output_1_transpose_y_0"), val = tensor(false)]; + tensor value_1_cast_fp16 = transpose(perm = value_1_perm_0, x = squeeze_2_cast_fp16)[name = tensor("transpose_105")]; + tensor attn_output_1_cast_fp16 = matmul(transpose_x = attn_output_1_transpose_x_0, transpose_y = attn_output_1_transpose_y_0, x = var_1198_cast_fp16, y = value_1_cast_fp16)[name = tensor("attn_output_1_cast_fp16")]; + tensor var_1202_perm_0 = const()[name = tensor("op_1202_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1204 = const()[name = tensor("op_1204"), val = tensor([1, 512, -1])]; + tensor var_1202_cast_fp16 = transpose(perm = var_1202_perm_0, x = attn_output_1_cast_fp16)[name = tensor("transpose_104")]; + tensor var_1205_cast_fp16 = reshape(shape = var_1204, x = var_1202_cast_fp16)[name = tensor("op_1205_cast_fp16")]; + tensor model_encoder_layers_0_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(202101440)))]; + tensor linear_1_bias_0_to_fp16 = const()[name = tensor("linear_1_bias_0_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(203281152)))]; + tensor linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_0_attn_Wo_weight_to_fp16, x = var_1205_cast_fp16)[name = tensor("linear_1_cast_fp16")]; + tensor input_13_cast_fp16 = add(x = input_5_cast_fp16, y = linear_1_cast_fp16)[name = tensor("input_13_cast_fp16")]; + tensor input_15_axes_0 = const()[name = tensor("input_15_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_0_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(203282752)))]; + tensor var_1126_to_fp16 = const()[name = tensor("op_1126_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_15_cast_fp16 = layer_norm(axes = input_15_axes_0, epsilon = var_1126_to_fp16, gamma = model_encoder_layers_0_mlp_norm_weight_to_fp16, x = input_13_cast_fp16)[name = tensor("input_15_cast_fp16")]; + tensor model_encoder_layers_0_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(203284352)))]; + tensor linear_2_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_0_mlp_Wi_weight_to_fp16, x = input_15_cast_fp16)[name = tensor("linear_2_cast_fp16")]; + tensor var_1212_split_sizes_0 = const()[name = tensor("op_1212_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1212_axis_0 = const()[name = tensor("op_1212_axis_0"), val = tensor(-1)]; + tensor var_1212_cast_fp16_0, tensor var_1212_cast_fp16_1 = split(axis = var_1212_axis_0, split_sizes = var_1212_split_sizes_0, x = linear_2_cast_fp16)[name = tensor("op_1212_cast_fp16")]; + tensor var_1214_mode_0 = const()[name = tensor("op_1214_mode_0"), val = tensor("EXACT")]; + tensor var_1214_cast_fp16 = gelu(mode = var_1214_mode_0, x = var_1212_cast_fp16_0)[name = tensor("op_1214_cast_fp16")]; + tensor input_19_cast_fp16 = mul(x = var_1214_cast_fp16, y = var_1212_cast_fp16_1)[name = tensor("input_19_cast_fp16")]; + tensor model_encoder_layers_0_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_0_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(206823360)))]; + tensor linear_3_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_0_mlp_Wo_weight_to_fp16, x = input_19_cast_fp16)[name = tensor("linear_3_cast_fp16")]; + tensor input_23_cast_fp16 = add(x = input_13_cast_fp16, y = linear_3_cast_fp16)[name = tensor("input_23_cast_fp16")]; + tensor var_1223 = const()[name = tensor("op_1223"), val = tensor(-1)]; + tensor hidden_states_3_axes_0 = const()[name = tensor("hidden_states_3_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_1_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(208592896)))]; + tensor var_1234_to_fp16 = const()[name = tensor("op_1234_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_3_cast_fp16 = layer_norm(axes = hidden_states_3_axes_0, epsilon = var_1234_to_fp16, gamma = model_encoder_layers_1_attn_norm_weight_to_fp16, x = input_23_cast_fp16)[name = tensor("hidden_states_3_cast_fp16")]; + tensor model_encoder_layers_1_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(208594496)))]; + tensor linear_4_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_1_attn_Wqkv_weight_to_fp16, x = hidden_states_3_cast_fp16)[name = tensor("linear_4_cast_fp16")]; + tensor var_1241 = const()[name = tensor("op_1241"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_7_cast_fp16 = reshape(shape = var_1241, x = linear_4_cast_fp16)[name = tensor("qkv_7_cast_fp16")]; + tensor var_1243_split_sizes_0 = const()[name = tensor("op_1243_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1243_axis_0 = const()[name = tensor("op_1243_axis_0"), val = tensor(-3)]; + tensor var_1243_cast_fp16_0, tensor var_1243_cast_fp16_1, tensor var_1243_cast_fp16_2 = split(axis = var_1243_axis_0, split_sizes = var_1243_split_sizes_0, x = qkv_7_cast_fp16)[name = tensor("op_1243_cast_fp16")]; + tensor squeeze_3_axes_0 = const()[name = tensor("squeeze_3_axes_0"), val = tensor([-3])]; + tensor squeeze_3_cast_fp16 = squeeze(axes = squeeze_3_axes_0, x = var_1243_cast_fp16_0)[name = tensor("squeeze_3_cast_fp16")]; + tensor squeeze_4_axes_0 = const()[name = tensor("squeeze_4_axes_0"), val = tensor([-3])]; + tensor squeeze_4_cast_fp16 = squeeze(axes = squeeze_4_axes_0, x = var_1243_cast_fp16_1)[name = tensor("squeeze_4_cast_fp16")]; + tensor squeeze_5_axes_0 = const()[name = tensor("squeeze_5_axes_0"), val = tensor([-3])]; + tensor squeeze_5_cast_fp16 = squeeze(axes = squeeze_5_axes_0, x = var_1243_cast_fp16_2)[name = tensor("squeeze_5_cast_fp16")]; + tensor q_5_perm_0 = const()[name = tensor("q_5_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_5_perm_0 = const()[name = tensor("k_5_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_3_perm_0 = const()[name = tensor("value_3_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_5_cast_fp16 = transpose(perm = q_5_perm_0, x = squeeze_3_cast_fp16)[name = tensor("transpose_103")]; + tensor var_1253_cast_fp16 = mul(x = q_5_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1253_cast_fp16")]; + tensor x1_5_begin_0 = const()[name = tensor("x1_5_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_5_end_0 = const()[name = tensor("x1_5_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_5_end_mask_0 = const()[name = tensor("x1_5_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_5_cast_fp16 = slice_by_index(begin = x1_5_begin_0, end = x1_5_end_0, end_mask = x1_5_end_mask_0, x = q_5_cast_fp16)[name = tensor("x1_5_cast_fp16")]; + tensor x2_5_begin_0 = const()[name = tensor("x2_5_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_5_end_0 = const()[name = tensor("x2_5_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_5_end_mask_0 = const()[name = tensor("x2_5_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_5_cast_fp16 = slice_by_index(begin = x2_5_begin_0, end = x2_5_end_0, end_mask = x2_5_end_mask_0, x = q_5_cast_fp16)[name = tensor("x2_5_cast_fp16")]; + tensor const_12_promoted_to_fp16 = const()[name = tensor("const_12_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1265_cast_fp16 = mul(x = x2_5_cast_fp16, y = const_12_promoted_to_fp16)[name = tensor("op_1265_cast_fp16")]; + tensor var_1267_interleave_0 = const()[name = tensor("op_1267_interleave_0"), val = tensor(false)]; + tensor var_1267_cast_fp16 = concat(axis = var_1223, interleave = var_1267_interleave_0, values = (var_1265_cast_fp16, x1_5_cast_fp16))[name = tensor("op_1267_cast_fp16")]; + tensor var_1268_cast_fp16 = mul(x = var_1267_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1268_cast_fp16")]; + tensor q_embed_3_cast_fp16 = add(x = var_1253_cast_fp16, y = var_1268_cast_fp16)[name = tensor("q_embed_3_cast_fp16")]; + tensor k_5_cast_fp16 = transpose(perm = k_5_perm_0, x = squeeze_4_cast_fp16)[name = tensor("transpose_102")]; + tensor var_1271_cast_fp16 = mul(x = k_5_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1271_cast_fp16")]; + tensor x1_7_begin_0 = const()[name = tensor("x1_7_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_7_end_0 = const()[name = tensor("x1_7_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_7_end_mask_0 = const()[name = tensor("x1_7_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_7_cast_fp16 = slice_by_index(begin = x1_7_begin_0, end = x1_7_end_0, end_mask = x1_7_end_mask_0, x = k_5_cast_fp16)[name = tensor("x1_7_cast_fp16")]; + tensor x2_7_begin_0 = const()[name = tensor("x2_7_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_7_end_0 = const()[name = tensor("x2_7_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_7_end_mask_0 = const()[name = tensor("x2_7_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_7_cast_fp16 = slice_by_index(begin = x2_7_begin_0, end = x2_7_end_0, end_mask = x2_7_end_mask_0, x = k_5_cast_fp16)[name = tensor("x2_7_cast_fp16")]; + tensor const_15_promoted_to_fp16 = const()[name = tensor("const_15_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1283_cast_fp16 = mul(x = x2_7_cast_fp16, y = const_15_promoted_to_fp16)[name = tensor("op_1283_cast_fp16")]; + tensor var_1285_interleave_0 = const()[name = tensor("op_1285_interleave_0"), val = tensor(false)]; + tensor var_1285_cast_fp16 = concat(axis = var_1223, interleave = var_1285_interleave_0, values = (var_1283_cast_fp16, x1_7_cast_fp16))[name = tensor("op_1285_cast_fp16")]; + tensor var_1286_cast_fp16 = mul(x = var_1285_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1286_cast_fp16")]; + tensor k_embed_3_cast_fp16 = add(x = var_1271_cast_fp16, y = var_1286_cast_fp16)[name = tensor("k_embed_3_cast_fp16")]; + tensor var_1291_transpose_x_1 = const()[name = tensor("op_1291_transpose_x_1"), val = tensor(false)]; + tensor var_1291_transpose_y_1 = const()[name = tensor("op_1291_transpose_y_1"), val = tensor(true)]; + tensor var_1291_cast_fp16 = matmul(transpose_x = var_1291_transpose_x_1, transpose_y = var_1291_transpose_y_1, x = q_embed_3_cast_fp16, y = k_embed_3_cast_fp16)[name = tensor("op_1291_cast_fp16")]; + tensor var_1292_to_fp16 = const()[name = tensor("op_1292_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_5_cast_fp16 = mul(x = var_1291_cast_fp16, y = var_1292_to_fp16)[name = tensor("attn_weights_5_cast_fp16")]; + tensor input_25_cast_fp16 = add(x = attn_weights_5_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_25_cast_fp16")]; + tensor var_1295_cast_fp16 = softmax(axis = var_1223, x = input_25_cast_fp16)[name = tensor("op_1295_cast_fp16")]; + tensor attn_output_7_transpose_x_0 = const()[name = tensor("attn_output_7_transpose_x_0"), val = tensor(false)]; + tensor attn_output_7_transpose_y_0 = const()[name = tensor("attn_output_7_transpose_y_0"), val = tensor(false)]; + tensor value_3_cast_fp16 = transpose(perm = value_3_perm_0, x = squeeze_5_cast_fp16)[name = tensor("transpose_101")]; + tensor attn_output_7_cast_fp16 = matmul(transpose_x = attn_output_7_transpose_x_0, transpose_y = attn_output_7_transpose_y_0, x = var_1295_cast_fp16, y = value_3_cast_fp16)[name = tensor("attn_output_7_cast_fp16")]; + tensor var_1299_perm_0 = const()[name = tensor("op_1299_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1301 = const()[name = tensor("op_1301"), val = tensor([1, 512, -1])]; + tensor var_1299_cast_fp16 = transpose(perm = var_1299_perm_0, x = attn_output_7_cast_fp16)[name = tensor("transpose_100")]; + tensor var_1302_cast_fp16 = reshape(shape = var_1301, x = var_1299_cast_fp16)[name = tensor("op_1302_cast_fp16")]; + tensor model_encoder_layers_1_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(212133504)))]; + tensor linear_5_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_1_attn_Wo_weight_to_fp16, x = var_1302_cast_fp16)[name = tensor("linear_5_cast_fp16")]; + tensor input_31_cast_fp16 = add(x = input_23_cast_fp16, y = linear_5_cast_fp16)[name = tensor("input_31_cast_fp16")]; + tensor input_33_axes_0 = const()[name = tensor("input_33_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_1_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(213313216)))]; + tensor input_33_cast_fp16 = layer_norm(axes = input_33_axes_0, epsilon = var_1234_to_fp16, gamma = model_encoder_layers_1_mlp_norm_weight_to_fp16, x = input_31_cast_fp16)[name = tensor("input_33_cast_fp16")]; + tensor model_encoder_layers_1_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(213314816)))]; + tensor linear_6_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_1_mlp_Wi_weight_to_fp16, x = input_33_cast_fp16)[name = tensor("linear_6_cast_fp16")]; + tensor var_1309_split_sizes_0 = const()[name = tensor("op_1309_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1309_axis_0 = const()[name = tensor("op_1309_axis_0"), val = tensor(-1)]; + tensor var_1309_cast_fp16_0, tensor var_1309_cast_fp16_1 = split(axis = var_1309_axis_0, split_sizes = var_1309_split_sizes_0, x = linear_6_cast_fp16)[name = tensor("op_1309_cast_fp16")]; + tensor var_1311_mode_0 = const()[name = tensor("op_1311_mode_0"), val = tensor("EXACT")]; + tensor var_1311_cast_fp16 = gelu(mode = var_1311_mode_0, x = var_1309_cast_fp16_0)[name = tensor("op_1311_cast_fp16")]; + tensor input_37_cast_fp16 = mul(x = var_1311_cast_fp16, y = var_1309_cast_fp16_1)[name = tensor("input_37_cast_fp16")]; + tensor model_encoder_layers_1_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_1_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(216853824)))]; + tensor linear_7_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_1_mlp_Wo_weight_to_fp16, x = input_37_cast_fp16)[name = tensor("linear_7_cast_fp16")]; + tensor input_41_cast_fp16 = add(x = input_31_cast_fp16, y = linear_7_cast_fp16)[name = tensor("input_41_cast_fp16")]; + tensor var_1320 = const()[name = tensor("op_1320"), val = tensor(-1)]; + tensor hidden_states_5_axes_0 = const()[name = tensor("hidden_states_5_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_2_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_2_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(218623360)))]; + tensor var_1331_to_fp16 = const()[name = tensor("op_1331_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_5_cast_fp16 = layer_norm(axes = hidden_states_5_axes_0, epsilon = var_1331_to_fp16, gamma = model_encoder_layers_2_attn_norm_weight_to_fp16, x = input_41_cast_fp16)[name = tensor("hidden_states_5_cast_fp16")]; + tensor model_encoder_layers_2_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_2_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(218624960)))]; + tensor linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_2_attn_Wqkv_weight_to_fp16, x = hidden_states_5_cast_fp16)[name = tensor("linear_8_cast_fp16")]; + tensor var_1338 = const()[name = tensor("op_1338"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_11_cast_fp16 = reshape(shape = var_1338, x = linear_8_cast_fp16)[name = tensor("qkv_11_cast_fp16")]; + tensor var_1340_split_sizes_0 = const()[name = tensor("op_1340_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1340_axis_0 = const()[name = tensor("op_1340_axis_0"), val = tensor(-3)]; + tensor var_1340_cast_fp16_0, tensor var_1340_cast_fp16_1, tensor var_1340_cast_fp16_2 = split(axis = var_1340_axis_0, split_sizes = var_1340_split_sizes_0, x = qkv_11_cast_fp16)[name = tensor("op_1340_cast_fp16")]; + tensor squeeze_6_axes_0 = const()[name = tensor("squeeze_6_axes_0"), val = tensor([-3])]; + tensor squeeze_6_cast_fp16 = squeeze(axes = squeeze_6_axes_0, x = var_1340_cast_fp16_0)[name = tensor("squeeze_6_cast_fp16")]; + tensor squeeze_7_axes_0 = const()[name = tensor("squeeze_7_axes_0"), val = tensor([-3])]; + tensor squeeze_7_cast_fp16 = squeeze(axes = squeeze_7_axes_0, x = var_1340_cast_fp16_1)[name = tensor("squeeze_7_cast_fp16")]; + tensor squeeze_8_axes_0 = const()[name = tensor("squeeze_8_axes_0"), val = tensor([-3])]; + tensor squeeze_8_cast_fp16 = squeeze(axes = squeeze_8_axes_0, x = var_1340_cast_fp16_2)[name = tensor("squeeze_8_cast_fp16")]; + tensor q_9_perm_0 = const()[name = tensor("q_9_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_9_perm_0 = const()[name = tensor("k_9_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_5_perm_0 = const()[name = tensor("value_5_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_9_cast_fp16 = transpose(perm = q_9_perm_0, x = squeeze_6_cast_fp16)[name = tensor("transpose_99")]; + tensor var_1350_cast_fp16 = mul(x = q_9_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1350_cast_fp16")]; + tensor x1_9_begin_0 = const()[name = tensor("x1_9_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_9_end_0 = const()[name = tensor("x1_9_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_9_end_mask_0 = const()[name = tensor("x1_9_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_9_cast_fp16 = slice_by_index(begin = x1_9_begin_0, end = x1_9_end_0, end_mask = x1_9_end_mask_0, x = q_9_cast_fp16)[name = tensor("x1_9_cast_fp16")]; + tensor x2_9_begin_0 = const()[name = tensor("x2_9_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_9_end_0 = const()[name = tensor("x2_9_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_9_end_mask_0 = const()[name = tensor("x2_9_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_9_cast_fp16 = slice_by_index(begin = x2_9_begin_0, end = x2_9_end_0, end_mask = x2_9_end_mask_0, x = q_9_cast_fp16)[name = tensor("x2_9_cast_fp16")]; + tensor const_20_promoted_to_fp16 = const()[name = tensor("const_20_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1362_cast_fp16 = mul(x = x2_9_cast_fp16, y = const_20_promoted_to_fp16)[name = tensor("op_1362_cast_fp16")]; + tensor var_1364_interleave_0 = const()[name = tensor("op_1364_interleave_0"), val = tensor(false)]; + tensor var_1364_cast_fp16 = concat(axis = var_1320, interleave = var_1364_interleave_0, values = (var_1362_cast_fp16, x1_9_cast_fp16))[name = tensor("op_1364_cast_fp16")]; + tensor var_1365_cast_fp16 = mul(x = var_1364_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1365_cast_fp16")]; + tensor q_embed_5_cast_fp16 = add(x = var_1350_cast_fp16, y = var_1365_cast_fp16)[name = tensor("q_embed_5_cast_fp16")]; + tensor k_9_cast_fp16 = transpose(perm = k_9_perm_0, x = squeeze_7_cast_fp16)[name = tensor("transpose_98")]; + tensor var_1368_cast_fp16 = mul(x = k_9_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1368_cast_fp16")]; + tensor x1_11_begin_0 = const()[name = tensor("x1_11_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_11_end_0 = const()[name = tensor("x1_11_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_11_end_mask_0 = const()[name = tensor("x1_11_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_11_cast_fp16 = slice_by_index(begin = x1_11_begin_0, end = x1_11_end_0, end_mask = x1_11_end_mask_0, x = k_9_cast_fp16)[name = tensor("x1_11_cast_fp16")]; + tensor x2_11_begin_0 = const()[name = tensor("x2_11_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_11_end_0 = const()[name = tensor("x2_11_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_11_end_mask_0 = const()[name = tensor("x2_11_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_11_cast_fp16 = slice_by_index(begin = x2_11_begin_0, end = x2_11_end_0, end_mask = x2_11_end_mask_0, x = k_9_cast_fp16)[name = tensor("x2_11_cast_fp16")]; + tensor const_23_promoted_to_fp16 = const()[name = tensor("const_23_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1380_cast_fp16 = mul(x = x2_11_cast_fp16, y = const_23_promoted_to_fp16)[name = tensor("op_1380_cast_fp16")]; + tensor var_1382_interleave_0 = const()[name = tensor("op_1382_interleave_0"), val = tensor(false)]; + tensor var_1382_cast_fp16 = concat(axis = var_1320, interleave = var_1382_interleave_0, values = (var_1380_cast_fp16, x1_11_cast_fp16))[name = tensor("op_1382_cast_fp16")]; + tensor var_1383_cast_fp16 = mul(x = var_1382_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1383_cast_fp16")]; + tensor k_embed_5_cast_fp16 = add(x = var_1368_cast_fp16, y = var_1383_cast_fp16)[name = tensor("k_embed_5_cast_fp16")]; + tensor var_1388_transpose_x_1 = const()[name = tensor("op_1388_transpose_x_1"), val = tensor(false)]; + tensor var_1388_transpose_y_1 = const()[name = tensor("op_1388_transpose_y_1"), val = tensor(true)]; + tensor var_1388_cast_fp16 = matmul(transpose_x = var_1388_transpose_x_1, transpose_y = var_1388_transpose_y_1, x = q_embed_5_cast_fp16, y = k_embed_5_cast_fp16)[name = tensor("op_1388_cast_fp16")]; + tensor var_1389_to_fp16 = const()[name = tensor("op_1389_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_9_cast_fp16 = mul(x = var_1388_cast_fp16, y = var_1389_to_fp16)[name = tensor("attn_weights_9_cast_fp16")]; + tensor input_43_cast_fp16 = add(x = attn_weights_9_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_43_cast_fp16")]; + tensor var_1392_cast_fp16 = softmax(axis = var_1320, x = input_43_cast_fp16)[name = tensor("op_1392_cast_fp16")]; + tensor attn_output_13_transpose_x_0 = const()[name = tensor("attn_output_13_transpose_x_0"), val = tensor(false)]; + tensor attn_output_13_transpose_y_0 = const()[name = tensor("attn_output_13_transpose_y_0"), val = tensor(false)]; + tensor value_5_cast_fp16 = transpose(perm = value_5_perm_0, x = squeeze_8_cast_fp16)[name = tensor("transpose_97")]; + tensor attn_output_13_cast_fp16 = matmul(transpose_x = attn_output_13_transpose_x_0, transpose_y = attn_output_13_transpose_y_0, x = var_1392_cast_fp16, y = value_5_cast_fp16)[name = tensor("attn_output_13_cast_fp16")]; + tensor var_1396_perm_0 = const()[name = tensor("op_1396_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1398 = const()[name = tensor("op_1398"), val = tensor([1, 512, -1])]; + tensor var_1396_cast_fp16 = transpose(perm = var_1396_perm_0, x = attn_output_13_cast_fp16)[name = tensor("transpose_96")]; + tensor var_1399_cast_fp16 = reshape(shape = var_1398, x = var_1396_cast_fp16)[name = tensor("op_1399_cast_fp16")]; + tensor model_encoder_layers_2_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_2_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(222163968)))]; + tensor linear_9_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_2_attn_Wo_weight_to_fp16, x = var_1399_cast_fp16)[name = tensor("linear_9_cast_fp16")]; + tensor input_49_cast_fp16 = add(x = input_41_cast_fp16, y = linear_9_cast_fp16)[name = tensor("input_49_cast_fp16")]; + tensor input_51_axes_0 = const()[name = tensor("input_51_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_2_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_2_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(223343680)))]; + tensor input_51_cast_fp16 = layer_norm(axes = input_51_axes_0, epsilon = var_1331_to_fp16, gamma = model_encoder_layers_2_mlp_norm_weight_to_fp16, x = input_49_cast_fp16)[name = tensor("input_51_cast_fp16")]; + tensor model_encoder_layers_2_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_2_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(223345280)))]; + tensor linear_10_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_2_mlp_Wi_weight_to_fp16, x = input_51_cast_fp16)[name = tensor("linear_10_cast_fp16")]; + tensor var_1406_split_sizes_0 = const()[name = tensor("op_1406_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1406_axis_0 = const()[name = tensor("op_1406_axis_0"), val = tensor(-1)]; + tensor var_1406_cast_fp16_0, tensor var_1406_cast_fp16_1 = split(axis = var_1406_axis_0, split_sizes = var_1406_split_sizes_0, x = linear_10_cast_fp16)[name = tensor("op_1406_cast_fp16")]; + tensor var_1408_mode_0 = const()[name = tensor("op_1408_mode_0"), val = tensor("EXACT")]; + tensor var_1408_cast_fp16 = gelu(mode = var_1408_mode_0, x = var_1406_cast_fp16_0)[name = tensor("op_1408_cast_fp16")]; + tensor input_55_cast_fp16 = mul(x = var_1408_cast_fp16, y = var_1406_cast_fp16_1)[name = tensor("input_55_cast_fp16")]; + tensor model_encoder_layers_2_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_2_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(226884288)))]; + tensor linear_11_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_2_mlp_Wo_weight_to_fp16, x = input_55_cast_fp16)[name = tensor("linear_11_cast_fp16")]; + tensor input_59_cast_fp16 = add(x = input_49_cast_fp16, y = linear_11_cast_fp16)[name = tensor("input_59_cast_fp16")]; + tensor var_1417 = const()[name = tensor("op_1417"), val = tensor(-1)]; + tensor hidden_states_7_axes_0 = const()[name = tensor("hidden_states_7_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_3_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_3_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(228653824)))]; + tensor var_1428_to_fp16 = const()[name = tensor("op_1428_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_7_cast_fp16 = layer_norm(axes = hidden_states_7_axes_0, epsilon = var_1428_to_fp16, gamma = model_encoder_layers_3_attn_norm_weight_to_fp16, x = input_59_cast_fp16)[name = tensor("hidden_states_7_cast_fp16")]; + tensor model_encoder_layers_3_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_3_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(228655424)))]; + tensor linear_12_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_3_attn_Wqkv_weight_to_fp16, x = hidden_states_7_cast_fp16)[name = tensor("linear_12_cast_fp16")]; + tensor var_1435 = const()[name = tensor("op_1435"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_15_cast_fp16 = reshape(shape = var_1435, x = linear_12_cast_fp16)[name = tensor("qkv_15_cast_fp16")]; + tensor var_1437_split_sizes_0 = const()[name = tensor("op_1437_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1437_axis_0 = const()[name = tensor("op_1437_axis_0"), val = tensor(-3)]; + tensor var_1437_cast_fp16_0, tensor var_1437_cast_fp16_1, tensor var_1437_cast_fp16_2 = split(axis = var_1437_axis_0, split_sizes = var_1437_split_sizes_0, x = qkv_15_cast_fp16)[name = tensor("op_1437_cast_fp16")]; + tensor squeeze_9_axes_0 = const()[name = tensor("squeeze_9_axes_0"), val = tensor([-3])]; + tensor squeeze_9_cast_fp16 = squeeze(axes = squeeze_9_axes_0, x = var_1437_cast_fp16_0)[name = tensor("squeeze_9_cast_fp16")]; + tensor squeeze_10_axes_0 = const()[name = tensor("squeeze_10_axes_0"), val = tensor([-3])]; + tensor squeeze_10_cast_fp16 = squeeze(axes = squeeze_10_axes_0, x = var_1437_cast_fp16_1)[name = tensor("squeeze_10_cast_fp16")]; + tensor squeeze_11_axes_0 = const()[name = tensor("squeeze_11_axes_0"), val = tensor([-3])]; + tensor squeeze_11_cast_fp16 = squeeze(axes = squeeze_11_axes_0, x = var_1437_cast_fp16_2)[name = tensor("squeeze_11_cast_fp16")]; + tensor q_13_perm_0 = const()[name = tensor("q_13_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_13_perm_0 = const()[name = tensor("k_13_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_7_perm_0 = const()[name = tensor("value_7_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_13_cast_fp16 = transpose(perm = q_13_perm_0, x = squeeze_9_cast_fp16)[name = tensor("transpose_95")]; + tensor var_1447_cast_fp16 = mul(x = q_13_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1447_cast_fp16")]; + tensor x1_13_begin_0 = const()[name = tensor("x1_13_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_13_end_0 = const()[name = tensor("x1_13_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_13_end_mask_0 = const()[name = tensor("x1_13_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_13_cast_fp16 = slice_by_index(begin = x1_13_begin_0, end = x1_13_end_0, end_mask = x1_13_end_mask_0, x = q_13_cast_fp16)[name = tensor("x1_13_cast_fp16")]; + tensor x2_13_begin_0 = const()[name = tensor("x2_13_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_13_end_0 = const()[name = tensor("x2_13_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_13_end_mask_0 = const()[name = tensor("x2_13_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_13_cast_fp16 = slice_by_index(begin = x2_13_begin_0, end = x2_13_end_0, end_mask = x2_13_end_mask_0, x = q_13_cast_fp16)[name = tensor("x2_13_cast_fp16")]; + tensor const_28_promoted_to_fp16 = const()[name = tensor("const_28_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1459_cast_fp16 = mul(x = x2_13_cast_fp16, y = const_28_promoted_to_fp16)[name = tensor("op_1459_cast_fp16")]; + tensor var_1461_interleave_0 = const()[name = tensor("op_1461_interleave_0"), val = tensor(false)]; + tensor var_1461_cast_fp16 = concat(axis = var_1417, interleave = var_1461_interleave_0, values = (var_1459_cast_fp16, x1_13_cast_fp16))[name = tensor("op_1461_cast_fp16")]; + tensor var_1462_cast_fp16 = mul(x = var_1461_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1462_cast_fp16")]; + tensor q_embed_7_cast_fp16 = add(x = var_1447_cast_fp16, y = var_1462_cast_fp16)[name = tensor("q_embed_7_cast_fp16")]; + tensor k_13_cast_fp16 = transpose(perm = k_13_perm_0, x = squeeze_10_cast_fp16)[name = tensor("transpose_94")]; + tensor var_1465_cast_fp16 = mul(x = k_13_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1465_cast_fp16")]; + tensor x1_15_begin_0 = const()[name = tensor("x1_15_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_15_end_0 = const()[name = tensor("x1_15_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_15_end_mask_0 = const()[name = tensor("x1_15_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_15_cast_fp16 = slice_by_index(begin = x1_15_begin_0, end = x1_15_end_0, end_mask = x1_15_end_mask_0, x = k_13_cast_fp16)[name = tensor("x1_15_cast_fp16")]; + tensor x2_15_begin_0 = const()[name = tensor("x2_15_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_15_end_0 = const()[name = tensor("x2_15_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_15_end_mask_0 = const()[name = tensor("x2_15_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_15_cast_fp16 = slice_by_index(begin = x2_15_begin_0, end = x2_15_end_0, end_mask = x2_15_end_mask_0, x = k_13_cast_fp16)[name = tensor("x2_15_cast_fp16")]; + tensor const_31_promoted_to_fp16 = const()[name = tensor("const_31_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1477_cast_fp16 = mul(x = x2_15_cast_fp16, y = const_31_promoted_to_fp16)[name = tensor("op_1477_cast_fp16")]; + tensor var_1479_interleave_0 = const()[name = tensor("op_1479_interleave_0"), val = tensor(false)]; + tensor var_1479_cast_fp16 = concat(axis = var_1417, interleave = var_1479_interleave_0, values = (var_1477_cast_fp16, x1_15_cast_fp16))[name = tensor("op_1479_cast_fp16")]; + tensor var_1480_cast_fp16 = mul(x = var_1479_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1480_cast_fp16")]; + tensor k_embed_7_cast_fp16 = add(x = var_1465_cast_fp16, y = var_1480_cast_fp16)[name = tensor("k_embed_7_cast_fp16")]; + tensor var_1485_transpose_x_1 = const()[name = tensor("op_1485_transpose_x_1"), val = tensor(false)]; + tensor var_1485_transpose_y_1 = const()[name = tensor("op_1485_transpose_y_1"), val = tensor(true)]; + tensor var_1485_cast_fp16 = matmul(transpose_x = var_1485_transpose_x_1, transpose_y = var_1485_transpose_y_1, x = q_embed_7_cast_fp16, y = k_embed_7_cast_fp16)[name = tensor("op_1485_cast_fp16")]; + tensor var_1486_to_fp16 = const()[name = tensor("op_1486_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_13_cast_fp16 = mul(x = var_1485_cast_fp16, y = var_1486_to_fp16)[name = tensor("attn_weights_13_cast_fp16")]; + tensor input_61_cast_fp16 = add(x = attn_weights_13_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_61_cast_fp16")]; + tensor var_1489_cast_fp16 = softmax(axis = var_1417, x = input_61_cast_fp16)[name = tensor("op_1489_cast_fp16")]; + tensor attn_output_19_transpose_x_0 = const()[name = tensor("attn_output_19_transpose_x_0"), val = tensor(false)]; + tensor attn_output_19_transpose_y_0 = const()[name = tensor("attn_output_19_transpose_y_0"), val = tensor(false)]; + tensor value_7_cast_fp16 = transpose(perm = value_7_perm_0, x = squeeze_11_cast_fp16)[name = tensor("transpose_93")]; + tensor attn_output_19_cast_fp16 = matmul(transpose_x = attn_output_19_transpose_x_0, transpose_y = attn_output_19_transpose_y_0, x = var_1489_cast_fp16, y = value_7_cast_fp16)[name = tensor("attn_output_19_cast_fp16")]; + tensor var_1493_perm_0 = const()[name = tensor("op_1493_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1495 = const()[name = tensor("op_1495"), val = tensor([1, 512, -1])]; + tensor var_1493_cast_fp16 = transpose(perm = var_1493_perm_0, x = attn_output_19_cast_fp16)[name = tensor("transpose_92")]; + tensor var_1496_cast_fp16 = reshape(shape = var_1495, x = var_1493_cast_fp16)[name = tensor("op_1496_cast_fp16")]; + tensor model_encoder_layers_3_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_3_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(232194432)))]; + tensor linear_13_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_3_attn_Wo_weight_to_fp16, x = var_1496_cast_fp16)[name = tensor("linear_13_cast_fp16")]; + tensor input_67_cast_fp16 = add(x = input_59_cast_fp16, y = linear_13_cast_fp16)[name = tensor("input_67_cast_fp16")]; + tensor input_69_axes_0 = const()[name = tensor("input_69_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_3_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_3_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(233374144)))]; + tensor input_69_cast_fp16 = layer_norm(axes = input_69_axes_0, epsilon = var_1428_to_fp16, gamma = model_encoder_layers_3_mlp_norm_weight_to_fp16, x = input_67_cast_fp16)[name = tensor("input_69_cast_fp16")]; + tensor model_encoder_layers_3_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_3_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(233375744)))]; + tensor linear_14_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_3_mlp_Wi_weight_to_fp16, x = input_69_cast_fp16)[name = tensor("linear_14_cast_fp16")]; + tensor var_1503_split_sizes_0 = const()[name = tensor("op_1503_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1503_axis_0 = const()[name = tensor("op_1503_axis_0"), val = tensor(-1)]; + tensor var_1503_cast_fp16_0, tensor var_1503_cast_fp16_1 = split(axis = var_1503_axis_0, split_sizes = var_1503_split_sizes_0, x = linear_14_cast_fp16)[name = tensor("op_1503_cast_fp16")]; + tensor var_1505_mode_0 = const()[name = tensor("op_1505_mode_0"), val = tensor("EXACT")]; + tensor var_1505_cast_fp16 = gelu(mode = var_1505_mode_0, x = var_1503_cast_fp16_0)[name = tensor("op_1505_cast_fp16")]; + tensor input_73_cast_fp16 = mul(x = var_1505_cast_fp16, y = var_1503_cast_fp16_1)[name = tensor("input_73_cast_fp16")]; + tensor model_encoder_layers_3_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_3_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(236914752)))]; + tensor linear_15_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_3_mlp_Wo_weight_to_fp16, x = input_73_cast_fp16)[name = tensor("linear_15_cast_fp16")]; + tensor input_77_cast_fp16 = add(x = input_67_cast_fp16, y = linear_15_cast_fp16)[name = tensor("input_77_cast_fp16")]; + tensor var_1514 = const()[name = tensor("op_1514"), val = tensor(-1)]; + tensor hidden_states_9_axes_0 = const()[name = tensor("hidden_states_9_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_4_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_4_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(238684288)))]; + tensor var_1525_to_fp16 = const()[name = tensor("op_1525_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_9_cast_fp16 = layer_norm(axes = hidden_states_9_axes_0, epsilon = var_1525_to_fp16, gamma = model_encoder_layers_4_attn_norm_weight_to_fp16, x = input_77_cast_fp16)[name = tensor("hidden_states_9_cast_fp16")]; + tensor model_encoder_layers_4_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_4_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(238685888)))]; + tensor linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_4_attn_Wqkv_weight_to_fp16, x = hidden_states_9_cast_fp16)[name = tensor("linear_16_cast_fp16")]; + tensor var_1532 = const()[name = tensor("op_1532"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_19_cast_fp16 = reshape(shape = var_1532, x = linear_16_cast_fp16)[name = tensor("qkv_19_cast_fp16")]; + tensor var_1534_split_sizes_0 = const()[name = tensor("op_1534_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1534_axis_0 = const()[name = tensor("op_1534_axis_0"), val = tensor(-3)]; + tensor var_1534_cast_fp16_0, tensor var_1534_cast_fp16_1, tensor var_1534_cast_fp16_2 = split(axis = var_1534_axis_0, split_sizes = var_1534_split_sizes_0, x = qkv_19_cast_fp16)[name = tensor("op_1534_cast_fp16")]; + tensor squeeze_12_axes_0 = const()[name = tensor("squeeze_12_axes_0"), val = tensor([-3])]; + tensor squeeze_12_cast_fp16 = squeeze(axes = squeeze_12_axes_0, x = var_1534_cast_fp16_0)[name = tensor("squeeze_12_cast_fp16")]; + tensor squeeze_13_axes_0 = const()[name = tensor("squeeze_13_axes_0"), val = tensor([-3])]; + tensor squeeze_13_cast_fp16 = squeeze(axes = squeeze_13_axes_0, x = var_1534_cast_fp16_1)[name = tensor("squeeze_13_cast_fp16")]; + tensor squeeze_14_axes_0 = const()[name = tensor("squeeze_14_axes_0"), val = tensor([-3])]; + tensor squeeze_14_cast_fp16 = squeeze(axes = squeeze_14_axes_0, x = var_1534_cast_fp16_2)[name = tensor("squeeze_14_cast_fp16")]; + tensor q_17_perm_0 = const()[name = tensor("q_17_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_17_perm_0 = const()[name = tensor("k_17_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_9_perm_0 = const()[name = tensor("value_9_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_17_cast_fp16 = transpose(perm = q_17_perm_0, x = squeeze_12_cast_fp16)[name = tensor("transpose_91")]; + tensor var_1544_cast_fp16 = mul(x = q_17_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1544_cast_fp16")]; + tensor x1_17_begin_0 = const()[name = tensor("x1_17_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_17_end_0 = const()[name = tensor("x1_17_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_17_end_mask_0 = const()[name = tensor("x1_17_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_17_cast_fp16 = slice_by_index(begin = x1_17_begin_0, end = x1_17_end_0, end_mask = x1_17_end_mask_0, x = q_17_cast_fp16)[name = tensor("x1_17_cast_fp16")]; + tensor x2_17_begin_0 = const()[name = tensor("x2_17_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_17_end_0 = const()[name = tensor("x2_17_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_17_end_mask_0 = const()[name = tensor("x2_17_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_17_cast_fp16 = slice_by_index(begin = x2_17_begin_0, end = x2_17_end_0, end_mask = x2_17_end_mask_0, x = q_17_cast_fp16)[name = tensor("x2_17_cast_fp16")]; + tensor const_36_promoted_to_fp16 = const()[name = tensor("const_36_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1556_cast_fp16 = mul(x = x2_17_cast_fp16, y = const_36_promoted_to_fp16)[name = tensor("op_1556_cast_fp16")]; + tensor var_1558_interleave_0 = const()[name = tensor("op_1558_interleave_0"), val = tensor(false)]; + tensor var_1558_cast_fp16 = concat(axis = var_1514, interleave = var_1558_interleave_0, values = (var_1556_cast_fp16, x1_17_cast_fp16))[name = tensor("op_1558_cast_fp16")]; + tensor var_1559_cast_fp16 = mul(x = var_1558_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1559_cast_fp16")]; + tensor q_embed_9_cast_fp16 = add(x = var_1544_cast_fp16, y = var_1559_cast_fp16)[name = tensor("q_embed_9_cast_fp16")]; + tensor k_17_cast_fp16 = transpose(perm = k_17_perm_0, x = squeeze_13_cast_fp16)[name = tensor("transpose_90")]; + tensor var_1562_cast_fp16 = mul(x = k_17_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1562_cast_fp16")]; + tensor x1_19_begin_0 = const()[name = tensor("x1_19_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_19_end_0 = const()[name = tensor("x1_19_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_19_end_mask_0 = const()[name = tensor("x1_19_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_19_cast_fp16 = slice_by_index(begin = x1_19_begin_0, end = x1_19_end_0, end_mask = x1_19_end_mask_0, x = k_17_cast_fp16)[name = tensor("x1_19_cast_fp16")]; + tensor x2_19_begin_0 = const()[name = tensor("x2_19_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_19_end_0 = const()[name = tensor("x2_19_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_19_end_mask_0 = const()[name = tensor("x2_19_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_19_cast_fp16 = slice_by_index(begin = x2_19_begin_0, end = x2_19_end_0, end_mask = x2_19_end_mask_0, x = k_17_cast_fp16)[name = tensor("x2_19_cast_fp16")]; + tensor const_39_promoted_to_fp16 = const()[name = tensor("const_39_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1574_cast_fp16 = mul(x = x2_19_cast_fp16, y = const_39_promoted_to_fp16)[name = tensor("op_1574_cast_fp16")]; + tensor var_1576_interleave_0 = const()[name = tensor("op_1576_interleave_0"), val = tensor(false)]; + tensor var_1576_cast_fp16 = concat(axis = var_1514, interleave = var_1576_interleave_0, values = (var_1574_cast_fp16, x1_19_cast_fp16))[name = tensor("op_1576_cast_fp16")]; + tensor var_1577_cast_fp16 = mul(x = var_1576_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1577_cast_fp16")]; + tensor k_embed_9_cast_fp16 = add(x = var_1562_cast_fp16, y = var_1577_cast_fp16)[name = tensor("k_embed_9_cast_fp16")]; + tensor var_1582_transpose_x_1 = const()[name = tensor("op_1582_transpose_x_1"), val = tensor(false)]; + tensor var_1582_transpose_y_1 = const()[name = tensor("op_1582_transpose_y_1"), val = tensor(true)]; + tensor var_1582_cast_fp16 = matmul(transpose_x = var_1582_transpose_x_1, transpose_y = var_1582_transpose_y_1, x = q_embed_9_cast_fp16, y = k_embed_9_cast_fp16)[name = tensor("op_1582_cast_fp16")]; + tensor var_1583_to_fp16 = const()[name = tensor("op_1583_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_17_cast_fp16 = mul(x = var_1582_cast_fp16, y = var_1583_to_fp16)[name = tensor("attn_weights_17_cast_fp16")]; + tensor input_79_cast_fp16 = add(x = attn_weights_17_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_79_cast_fp16")]; + tensor var_1586_cast_fp16 = softmax(axis = var_1514, x = input_79_cast_fp16)[name = tensor("op_1586_cast_fp16")]; + tensor attn_output_25_transpose_x_0 = const()[name = tensor("attn_output_25_transpose_x_0"), val = tensor(false)]; + tensor attn_output_25_transpose_y_0 = const()[name = tensor("attn_output_25_transpose_y_0"), val = tensor(false)]; + tensor value_9_cast_fp16 = transpose(perm = value_9_perm_0, x = squeeze_14_cast_fp16)[name = tensor("transpose_89")]; + tensor attn_output_25_cast_fp16 = matmul(transpose_x = attn_output_25_transpose_x_0, transpose_y = attn_output_25_transpose_y_0, x = var_1586_cast_fp16, y = value_9_cast_fp16)[name = tensor("attn_output_25_cast_fp16")]; + tensor var_1590_perm_0 = const()[name = tensor("op_1590_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1592 = const()[name = tensor("op_1592"), val = tensor([1, 512, -1])]; + tensor var_1590_cast_fp16 = transpose(perm = var_1590_perm_0, x = attn_output_25_cast_fp16)[name = tensor("transpose_88")]; + tensor var_1593_cast_fp16 = reshape(shape = var_1592, x = var_1590_cast_fp16)[name = tensor("op_1593_cast_fp16")]; + tensor model_encoder_layers_4_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_4_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(242224896)))]; + tensor linear_17_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_4_attn_Wo_weight_to_fp16, x = var_1593_cast_fp16)[name = tensor("linear_17_cast_fp16")]; + tensor input_85_cast_fp16 = add(x = input_77_cast_fp16, y = linear_17_cast_fp16)[name = tensor("input_85_cast_fp16")]; + tensor input_87_axes_0 = const()[name = tensor("input_87_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_4_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_4_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(243404608)))]; + tensor input_87_cast_fp16 = layer_norm(axes = input_87_axes_0, epsilon = var_1525_to_fp16, gamma = model_encoder_layers_4_mlp_norm_weight_to_fp16, x = input_85_cast_fp16)[name = tensor("input_87_cast_fp16")]; + tensor model_encoder_layers_4_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_4_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(243406208)))]; + tensor linear_18_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_4_mlp_Wi_weight_to_fp16, x = input_87_cast_fp16)[name = tensor("linear_18_cast_fp16")]; + tensor var_1600_split_sizes_0 = const()[name = tensor("op_1600_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1600_axis_0 = const()[name = tensor("op_1600_axis_0"), val = tensor(-1)]; + tensor var_1600_cast_fp16_0, tensor var_1600_cast_fp16_1 = split(axis = var_1600_axis_0, split_sizes = var_1600_split_sizes_0, x = linear_18_cast_fp16)[name = tensor("op_1600_cast_fp16")]; + tensor var_1602_mode_0 = const()[name = tensor("op_1602_mode_0"), val = tensor("EXACT")]; + tensor var_1602_cast_fp16 = gelu(mode = var_1602_mode_0, x = var_1600_cast_fp16_0)[name = tensor("op_1602_cast_fp16")]; + tensor input_91_cast_fp16 = mul(x = var_1602_cast_fp16, y = var_1600_cast_fp16_1)[name = tensor("input_91_cast_fp16")]; + tensor model_encoder_layers_4_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_4_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(246945216)))]; + tensor linear_19_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_4_mlp_Wo_weight_to_fp16, x = input_91_cast_fp16)[name = tensor("linear_19_cast_fp16")]; + tensor input_95_cast_fp16 = add(x = input_85_cast_fp16, y = linear_19_cast_fp16)[name = tensor("input_95_cast_fp16")]; + tensor var_1611 = const()[name = tensor("op_1611"), val = tensor(-1)]; + tensor hidden_states_11_axes_0 = const()[name = tensor("hidden_states_11_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_5_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_5_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(248714752)))]; + tensor var_1622_to_fp16 = const()[name = tensor("op_1622_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_11_cast_fp16 = layer_norm(axes = hidden_states_11_axes_0, epsilon = var_1622_to_fp16, gamma = model_encoder_layers_5_attn_norm_weight_to_fp16, x = input_95_cast_fp16)[name = tensor("hidden_states_11_cast_fp16")]; + tensor model_encoder_layers_5_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_5_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(248716352)))]; + tensor linear_20_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_5_attn_Wqkv_weight_to_fp16, x = hidden_states_11_cast_fp16)[name = tensor("linear_20_cast_fp16")]; + tensor var_1629 = const()[name = tensor("op_1629"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_23_cast_fp16 = reshape(shape = var_1629, x = linear_20_cast_fp16)[name = tensor("qkv_23_cast_fp16")]; + tensor var_1631_split_sizes_0 = const()[name = tensor("op_1631_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1631_axis_0 = const()[name = tensor("op_1631_axis_0"), val = tensor(-3)]; + tensor var_1631_cast_fp16_0, tensor var_1631_cast_fp16_1, tensor var_1631_cast_fp16_2 = split(axis = var_1631_axis_0, split_sizes = var_1631_split_sizes_0, x = qkv_23_cast_fp16)[name = tensor("op_1631_cast_fp16")]; + tensor squeeze_15_axes_0 = const()[name = tensor("squeeze_15_axes_0"), val = tensor([-3])]; + tensor squeeze_15_cast_fp16 = squeeze(axes = squeeze_15_axes_0, x = var_1631_cast_fp16_0)[name = tensor("squeeze_15_cast_fp16")]; + tensor squeeze_16_axes_0 = const()[name = tensor("squeeze_16_axes_0"), val = tensor([-3])]; + tensor squeeze_16_cast_fp16 = squeeze(axes = squeeze_16_axes_0, x = var_1631_cast_fp16_1)[name = tensor("squeeze_16_cast_fp16")]; + tensor squeeze_17_axes_0 = const()[name = tensor("squeeze_17_axes_0"), val = tensor([-3])]; + tensor squeeze_17_cast_fp16 = squeeze(axes = squeeze_17_axes_0, x = var_1631_cast_fp16_2)[name = tensor("squeeze_17_cast_fp16")]; + tensor q_21_perm_0 = const()[name = tensor("q_21_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_21_perm_0 = const()[name = tensor("k_21_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_11_perm_0 = const()[name = tensor("value_11_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_21_cast_fp16 = transpose(perm = q_21_perm_0, x = squeeze_15_cast_fp16)[name = tensor("transpose_87")]; + tensor var_1641_cast_fp16 = mul(x = q_21_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1641_cast_fp16")]; + tensor x1_21_begin_0 = const()[name = tensor("x1_21_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_21_end_0 = const()[name = tensor("x1_21_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_21_end_mask_0 = const()[name = tensor("x1_21_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_21_cast_fp16 = slice_by_index(begin = x1_21_begin_0, end = x1_21_end_0, end_mask = x1_21_end_mask_0, x = q_21_cast_fp16)[name = tensor("x1_21_cast_fp16")]; + tensor x2_21_begin_0 = const()[name = tensor("x2_21_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_21_end_0 = const()[name = tensor("x2_21_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_21_end_mask_0 = const()[name = tensor("x2_21_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_21_cast_fp16 = slice_by_index(begin = x2_21_begin_0, end = x2_21_end_0, end_mask = x2_21_end_mask_0, x = q_21_cast_fp16)[name = tensor("x2_21_cast_fp16")]; + tensor const_44_promoted_to_fp16 = const()[name = tensor("const_44_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1653_cast_fp16 = mul(x = x2_21_cast_fp16, y = const_44_promoted_to_fp16)[name = tensor("op_1653_cast_fp16")]; + tensor var_1655_interleave_0 = const()[name = tensor("op_1655_interleave_0"), val = tensor(false)]; + tensor var_1655_cast_fp16 = concat(axis = var_1611, interleave = var_1655_interleave_0, values = (var_1653_cast_fp16, x1_21_cast_fp16))[name = tensor("op_1655_cast_fp16")]; + tensor var_1656_cast_fp16 = mul(x = var_1655_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1656_cast_fp16")]; + tensor q_embed_11_cast_fp16 = add(x = var_1641_cast_fp16, y = var_1656_cast_fp16)[name = tensor("q_embed_11_cast_fp16")]; + tensor k_21_cast_fp16 = transpose(perm = k_21_perm_0, x = squeeze_16_cast_fp16)[name = tensor("transpose_86")]; + tensor var_1659_cast_fp16 = mul(x = k_21_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1659_cast_fp16")]; + tensor x1_23_begin_0 = const()[name = tensor("x1_23_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_23_end_0 = const()[name = tensor("x1_23_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_23_end_mask_0 = const()[name = tensor("x1_23_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_23_cast_fp16 = slice_by_index(begin = x1_23_begin_0, end = x1_23_end_0, end_mask = x1_23_end_mask_0, x = k_21_cast_fp16)[name = tensor("x1_23_cast_fp16")]; + tensor x2_23_begin_0 = const()[name = tensor("x2_23_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_23_end_0 = const()[name = tensor("x2_23_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_23_end_mask_0 = const()[name = tensor("x2_23_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_23_cast_fp16 = slice_by_index(begin = x2_23_begin_0, end = x2_23_end_0, end_mask = x2_23_end_mask_0, x = k_21_cast_fp16)[name = tensor("x2_23_cast_fp16")]; + tensor const_47_promoted_to_fp16 = const()[name = tensor("const_47_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1671_cast_fp16 = mul(x = x2_23_cast_fp16, y = const_47_promoted_to_fp16)[name = tensor("op_1671_cast_fp16")]; + tensor var_1673_interleave_0 = const()[name = tensor("op_1673_interleave_0"), val = tensor(false)]; + tensor var_1673_cast_fp16 = concat(axis = var_1611, interleave = var_1673_interleave_0, values = (var_1671_cast_fp16, x1_23_cast_fp16))[name = tensor("op_1673_cast_fp16")]; + tensor var_1674_cast_fp16 = mul(x = var_1673_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1674_cast_fp16")]; + tensor k_embed_11_cast_fp16 = add(x = var_1659_cast_fp16, y = var_1674_cast_fp16)[name = tensor("k_embed_11_cast_fp16")]; + tensor var_1679_transpose_x_1 = const()[name = tensor("op_1679_transpose_x_1"), val = tensor(false)]; + tensor var_1679_transpose_y_1 = const()[name = tensor("op_1679_transpose_y_1"), val = tensor(true)]; + tensor var_1679_cast_fp16 = matmul(transpose_x = var_1679_transpose_x_1, transpose_y = var_1679_transpose_y_1, x = q_embed_11_cast_fp16, y = k_embed_11_cast_fp16)[name = tensor("op_1679_cast_fp16")]; + tensor var_1680_to_fp16 = const()[name = tensor("op_1680_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_21_cast_fp16 = mul(x = var_1679_cast_fp16, y = var_1680_to_fp16)[name = tensor("attn_weights_21_cast_fp16")]; + tensor input_97_cast_fp16 = add(x = attn_weights_21_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_97_cast_fp16")]; + tensor var_1683_cast_fp16 = softmax(axis = var_1611, x = input_97_cast_fp16)[name = tensor("op_1683_cast_fp16")]; + tensor attn_output_31_transpose_x_0 = const()[name = tensor("attn_output_31_transpose_x_0"), val = tensor(false)]; + tensor attn_output_31_transpose_y_0 = const()[name = tensor("attn_output_31_transpose_y_0"), val = tensor(false)]; + tensor value_11_cast_fp16 = transpose(perm = value_11_perm_0, x = squeeze_17_cast_fp16)[name = tensor("transpose_85")]; + tensor attn_output_31_cast_fp16 = matmul(transpose_x = attn_output_31_transpose_x_0, transpose_y = attn_output_31_transpose_y_0, x = var_1683_cast_fp16, y = value_11_cast_fp16)[name = tensor("attn_output_31_cast_fp16")]; + tensor var_1687_perm_0 = const()[name = tensor("op_1687_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1689 = const()[name = tensor("op_1689"), val = tensor([1, 512, -1])]; + tensor var_1687_cast_fp16 = transpose(perm = var_1687_perm_0, x = attn_output_31_cast_fp16)[name = tensor("transpose_84")]; + tensor var_1690_cast_fp16 = reshape(shape = var_1689, x = var_1687_cast_fp16)[name = tensor("op_1690_cast_fp16")]; + tensor model_encoder_layers_5_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_5_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(252255360)))]; + tensor linear_21_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_5_attn_Wo_weight_to_fp16, x = var_1690_cast_fp16)[name = tensor("linear_21_cast_fp16")]; + tensor input_103_cast_fp16 = add(x = input_95_cast_fp16, y = linear_21_cast_fp16)[name = tensor("input_103_cast_fp16")]; + tensor input_105_axes_0 = const()[name = tensor("input_105_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_5_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_5_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(253435072)))]; + tensor input_105_cast_fp16 = layer_norm(axes = input_105_axes_0, epsilon = var_1622_to_fp16, gamma = model_encoder_layers_5_mlp_norm_weight_to_fp16, x = input_103_cast_fp16)[name = tensor("input_105_cast_fp16")]; + tensor model_encoder_layers_5_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_5_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(253436672)))]; + tensor linear_22_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_5_mlp_Wi_weight_to_fp16, x = input_105_cast_fp16)[name = tensor("linear_22_cast_fp16")]; + tensor var_1697_split_sizes_0 = const()[name = tensor("op_1697_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1697_axis_0 = const()[name = tensor("op_1697_axis_0"), val = tensor(-1)]; + tensor var_1697_cast_fp16_0, tensor var_1697_cast_fp16_1 = split(axis = var_1697_axis_0, split_sizes = var_1697_split_sizes_0, x = linear_22_cast_fp16)[name = tensor("op_1697_cast_fp16")]; + tensor var_1699_mode_0 = const()[name = tensor("op_1699_mode_0"), val = tensor("EXACT")]; + tensor var_1699_cast_fp16 = gelu(mode = var_1699_mode_0, x = var_1697_cast_fp16_0)[name = tensor("op_1699_cast_fp16")]; + tensor input_109_cast_fp16 = mul(x = var_1699_cast_fp16, y = var_1697_cast_fp16_1)[name = tensor("input_109_cast_fp16")]; + tensor model_encoder_layers_5_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_5_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(256975680)))]; + tensor linear_23_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_5_mlp_Wo_weight_to_fp16, x = input_109_cast_fp16)[name = tensor("linear_23_cast_fp16")]; + tensor input_113_cast_fp16 = add(x = input_103_cast_fp16, y = linear_23_cast_fp16)[name = tensor("input_113_cast_fp16")]; + tensor var_1708 = const()[name = tensor("op_1708"), val = tensor(-1)]; + tensor hidden_states_13_axes_0 = const()[name = tensor("hidden_states_13_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_6_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_6_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(258745216)))]; + tensor var_1719_to_fp16 = const()[name = tensor("op_1719_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_13_cast_fp16 = layer_norm(axes = hidden_states_13_axes_0, epsilon = var_1719_to_fp16, gamma = model_encoder_layers_6_attn_norm_weight_to_fp16, x = input_113_cast_fp16)[name = tensor("hidden_states_13_cast_fp16")]; + tensor model_encoder_layers_6_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_6_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(258746816)))]; + tensor linear_24_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_6_attn_Wqkv_weight_to_fp16, x = hidden_states_13_cast_fp16)[name = tensor("linear_24_cast_fp16")]; + tensor var_1726 = const()[name = tensor("op_1726"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_27_cast_fp16 = reshape(shape = var_1726, x = linear_24_cast_fp16)[name = tensor("qkv_27_cast_fp16")]; + tensor var_1728_split_sizes_0 = const()[name = tensor("op_1728_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1728_axis_0 = const()[name = tensor("op_1728_axis_0"), val = tensor(-3)]; + tensor var_1728_cast_fp16_0, tensor var_1728_cast_fp16_1, tensor var_1728_cast_fp16_2 = split(axis = var_1728_axis_0, split_sizes = var_1728_split_sizes_0, x = qkv_27_cast_fp16)[name = tensor("op_1728_cast_fp16")]; + tensor squeeze_18_axes_0 = const()[name = tensor("squeeze_18_axes_0"), val = tensor([-3])]; + tensor squeeze_18_cast_fp16 = squeeze(axes = squeeze_18_axes_0, x = var_1728_cast_fp16_0)[name = tensor("squeeze_18_cast_fp16")]; + tensor squeeze_19_axes_0 = const()[name = tensor("squeeze_19_axes_0"), val = tensor([-3])]; + tensor squeeze_19_cast_fp16 = squeeze(axes = squeeze_19_axes_0, x = var_1728_cast_fp16_1)[name = tensor("squeeze_19_cast_fp16")]; + tensor squeeze_20_axes_0 = const()[name = tensor("squeeze_20_axes_0"), val = tensor([-3])]; + tensor squeeze_20_cast_fp16 = squeeze(axes = squeeze_20_axes_0, x = var_1728_cast_fp16_2)[name = tensor("squeeze_20_cast_fp16")]; + tensor q_25_perm_0 = const()[name = tensor("q_25_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_25_perm_0 = const()[name = tensor("k_25_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_13_perm_0 = const()[name = tensor("value_13_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_25_cast_fp16 = transpose(perm = q_25_perm_0, x = squeeze_18_cast_fp16)[name = tensor("transpose_83")]; + tensor var_1738_cast_fp16 = mul(x = q_25_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1738_cast_fp16")]; + tensor x1_25_begin_0 = const()[name = tensor("x1_25_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_25_end_0 = const()[name = tensor("x1_25_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_25_end_mask_0 = const()[name = tensor("x1_25_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_25_cast_fp16 = slice_by_index(begin = x1_25_begin_0, end = x1_25_end_0, end_mask = x1_25_end_mask_0, x = q_25_cast_fp16)[name = tensor("x1_25_cast_fp16")]; + tensor x2_25_begin_0 = const()[name = tensor("x2_25_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_25_end_0 = const()[name = tensor("x2_25_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_25_end_mask_0 = const()[name = tensor("x2_25_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_25_cast_fp16 = slice_by_index(begin = x2_25_begin_0, end = x2_25_end_0, end_mask = x2_25_end_mask_0, x = q_25_cast_fp16)[name = tensor("x2_25_cast_fp16")]; + tensor const_52_promoted_to_fp16 = const()[name = tensor("const_52_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1750_cast_fp16 = mul(x = x2_25_cast_fp16, y = const_52_promoted_to_fp16)[name = tensor("op_1750_cast_fp16")]; + tensor var_1752_interleave_0 = const()[name = tensor("op_1752_interleave_0"), val = tensor(false)]; + tensor var_1752_cast_fp16 = concat(axis = var_1708, interleave = var_1752_interleave_0, values = (var_1750_cast_fp16, x1_25_cast_fp16))[name = tensor("op_1752_cast_fp16")]; + tensor var_1753_cast_fp16 = mul(x = var_1752_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1753_cast_fp16")]; + tensor q_embed_13_cast_fp16 = add(x = var_1738_cast_fp16, y = var_1753_cast_fp16)[name = tensor("q_embed_13_cast_fp16")]; + tensor k_25_cast_fp16 = transpose(perm = k_25_perm_0, x = squeeze_19_cast_fp16)[name = tensor("transpose_82")]; + tensor var_1756_cast_fp16 = mul(x = k_25_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1756_cast_fp16")]; + tensor x1_27_begin_0 = const()[name = tensor("x1_27_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_27_end_0 = const()[name = tensor("x1_27_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_27_end_mask_0 = const()[name = tensor("x1_27_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_27_cast_fp16 = slice_by_index(begin = x1_27_begin_0, end = x1_27_end_0, end_mask = x1_27_end_mask_0, x = k_25_cast_fp16)[name = tensor("x1_27_cast_fp16")]; + tensor x2_27_begin_0 = const()[name = tensor("x2_27_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_27_end_0 = const()[name = tensor("x2_27_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_27_end_mask_0 = const()[name = tensor("x2_27_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_27_cast_fp16 = slice_by_index(begin = x2_27_begin_0, end = x2_27_end_0, end_mask = x2_27_end_mask_0, x = k_25_cast_fp16)[name = tensor("x2_27_cast_fp16")]; + tensor const_55_promoted_to_fp16 = const()[name = tensor("const_55_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1768_cast_fp16 = mul(x = x2_27_cast_fp16, y = const_55_promoted_to_fp16)[name = tensor("op_1768_cast_fp16")]; + tensor var_1770_interleave_0 = const()[name = tensor("op_1770_interleave_0"), val = tensor(false)]; + tensor var_1770_cast_fp16 = concat(axis = var_1708, interleave = var_1770_interleave_0, values = (var_1768_cast_fp16, x1_27_cast_fp16))[name = tensor("op_1770_cast_fp16")]; + tensor var_1771_cast_fp16 = mul(x = var_1770_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1771_cast_fp16")]; + tensor k_embed_13_cast_fp16 = add(x = var_1756_cast_fp16, y = var_1771_cast_fp16)[name = tensor("k_embed_13_cast_fp16")]; + tensor var_1776_transpose_x_1 = const()[name = tensor("op_1776_transpose_x_1"), val = tensor(false)]; + tensor var_1776_transpose_y_1 = const()[name = tensor("op_1776_transpose_y_1"), val = tensor(true)]; + tensor var_1776_cast_fp16 = matmul(transpose_x = var_1776_transpose_x_1, transpose_y = var_1776_transpose_y_1, x = q_embed_13_cast_fp16, y = k_embed_13_cast_fp16)[name = tensor("op_1776_cast_fp16")]; + tensor var_1777_to_fp16 = const()[name = tensor("op_1777_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_25_cast_fp16 = mul(x = var_1776_cast_fp16, y = var_1777_to_fp16)[name = tensor("attn_weights_25_cast_fp16")]; + tensor input_115_cast_fp16 = add(x = attn_weights_25_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_115_cast_fp16")]; + tensor var_1780_cast_fp16 = softmax(axis = var_1708, x = input_115_cast_fp16)[name = tensor("op_1780_cast_fp16")]; + tensor attn_output_37_transpose_x_0 = const()[name = tensor("attn_output_37_transpose_x_0"), val = tensor(false)]; + tensor attn_output_37_transpose_y_0 = const()[name = tensor("attn_output_37_transpose_y_0"), val = tensor(false)]; + tensor value_13_cast_fp16 = transpose(perm = value_13_perm_0, x = squeeze_20_cast_fp16)[name = tensor("transpose_81")]; + tensor attn_output_37_cast_fp16 = matmul(transpose_x = attn_output_37_transpose_x_0, transpose_y = attn_output_37_transpose_y_0, x = var_1780_cast_fp16, y = value_13_cast_fp16)[name = tensor("attn_output_37_cast_fp16")]; + tensor var_1784_perm_0 = const()[name = tensor("op_1784_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1786 = const()[name = tensor("op_1786"), val = tensor([1, 512, -1])]; + tensor var_1784_cast_fp16 = transpose(perm = var_1784_perm_0, x = attn_output_37_cast_fp16)[name = tensor("transpose_80")]; + tensor var_1787_cast_fp16 = reshape(shape = var_1786, x = var_1784_cast_fp16)[name = tensor("op_1787_cast_fp16")]; + tensor model_encoder_layers_6_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_6_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(262285824)))]; + tensor linear_25_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_6_attn_Wo_weight_to_fp16, x = var_1787_cast_fp16)[name = tensor("linear_25_cast_fp16")]; + tensor input_121_cast_fp16 = add(x = input_113_cast_fp16, y = linear_25_cast_fp16)[name = tensor("input_121_cast_fp16")]; + tensor input_123_axes_0 = const()[name = tensor("input_123_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_6_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_6_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263465536)))]; + tensor input_123_cast_fp16 = layer_norm(axes = input_123_axes_0, epsilon = var_1719_to_fp16, gamma = model_encoder_layers_6_mlp_norm_weight_to_fp16, x = input_121_cast_fp16)[name = tensor("input_123_cast_fp16")]; + tensor model_encoder_layers_6_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_6_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(263467136)))]; + tensor linear_26_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_6_mlp_Wi_weight_to_fp16, x = input_123_cast_fp16)[name = tensor("linear_26_cast_fp16")]; + tensor var_1794_split_sizes_0 = const()[name = tensor("op_1794_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1794_axis_0 = const()[name = tensor("op_1794_axis_0"), val = tensor(-1)]; + tensor var_1794_cast_fp16_0, tensor var_1794_cast_fp16_1 = split(axis = var_1794_axis_0, split_sizes = var_1794_split_sizes_0, x = linear_26_cast_fp16)[name = tensor("op_1794_cast_fp16")]; + tensor var_1796_mode_0 = const()[name = tensor("op_1796_mode_0"), val = tensor("EXACT")]; + tensor var_1796_cast_fp16 = gelu(mode = var_1796_mode_0, x = var_1794_cast_fp16_0)[name = tensor("op_1796_cast_fp16")]; + tensor input_127_cast_fp16 = mul(x = var_1796_cast_fp16, y = var_1794_cast_fp16_1)[name = tensor("input_127_cast_fp16")]; + tensor model_encoder_layers_6_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_6_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(267006144)))]; + tensor linear_27_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_6_mlp_Wo_weight_to_fp16, x = input_127_cast_fp16)[name = tensor("linear_27_cast_fp16")]; + tensor input_131_cast_fp16 = add(x = input_121_cast_fp16, y = linear_27_cast_fp16)[name = tensor("input_131_cast_fp16")]; + tensor var_1805 = const()[name = tensor("op_1805"), val = tensor(-1)]; + tensor hidden_states_15_axes_0 = const()[name = tensor("hidden_states_15_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_7_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_7_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(268775680)))]; + tensor var_1816_to_fp16 = const()[name = tensor("op_1816_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_15_cast_fp16 = layer_norm(axes = hidden_states_15_axes_0, epsilon = var_1816_to_fp16, gamma = model_encoder_layers_7_attn_norm_weight_to_fp16, x = input_131_cast_fp16)[name = tensor("hidden_states_15_cast_fp16")]; + tensor model_encoder_layers_7_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_7_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(268777280)))]; + tensor linear_28_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_7_attn_Wqkv_weight_to_fp16, x = hidden_states_15_cast_fp16)[name = tensor("linear_28_cast_fp16")]; + tensor var_1823 = const()[name = tensor("op_1823"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_31_cast_fp16 = reshape(shape = var_1823, x = linear_28_cast_fp16)[name = tensor("qkv_31_cast_fp16")]; + tensor var_1825_split_sizes_0 = const()[name = tensor("op_1825_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1825_axis_0 = const()[name = tensor("op_1825_axis_0"), val = tensor(-3)]; + tensor var_1825_cast_fp16_0, tensor var_1825_cast_fp16_1, tensor var_1825_cast_fp16_2 = split(axis = var_1825_axis_0, split_sizes = var_1825_split_sizes_0, x = qkv_31_cast_fp16)[name = tensor("op_1825_cast_fp16")]; + tensor squeeze_21_axes_0 = const()[name = tensor("squeeze_21_axes_0"), val = tensor([-3])]; + tensor squeeze_21_cast_fp16 = squeeze(axes = squeeze_21_axes_0, x = var_1825_cast_fp16_0)[name = tensor("squeeze_21_cast_fp16")]; + tensor squeeze_22_axes_0 = const()[name = tensor("squeeze_22_axes_0"), val = tensor([-3])]; + tensor squeeze_22_cast_fp16 = squeeze(axes = squeeze_22_axes_0, x = var_1825_cast_fp16_1)[name = tensor("squeeze_22_cast_fp16")]; + tensor squeeze_23_axes_0 = const()[name = tensor("squeeze_23_axes_0"), val = tensor([-3])]; + tensor squeeze_23_cast_fp16 = squeeze(axes = squeeze_23_axes_0, x = var_1825_cast_fp16_2)[name = tensor("squeeze_23_cast_fp16")]; + tensor q_29_perm_0 = const()[name = tensor("q_29_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_29_perm_0 = const()[name = tensor("k_29_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_15_perm_0 = const()[name = tensor("value_15_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_29_cast_fp16 = transpose(perm = q_29_perm_0, x = squeeze_21_cast_fp16)[name = tensor("transpose_79")]; + tensor var_1835_cast_fp16 = mul(x = q_29_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1835_cast_fp16")]; + tensor x1_29_begin_0 = const()[name = tensor("x1_29_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_29_end_0 = const()[name = tensor("x1_29_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_29_end_mask_0 = const()[name = tensor("x1_29_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_29_cast_fp16 = slice_by_index(begin = x1_29_begin_0, end = x1_29_end_0, end_mask = x1_29_end_mask_0, x = q_29_cast_fp16)[name = tensor("x1_29_cast_fp16")]; + tensor x2_29_begin_0 = const()[name = tensor("x2_29_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_29_end_0 = const()[name = tensor("x2_29_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_29_end_mask_0 = const()[name = tensor("x2_29_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_29_cast_fp16 = slice_by_index(begin = x2_29_begin_0, end = x2_29_end_0, end_mask = x2_29_end_mask_0, x = q_29_cast_fp16)[name = tensor("x2_29_cast_fp16")]; + tensor const_60_promoted_to_fp16 = const()[name = tensor("const_60_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1847_cast_fp16 = mul(x = x2_29_cast_fp16, y = const_60_promoted_to_fp16)[name = tensor("op_1847_cast_fp16")]; + tensor var_1849_interleave_0 = const()[name = tensor("op_1849_interleave_0"), val = tensor(false)]; + tensor var_1849_cast_fp16 = concat(axis = var_1805, interleave = var_1849_interleave_0, values = (var_1847_cast_fp16, x1_29_cast_fp16))[name = tensor("op_1849_cast_fp16")]; + tensor var_1850_cast_fp16 = mul(x = var_1849_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1850_cast_fp16")]; + tensor q_embed_15_cast_fp16 = add(x = var_1835_cast_fp16, y = var_1850_cast_fp16)[name = tensor("q_embed_15_cast_fp16")]; + tensor k_29_cast_fp16 = transpose(perm = k_29_perm_0, x = squeeze_22_cast_fp16)[name = tensor("transpose_78")]; + tensor var_1853_cast_fp16 = mul(x = k_29_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1853_cast_fp16")]; + tensor x1_31_begin_0 = const()[name = tensor("x1_31_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_31_end_0 = const()[name = tensor("x1_31_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_31_end_mask_0 = const()[name = tensor("x1_31_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_31_cast_fp16 = slice_by_index(begin = x1_31_begin_0, end = x1_31_end_0, end_mask = x1_31_end_mask_0, x = k_29_cast_fp16)[name = tensor("x1_31_cast_fp16")]; + tensor x2_31_begin_0 = const()[name = tensor("x2_31_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_31_end_0 = const()[name = tensor("x2_31_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_31_end_mask_0 = const()[name = tensor("x2_31_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_31_cast_fp16 = slice_by_index(begin = x2_31_begin_0, end = x2_31_end_0, end_mask = x2_31_end_mask_0, x = k_29_cast_fp16)[name = tensor("x2_31_cast_fp16")]; + tensor const_63_promoted_to_fp16 = const()[name = tensor("const_63_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1865_cast_fp16 = mul(x = x2_31_cast_fp16, y = const_63_promoted_to_fp16)[name = tensor("op_1865_cast_fp16")]; + tensor var_1867_interleave_0 = const()[name = tensor("op_1867_interleave_0"), val = tensor(false)]; + tensor var_1867_cast_fp16 = concat(axis = var_1805, interleave = var_1867_interleave_0, values = (var_1865_cast_fp16, x1_31_cast_fp16))[name = tensor("op_1867_cast_fp16")]; + tensor var_1868_cast_fp16 = mul(x = var_1867_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1868_cast_fp16")]; + tensor k_embed_15_cast_fp16 = add(x = var_1853_cast_fp16, y = var_1868_cast_fp16)[name = tensor("k_embed_15_cast_fp16")]; + tensor var_1873_transpose_x_1 = const()[name = tensor("op_1873_transpose_x_1"), val = tensor(false)]; + tensor var_1873_transpose_y_1 = const()[name = tensor("op_1873_transpose_y_1"), val = tensor(true)]; + tensor var_1873_cast_fp16 = matmul(transpose_x = var_1873_transpose_x_1, transpose_y = var_1873_transpose_y_1, x = q_embed_15_cast_fp16, y = k_embed_15_cast_fp16)[name = tensor("op_1873_cast_fp16")]; + tensor var_1874_to_fp16 = const()[name = tensor("op_1874_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_29_cast_fp16 = mul(x = var_1873_cast_fp16, y = var_1874_to_fp16)[name = tensor("attn_weights_29_cast_fp16")]; + tensor input_133_cast_fp16 = add(x = attn_weights_29_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_133_cast_fp16")]; + tensor var_1877_cast_fp16 = softmax(axis = var_1805, x = input_133_cast_fp16)[name = tensor("op_1877_cast_fp16")]; + tensor attn_output_43_transpose_x_0 = const()[name = tensor("attn_output_43_transpose_x_0"), val = tensor(false)]; + tensor attn_output_43_transpose_y_0 = const()[name = tensor("attn_output_43_transpose_y_0"), val = tensor(false)]; + tensor value_15_cast_fp16 = transpose(perm = value_15_perm_0, x = squeeze_23_cast_fp16)[name = tensor("transpose_77")]; + tensor attn_output_43_cast_fp16 = matmul(transpose_x = attn_output_43_transpose_x_0, transpose_y = attn_output_43_transpose_y_0, x = var_1877_cast_fp16, y = value_15_cast_fp16)[name = tensor("attn_output_43_cast_fp16")]; + tensor var_1881_perm_0 = const()[name = tensor("op_1881_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1883 = const()[name = tensor("op_1883"), val = tensor([1, 512, -1])]; + tensor var_1881_cast_fp16 = transpose(perm = var_1881_perm_0, x = attn_output_43_cast_fp16)[name = tensor("transpose_76")]; + tensor var_1884_cast_fp16 = reshape(shape = var_1883, x = var_1881_cast_fp16)[name = tensor("op_1884_cast_fp16")]; + tensor model_encoder_layers_7_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_7_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(272316288)))]; + tensor linear_29_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_7_attn_Wo_weight_to_fp16, x = var_1884_cast_fp16)[name = tensor("linear_29_cast_fp16")]; + tensor input_139_cast_fp16 = add(x = input_131_cast_fp16, y = linear_29_cast_fp16)[name = tensor("input_139_cast_fp16")]; + tensor input_141_axes_0 = const()[name = tensor("input_141_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_7_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_7_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(273496000)))]; + tensor input_141_cast_fp16 = layer_norm(axes = input_141_axes_0, epsilon = var_1816_to_fp16, gamma = model_encoder_layers_7_mlp_norm_weight_to_fp16, x = input_139_cast_fp16)[name = tensor("input_141_cast_fp16")]; + tensor model_encoder_layers_7_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_7_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(273497600)))]; + tensor linear_30_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_7_mlp_Wi_weight_to_fp16, x = input_141_cast_fp16)[name = tensor("linear_30_cast_fp16")]; + tensor var_1891_split_sizes_0 = const()[name = tensor("op_1891_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1891_axis_0 = const()[name = tensor("op_1891_axis_0"), val = tensor(-1)]; + tensor var_1891_cast_fp16_0, tensor var_1891_cast_fp16_1 = split(axis = var_1891_axis_0, split_sizes = var_1891_split_sizes_0, x = linear_30_cast_fp16)[name = tensor("op_1891_cast_fp16")]; + tensor var_1893_mode_0 = const()[name = tensor("op_1893_mode_0"), val = tensor("EXACT")]; + tensor var_1893_cast_fp16 = gelu(mode = var_1893_mode_0, x = var_1891_cast_fp16_0)[name = tensor("op_1893_cast_fp16")]; + tensor input_145_cast_fp16 = mul(x = var_1893_cast_fp16, y = var_1891_cast_fp16_1)[name = tensor("input_145_cast_fp16")]; + tensor model_encoder_layers_7_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_7_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(277036608)))]; + tensor linear_31_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_7_mlp_Wo_weight_to_fp16, x = input_145_cast_fp16)[name = tensor("linear_31_cast_fp16")]; + tensor input_149_cast_fp16 = add(x = input_139_cast_fp16, y = linear_31_cast_fp16)[name = tensor("input_149_cast_fp16")]; + tensor var_1902 = const()[name = tensor("op_1902"), val = tensor(-1)]; + tensor hidden_states_17_axes_0 = const()[name = tensor("hidden_states_17_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_8_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_8_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(278806144)))]; + tensor var_1913_to_fp16 = const()[name = tensor("op_1913_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_17_cast_fp16 = layer_norm(axes = hidden_states_17_axes_0, epsilon = var_1913_to_fp16, gamma = model_encoder_layers_8_attn_norm_weight_to_fp16, x = input_149_cast_fp16)[name = tensor("hidden_states_17_cast_fp16")]; + tensor model_encoder_layers_8_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_8_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(278807744)))]; + tensor linear_32_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_8_attn_Wqkv_weight_to_fp16, x = hidden_states_17_cast_fp16)[name = tensor("linear_32_cast_fp16")]; + tensor var_1920 = const()[name = tensor("op_1920"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_35_cast_fp16 = reshape(shape = var_1920, x = linear_32_cast_fp16)[name = tensor("qkv_35_cast_fp16")]; + tensor var_1922_split_sizes_0 = const()[name = tensor("op_1922_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_1922_axis_0 = const()[name = tensor("op_1922_axis_0"), val = tensor(-3)]; + tensor var_1922_cast_fp16_0, tensor var_1922_cast_fp16_1, tensor var_1922_cast_fp16_2 = split(axis = var_1922_axis_0, split_sizes = var_1922_split_sizes_0, x = qkv_35_cast_fp16)[name = tensor("op_1922_cast_fp16")]; + tensor squeeze_24_axes_0 = const()[name = tensor("squeeze_24_axes_0"), val = tensor([-3])]; + tensor squeeze_24_cast_fp16 = squeeze(axes = squeeze_24_axes_0, x = var_1922_cast_fp16_0)[name = tensor("squeeze_24_cast_fp16")]; + tensor squeeze_25_axes_0 = const()[name = tensor("squeeze_25_axes_0"), val = tensor([-3])]; + tensor squeeze_25_cast_fp16 = squeeze(axes = squeeze_25_axes_0, x = var_1922_cast_fp16_1)[name = tensor("squeeze_25_cast_fp16")]; + tensor squeeze_26_axes_0 = const()[name = tensor("squeeze_26_axes_0"), val = tensor([-3])]; + tensor squeeze_26_cast_fp16 = squeeze(axes = squeeze_26_axes_0, x = var_1922_cast_fp16_2)[name = tensor("squeeze_26_cast_fp16")]; + tensor q_33_perm_0 = const()[name = tensor("q_33_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_33_perm_0 = const()[name = tensor("k_33_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_17_perm_0 = const()[name = tensor("value_17_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_33_cast_fp16 = transpose(perm = q_33_perm_0, x = squeeze_24_cast_fp16)[name = tensor("transpose_75")]; + tensor var_1932_cast_fp16 = mul(x = q_33_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1932_cast_fp16")]; + tensor x1_33_begin_0 = const()[name = tensor("x1_33_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_33_end_0 = const()[name = tensor("x1_33_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_33_end_mask_0 = const()[name = tensor("x1_33_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_33_cast_fp16 = slice_by_index(begin = x1_33_begin_0, end = x1_33_end_0, end_mask = x1_33_end_mask_0, x = q_33_cast_fp16)[name = tensor("x1_33_cast_fp16")]; + tensor x2_33_begin_0 = const()[name = tensor("x2_33_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_33_end_0 = const()[name = tensor("x2_33_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_33_end_mask_0 = const()[name = tensor("x2_33_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_33_cast_fp16 = slice_by_index(begin = x2_33_begin_0, end = x2_33_end_0, end_mask = x2_33_end_mask_0, x = q_33_cast_fp16)[name = tensor("x2_33_cast_fp16")]; + tensor const_68_promoted_to_fp16 = const()[name = tensor("const_68_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1944_cast_fp16 = mul(x = x2_33_cast_fp16, y = const_68_promoted_to_fp16)[name = tensor("op_1944_cast_fp16")]; + tensor var_1946_interleave_0 = const()[name = tensor("op_1946_interleave_0"), val = tensor(false)]; + tensor var_1946_cast_fp16 = concat(axis = var_1902, interleave = var_1946_interleave_0, values = (var_1944_cast_fp16, x1_33_cast_fp16))[name = tensor("op_1946_cast_fp16")]; + tensor var_1947_cast_fp16 = mul(x = var_1946_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1947_cast_fp16")]; + tensor q_embed_17_cast_fp16 = add(x = var_1932_cast_fp16, y = var_1947_cast_fp16)[name = tensor("q_embed_17_cast_fp16")]; + tensor k_33_cast_fp16 = transpose(perm = k_33_perm_0, x = squeeze_25_cast_fp16)[name = tensor("transpose_74")]; + tensor var_1950_cast_fp16 = mul(x = k_33_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_1950_cast_fp16")]; + tensor x1_35_begin_0 = const()[name = tensor("x1_35_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_35_end_0 = const()[name = tensor("x1_35_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_35_end_mask_0 = const()[name = tensor("x1_35_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_35_cast_fp16 = slice_by_index(begin = x1_35_begin_0, end = x1_35_end_0, end_mask = x1_35_end_mask_0, x = k_33_cast_fp16)[name = tensor("x1_35_cast_fp16")]; + tensor x2_35_begin_0 = const()[name = tensor("x2_35_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_35_end_0 = const()[name = tensor("x2_35_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_35_end_mask_0 = const()[name = tensor("x2_35_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_35_cast_fp16 = slice_by_index(begin = x2_35_begin_0, end = x2_35_end_0, end_mask = x2_35_end_mask_0, x = k_33_cast_fp16)[name = tensor("x2_35_cast_fp16")]; + tensor const_71_promoted_to_fp16 = const()[name = tensor("const_71_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_1962_cast_fp16 = mul(x = x2_35_cast_fp16, y = const_71_promoted_to_fp16)[name = tensor("op_1962_cast_fp16")]; + tensor var_1964_interleave_0 = const()[name = tensor("op_1964_interleave_0"), val = tensor(false)]; + tensor var_1964_cast_fp16 = concat(axis = var_1902, interleave = var_1964_interleave_0, values = (var_1962_cast_fp16, x1_35_cast_fp16))[name = tensor("op_1964_cast_fp16")]; + tensor var_1965_cast_fp16 = mul(x = var_1964_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_1965_cast_fp16")]; + tensor k_embed_17_cast_fp16 = add(x = var_1950_cast_fp16, y = var_1965_cast_fp16)[name = tensor("k_embed_17_cast_fp16")]; + tensor var_1970_transpose_x_1 = const()[name = tensor("op_1970_transpose_x_1"), val = tensor(false)]; + tensor var_1970_transpose_y_1 = const()[name = tensor("op_1970_transpose_y_1"), val = tensor(true)]; + tensor var_1970_cast_fp16 = matmul(transpose_x = var_1970_transpose_x_1, transpose_y = var_1970_transpose_y_1, x = q_embed_17_cast_fp16, y = k_embed_17_cast_fp16)[name = tensor("op_1970_cast_fp16")]; + tensor var_1971_to_fp16 = const()[name = tensor("op_1971_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_33_cast_fp16 = mul(x = var_1970_cast_fp16, y = var_1971_to_fp16)[name = tensor("attn_weights_33_cast_fp16")]; + tensor input_151_cast_fp16 = add(x = attn_weights_33_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_151_cast_fp16")]; + tensor var_1974_cast_fp16 = softmax(axis = var_1902, x = input_151_cast_fp16)[name = tensor("op_1974_cast_fp16")]; + tensor attn_output_49_transpose_x_0 = const()[name = tensor("attn_output_49_transpose_x_0"), val = tensor(false)]; + tensor attn_output_49_transpose_y_0 = const()[name = tensor("attn_output_49_transpose_y_0"), val = tensor(false)]; + tensor value_17_cast_fp16 = transpose(perm = value_17_perm_0, x = squeeze_26_cast_fp16)[name = tensor("transpose_73")]; + tensor attn_output_49_cast_fp16 = matmul(transpose_x = attn_output_49_transpose_x_0, transpose_y = attn_output_49_transpose_y_0, x = var_1974_cast_fp16, y = value_17_cast_fp16)[name = tensor("attn_output_49_cast_fp16")]; + tensor var_1978_perm_0 = const()[name = tensor("op_1978_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_1980 = const()[name = tensor("op_1980"), val = tensor([1, 512, -1])]; + tensor var_1978_cast_fp16 = transpose(perm = var_1978_perm_0, x = attn_output_49_cast_fp16)[name = tensor("transpose_72")]; + tensor var_1981_cast_fp16 = reshape(shape = var_1980, x = var_1978_cast_fp16)[name = tensor("op_1981_cast_fp16")]; + tensor model_encoder_layers_8_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_8_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(282346752)))]; + tensor linear_33_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_8_attn_Wo_weight_to_fp16, x = var_1981_cast_fp16)[name = tensor("linear_33_cast_fp16")]; + tensor input_157_cast_fp16 = add(x = input_149_cast_fp16, y = linear_33_cast_fp16)[name = tensor("input_157_cast_fp16")]; + tensor input_159_axes_0 = const()[name = tensor("input_159_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_8_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_8_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(283526464)))]; + tensor input_159_cast_fp16 = layer_norm(axes = input_159_axes_0, epsilon = var_1913_to_fp16, gamma = model_encoder_layers_8_mlp_norm_weight_to_fp16, x = input_157_cast_fp16)[name = tensor("input_159_cast_fp16")]; + tensor model_encoder_layers_8_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_8_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(283528064)))]; + tensor linear_34_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_8_mlp_Wi_weight_to_fp16, x = input_159_cast_fp16)[name = tensor("linear_34_cast_fp16")]; + tensor var_1988_split_sizes_0 = const()[name = tensor("op_1988_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_1988_axis_0 = const()[name = tensor("op_1988_axis_0"), val = tensor(-1)]; + tensor var_1988_cast_fp16_0, tensor var_1988_cast_fp16_1 = split(axis = var_1988_axis_0, split_sizes = var_1988_split_sizes_0, x = linear_34_cast_fp16)[name = tensor("op_1988_cast_fp16")]; + tensor var_1990_mode_0 = const()[name = tensor("op_1990_mode_0"), val = tensor("EXACT")]; + tensor var_1990_cast_fp16 = gelu(mode = var_1990_mode_0, x = var_1988_cast_fp16_0)[name = tensor("op_1990_cast_fp16")]; + tensor input_163_cast_fp16 = mul(x = var_1990_cast_fp16, y = var_1988_cast_fp16_1)[name = tensor("input_163_cast_fp16")]; + tensor model_encoder_layers_8_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_8_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(287067072)))]; + tensor linear_35_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_8_mlp_Wo_weight_to_fp16, x = input_163_cast_fp16)[name = tensor("linear_35_cast_fp16")]; + tensor input_167_cast_fp16 = add(x = input_157_cast_fp16, y = linear_35_cast_fp16)[name = tensor("input_167_cast_fp16")]; + tensor var_1999 = const()[name = tensor("op_1999"), val = tensor(-1)]; + tensor hidden_states_19_axes_0 = const()[name = tensor("hidden_states_19_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_9_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_9_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(288836608)))]; + tensor var_2010_to_fp16 = const()[name = tensor("op_2010_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_19_cast_fp16 = layer_norm(axes = hidden_states_19_axes_0, epsilon = var_2010_to_fp16, gamma = model_encoder_layers_9_attn_norm_weight_to_fp16, x = input_167_cast_fp16)[name = tensor("hidden_states_19_cast_fp16")]; + tensor model_encoder_layers_9_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_9_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(288838208)))]; + tensor linear_36_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_9_attn_Wqkv_weight_to_fp16, x = hidden_states_19_cast_fp16)[name = tensor("linear_36_cast_fp16")]; + tensor var_2017 = const()[name = tensor("op_2017"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_39_cast_fp16 = reshape(shape = var_2017, x = linear_36_cast_fp16)[name = tensor("qkv_39_cast_fp16")]; + tensor var_2019_split_sizes_0 = const()[name = tensor("op_2019_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2019_axis_0 = const()[name = tensor("op_2019_axis_0"), val = tensor(-3)]; + tensor var_2019_cast_fp16_0, tensor var_2019_cast_fp16_1, tensor var_2019_cast_fp16_2 = split(axis = var_2019_axis_0, split_sizes = var_2019_split_sizes_0, x = qkv_39_cast_fp16)[name = tensor("op_2019_cast_fp16")]; + tensor squeeze_27_axes_0 = const()[name = tensor("squeeze_27_axes_0"), val = tensor([-3])]; + tensor squeeze_27_cast_fp16 = squeeze(axes = squeeze_27_axes_0, x = var_2019_cast_fp16_0)[name = tensor("squeeze_27_cast_fp16")]; + tensor squeeze_28_axes_0 = const()[name = tensor("squeeze_28_axes_0"), val = tensor([-3])]; + tensor squeeze_28_cast_fp16 = squeeze(axes = squeeze_28_axes_0, x = var_2019_cast_fp16_1)[name = tensor("squeeze_28_cast_fp16")]; + tensor squeeze_29_axes_0 = const()[name = tensor("squeeze_29_axes_0"), val = tensor([-3])]; + tensor squeeze_29_cast_fp16 = squeeze(axes = squeeze_29_axes_0, x = var_2019_cast_fp16_2)[name = tensor("squeeze_29_cast_fp16")]; + tensor q_37_perm_0 = const()[name = tensor("q_37_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_37_perm_0 = const()[name = tensor("k_37_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_19_perm_0 = const()[name = tensor("value_19_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_37_cast_fp16 = transpose(perm = q_37_perm_0, x = squeeze_27_cast_fp16)[name = tensor("transpose_71")]; + tensor var_2029_cast_fp16 = mul(x = q_37_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2029_cast_fp16")]; + tensor x1_37_begin_0 = const()[name = tensor("x1_37_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_37_end_0 = const()[name = tensor("x1_37_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_37_end_mask_0 = const()[name = tensor("x1_37_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_37_cast_fp16 = slice_by_index(begin = x1_37_begin_0, end = x1_37_end_0, end_mask = x1_37_end_mask_0, x = q_37_cast_fp16)[name = tensor("x1_37_cast_fp16")]; + tensor x2_37_begin_0 = const()[name = tensor("x2_37_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_37_end_0 = const()[name = tensor("x2_37_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_37_end_mask_0 = const()[name = tensor("x2_37_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_37_cast_fp16 = slice_by_index(begin = x2_37_begin_0, end = x2_37_end_0, end_mask = x2_37_end_mask_0, x = q_37_cast_fp16)[name = tensor("x2_37_cast_fp16")]; + tensor const_76_promoted_to_fp16 = const()[name = tensor("const_76_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2041_cast_fp16 = mul(x = x2_37_cast_fp16, y = const_76_promoted_to_fp16)[name = tensor("op_2041_cast_fp16")]; + tensor var_2043_interleave_0 = const()[name = tensor("op_2043_interleave_0"), val = tensor(false)]; + tensor var_2043_cast_fp16 = concat(axis = var_1999, interleave = var_2043_interleave_0, values = (var_2041_cast_fp16, x1_37_cast_fp16))[name = tensor("op_2043_cast_fp16")]; + tensor var_2044_cast_fp16 = mul(x = var_2043_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2044_cast_fp16")]; + tensor q_embed_19_cast_fp16 = add(x = var_2029_cast_fp16, y = var_2044_cast_fp16)[name = tensor("q_embed_19_cast_fp16")]; + tensor k_37_cast_fp16 = transpose(perm = k_37_perm_0, x = squeeze_28_cast_fp16)[name = tensor("transpose_70")]; + tensor var_2047_cast_fp16 = mul(x = k_37_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2047_cast_fp16")]; + tensor x1_39_begin_0 = const()[name = tensor("x1_39_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_39_end_0 = const()[name = tensor("x1_39_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_39_end_mask_0 = const()[name = tensor("x1_39_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_39_cast_fp16 = slice_by_index(begin = x1_39_begin_0, end = x1_39_end_0, end_mask = x1_39_end_mask_0, x = k_37_cast_fp16)[name = tensor("x1_39_cast_fp16")]; + tensor x2_39_begin_0 = const()[name = tensor("x2_39_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_39_end_0 = const()[name = tensor("x2_39_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_39_end_mask_0 = const()[name = tensor("x2_39_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_39_cast_fp16 = slice_by_index(begin = x2_39_begin_0, end = x2_39_end_0, end_mask = x2_39_end_mask_0, x = k_37_cast_fp16)[name = tensor("x2_39_cast_fp16")]; + tensor const_79_promoted_to_fp16 = const()[name = tensor("const_79_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2059_cast_fp16 = mul(x = x2_39_cast_fp16, y = const_79_promoted_to_fp16)[name = tensor("op_2059_cast_fp16")]; + tensor var_2061_interleave_0 = const()[name = tensor("op_2061_interleave_0"), val = tensor(false)]; + tensor var_2061_cast_fp16 = concat(axis = var_1999, interleave = var_2061_interleave_0, values = (var_2059_cast_fp16, x1_39_cast_fp16))[name = tensor("op_2061_cast_fp16")]; + tensor var_2062_cast_fp16 = mul(x = var_2061_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2062_cast_fp16")]; + tensor k_embed_19_cast_fp16 = add(x = var_2047_cast_fp16, y = var_2062_cast_fp16)[name = tensor("k_embed_19_cast_fp16")]; + tensor var_2067_transpose_x_1 = const()[name = tensor("op_2067_transpose_x_1"), val = tensor(false)]; + tensor var_2067_transpose_y_1 = const()[name = tensor("op_2067_transpose_y_1"), val = tensor(true)]; + tensor var_2067_cast_fp16 = matmul(transpose_x = var_2067_transpose_x_1, transpose_y = var_2067_transpose_y_1, x = q_embed_19_cast_fp16, y = k_embed_19_cast_fp16)[name = tensor("op_2067_cast_fp16")]; + tensor var_2068_to_fp16 = const()[name = tensor("op_2068_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_37_cast_fp16 = mul(x = var_2067_cast_fp16, y = var_2068_to_fp16)[name = tensor("attn_weights_37_cast_fp16")]; + tensor input_169_cast_fp16 = add(x = attn_weights_37_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_169_cast_fp16")]; + tensor var_2071_cast_fp16 = softmax(axis = var_1999, x = input_169_cast_fp16)[name = tensor("op_2071_cast_fp16")]; + tensor attn_output_55_transpose_x_0 = const()[name = tensor("attn_output_55_transpose_x_0"), val = tensor(false)]; + tensor attn_output_55_transpose_y_0 = const()[name = tensor("attn_output_55_transpose_y_0"), val = tensor(false)]; + tensor value_19_cast_fp16 = transpose(perm = value_19_perm_0, x = squeeze_29_cast_fp16)[name = tensor("transpose_69")]; + tensor attn_output_55_cast_fp16 = matmul(transpose_x = attn_output_55_transpose_x_0, transpose_y = attn_output_55_transpose_y_0, x = var_2071_cast_fp16, y = value_19_cast_fp16)[name = tensor("attn_output_55_cast_fp16")]; + tensor var_2075_perm_0 = const()[name = tensor("op_2075_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2077 = const()[name = tensor("op_2077"), val = tensor([1, 512, -1])]; + tensor var_2075_cast_fp16 = transpose(perm = var_2075_perm_0, x = attn_output_55_cast_fp16)[name = tensor("transpose_68")]; + tensor var_2078_cast_fp16 = reshape(shape = var_2077, x = var_2075_cast_fp16)[name = tensor("op_2078_cast_fp16")]; + tensor model_encoder_layers_9_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_9_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(292377216)))]; + tensor linear_37_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_9_attn_Wo_weight_to_fp16, x = var_2078_cast_fp16)[name = tensor("linear_37_cast_fp16")]; + tensor input_175_cast_fp16 = add(x = input_167_cast_fp16, y = linear_37_cast_fp16)[name = tensor("input_175_cast_fp16")]; + tensor input_177_axes_0 = const()[name = tensor("input_177_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_9_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_9_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(293556928)))]; + tensor input_177_cast_fp16 = layer_norm(axes = input_177_axes_0, epsilon = var_2010_to_fp16, gamma = model_encoder_layers_9_mlp_norm_weight_to_fp16, x = input_175_cast_fp16)[name = tensor("input_177_cast_fp16")]; + tensor model_encoder_layers_9_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_9_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(293558528)))]; + tensor linear_38_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_9_mlp_Wi_weight_to_fp16, x = input_177_cast_fp16)[name = tensor("linear_38_cast_fp16")]; + tensor var_2085_split_sizes_0 = const()[name = tensor("op_2085_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2085_axis_0 = const()[name = tensor("op_2085_axis_0"), val = tensor(-1)]; + tensor var_2085_cast_fp16_0, tensor var_2085_cast_fp16_1 = split(axis = var_2085_axis_0, split_sizes = var_2085_split_sizes_0, x = linear_38_cast_fp16)[name = tensor("op_2085_cast_fp16")]; + tensor var_2087_mode_0 = const()[name = tensor("op_2087_mode_0"), val = tensor("EXACT")]; + tensor var_2087_cast_fp16 = gelu(mode = var_2087_mode_0, x = var_2085_cast_fp16_0)[name = tensor("op_2087_cast_fp16")]; + tensor input_181_cast_fp16 = mul(x = var_2087_cast_fp16, y = var_2085_cast_fp16_1)[name = tensor("input_181_cast_fp16")]; + tensor model_encoder_layers_9_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_9_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(297097536)))]; + tensor linear_39_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_9_mlp_Wo_weight_to_fp16, x = input_181_cast_fp16)[name = tensor("linear_39_cast_fp16")]; + tensor input_185_cast_fp16 = add(x = input_175_cast_fp16, y = linear_39_cast_fp16)[name = tensor("input_185_cast_fp16")]; + tensor var_2096 = const()[name = tensor("op_2096"), val = tensor(-1)]; + tensor hidden_states_21_axes_0 = const()[name = tensor("hidden_states_21_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_10_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_10_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(298867072)))]; + tensor var_2107_to_fp16 = const()[name = tensor("op_2107_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_21_cast_fp16 = layer_norm(axes = hidden_states_21_axes_0, epsilon = var_2107_to_fp16, gamma = model_encoder_layers_10_attn_norm_weight_to_fp16, x = input_185_cast_fp16)[name = tensor("hidden_states_21_cast_fp16")]; + tensor model_encoder_layers_10_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_10_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(298868672)))]; + tensor linear_40_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_10_attn_Wqkv_weight_to_fp16, x = hidden_states_21_cast_fp16)[name = tensor("linear_40_cast_fp16")]; + tensor var_2114 = const()[name = tensor("op_2114"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_43_cast_fp16 = reshape(shape = var_2114, x = linear_40_cast_fp16)[name = tensor("qkv_43_cast_fp16")]; + tensor var_2116_split_sizes_0 = const()[name = tensor("op_2116_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2116_axis_0 = const()[name = tensor("op_2116_axis_0"), val = tensor(-3)]; + tensor var_2116_cast_fp16_0, tensor var_2116_cast_fp16_1, tensor var_2116_cast_fp16_2 = split(axis = var_2116_axis_0, split_sizes = var_2116_split_sizes_0, x = qkv_43_cast_fp16)[name = tensor("op_2116_cast_fp16")]; + tensor squeeze_30_axes_0 = const()[name = tensor("squeeze_30_axes_0"), val = tensor([-3])]; + tensor squeeze_30_cast_fp16 = squeeze(axes = squeeze_30_axes_0, x = var_2116_cast_fp16_0)[name = tensor("squeeze_30_cast_fp16")]; + tensor squeeze_31_axes_0 = const()[name = tensor("squeeze_31_axes_0"), val = tensor([-3])]; + tensor squeeze_31_cast_fp16 = squeeze(axes = squeeze_31_axes_0, x = var_2116_cast_fp16_1)[name = tensor("squeeze_31_cast_fp16")]; + tensor squeeze_32_axes_0 = const()[name = tensor("squeeze_32_axes_0"), val = tensor([-3])]; + tensor squeeze_32_cast_fp16 = squeeze(axes = squeeze_32_axes_0, x = var_2116_cast_fp16_2)[name = tensor("squeeze_32_cast_fp16")]; + tensor q_41_perm_0 = const()[name = tensor("q_41_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_41_perm_0 = const()[name = tensor("k_41_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_21_perm_0 = const()[name = tensor("value_21_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_41_cast_fp16 = transpose(perm = q_41_perm_0, x = squeeze_30_cast_fp16)[name = tensor("transpose_67")]; + tensor var_2126_cast_fp16 = mul(x = q_41_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2126_cast_fp16")]; + tensor x1_41_begin_0 = const()[name = tensor("x1_41_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_41_end_0 = const()[name = tensor("x1_41_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_41_end_mask_0 = const()[name = tensor("x1_41_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_41_cast_fp16 = slice_by_index(begin = x1_41_begin_0, end = x1_41_end_0, end_mask = x1_41_end_mask_0, x = q_41_cast_fp16)[name = tensor("x1_41_cast_fp16")]; + tensor x2_41_begin_0 = const()[name = tensor("x2_41_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_41_end_0 = const()[name = tensor("x2_41_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_41_end_mask_0 = const()[name = tensor("x2_41_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_41_cast_fp16 = slice_by_index(begin = x2_41_begin_0, end = x2_41_end_0, end_mask = x2_41_end_mask_0, x = q_41_cast_fp16)[name = tensor("x2_41_cast_fp16")]; + tensor const_84_promoted_to_fp16 = const()[name = tensor("const_84_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2138_cast_fp16 = mul(x = x2_41_cast_fp16, y = const_84_promoted_to_fp16)[name = tensor("op_2138_cast_fp16")]; + tensor var_2140_interleave_0 = const()[name = tensor("op_2140_interleave_0"), val = tensor(false)]; + tensor var_2140_cast_fp16 = concat(axis = var_2096, interleave = var_2140_interleave_0, values = (var_2138_cast_fp16, x1_41_cast_fp16))[name = tensor("op_2140_cast_fp16")]; + tensor var_2141_cast_fp16 = mul(x = var_2140_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2141_cast_fp16")]; + tensor q_embed_21_cast_fp16 = add(x = var_2126_cast_fp16, y = var_2141_cast_fp16)[name = tensor("q_embed_21_cast_fp16")]; + tensor k_41_cast_fp16 = transpose(perm = k_41_perm_0, x = squeeze_31_cast_fp16)[name = tensor("transpose_66")]; + tensor var_2144_cast_fp16 = mul(x = k_41_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2144_cast_fp16")]; + tensor x1_43_begin_0 = const()[name = tensor("x1_43_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_43_end_0 = const()[name = tensor("x1_43_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_43_end_mask_0 = const()[name = tensor("x1_43_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_43_cast_fp16 = slice_by_index(begin = x1_43_begin_0, end = x1_43_end_0, end_mask = x1_43_end_mask_0, x = k_41_cast_fp16)[name = tensor("x1_43_cast_fp16")]; + tensor x2_43_begin_0 = const()[name = tensor("x2_43_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_43_end_0 = const()[name = tensor("x2_43_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_43_end_mask_0 = const()[name = tensor("x2_43_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_43_cast_fp16 = slice_by_index(begin = x2_43_begin_0, end = x2_43_end_0, end_mask = x2_43_end_mask_0, x = k_41_cast_fp16)[name = tensor("x2_43_cast_fp16")]; + tensor const_87_promoted_to_fp16 = const()[name = tensor("const_87_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2156_cast_fp16 = mul(x = x2_43_cast_fp16, y = const_87_promoted_to_fp16)[name = tensor("op_2156_cast_fp16")]; + tensor var_2158_interleave_0 = const()[name = tensor("op_2158_interleave_0"), val = tensor(false)]; + tensor var_2158_cast_fp16 = concat(axis = var_2096, interleave = var_2158_interleave_0, values = (var_2156_cast_fp16, x1_43_cast_fp16))[name = tensor("op_2158_cast_fp16")]; + tensor var_2159_cast_fp16 = mul(x = var_2158_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2159_cast_fp16")]; + tensor k_embed_21_cast_fp16 = add(x = var_2144_cast_fp16, y = var_2159_cast_fp16)[name = tensor("k_embed_21_cast_fp16")]; + tensor var_2164_transpose_x_1 = const()[name = tensor("op_2164_transpose_x_1"), val = tensor(false)]; + tensor var_2164_transpose_y_1 = const()[name = tensor("op_2164_transpose_y_1"), val = tensor(true)]; + tensor var_2164_cast_fp16 = matmul(transpose_x = var_2164_transpose_x_1, transpose_y = var_2164_transpose_y_1, x = q_embed_21_cast_fp16, y = k_embed_21_cast_fp16)[name = tensor("op_2164_cast_fp16")]; + tensor var_2165_to_fp16 = const()[name = tensor("op_2165_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_41_cast_fp16 = mul(x = var_2164_cast_fp16, y = var_2165_to_fp16)[name = tensor("attn_weights_41_cast_fp16")]; + tensor input_187_cast_fp16 = add(x = attn_weights_41_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_187_cast_fp16")]; + tensor var_2168_cast_fp16 = softmax(axis = var_2096, x = input_187_cast_fp16)[name = tensor("op_2168_cast_fp16")]; + tensor attn_output_61_transpose_x_0 = const()[name = tensor("attn_output_61_transpose_x_0"), val = tensor(false)]; + tensor attn_output_61_transpose_y_0 = const()[name = tensor("attn_output_61_transpose_y_0"), val = tensor(false)]; + tensor value_21_cast_fp16 = transpose(perm = value_21_perm_0, x = squeeze_32_cast_fp16)[name = tensor("transpose_65")]; + tensor attn_output_61_cast_fp16 = matmul(transpose_x = attn_output_61_transpose_x_0, transpose_y = attn_output_61_transpose_y_0, x = var_2168_cast_fp16, y = value_21_cast_fp16)[name = tensor("attn_output_61_cast_fp16")]; + tensor var_2172_perm_0 = const()[name = tensor("op_2172_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2174 = const()[name = tensor("op_2174"), val = tensor([1, 512, -1])]; + tensor var_2172_cast_fp16 = transpose(perm = var_2172_perm_0, x = attn_output_61_cast_fp16)[name = tensor("transpose_64")]; + tensor var_2175_cast_fp16 = reshape(shape = var_2174, x = var_2172_cast_fp16)[name = tensor("op_2175_cast_fp16")]; + tensor model_encoder_layers_10_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_10_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(302407680)))]; + tensor linear_41_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_10_attn_Wo_weight_to_fp16, x = var_2175_cast_fp16)[name = tensor("linear_41_cast_fp16")]; + tensor input_193_cast_fp16 = add(x = input_185_cast_fp16, y = linear_41_cast_fp16)[name = tensor("input_193_cast_fp16")]; + tensor input_195_axes_0 = const()[name = tensor("input_195_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_10_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_10_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(303587392)))]; + tensor input_195_cast_fp16 = layer_norm(axes = input_195_axes_0, epsilon = var_2107_to_fp16, gamma = model_encoder_layers_10_mlp_norm_weight_to_fp16, x = input_193_cast_fp16)[name = tensor("input_195_cast_fp16")]; + tensor model_encoder_layers_10_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_10_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(303588992)))]; + tensor linear_42_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_10_mlp_Wi_weight_to_fp16, x = input_195_cast_fp16)[name = tensor("linear_42_cast_fp16")]; + tensor var_2182_split_sizes_0 = const()[name = tensor("op_2182_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2182_axis_0 = const()[name = tensor("op_2182_axis_0"), val = tensor(-1)]; + tensor var_2182_cast_fp16_0, tensor var_2182_cast_fp16_1 = split(axis = var_2182_axis_0, split_sizes = var_2182_split_sizes_0, x = linear_42_cast_fp16)[name = tensor("op_2182_cast_fp16")]; + tensor var_2184_mode_0 = const()[name = tensor("op_2184_mode_0"), val = tensor("EXACT")]; + tensor var_2184_cast_fp16 = gelu(mode = var_2184_mode_0, x = var_2182_cast_fp16_0)[name = tensor("op_2184_cast_fp16")]; + tensor input_199_cast_fp16 = mul(x = var_2184_cast_fp16, y = var_2182_cast_fp16_1)[name = tensor("input_199_cast_fp16")]; + tensor model_encoder_layers_10_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_10_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(307128000)))]; + tensor linear_43_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_10_mlp_Wo_weight_to_fp16, x = input_199_cast_fp16)[name = tensor("linear_43_cast_fp16")]; + tensor input_203_cast_fp16 = add(x = input_193_cast_fp16, y = linear_43_cast_fp16)[name = tensor("input_203_cast_fp16")]; + tensor var_2193 = const()[name = tensor("op_2193"), val = tensor(-1)]; + tensor hidden_states_23_axes_0 = const()[name = tensor("hidden_states_23_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_11_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_11_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(308897536)))]; + tensor var_2204_to_fp16 = const()[name = tensor("op_2204_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_23_cast_fp16 = layer_norm(axes = hidden_states_23_axes_0, epsilon = var_2204_to_fp16, gamma = model_encoder_layers_11_attn_norm_weight_to_fp16, x = input_203_cast_fp16)[name = tensor("hidden_states_23_cast_fp16")]; + tensor model_encoder_layers_11_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_11_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(308899136)))]; + tensor linear_44_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_11_attn_Wqkv_weight_to_fp16, x = hidden_states_23_cast_fp16)[name = tensor("linear_44_cast_fp16")]; + tensor var_2211 = const()[name = tensor("op_2211"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_47_cast_fp16 = reshape(shape = var_2211, x = linear_44_cast_fp16)[name = tensor("qkv_47_cast_fp16")]; + tensor var_2213_split_sizes_0 = const()[name = tensor("op_2213_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2213_axis_0 = const()[name = tensor("op_2213_axis_0"), val = tensor(-3)]; + tensor var_2213_cast_fp16_0, tensor var_2213_cast_fp16_1, tensor var_2213_cast_fp16_2 = split(axis = var_2213_axis_0, split_sizes = var_2213_split_sizes_0, x = qkv_47_cast_fp16)[name = tensor("op_2213_cast_fp16")]; + tensor squeeze_33_axes_0 = const()[name = tensor("squeeze_33_axes_0"), val = tensor([-3])]; + tensor squeeze_33_cast_fp16 = squeeze(axes = squeeze_33_axes_0, x = var_2213_cast_fp16_0)[name = tensor("squeeze_33_cast_fp16")]; + tensor squeeze_34_axes_0 = const()[name = tensor("squeeze_34_axes_0"), val = tensor([-3])]; + tensor squeeze_34_cast_fp16 = squeeze(axes = squeeze_34_axes_0, x = var_2213_cast_fp16_1)[name = tensor("squeeze_34_cast_fp16")]; + tensor squeeze_35_axes_0 = const()[name = tensor("squeeze_35_axes_0"), val = tensor([-3])]; + tensor squeeze_35_cast_fp16 = squeeze(axes = squeeze_35_axes_0, x = var_2213_cast_fp16_2)[name = tensor("squeeze_35_cast_fp16")]; + tensor q_45_perm_0 = const()[name = tensor("q_45_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_45_perm_0 = const()[name = tensor("k_45_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_23_perm_0 = const()[name = tensor("value_23_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_45_cast_fp16 = transpose(perm = q_45_perm_0, x = squeeze_33_cast_fp16)[name = tensor("transpose_63")]; + tensor var_2223_cast_fp16 = mul(x = q_45_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2223_cast_fp16")]; + tensor x1_45_begin_0 = const()[name = tensor("x1_45_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_45_end_0 = const()[name = tensor("x1_45_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_45_end_mask_0 = const()[name = tensor("x1_45_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_45_cast_fp16 = slice_by_index(begin = x1_45_begin_0, end = x1_45_end_0, end_mask = x1_45_end_mask_0, x = q_45_cast_fp16)[name = tensor("x1_45_cast_fp16")]; + tensor x2_45_begin_0 = const()[name = tensor("x2_45_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_45_end_0 = const()[name = tensor("x2_45_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_45_end_mask_0 = const()[name = tensor("x2_45_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_45_cast_fp16 = slice_by_index(begin = x2_45_begin_0, end = x2_45_end_0, end_mask = x2_45_end_mask_0, x = q_45_cast_fp16)[name = tensor("x2_45_cast_fp16")]; + tensor const_92_promoted_to_fp16 = const()[name = tensor("const_92_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2235_cast_fp16 = mul(x = x2_45_cast_fp16, y = const_92_promoted_to_fp16)[name = tensor("op_2235_cast_fp16")]; + tensor var_2237_interleave_0 = const()[name = tensor("op_2237_interleave_0"), val = tensor(false)]; + tensor var_2237_cast_fp16 = concat(axis = var_2193, interleave = var_2237_interleave_0, values = (var_2235_cast_fp16, x1_45_cast_fp16))[name = tensor("op_2237_cast_fp16")]; + tensor var_2238_cast_fp16 = mul(x = var_2237_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2238_cast_fp16")]; + tensor q_embed_23_cast_fp16 = add(x = var_2223_cast_fp16, y = var_2238_cast_fp16)[name = tensor("q_embed_23_cast_fp16")]; + tensor k_45_cast_fp16 = transpose(perm = k_45_perm_0, x = squeeze_34_cast_fp16)[name = tensor("transpose_62")]; + tensor var_2241_cast_fp16 = mul(x = k_45_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2241_cast_fp16")]; + tensor x1_47_begin_0 = const()[name = tensor("x1_47_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_47_end_0 = const()[name = tensor("x1_47_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_47_end_mask_0 = const()[name = tensor("x1_47_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_47_cast_fp16 = slice_by_index(begin = x1_47_begin_0, end = x1_47_end_0, end_mask = x1_47_end_mask_0, x = k_45_cast_fp16)[name = tensor("x1_47_cast_fp16")]; + tensor x2_47_begin_0 = const()[name = tensor("x2_47_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_47_end_0 = const()[name = tensor("x2_47_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_47_end_mask_0 = const()[name = tensor("x2_47_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_47_cast_fp16 = slice_by_index(begin = x2_47_begin_0, end = x2_47_end_0, end_mask = x2_47_end_mask_0, x = k_45_cast_fp16)[name = tensor("x2_47_cast_fp16")]; + tensor const_95_promoted_to_fp16 = const()[name = tensor("const_95_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2253_cast_fp16 = mul(x = x2_47_cast_fp16, y = const_95_promoted_to_fp16)[name = tensor("op_2253_cast_fp16")]; + tensor var_2255_interleave_0 = const()[name = tensor("op_2255_interleave_0"), val = tensor(false)]; + tensor var_2255_cast_fp16 = concat(axis = var_2193, interleave = var_2255_interleave_0, values = (var_2253_cast_fp16, x1_47_cast_fp16))[name = tensor("op_2255_cast_fp16")]; + tensor var_2256_cast_fp16 = mul(x = var_2255_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2256_cast_fp16")]; + tensor k_embed_23_cast_fp16 = add(x = var_2241_cast_fp16, y = var_2256_cast_fp16)[name = tensor("k_embed_23_cast_fp16")]; + tensor var_2261_transpose_x_1 = const()[name = tensor("op_2261_transpose_x_1"), val = tensor(false)]; + tensor var_2261_transpose_y_1 = const()[name = tensor("op_2261_transpose_y_1"), val = tensor(true)]; + tensor var_2261_cast_fp16 = matmul(transpose_x = var_2261_transpose_x_1, transpose_y = var_2261_transpose_y_1, x = q_embed_23_cast_fp16, y = k_embed_23_cast_fp16)[name = tensor("op_2261_cast_fp16")]; + tensor var_2262_to_fp16 = const()[name = tensor("op_2262_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_45_cast_fp16 = mul(x = var_2261_cast_fp16, y = var_2262_to_fp16)[name = tensor("attn_weights_45_cast_fp16")]; + tensor input_205_cast_fp16 = add(x = attn_weights_45_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_205_cast_fp16")]; + tensor var_2265_cast_fp16 = softmax(axis = var_2193, x = input_205_cast_fp16)[name = tensor("op_2265_cast_fp16")]; + tensor attn_output_67_transpose_x_0 = const()[name = tensor("attn_output_67_transpose_x_0"), val = tensor(false)]; + tensor attn_output_67_transpose_y_0 = const()[name = tensor("attn_output_67_transpose_y_0"), val = tensor(false)]; + tensor value_23_cast_fp16 = transpose(perm = value_23_perm_0, x = squeeze_35_cast_fp16)[name = tensor("transpose_61")]; + tensor attn_output_67_cast_fp16 = matmul(transpose_x = attn_output_67_transpose_x_0, transpose_y = attn_output_67_transpose_y_0, x = var_2265_cast_fp16, y = value_23_cast_fp16)[name = tensor("attn_output_67_cast_fp16")]; + tensor var_2269_perm_0 = const()[name = tensor("op_2269_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2271 = const()[name = tensor("op_2271"), val = tensor([1, 512, -1])]; + tensor var_2269_cast_fp16 = transpose(perm = var_2269_perm_0, x = attn_output_67_cast_fp16)[name = tensor("transpose_60")]; + tensor var_2272_cast_fp16 = reshape(shape = var_2271, x = var_2269_cast_fp16)[name = tensor("op_2272_cast_fp16")]; + tensor model_encoder_layers_11_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_11_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(312438144)))]; + tensor linear_45_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_11_attn_Wo_weight_to_fp16, x = var_2272_cast_fp16)[name = tensor("linear_45_cast_fp16")]; + tensor input_211_cast_fp16 = add(x = input_203_cast_fp16, y = linear_45_cast_fp16)[name = tensor("input_211_cast_fp16")]; + tensor input_213_axes_0 = const()[name = tensor("input_213_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_11_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_11_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(313617856)))]; + tensor input_213_cast_fp16 = layer_norm(axes = input_213_axes_0, epsilon = var_2204_to_fp16, gamma = model_encoder_layers_11_mlp_norm_weight_to_fp16, x = input_211_cast_fp16)[name = tensor("input_213_cast_fp16")]; + tensor model_encoder_layers_11_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_11_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(313619456)))]; + tensor linear_46_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_11_mlp_Wi_weight_to_fp16, x = input_213_cast_fp16)[name = tensor("linear_46_cast_fp16")]; + tensor var_2279_split_sizes_0 = const()[name = tensor("op_2279_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2279_axis_0 = const()[name = tensor("op_2279_axis_0"), val = tensor(-1)]; + tensor var_2279_cast_fp16_0, tensor var_2279_cast_fp16_1 = split(axis = var_2279_axis_0, split_sizes = var_2279_split_sizes_0, x = linear_46_cast_fp16)[name = tensor("op_2279_cast_fp16")]; + tensor var_2281_mode_0 = const()[name = tensor("op_2281_mode_0"), val = tensor("EXACT")]; + tensor var_2281_cast_fp16 = gelu(mode = var_2281_mode_0, x = var_2279_cast_fp16_0)[name = tensor("op_2281_cast_fp16")]; + tensor input_217_cast_fp16 = mul(x = var_2281_cast_fp16, y = var_2279_cast_fp16_1)[name = tensor("input_217_cast_fp16")]; + tensor model_encoder_layers_11_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_11_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(317158464)))]; + tensor linear_47_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_11_mlp_Wo_weight_to_fp16, x = input_217_cast_fp16)[name = tensor("linear_47_cast_fp16")]; + tensor input_221_cast_fp16 = add(x = input_211_cast_fp16, y = linear_47_cast_fp16)[name = tensor("input_221_cast_fp16")]; + tensor var_2290 = const()[name = tensor("op_2290"), val = tensor(-1)]; + tensor hidden_states_25_axes_0 = const()[name = tensor("hidden_states_25_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_12_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_12_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(318928000)))]; + tensor var_2301_to_fp16 = const()[name = tensor("op_2301_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_25_cast_fp16 = layer_norm(axes = hidden_states_25_axes_0, epsilon = var_2301_to_fp16, gamma = model_encoder_layers_12_attn_norm_weight_to_fp16, x = input_221_cast_fp16)[name = tensor("hidden_states_25_cast_fp16")]; + tensor model_encoder_layers_12_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_12_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(318929600)))]; + tensor linear_48_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_12_attn_Wqkv_weight_to_fp16, x = hidden_states_25_cast_fp16)[name = tensor("linear_48_cast_fp16")]; + tensor var_2308 = const()[name = tensor("op_2308"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_51_cast_fp16 = reshape(shape = var_2308, x = linear_48_cast_fp16)[name = tensor("qkv_51_cast_fp16")]; + tensor var_2310_split_sizes_0 = const()[name = tensor("op_2310_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2310_axis_0 = const()[name = tensor("op_2310_axis_0"), val = tensor(-3)]; + tensor var_2310_cast_fp16_0, tensor var_2310_cast_fp16_1, tensor var_2310_cast_fp16_2 = split(axis = var_2310_axis_0, split_sizes = var_2310_split_sizes_0, x = qkv_51_cast_fp16)[name = tensor("op_2310_cast_fp16")]; + tensor squeeze_36_axes_0 = const()[name = tensor("squeeze_36_axes_0"), val = tensor([-3])]; + tensor squeeze_36_cast_fp16 = squeeze(axes = squeeze_36_axes_0, x = var_2310_cast_fp16_0)[name = tensor("squeeze_36_cast_fp16")]; + tensor squeeze_37_axes_0 = const()[name = tensor("squeeze_37_axes_0"), val = tensor([-3])]; + tensor squeeze_37_cast_fp16 = squeeze(axes = squeeze_37_axes_0, x = var_2310_cast_fp16_1)[name = tensor("squeeze_37_cast_fp16")]; + tensor squeeze_38_axes_0 = const()[name = tensor("squeeze_38_axes_0"), val = tensor([-3])]; + tensor squeeze_38_cast_fp16 = squeeze(axes = squeeze_38_axes_0, x = var_2310_cast_fp16_2)[name = tensor("squeeze_38_cast_fp16")]; + tensor q_49_perm_0 = const()[name = tensor("q_49_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_49_perm_0 = const()[name = tensor("k_49_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_25_perm_0 = const()[name = tensor("value_25_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_49_cast_fp16 = transpose(perm = q_49_perm_0, x = squeeze_36_cast_fp16)[name = tensor("transpose_59")]; + tensor var_2320_cast_fp16 = mul(x = q_49_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2320_cast_fp16")]; + tensor x1_49_begin_0 = const()[name = tensor("x1_49_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_49_end_0 = const()[name = tensor("x1_49_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_49_end_mask_0 = const()[name = tensor("x1_49_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_49_cast_fp16 = slice_by_index(begin = x1_49_begin_0, end = x1_49_end_0, end_mask = x1_49_end_mask_0, x = q_49_cast_fp16)[name = tensor("x1_49_cast_fp16")]; + tensor x2_49_begin_0 = const()[name = tensor("x2_49_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_49_end_0 = const()[name = tensor("x2_49_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_49_end_mask_0 = const()[name = tensor("x2_49_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_49_cast_fp16 = slice_by_index(begin = x2_49_begin_0, end = x2_49_end_0, end_mask = x2_49_end_mask_0, x = q_49_cast_fp16)[name = tensor("x2_49_cast_fp16")]; + tensor const_100_promoted_to_fp16 = const()[name = tensor("const_100_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2332_cast_fp16 = mul(x = x2_49_cast_fp16, y = const_100_promoted_to_fp16)[name = tensor("op_2332_cast_fp16")]; + tensor var_2334_interleave_0 = const()[name = tensor("op_2334_interleave_0"), val = tensor(false)]; + tensor var_2334_cast_fp16 = concat(axis = var_2290, interleave = var_2334_interleave_0, values = (var_2332_cast_fp16, x1_49_cast_fp16))[name = tensor("op_2334_cast_fp16")]; + tensor var_2335_cast_fp16 = mul(x = var_2334_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2335_cast_fp16")]; + tensor q_embed_25_cast_fp16 = add(x = var_2320_cast_fp16, y = var_2335_cast_fp16)[name = tensor("q_embed_25_cast_fp16")]; + tensor k_49_cast_fp16 = transpose(perm = k_49_perm_0, x = squeeze_37_cast_fp16)[name = tensor("transpose_58")]; + tensor var_2338_cast_fp16 = mul(x = k_49_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2338_cast_fp16")]; + tensor x1_51_begin_0 = const()[name = tensor("x1_51_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_51_end_0 = const()[name = tensor("x1_51_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_51_end_mask_0 = const()[name = tensor("x1_51_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_51_cast_fp16 = slice_by_index(begin = x1_51_begin_0, end = x1_51_end_0, end_mask = x1_51_end_mask_0, x = k_49_cast_fp16)[name = tensor("x1_51_cast_fp16")]; + tensor x2_51_begin_0 = const()[name = tensor("x2_51_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_51_end_0 = const()[name = tensor("x2_51_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_51_end_mask_0 = const()[name = tensor("x2_51_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_51_cast_fp16 = slice_by_index(begin = x2_51_begin_0, end = x2_51_end_0, end_mask = x2_51_end_mask_0, x = k_49_cast_fp16)[name = tensor("x2_51_cast_fp16")]; + tensor const_103_promoted_to_fp16 = const()[name = tensor("const_103_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2350_cast_fp16 = mul(x = x2_51_cast_fp16, y = const_103_promoted_to_fp16)[name = tensor("op_2350_cast_fp16")]; + tensor var_2352_interleave_0 = const()[name = tensor("op_2352_interleave_0"), val = tensor(false)]; + tensor var_2352_cast_fp16 = concat(axis = var_2290, interleave = var_2352_interleave_0, values = (var_2350_cast_fp16, x1_51_cast_fp16))[name = tensor("op_2352_cast_fp16")]; + tensor var_2353_cast_fp16 = mul(x = var_2352_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2353_cast_fp16")]; + tensor k_embed_25_cast_fp16 = add(x = var_2338_cast_fp16, y = var_2353_cast_fp16)[name = tensor("k_embed_25_cast_fp16")]; + tensor var_2358_transpose_x_1 = const()[name = tensor("op_2358_transpose_x_1"), val = tensor(false)]; + tensor var_2358_transpose_y_1 = const()[name = tensor("op_2358_transpose_y_1"), val = tensor(true)]; + tensor var_2358_cast_fp16 = matmul(transpose_x = var_2358_transpose_x_1, transpose_y = var_2358_transpose_y_1, x = q_embed_25_cast_fp16, y = k_embed_25_cast_fp16)[name = tensor("op_2358_cast_fp16")]; + tensor var_2359_to_fp16 = const()[name = tensor("op_2359_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_49_cast_fp16 = mul(x = var_2358_cast_fp16, y = var_2359_to_fp16)[name = tensor("attn_weights_49_cast_fp16")]; + tensor input_223_cast_fp16 = add(x = attn_weights_49_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_223_cast_fp16")]; + tensor var_2362_cast_fp16 = softmax(axis = var_2290, x = input_223_cast_fp16)[name = tensor("op_2362_cast_fp16")]; + tensor attn_output_73_transpose_x_0 = const()[name = tensor("attn_output_73_transpose_x_0"), val = tensor(false)]; + tensor attn_output_73_transpose_y_0 = const()[name = tensor("attn_output_73_transpose_y_0"), val = tensor(false)]; + tensor value_25_cast_fp16 = transpose(perm = value_25_perm_0, x = squeeze_38_cast_fp16)[name = tensor("transpose_57")]; + tensor attn_output_73_cast_fp16 = matmul(transpose_x = attn_output_73_transpose_x_0, transpose_y = attn_output_73_transpose_y_0, x = var_2362_cast_fp16, y = value_25_cast_fp16)[name = tensor("attn_output_73_cast_fp16")]; + tensor var_2366_perm_0 = const()[name = tensor("op_2366_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2368 = const()[name = tensor("op_2368"), val = tensor([1, 512, -1])]; + tensor var_2366_cast_fp16 = transpose(perm = var_2366_perm_0, x = attn_output_73_cast_fp16)[name = tensor("transpose_56")]; + tensor var_2369_cast_fp16 = reshape(shape = var_2368, x = var_2366_cast_fp16)[name = tensor("op_2369_cast_fp16")]; + tensor model_encoder_layers_12_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_12_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(322468608)))]; + tensor linear_49_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_12_attn_Wo_weight_to_fp16, x = var_2369_cast_fp16)[name = tensor("linear_49_cast_fp16")]; + tensor input_229_cast_fp16 = add(x = input_221_cast_fp16, y = linear_49_cast_fp16)[name = tensor("input_229_cast_fp16")]; + tensor input_231_axes_0 = const()[name = tensor("input_231_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_12_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_12_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(323648320)))]; + tensor input_231_cast_fp16 = layer_norm(axes = input_231_axes_0, epsilon = var_2301_to_fp16, gamma = model_encoder_layers_12_mlp_norm_weight_to_fp16, x = input_229_cast_fp16)[name = tensor("input_231_cast_fp16")]; + tensor model_encoder_layers_12_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_12_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(323649920)))]; + tensor linear_50_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_12_mlp_Wi_weight_to_fp16, x = input_231_cast_fp16)[name = tensor("linear_50_cast_fp16")]; + tensor var_2376_split_sizes_0 = const()[name = tensor("op_2376_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2376_axis_0 = const()[name = tensor("op_2376_axis_0"), val = tensor(-1)]; + tensor var_2376_cast_fp16_0, tensor var_2376_cast_fp16_1 = split(axis = var_2376_axis_0, split_sizes = var_2376_split_sizes_0, x = linear_50_cast_fp16)[name = tensor("op_2376_cast_fp16")]; + tensor var_2378_mode_0 = const()[name = tensor("op_2378_mode_0"), val = tensor("EXACT")]; + tensor var_2378_cast_fp16 = gelu(mode = var_2378_mode_0, x = var_2376_cast_fp16_0)[name = tensor("op_2378_cast_fp16")]; + tensor input_235_cast_fp16 = mul(x = var_2378_cast_fp16, y = var_2376_cast_fp16_1)[name = tensor("input_235_cast_fp16")]; + tensor model_encoder_layers_12_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_12_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(327188928)))]; + tensor linear_51_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_12_mlp_Wo_weight_to_fp16, x = input_235_cast_fp16)[name = tensor("linear_51_cast_fp16")]; + tensor input_239_cast_fp16 = add(x = input_229_cast_fp16, y = linear_51_cast_fp16)[name = tensor("input_239_cast_fp16")]; + tensor var_2387 = const()[name = tensor("op_2387"), val = tensor(-1)]; + tensor hidden_states_27_axes_0 = const()[name = tensor("hidden_states_27_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_13_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_13_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(328958464)))]; + tensor var_2398_to_fp16 = const()[name = tensor("op_2398_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_27_cast_fp16 = layer_norm(axes = hidden_states_27_axes_0, epsilon = var_2398_to_fp16, gamma = model_encoder_layers_13_attn_norm_weight_to_fp16, x = input_239_cast_fp16)[name = tensor("hidden_states_27_cast_fp16")]; + tensor model_encoder_layers_13_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_13_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(328960064)))]; + tensor linear_52_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_13_attn_Wqkv_weight_to_fp16, x = hidden_states_27_cast_fp16)[name = tensor("linear_52_cast_fp16")]; + tensor var_2405 = const()[name = tensor("op_2405"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_55_cast_fp16 = reshape(shape = var_2405, x = linear_52_cast_fp16)[name = tensor("qkv_55_cast_fp16")]; + tensor var_2407_split_sizes_0 = const()[name = tensor("op_2407_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2407_axis_0 = const()[name = tensor("op_2407_axis_0"), val = tensor(-3)]; + tensor var_2407_cast_fp16_0, tensor var_2407_cast_fp16_1, tensor var_2407_cast_fp16_2 = split(axis = var_2407_axis_0, split_sizes = var_2407_split_sizes_0, x = qkv_55_cast_fp16)[name = tensor("op_2407_cast_fp16")]; + tensor squeeze_39_axes_0 = const()[name = tensor("squeeze_39_axes_0"), val = tensor([-3])]; + tensor squeeze_39_cast_fp16 = squeeze(axes = squeeze_39_axes_0, x = var_2407_cast_fp16_0)[name = tensor("squeeze_39_cast_fp16")]; + tensor squeeze_40_axes_0 = const()[name = tensor("squeeze_40_axes_0"), val = tensor([-3])]; + tensor squeeze_40_cast_fp16 = squeeze(axes = squeeze_40_axes_0, x = var_2407_cast_fp16_1)[name = tensor("squeeze_40_cast_fp16")]; + tensor squeeze_41_axes_0 = const()[name = tensor("squeeze_41_axes_0"), val = tensor([-3])]; + tensor squeeze_41_cast_fp16 = squeeze(axes = squeeze_41_axes_0, x = var_2407_cast_fp16_2)[name = tensor("squeeze_41_cast_fp16")]; + tensor q_53_perm_0 = const()[name = tensor("q_53_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_53_perm_0 = const()[name = tensor("k_53_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_27_perm_0 = const()[name = tensor("value_27_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_53_cast_fp16 = transpose(perm = q_53_perm_0, x = squeeze_39_cast_fp16)[name = tensor("transpose_55")]; + tensor var_2417_cast_fp16 = mul(x = q_53_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2417_cast_fp16")]; + tensor x1_53_begin_0 = const()[name = tensor("x1_53_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_53_end_0 = const()[name = tensor("x1_53_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_53_end_mask_0 = const()[name = tensor("x1_53_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_53_cast_fp16 = slice_by_index(begin = x1_53_begin_0, end = x1_53_end_0, end_mask = x1_53_end_mask_0, x = q_53_cast_fp16)[name = tensor("x1_53_cast_fp16")]; + tensor x2_53_begin_0 = const()[name = tensor("x2_53_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_53_end_0 = const()[name = tensor("x2_53_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_53_end_mask_0 = const()[name = tensor("x2_53_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_53_cast_fp16 = slice_by_index(begin = x2_53_begin_0, end = x2_53_end_0, end_mask = x2_53_end_mask_0, x = q_53_cast_fp16)[name = tensor("x2_53_cast_fp16")]; + tensor const_108_promoted_to_fp16 = const()[name = tensor("const_108_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2429_cast_fp16 = mul(x = x2_53_cast_fp16, y = const_108_promoted_to_fp16)[name = tensor("op_2429_cast_fp16")]; + tensor var_2431_interleave_0 = const()[name = tensor("op_2431_interleave_0"), val = tensor(false)]; + tensor var_2431_cast_fp16 = concat(axis = var_2387, interleave = var_2431_interleave_0, values = (var_2429_cast_fp16, x1_53_cast_fp16))[name = tensor("op_2431_cast_fp16")]; + tensor var_2432_cast_fp16 = mul(x = var_2431_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2432_cast_fp16")]; + tensor q_embed_27_cast_fp16 = add(x = var_2417_cast_fp16, y = var_2432_cast_fp16)[name = tensor("q_embed_27_cast_fp16")]; + tensor k_53_cast_fp16 = transpose(perm = k_53_perm_0, x = squeeze_40_cast_fp16)[name = tensor("transpose_54")]; + tensor var_2435_cast_fp16 = mul(x = k_53_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2435_cast_fp16")]; + tensor x1_55_begin_0 = const()[name = tensor("x1_55_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_55_end_0 = const()[name = tensor("x1_55_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_55_end_mask_0 = const()[name = tensor("x1_55_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_55_cast_fp16 = slice_by_index(begin = x1_55_begin_0, end = x1_55_end_0, end_mask = x1_55_end_mask_0, x = k_53_cast_fp16)[name = tensor("x1_55_cast_fp16")]; + tensor x2_55_begin_0 = const()[name = tensor("x2_55_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_55_end_0 = const()[name = tensor("x2_55_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_55_end_mask_0 = const()[name = tensor("x2_55_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_55_cast_fp16 = slice_by_index(begin = x2_55_begin_0, end = x2_55_end_0, end_mask = x2_55_end_mask_0, x = k_53_cast_fp16)[name = tensor("x2_55_cast_fp16")]; + tensor const_111_promoted_to_fp16 = const()[name = tensor("const_111_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2447_cast_fp16 = mul(x = x2_55_cast_fp16, y = const_111_promoted_to_fp16)[name = tensor("op_2447_cast_fp16")]; + tensor var_2449_interleave_0 = const()[name = tensor("op_2449_interleave_0"), val = tensor(false)]; + tensor var_2449_cast_fp16 = concat(axis = var_2387, interleave = var_2449_interleave_0, values = (var_2447_cast_fp16, x1_55_cast_fp16))[name = tensor("op_2449_cast_fp16")]; + tensor var_2450_cast_fp16 = mul(x = var_2449_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2450_cast_fp16")]; + tensor k_embed_27_cast_fp16 = add(x = var_2435_cast_fp16, y = var_2450_cast_fp16)[name = tensor("k_embed_27_cast_fp16")]; + tensor var_2455_transpose_x_1 = const()[name = tensor("op_2455_transpose_x_1"), val = tensor(false)]; + tensor var_2455_transpose_y_1 = const()[name = tensor("op_2455_transpose_y_1"), val = tensor(true)]; + tensor var_2455_cast_fp16 = matmul(transpose_x = var_2455_transpose_x_1, transpose_y = var_2455_transpose_y_1, x = q_embed_27_cast_fp16, y = k_embed_27_cast_fp16)[name = tensor("op_2455_cast_fp16")]; + tensor var_2456_to_fp16 = const()[name = tensor("op_2456_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_53_cast_fp16 = mul(x = var_2455_cast_fp16, y = var_2456_to_fp16)[name = tensor("attn_weights_53_cast_fp16")]; + tensor input_241_cast_fp16 = add(x = attn_weights_53_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_241_cast_fp16")]; + tensor var_2459_cast_fp16 = softmax(axis = var_2387, x = input_241_cast_fp16)[name = tensor("op_2459_cast_fp16")]; + tensor attn_output_79_transpose_x_0 = const()[name = tensor("attn_output_79_transpose_x_0"), val = tensor(false)]; + tensor attn_output_79_transpose_y_0 = const()[name = tensor("attn_output_79_transpose_y_0"), val = tensor(false)]; + tensor value_27_cast_fp16 = transpose(perm = value_27_perm_0, x = squeeze_41_cast_fp16)[name = tensor("transpose_53")]; + tensor attn_output_79_cast_fp16 = matmul(transpose_x = attn_output_79_transpose_x_0, transpose_y = attn_output_79_transpose_y_0, x = var_2459_cast_fp16, y = value_27_cast_fp16)[name = tensor("attn_output_79_cast_fp16")]; + tensor var_2463_perm_0 = const()[name = tensor("op_2463_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2465 = const()[name = tensor("op_2465"), val = tensor([1, 512, -1])]; + tensor var_2463_cast_fp16 = transpose(perm = var_2463_perm_0, x = attn_output_79_cast_fp16)[name = tensor("transpose_52")]; + tensor var_2466_cast_fp16 = reshape(shape = var_2465, x = var_2463_cast_fp16)[name = tensor("op_2466_cast_fp16")]; + tensor model_encoder_layers_13_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_13_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(332499072)))]; + tensor linear_53_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_13_attn_Wo_weight_to_fp16, x = var_2466_cast_fp16)[name = tensor("linear_53_cast_fp16")]; + tensor input_247_cast_fp16 = add(x = input_239_cast_fp16, y = linear_53_cast_fp16)[name = tensor("input_247_cast_fp16")]; + tensor input_249_axes_0 = const()[name = tensor("input_249_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_13_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_13_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(333678784)))]; + tensor input_249_cast_fp16 = layer_norm(axes = input_249_axes_0, epsilon = var_2398_to_fp16, gamma = model_encoder_layers_13_mlp_norm_weight_to_fp16, x = input_247_cast_fp16)[name = tensor("input_249_cast_fp16")]; + tensor model_encoder_layers_13_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_13_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(333680384)))]; + tensor linear_54_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_13_mlp_Wi_weight_to_fp16, x = input_249_cast_fp16)[name = tensor("linear_54_cast_fp16")]; + tensor var_2473_split_sizes_0 = const()[name = tensor("op_2473_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2473_axis_0 = const()[name = tensor("op_2473_axis_0"), val = tensor(-1)]; + tensor var_2473_cast_fp16_0, tensor var_2473_cast_fp16_1 = split(axis = var_2473_axis_0, split_sizes = var_2473_split_sizes_0, x = linear_54_cast_fp16)[name = tensor("op_2473_cast_fp16")]; + tensor var_2475_mode_0 = const()[name = tensor("op_2475_mode_0"), val = tensor("EXACT")]; + tensor var_2475_cast_fp16 = gelu(mode = var_2475_mode_0, x = var_2473_cast_fp16_0)[name = tensor("op_2475_cast_fp16")]; + tensor input_253_cast_fp16 = mul(x = var_2475_cast_fp16, y = var_2473_cast_fp16_1)[name = tensor("input_253_cast_fp16")]; + tensor model_encoder_layers_13_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_13_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(337219392)))]; + tensor linear_55_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_13_mlp_Wo_weight_to_fp16, x = input_253_cast_fp16)[name = tensor("linear_55_cast_fp16")]; + tensor input_257_cast_fp16 = add(x = input_247_cast_fp16, y = linear_55_cast_fp16)[name = tensor("input_257_cast_fp16")]; + tensor var_2484 = const()[name = tensor("op_2484"), val = tensor(-1)]; + tensor hidden_states_29_axes_0 = const()[name = tensor("hidden_states_29_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_14_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_14_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(338988928)))]; + tensor var_2495_to_fp16 = const()[name = tensor("op_2495_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_29_cast_fp16 = layer_norm(axes = hidden_states_29_axes_0, epsilon = var_2495_to_fp16, gamma = model_encoder_layers_14_attn_norm_weight_to_fp16, x = input_257_cast_fp16)[name = tensor("hidden_states_29_cast_fp16")]; + tensor model_encoder_layers_14_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_14_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(338990528)))]; + tensor linear_56_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_14_attn_Wqkv_weight_to_fp16, x = hidden_states_29_cast_fp16)[name = tensor("linear_56_cast_fp16")]; + tensor var_2502 = const()[name = tensor("op_2502"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_59_cast_fp16 = reshape(shape = var_2502, x = linear_56_cast_fp16)[name = tensor("qkv_59_cast_fp16")]; + tensor var_2504_split_sizes_0 = const()[name = tensor("op_2504_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2504_axis_0 = const()[name = tensor("op_2504_axis_0"), val = tensor(-3)]; + tensor var_2504_cast_fp16_0, tensor var_2504_cast_fp16_1, tensor var_2504_cast_fp16_2 = split(axis = var_2504_axis_0, split_sizes = var_2504_split_sizes_0, x = qkv_59_cast_fp16)[name = tensor("op_2504_cast_fp16")]; + tensor squeeze_42_axes_0 = const()[name = tensor("squeeze_42_axes_0"), val = tensor([-3])]; + tensor squeeze_42_cast_fp16 = squeeze(axes = squeeze_42_axes_0, x = var_2504_cast_fp16_0)[name = tensor("squeeze_42_cast_fp16")]; + tensor squeeze_43_axes_0 = const()[name = tensor("squeeze_43_axes_0"), val = tensor([-3])]; + tensor squeeze_43_cast_fp16 = squeeze(axes = squeeze_43_axes_0, x = var_2504_cast_fp16_1)[name = tensor("squeeze_43_cast_fp16")]; + tensor squeeze_44_axes_0 = const()[name = tensor("squeeze_44_axes_0"), val = tensor([-3])]; + tensor squeeze_44_cast_fp16 = squeeze(axes = squeeze_44_axes_0, x = var_2504_cast_fp16_2)[name = tensor("squeeze_44_cast_fp16")]; + tensor q_57_perm_0 = const()[name = tensor("q_57_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_57_perm_0 = const()[name = tensor("k_57_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_29_perm_0 = const()[name = tensor("value_29_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_57_cast_fp16 = transpose(perm = q_57_perm_0, x = squeeze_42_cast_fp16)[name = tensor("transpose_51")]; + tensor var_2514_cast_fp16 = mul(x = q_57_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2514_cast_fp16")]; + tensor x1_57_begin_0 = const()[name = tensor("x1_57_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_57_end_0 = const()[name = tensor("x1_57_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_57_end_mask_0 = const()[name = tensor("x1_57_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_57_cast_fp16 = slice_by_index(begin = x1_57_begin_0, end = x1_57_end_0, end_mask = x1_57_end_mask_0, x = q_57_cast_fp16)[name = tensor("x1_57_cast_fp16")]; + tensor x2_57_begin_0 = const()[name = tensor("x2_57_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_57_end_0 = const()[name = tensor("x2_57_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_57_end_mask_0 = const()[name = tensor("x2_57_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_57_cast_fp16 = slice_by_index(begin = x2_57_begin_0, end = x2_57_end_0, end_mask = x2_57_end_mask_0, x = q_57_cast_fp16)[name = tensor("x2_57_cast_fp16")]; + tensor const_116_promoted_to_fp16 = const()[name = tensor("const_116_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2526_cast_fp16 = mul(x = x2_57_cast_fp16, y = const_116_promoted_to_fp16)[name = tensor("op_2526_cast_fp16")]; + tensor var_2528_interleave_0 = const()[name = tensor("op_2528_interleave_0"), val = tensor(false)]; + tensor var_2528_cast_fp16 = concat(axis = var_2484, interleave = var_2528_interleave_0, values = (var_2526_cast_fp16, x1_57_cast_fp16))[name = tensor("op_2528_cast_fp16")]; + tensor var_2529_cast_fp16 = mul(x = var_2528_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2529_cast_fp16")]; + tensor q_embed_29_cast_fp16 = add(x = var_2514_cast_fp16, y = var_2529_cast_fp16)[name = tensor("q_embed_29_cast_fp16")]; + tensor k_57_cast_fp16 = transpose(perm = k_57_perm_0, x = squeeze_43_cast_fp16)[name = tensor("transpose_50")]; + tensor var_2532_cast_fp16 = mul(x = k_57_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2532_cast_fp16")]; + tensor x1_59_begin_0 = const()[name = tensor("x1_59_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_59_end_0 = const()[name = tensor("x1_59_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_59_end_mask_0 = const()[name = tensor("x1_59_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_59_cast_fp16 = slice_by_index(begin = x1_59_begin_0, end = x1_59_end_0, end_mask = x1_59_end_mask_0, x = k_57_cast_fp16)[name = tensor("x1_59_cast_fp16")]; + tensor x2_59_begin_0 = const()[name = tensor("x2_59_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_59_end_0 = const()[name = tensor("x2_59_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_59_end_mask_0 = const()[name = tensor("x2_59_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_59_cast_fp16 = slice_by_index(begin = x2_59_begin_0, end = x2_59_end_0, end_mask = x2_59_end_mask_0, x = k_57_cast_fp16)[name = tensor("x2_59_cast_fp16")]; + tensor const_119_promoted_to_fp16 = const()[name = tensor("const_119_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2544_cast_fp16 = mul(x = x2_59_cast_fp16, y = const_119_promoted_to_fp16)[name = tensor("op_2544_cast_fp16")]; + tensor var_2546_interleave_0 = const()[name = tensor("op_2546_interleave_0"), val = tensor(false)]; + tensor var_2546_cast_fp16 = concat(axis = var_2484, interleave = var_2546_interleave_0, values = (var_2544_cast_fp16, x1_59_cast_fp16))[name = tensor("op_2546_cast_fp16")]; + tensor var_2547_cast_fp16 = mul(x = var_2546_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2547_cast_fp16")]; + tensor k_embed_29_cast_fp16 = add(x = var_2532_cast_fp16, y = var_2547_cast_fp16)[name = tensor("k_embed_29_cast_fp16")]; + tensor var_2552_transpose_x_1 = const()[name = tensor("op_2552_transpose_x_1"), val = tensor(false)]; + tensor var_2552_transpose_y_1 = const()[name = tensor("op_2552_transpose_y_1"), val = tensor(true)]; + tensor var_2552_cast_fp16 = matmul(transpose_x = var_2552_transpose_x_1, transpose_y = var_2552_transpose_y_1, x = q_embed_29_cast_fp16, y = k_embed_29_cast_fp16)[name = tensor("op_2552_cast_fp16")]; + tensor var_2553_to_fp16 = const()[name = tensor("op_2553_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_57_cast_fp16 = mul(x = var_2552_cast_fp16, y = var_2553_to_fp16)[name = tensor("attn_weights_57_cast_fp16")]; + tensor input_259_cast_fp16 = add(x = attn_weights_57_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_259_cast_fp16")]; + tensor var_2556_cast_fp16 = softmax(axis = var_2484, x = input_259_cast_fp16)[name = tensor("op_2556_cast_fp16")]; + tensor attn_output_85_transpose_x_0 = const()[name = tensor("attn_output_85_transpose_x_0"), val = tensor(false)]; + tensor attn_output_85_transpose_y_0 = const()[name = tensor("attn_output_85_transpose_y_0"), val = tensor(false)]; + tensor value_29_cast_fp16 = transpose(perm = value_29_perm_0, x = squeeze_44_cast_fp16)[name = tensor("transpose_49")]; + tensor attn_output_85_cast_fp16 = matmul(transpose_x = attn_output_85_transpose_x_0, transpose_y = attn_output_85_transpose_y_0, x = var_2556_cast_fp16, y = value_29_cast_fp16)[name = tensor("attn_output_85_cast_fp16")]; + tensor var_2560_perm_0 = const()[name = tensor("op_2560_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2562 = const()[name = tensor("op_2562"), val = tensor([1, 512, -1])]; + tensor var_2560_cast_fp16 = transpose(perm = var_2560_perm_0, x = attn_output_85_cast_fp16)[name = tensor("transpose_48")]; + tensor var_2563_cast_fp16 = reshape(shape = var_2562, x = var_2560_cast_fp16)[name = tensor("op_2563_cast_fp16")]; + tensor model_encoder_layers_14_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_14_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(342529536)))]; + tensor linear_57_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_14_attn_Wo_weight_to_fp16, x = var_2563_cast_fp16)[name = tensor("linear_57_cast_fp16")]; + tensor input_265_cast_fp16 = add(x = input_257_cast_fp16, y = linear_57_cast_fp16)[name = tensor("input_265_cast_fp16")]; + tensor input_267_axes_0 = const()[name = tensor("input_267_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_14_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_14_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(343709248)))]; + tensor input_267_cast_fp16 = layer_norm(axes = input_267_axes_0, epsilon = var_2495_to_fp16, gamma = model_encoder_layers_14_mlp_norm_weight_to_fp16, x = input_265_cast_fp16)[name = tensor("input_267_cast_fp16")]; + tensor model_encoder_layers_14_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_14_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(343710848)))]; + tensor linear_58_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_14_mlp_Wi_weight_to_fp16, x = input_267_cast_fp16)[name = tensor("linear_58_cast_fp16")]; + tensor var_2570_split_sizes_0 = const()[name = tensor("op_2570_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2570_axis_0 = const()[name = tensor("op_2570_axis_0"), val = tensor(-1)]; + tensor var_2570_cast_fp16_0, tensor var_2570_cast_fp16_1 = split(axis = var_2570_axis_0, split_sizes = var_2570_split_sizes_0, x = linear_58_cast_fp16)[name = tensor("op_2570_cast_fp16")]; + tensor var_2572_mode_0 = const()[name = tensor("op_2572_mode_0"), val = tensor("EXACT")]; + tensor var_2572_cast_fp16 = gelu(mode = var_2572_mode_0, x = var_2570_cast_fp16_0)[name = tensor("op_2572_cast_fp16")]; + tensor input_271_cast_fp16 = mul(x = var_2572_cast_fp16, y = var_2570_cast_fp16_1)[name = tensor("input_271_cast_fp16")]; + tensor model_encoder_layers_14_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_14_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(347249856)))]; + tensor linear_59_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_14_mlp_Wo_weight_to_fp16, x = input_271_cast_fp16)[name = tensor("linear_59_cast_fp16")]; + tensor input_275_cast_fp16 = add(x = input_265_cast_fp16, y = linear_59_cast_fp16)[name = tensor("input_275_cast_fp16")]; + tensor var_2581 = const()[name = tensor("op_2581"), val = tensor(-1)]; + tensor hidden_states_31_axes_0 = const()[name = tensor("hidden_states_31_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_15_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_15_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(349019392)))]; + tensor var_2592_to_fp16 = const()[name = tensor("op_2592_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_31_cast_fp16 = layer_norm(axes = hidden_states_31_axes_0, epsilon = var_2592_to_fp16, gamma = model_encoder_layers_15_attn_norm_weight_to_fp16, x = input_275_cast_fp16)[name = tensor("hidden_states_31_cast_fp16")]; + tensor model_encoder_layers_15_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_15_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(349020992)))]; + tensor linear_60_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_15_attn_Wqkv_weight_to_fp16, x = hidden_states_31_cast_fp16)[name = tensor("linear_60_cast_fp16")]; + tensor var_2599 = const()[name = tensor("op_2599"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_63_cast_fp16 = reshape(shape = var_2599, x = linear_60_cast_fp16)[name = tensor("qkv_63_cast_fp16")]; + tensor var_2601_split_sizes_0 = const()[name = tensor("op_2601_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2601_axis_0 = const()[name = tensor("op_2601_axis_0"), val = tensor(-3)]; + tensor var_2601_cast_fp16_0, tensor var_2601_cast_fp16_1, tensor var_2601_cast_fp16_2 = split(axis = var_2601_axis_0, split_sizes = var_2601_split_sizes_0, x = qkv_63_cast_fp16)[name = tensor("op_2601_cast_fp16")]; + tensor squeeze_45_axes_0 = const()[name = tensor("squeeze_45_axes_0"), val = tensor([-3])]; + tensor squeeze_45_cast_fp16 = squeeze(axes = squeeze_45_axes_0, x = var_2601_cast_fp16_0)[name = tensor("squeeze_45_cast_fp16")]; + tensor squeeze_46_axes_0 = const()[name = tensor("squeeze_46_axes_0"), val = tensor([-3])]; + tensor squeeze_46_cast_fp16 = squeeze(axes = squeeze_46_axes_0, x = var_2601_cast_fp16_1)[name = tensor("squeeze_46_cast_fp16")]; + tensor squeeze_47_axes_0 = const()[name = tensor("squeeze_47_axes_0"), val = tensor([-3])]; + tensor squeeze_47_cast_fp16 = squeeze(axes = squeeze_47_axes_0, x = var_2601_cast_fp16_2)[name = tensor("squeeze_47_cast_fp16")]; + tensor q_61_perm_0 = const()[name = tensor("q_61_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_61_perm_0 = const()[name = tensor("k_61_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_31_perm_0 = const()[name = tensor("value_31_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_61_cast_fp16 = transpose(perm = q_61_perm_0, x = squeeze_45_cast_fp16)[name = tensor("transpose_47")]; + tensor var_2611_cast_fp16 = mul(x = q_61_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2611_cast_fp16")]; + tensor x1_61_begin_0 = const()[name = tensor("x1_61_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_61_end_0 = const()[name = tensor("x1_61_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_61_end_mask_0 = const()[name = tensor("x1_61_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_61_cast_fp16 = slice_by_index(begin = x1_61_begin_0, end = x1_61_end_0, end_mask = x1_61_end_mask_0, x = q_61_cast_fp16)[name = tensor("x1_61_cast_fp16")]; + tensor x2_61_begin_0 = const()[name = tensor("x2_61_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_61_end_0 = const()[name = tensor("x2_61_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_61_end_mask_0 = const()[name = tensor("x2_61_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_61_cast_fp16 = slice_by_index(begin = x2_61_begin_0, end = x2_61_end_0, end_mask = x2_61_end_mask_0, x = q_61_cast_fp16)[name = tensor("x2_61_cast_fp16")]; + tensor const_124_promoted_to_fp16 = const()[name = tensor("const_124_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2623_cast_fp16 = mul(x = x2_61_cast_fp16, y = const_124_promoted_to_fp16)[name = tensor("op_2623_cast_fp16")]; + tensor var_2625_interleave_0 = const()[name = tensor("op_2625_interleave_0"), val = tensor(false)]; + tensor var_2625_cast_fp16 = concat(axis = var_2581, interleave = var_2625_interleave_0, values = (var_2623_cast_fp16, x1_61_cast_fp16))[name = tensor("op_2625_cast_fp16")]; + tensor var_2626_cast_fp16 = mul(x = var_2625_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2626_cast_fp16")]; + tensor q_embed_31_cast_fp16 = add(x = var_2611_cast_fp16, y = var_2626_cast_fp16)[name = tensor("q_embed_31_cast_fp16")]; + tensor k_61_cast_fp16 = transpose(perm = k_61_perm_0, x = squeeze_46_cast_fp16)[name = tensor("transpose_46")]; + tensor var_2629_cast_fp16 = mul(x = k_61_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2629_cast_fp16")]; + tensor x1_63_begin_0 = const()[name = tensor("x1_63_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_63_end_0 = const()[name = tensor("x1_63_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_63_end_mask_0 = const()[name = tensor("x1_63_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_63_cast_fp16 = slice_by_index(begin = x1_63_begin_0, end = x1_63_end_0, end_mask = x1_63_end_mask_0, x = k_61_cast_fp16)[name = tensor("x1_63_cast_fp16")]; + tensor x2_63_begin_0 = const()[name = tensor("x2_63_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_63_end_0 = const()[name = tensor("x2_63_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_63_end_mask_0 = const()[name = tensor("x2_63_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_63_cast_fp16 = slice_by_index(begin = x2_63_begin_0, end = x2_63_end_0, end_mask = x2_63_end_mask_0, x = k_61_cast_fp16)[name = tensor("x2_63_cast_fp16")]; + tensor const_127_promoted_to_fp16 = const()[name = tensor("const_127_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2641_cast_fp16 = mul(x = x2_63_cast_fp16, y = const_127_promoted_to_fp16)[name = tensor("op_2641_cast_fp16")]; + tensor var_2643_interleave_0 = const()[name = tensor("op_2643_interleave_0"), val = tensor(false)]; + tensor var_2643_cast_fp16 = concat(axis = var_2581, interleave = var_2643_interleave_0, values = (var_2641_cast_fp16, x1_63_cast_fp16))[name = tensor("op_2643_cast_fp16")]; + tensor var_2644_cast_fp16 = mul(x = var_2643_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2644_cast_fp16")]; + tensor k_embed_31_cast_fp16 = add(x = var_2629_cast_fp16, y = var_2644_cast_fp16)[name = tensor("k_embed_31_cast_fp16")]; + tensor var_2649_transpose_x_1 = const()[name = tensor("op_2649_transpose_x_1"), val = tensor(false)]; + tensor var_2649_transpose_y_1 = const()[name = tensor("op_2649_transpose_y_1"), val = tensor(true)]; + tensor var_2649_cast_fp16 = matmul(transpose_x = var_2649_transpose_x_1, transpose_y = var_2649_transpose_y_1, x = q_embed_31_cast_fp16, y = k_embed_31_cast_fp16)[name = tensor("op_2649_cast_fp16")]; + tensor var_2650_to_fp16 = const()[name = tensor("op_2650_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_61_cast_fp16 = mul(x = var_2649_cast_fp16, y = var_2650_to_fp16)[name = tensor("attn_weights_61_cast_fp16")]; + tensor input_277_cast_fp16 = add(x = attn_weights_61_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_277_cast_fp16")]; + tensor var_2653_cast_fp16 = softmax(axis = var_2581, x = input_277_cast_fp16)[name = tensor("op_2653_cast_fp16")]; + tensor attn_output_91_transpose_x_0 = const()[name = tensor("attn_output_91_transpose_x_0"), val = tensor(false)]; + tensor attn_output_91_transpose_y_0 = const()[name = tensor("attn_output_91_transpose_y_0"), val = tensor(false)]; + tensor value_31_cast_fp16 = transpose(perm = value_31_perm_0, x = squeeze_47_cast_fp16)[name = tensor("transpose_45")]; + tensor attn_output_91_cast_fp16 = matmul(transpose_x = attn_output_91_transpose_x_0, transpose_y = attn_output_91_transpose_y_0, x = var_2653_cast_fp16, y = value_31_cast_fp16)[name = tensor("attn_output_91_cast_fp16")]; + tensor var_2657_perm_0 = const()[name = tensor("op_2657_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2659 = const()[name = tensor("op_2659"), val = tensor([1, 512, -1])]; + tensor var_2657_cast_fp16 = transpose(perm = var_2657_perm_0, x = attn_output_91_cast_fp16)[name = tensor("transpose_44")]; + tensor var_2660_cast_fp16 = reshape(shape = var_2659, x = var_2657_cast_fp16)[name = tensor("op_2660_cast_fp16")]; + tensor model_encoder_layers_15_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_15_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(352560000)))]; + tensor linear_61_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_15_attn_Wo_weight_to_fp16, x = var_2660_cast_fp16)[name = tensor("linear_61_cast_fp16")]; + tensor input_283_cast_fp16 = add(x = input_275_cast_fp16, y = linear_61_cast_fp16)[name = tensor("input_283_cast_fp16")]; + tensor input_285_axes_0 = const()[name = tensor("input_285_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_15_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_15_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(353739712)))]; + tensor input_285_cast_fp16 = layer_norm(axes = input_285_axes_0, epsilon = var_2592_to_fp16, gamma = model_encoder_layers_15_mlp_norm_weight_to_fp16, x = input_283_cast_fp16)[name = tensor("input_285_cast_fp16")]; + tensor model_encoder_layers_15_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_15_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(353741312)))]; + tensor linear_62_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_15_mlp_Wi_weight_to_fp16, x = input_285_cast_fp16)[name = tensor("linear_62_cast_fp16")]; + tensor var_2667_split_sizes_0 = const()[name = tensor("op_2667_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2667_axis_0 = const()[name = tensor("op_2667_axis_0"), val = tensor(-1)]; + tensor var_2667_cast_fp16_0, tensor var_2667_cast_fp16_1 = split(axis = var_2667_axis_0, split_sizes = var_2667_split_sizes_0, x = linear_62_cast_fp16)[name = tensor("op_2667_cast_fp16")]; + tensor var_2669_mode_0 = const()[name = tensor("op_2669_mode_0"), val = tensor("EXACT")]; + tensor var_2669_cast_fp16 = gelu(mode = var_2669_mode_0, x = var_2667_cast_fp16_0)[name = tensor("op_2669_cast_fp16")]; + tensor input_289_cast_fp16 = mul(x = var_2669_cast_fp16, y = var_2667_cast_fp16_1)[name = tensor("input_289_cast_fp16")]; + tensor model_encoder_layers_15_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_15_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(357280320)))]; + tensor linear_63_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_15_mlp_Wo_weight_to_fp16, x = input_289_cast_fp16)[name = tensor("linear_63_cast_fp16")]; + tensor input_293_cast_fp16 = add(x = input_283_cast_fp16, y = linear_63_cast_fp16)[name = tensor("input_293_cast_fp16")]; + tensor var_2678 = const()[name = tensor("op_2678"), val = tensor(-1)]; + tensor hidden_states_33_axes_0 = const()[name = tensor("hidden_states_33_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_16_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_16_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(359049856)))]; + tensor var_2689_to_fp16 = const()[name = tensor("op_2689_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_33_cast_fp16 = layer_norm(axes = hidden_states_33_axes_0, epsilon = var_2689_to_fp16, gamma = model_encoder_layers_16_attn_norm_weight_to_fp16, x = input_293_cast_fp16)[name = tensor("hidden_states_33_cast_fp16")]; + tensor model_encoder_layers_16_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_16_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(359051456)))]; + tensor linear_64_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_16_attn_Wqkv_weight_to_fp16, x = hidden_states_33_cast_fp16)[name = tensor("linear_64_cast_fp16")]; + tensor var_2696 = const()[name = tensor("op_2696"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_67_cast_fp16 = reshape(shape = var_2696, x = linear_64_cast_fp16)[name = tensor("qkv_67_cast_fp16")]; + tensor var_2698_split_sizes_0 = const()[name = tensor("op_2698_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2698_axis_0 = const()[name = tensor("op_2698_axis_0"), val = tensor(-3)]; + tensor var_2698_cast_fp16_0, tensor var_2698_cast_fp16_1, tensor var_2698_cast_fp16_2 = split(axis = var_2698_axis_0, split_sizes = var_2698_split_sizes_0, x = qkv_67_cast_fp16)[name = tensor("op_2698_cast_fp16")]; + tensor squeeze_48_axes_0 = const()[name = tensor("squeeze_48_axes_0"), val = tensor([-3])]; + tensor squeeze_48_cast_fp16 = squeeze(axes = squeeze_48_axes_0, x = var_2698_cast_fp16_0)[name = tensor("squeeze_48_cast_fp16")]; + tensor squeeze_49_axes_0 = const()[name = tensor("squeeze_49_axes_0"), val = tensor([-3])]; + tensor squeeze_49_cast_fp16 = squeeze(axes = squeeze_49_axes_0, x = var_2698_cast_fp16_1)[name = tensor("squeeze_49_cast_fp16")]; + tensor squeeze_50_axes_0 = const()[name = tensor("squeeze_50_axes_0"), val = tensor([-3])]; + tensor squeeze_50_cast_fp16 = squeeze(axes = squeeze_50_axes_0, x = var_2698_cast_fp16_2)[name = tensor("squeeze_50_cast_fp16")]; + tensor q_65_perm_0 = const()[name = tensor("q_65_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_65_perm_0 = const()[name = tensor("k_65_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_33_perm_0 = const()[name = tensor("value_33_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_65_cast_fp16 = transpose(perm = q_65_perm_0, x = squeeze_48_cast_fp16)[name = tensor("transpose_43")]; + tensor var_2708_cast_fp16 = mul(x = q_65_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2708_cast_fp16")]; + tensor x1_65_begin_0 = const()[name = tensor("x1_65_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_65_end_0 = const()[name = tensor("x1_65_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_65_end_mask_0 = const()[name = tensor("x1_65_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_65_cast_fp16 = slice_by_index(begin = x1_65_begin_0, end = x1_65_end_0, end_mask = x1_65_end_mask_0, x = q_65_cast_fp16)[name = tensor("x1_65_cast_fp16")]; + tensor x2_65_begin_0 = const()[name = tensor("x2_65_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_65_end_0 = const()[name = tensor("x2_65_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_65_end_mask_0 = const()[name = tensor("x2_65_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_65_cast_fp16 = slice_by_index(begin = x2_65_begin_0, end = x2_65_end_0, end_mask = x2_65_end_mask_0, x = q_65_cast_fp16)[name = tensor("x2_65_cast_fp16")]; + tensor const_132_promoted_to_fp16 = const()[name = tensor("const_132_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2720_cast_fp16 = mul(x = x2_65_cast_fp16, y = const_132_promoted_to_fp16)[name = tensor("op_2720_cast_fp16")]; + tensor var_2722_interleave_0 = const()[name = tensor("op_2722_interleave_0"), val = tensor(false)]; + tensor var_2722_cast_fp16 = concat(axis = var_2678, interleave = var_2722_interleave_0, values = (var_2720_cast_fp16, x1_65_cast_fp16))[name = tensor("op_2722_cast_fp16")]; + tensor var_2723_cast_fp16 = mul(x = var_2722_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2723_cast_fp16")]; + tensor q_embed_33_cast_fp16 = add(x = var_2708_cast_fp16, y = var_2723_cast_fp16)[name = tensor("q_embed_33_cast_fp16")]; + tensor k_65_cast_fp16 = transpose(perm = k_65_perm_0, x = squeeze_49_cast_fp16)[name = tensor("transpose_42")]; + tensor var_2726_cast_fp16 = mul(x = k_65_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2726_cast_fp16")]; + tensor x1_67_begin_0 = const()[name = tensor("x1_67_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_67_end_0 = const()[name = tensor("x1_67_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_67_end_mask_0 = const()[name = tensor("x1_67_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_67_cast_fp16 = slice_by_index(begin = x1_67_begin_0, end = x1_67_end_0, end_mask = x1_67_end_mask_0, x = k_65_cast_fp16)[name = tensor("x1_67_cast_fp16")]; + tensor x2_67_begin_0 = const()[name = tensor("x2_67_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_67_end_0 = const()[name = tensor("x2_67_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_67_end_mask_0 = const()[name = tensor("x2_67_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_67_cast_fp16 = slice_by_index(begin = x2_67_begin_0, end = x2_67_end_0, end_mask = x2_67_end_mask_0, x = k_65_cast_fp16)[name = tensor("x2_67_cast_fp16")]; + tensor const_135_promoted_to_fp16 = const()[name = tensor("const_135_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2738_cast_fp16 = mul(x = x2_67_cast_fp16, y = const_135_promoted_to_fp16)[name = tensor("op_2738_cast_fp16")]; + tensor var_2740_interleave_0 = const()[name = tensor("op_2740_interleave_0"), val = tensor(false)]; + tensor var_2740_cast_fp16 = concat(axis = var_2678, interleave = var_2740_interleave_0, values = (var_2738_cast_fp16, x1_67_cast_fp16))[name = tensor("op_2740_cast_fp16")]; + tensor var_2741_cast_fp16 = mul(x = var_2740_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2741_cast_fp16")]; + tensor k_embed_33_cast_fp16 = add(x = var_2726_cast_fp16, y = var_2741_cast_fp16)[name = tensor("k_embed_33_cast_fp16")]; + tensor var_2746_transpose_x_1 = const()[name = tensor("op_2746_transpose_x_1"), val = tensor(false)]; + tensor var_2746_transpose_y_1 = const()[name = tensor("op_2746_transpose_y_1"), val = tensor(true)]; + tensor var_2746_cast_fp16 = matmul(transpose_x = var_2746_transpose_x_1, transpose_y = var_2746_transpose_y_1, x = q_embed_33_cast_fp16, y = k_embed_33_cast_fp16)[name = tensor("op_2746_cast_fp16")]; + tensor var_2747_to_fp16 = const()[name = tensor("op_2747_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_65_cast_fp16 = mul(x = var_2746_cast_fp16, y = var_2747_to_fp16)[name = tensor("attn_weights_65_cast_fp16")]; + tensor input_295_cast_fp16 = add(x = attn_weights_65_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_295_cast_fp16")]; + tensor var_2750_cast_fp16 = softmax(axis = var_2678, x = input_295_cast_fp16)[name = tensor("op_2750_cast_fp16")]; + tensor attn_output_97_transpose_x_0 = const()[name = tensor("attn_output_97_transpose_x_0"), val = tensor(false)]; + tensor attn_output_97_transpose_y_0 = const()[name = tensor("attn_output_97_transpose_y_0"), val = tensor(false)]; + tensor value_33_cast_fp16 = transpose(perm = value_33_perm_0, x = squeeze_50_cast_fp16)[name = tensor("transpose_41")]; + tensor attn_output_97_cast_fp16 = matmul(transpose_x = attn_output_97_transpose_x_0, transpose_y = attn_output_97_transpose_y_0, x = var_2750_cast_fp16, y = value_33_cast_fp16)[name = tensor("attn_output_97_cast_fp16")]; + tensor var_2754_perm_0 = const()[name = tensor("op_2754_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2756 = const()[name = tensor("op_2756"), val = tensor([1, 512, -1])]; + tensor var_2754_cast_fp16 = transpose(perm = var_2754_perm_0, x = attn_output_97_cast_fp16)[name = tensor("transpose_40")]; + tensor var_2757_cast_fp16 = reshape(shape = var_2756, x = var_2754_cast_fp16)[name = tensor("op_2757_cast_fp16")]; + tensor model_encoder_layers_16_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_16_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(362590464)))]; + tensor linear_65_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_16_attn_Wo_weight_to_fp16, x = var_2757_cast_fp16)[name = tensor("linear_65_cast_fp16")]; + tensor input_301_cast_fp16 = add(x = input_293_cast_fp16, y = linear_65_cast_fp16)[name = tensor("input_301_cast_fp16")]; + tensor input_303_axes_0 = const()[name = tensor("input_303_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_16_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_16_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(363770176)))]; + tensor input_303_cast_fp16 = layer_norm(axes = input_303_axes_0, epsilon = var_2689_to_fp16, gamma = model_encoder_layers_16_mlp_norm_weight_to_fp16, x = input_301_cast_fp16)[name = tensor("input_303_cast_fp16")]; + tensor model_encoder_layers_16_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_16_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(363771776)))]; + tensor linear_66_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_16_mlp_Wi_weight_to_fp16, x = input_303_cast_fp16)[name = tensor("linear_66_cast_fp16")]; + tensor var_2764_split_sizes_0 = const()[name = tensor("op_2764_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2764_axis_0 = const()[name = tensor("op_2764_axis_0"), val = tensor(-1)]; + tensor var_2764_cast_fp16_0, tensor var_2764_cast_fp16_1 = split(axis = var_2764_axis_0, split_sizes = var_2764_split_sizes_0, x = linear_66_cast_fp16)[name = tensor("op_2764_cast_fp16")]; + tensor var_2766_mode_0 = const()[name = tensor("op_2766_mode_0"), val = tensor("EXACT")]; + tensor var_2766_cast_fp16 = gelu(mode = var_2766_mode_0, x = var_2764_cast_fp16_0)[name = tensor("op_2766_cast_fp16")]; + tensor input_307_cast_fp16 = mul(x = var_2766_cast_fp16, y = var_2764_cast_fp16_1)[name = tensor("input_307_cast_fp16")]; + tensor model_encoder_layers_16_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_16_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(367310784)))]; + tensor linear_67_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_16_mlp_Wo_weight_to_fp16, x = input_307_cast_fp16)[name = tensor("linear_67_cast_fp16")]; + tensor input_311_cast_fp16 = add(x = input_301_cast_fp16, y = linear_67_cast_fp16)[name = tensor("input_311_cast_fp16")]; + tensor var_2775 = const()[name = tensor("op_2775"), val = tensor(-1)]; + tensor hidden_states_35_axes_0 = const()[name = tensor("hidden_states_35_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_17_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_17_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(369080320)))]; + tensor var_2786_to_fp16 = const()[name = tensor("op_2786_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_35_cast_fp16 = layer_norm(axes = hidden_states_35_axes_0, epsilon = var_2786_to_fp16, gamma = model_encoder_layers_17_attn_norm_weight_to_fp16, x = input_311_cast_fp16)[name = tensor("hidden_states_35_cast_fp16")]; + tensor model_encoder_layers_17_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_17_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(369081920)))]; + tensor linear_68_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_17_attn_Wqkv_weight_to_fp16, x = hidden_states_35_cast_fp16)[name = tensor("linear_68_cast_fp16")]; + tensor var_2793 = const()[name = tensor("op_2793"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_71_cast_fp16 = reshape(shape = var_2793, x = linear_68_cast_fp16)[name = tensor("qkv_71_cast_fp16")]; + tensor var_2795_split_sizes_0 = const()[name = tensor("op_2795_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2795_axis_0 = const()[name = tensor("op_2795_axis_0"), val = tensor(-3)]; + tensor var_2795_cast_fp16_0, tensor var_2795_cast_fp16_1, tensor var_2795_cast_fp16_2 = split(axis = var_2795_axis_0, split_sizes = var_2795_split_sizes_0, x = qkv_71_cast_fp16)[name = tensor("op_2795_cast_fp16")]; + tensor squeeze_51_axes_0 = const()[name = tensor("squeeze_51_axes_0"), val = tensor([-3])]; + tensor squeeze_51_cast_fp16 = squeeze(axes = squeeze_51_axes_0, x = var_2795_cast_fp16_0)[name = tensor("squeeze_51_cast_fp16")]; + tensor squeeze_52_axes_0 = const()[name = tensor("squeeze_52_axes_0"), val = tensor([-3])]; + tensor squeeze_52_cast_fp16 = squeeze(axes = squeeze_52_axes_0, x = var_2795_cast_fp16_1)[name = tensor("squeeze_52_cast_fp16")]; + tensor squeeze_53_axes_0 = const()[name = tensor("squeeze_53_axes_0"), val = tensor([-3])]; + tensor squeeze_53_cast_fp16 = squeeze(axes = squeeze_53_axes_0, x = var_2795_cast_fp16_2)[name = tensor("squeeze_53_cast_fp16")]; + tensor q_69_perm_0 = const()[name = tensor("q_69_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_69_perm_0 = const()[name = tensor("k_69_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_35_perm_0 = const()[name = tensor("value_35_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_69_cast_fp16 = transpose(perm = q_69_perm_0, x = squeeze_51_cast_fp16)[name = tensor("transpose_39")]; + tensor var_2805_cast_fp16 = mul(x = q_69_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2805_cast_fp16")]; + tensor x1_69_begin_0 = const()[name = tensor("x1_69_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_69_end_0 = const()[name = tensor("x1_69_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_69_end_mask_0 = const()[name = tensor("x1_69_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_69_cast_fp16 = slice_by_index(begin = x1_69_begin_0, end = x1_69_end_0, end_mask = x1_69_end_mask_0, x = q_69_cast_fp16)[name = tensor("x1_69_cast_fp16")]; + tensor x2_69_begin_0 = const()[name = tensor("x2_69_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_69_end_0 = const()[name = tensor("x2_69_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_69_end_mask_0 = const()[name = tensor("x2_69_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_69_cast_fp16 = slice_by_index(begin = x2_69_begin_0, end = x2_69_end_0, end_mask = x2_69_end_mask_0, x = q_69_cast_fp16)[name = tensor("x2_69_cast_fp16")]; + tensor const_140_promoted_to_fp16 = const()[name = tensor("const_140_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2817_cast_fp16 = mul(x = x2_69_cast_fp16, y = const_140_promoted_to_fp16)[name = tensor("op_2817_cast_fp16")]; + tensor var_2819_interleave_0 = const()[name = tensor("op_2819_interleave_0"), val = tensor(false)]; + tensor var_2819_cast_fp16 = concat(axis = var_2775, interleave = var_2819_interleave_0, values = (var_2817_cast_fp16, x1_69_cast_fp16))[name = tensor("op_2819_cast_fp16")]; + tensor var_2820_cast_fp16 = mul(x = var_2819_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2820_cast_fp16")]; + tensor q_embed_35_cast_fp16 = add(x = var_2805_cast_fp16, y = var_2820_cast_fp16)[name = tensor("q_embed_35_cast_fp16")]; + tensor k_69_cast_fp16 = transpose(perm = k_69_perm_0, x = squeeze_52_cast_fp16)[name = tensor("transpose_38")]; + tensor var_2823_cast_fp16 = mul(x = k_69_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2823_cast_fp16")]; + tensor x1_71_begin_0 = const()[name = tensor("x1_71_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_71_end_0 = const()[name = tensor("x1_71_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_71_end_mask_0 = const()[name = tensor("x1_71_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_71_cast_fp16 = slice_by_index(begin = x1_71_begin_0, end = x1_71_end_0, end_mask = x1_71_end_mask_0, x = k_69_cast_fp16)[name = tensor("x1_71_cast_fp16")]; + tensor x2_71_begin_0 = const()[name = tensor("x2_71_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_71_end_0 = const()[name = tensor("x2_71_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_71_end_mask_0 = const()[name = tensor("x2_71_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_71_cast_fp16 = slice_by_index(begin = x2_71_begin_0, end = x2_71_end_0, end_mask = x2_71_end_mask_0, x = k_69_cast_fp16)[name = tensor("x2_71_cast_fp16")]; + tensor const_143_promoted_to_fp16 = const()[name = tensor("const_143_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2835_cast_fp16 = mul(x = x2_71_cast_fp16, y = const_143_promoted_to_fp16)[name = tensor("op_2835_cast_fp16")]; + tensor var_2837_interleave_0 = const()[name = tensor("op_2837_interleave_0"), val = tensor(false)]; + tensor var_2837_cast_fp16 = concat(axis = var_2775, interleave = var_2837_interleave_0, values = (var_2835_cast_fp16, x1_71_cast_fp16))[name = tensor("op_2837_cast_fp16")]; + tensor var_2838_cast_fp16 = mul(x = var_2837_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2838_cast_fp16")]; + tensor k_embed_35_cast_fp16 = add(x = var_2823_cast_fp16, y = var_2838_cast_fp16)[name = tensor("k_embed_35_cast_fp16")]; + tensor var_2843_transpose_x_1 = const()[name = tensor("op_2843_transpose_x_1"), val = tensor(false)]; + tensor var_2843_transpose_y_1 = const()[name = tensor("op_2843_transpose_y_1"), val = tensor(true)]; + tensor var_2843_cast_fp16 = matmul(transpose_x = var_2843_transpose_x_1, transpose_y = var_2843_transpose_y_1, x = q_embed_35_cast_fp16, y = k_embed_35_cast_fp16)[name = tensor("op_2843_cast_fp16")]; + tensor var_2844_to_fp16 = const()[name = tensor("op_2844_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_69_cast_fp16 = mul(x = var_2843_cast_fp16, y = var_2844_to_fp16)[name = tensor("attn_weights_69_cast_fp16")]; + tensor input_313_cast_fp16 = add(x = attn_weights_69_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_313_cast_fp16")]; + tensor var_2847_cast_fp16 = softmax(axis = var_2775, x = input_313_cast_fp16)[name = tensor("op_2847_cast_fp16")]; + tensor attn_output_103_transpose_x_0 = const()[name = tensor("attn_output_103_transpose_x_0"), val = tensor(false)]; + tensor attn_output_103_transpose_y_0 = const()[name = tensor("attn_output_103_transpose_y_0"), val = tensor(false)]; + tensor value_35_cast_fp16 = transpose(perm = value_35_perm_0, x = squeeze_53_cast_fp16)[name = tensor("transpose_37")]; + tensor attn_output_103_cast_fp16 = matmul(transpose_x = attn_output_103_transpose_x_0, transpose_y = attn_output_103_transpose_y_0, x = var_2847_cast_fp16, y = value_35_cast_fp16)[name = tensor("attn_output_103_cast_fp16")]; + tensor var_2851_perm_0 = const()[name = tensor("op_2851_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2853 = const()[name = tensor("op_2853"), val = tensor([1, 512, -1])]; + tensor var_2851_cast_fp16 = transpose(perm = var_2851_perm_0, x = attn_output_103_cast_fp16)[name = tensor("transpose_36")]; + tensor var_2854_cast_fp16 = reshape(shape = var_2853, x = var_2851_cast_fp16)[name = tensor("op_2854_cast_fp16")]; + tensor model_encoder_layers_17_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_17_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(372620928)))]; + tensor linear_69_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_17_attn_Wo_weight_to_fp16, x = var_2854_cast_fp16)[name = tensor("linear_69_cast_fp16")]; + tensor input_319_cast_fp16 = add(x = input_311_cast_fp16, y = linear_69_cast_fp16)[name = tensor("input_319_cast_fp16")]; + tensor input_321_axes_0 = const()[name = tensor("input_321_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_17_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_17_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(373800640)))]; + tensor input_321_cast_fp16 = layer_norm(axes = input_321_axes_0, epsilon = var_2786_to_fp16, gamma = model_encoder_layers_17_mlp_norm_weight_to_fp16, x = input_319_cast_fp16)[name = tensor("input_321_cast_fp16")]; + tensor model_encoder_layers_17_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_17_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(373802240)))]; + tensor linear_70_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_17_mlp_Wi_weight_to_fp16, x = input_321_cast_fp16)[name = tensor("linear_70_cast_fp16")]; + tensor var_2861_split_sizes_0 = const()[name = tensor("op_2861_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2861_axis_0 = const()[name = tensor("op_2861_axis_0"), val = tensor(-1)]; + tensor var_2861_cast_fp16_0, tensor var_2861_cast_fp16_1 = split(axis = var_2861_axis_0, split_sizes = var_2861_split_sizes_0, x = linear_70_cast_fp16)[name = tensor("op_2861_cast_fp16")]; + tensor var_2863_mode_0 = const()[name = tensor("op_2863_mode_0"), val = tensor("EXACT")]; + tensor var_2863_cast_fp16 = gelu(mode = var_2863_mode_0, x = var_2861_cast_fp16_0)[name = tensor("op_2863_cast_fp16")]; + tensor input_325_cast_fp16 = mul(x = var_2863_cast_fp16, y = var_2861_cast_fp16_1)[name = tensor("input_325_cast_fp16")]; + tensor model_encoder_layers_17_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_17_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(377341248)))]; + tensor linear_71_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_17_mlp_Wo_weight_to_fp16, x = input_325_cast_fp16)[name = tensor("linear_71_cast_fp16")]; + tensor input_329_cast_fp16 = add(x = input_319_cast_fp16, y = linear_71_cast_fp16)[name = tensor("input_329_cast_fp16")]; + tensor var_2872 = const()[name = tensor("op_2872"), val = tensor(-1)]; + tensor hidden_states_37_axes_0 = const()[name = tensor("hidden_states_37_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_18_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_18_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(379110784)))]; + tensor var_2883_to_fp16 = const()[name = tensor("op_2883_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_37_cast_fp16 = layer_norm(axes = hidden_states_37_axes_0, epsilon = var_2883_to_fp16, gamma = model_encoder_layers_18_attn_norm_weight_to_fp16, x = input_329_cast_fp16)[name = tensor("hidden_states_37_cast_fp16")]; + tensor model_encoder_layers_18_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_18_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(379112384)))]; + tensor linear_72_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_18_attn_Wqkv_weight_to_fp16, x = hidden_states_37_cast_fp16)[name = tensor("linear_72_cast_fp16")]; + tensor var_2890 = const()[name = tensor("op_2890"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_75_cast_fp16 = reshape(shape = var_2890, x = linear_72_cast_fp16)[name = tensor("qkv_75_cast_fp16")]; + tensor var_2892_split_sizes_0 = const()[name = tensor("op_2892_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2892_axis_0 = const()[name = tensor("op_2892_axis_0"), val = tensor(-3)]; + tensor var_2892_cast_fp16_0, tensor var_2892_cast_fp16_1, tensor var_2892_cast_fp16_2 = split(axis = var_2892_axis_0, split_sizes = var_2892_split_sizes_0, x = qkv_75_cast_fp16)[name = tensor("op_2892_cast_fp16")]; + tensor squeeze_54_axes_0 = const()[name = tensor("squeeze_54_axes_0"), val = tensor([-3])]; + tensor squeeze_54_cast_fp16 = squeeze(axes = squeeze_54_axes_0, x = var_2892_cast_fp16_0)[name = tensor("squeeze_54_cast_fp16")]; + tensor squeeze_55_axes_0 = const()[name = tensor("squeeze_55_axes_0"), val = tensor([-3])]; + tensor squeeze_55_cast_fp16 = squeeze(axes = squeeze_55_axes_0, x = var_2892_cast_fp16_1)[name = tensor("squeeze_55_cast_fp16")]; + tensor squeeze_56_axes_0 = const()[name = tensor("squeeze_56_axes_0"), val = tensor([-3])]; + tensor squeeze_56_cast_fp16 = squeeze(axes = squeeze_56_axes_0, x = var_2892_cast_fp16_2)[name = tensor("squeeze_56_cast_fp16")]; + tensor q_73_perm_0 = const()[name = tensor("q_73_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_73_perm_0 = const()[name = tensor("k_73_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_37_perm_0 = const()[name = tensor("value_37_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_73_cast_fp16 = transpose(perm = q_73_perm_0, x = squeeze_54_cast_fp16)[name = tensor("transpose_35")]; + tensor var_2902_cast_fp16 = mul(x = q_73_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2902_cast_fp16")]; + tensor x1_73_begin_0 = const()[name = tensor("x1_73_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_73_end_0 = const()[name = tensor("x1_73_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_73_end_mask_0 = const()[name = tensor("x1_73_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_73_cast_fp16 = slice_by_index(begin = x1_73_begin_0, end = x1_73_end_0, end_mask = x1_73_end_mask_0, x = q_73_cast_fp16)[name = tensor("x1_73_cast_fp16")]; + tensor x2_73_begin_0 = const()[name = tensor("x2_73_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_73_end_0 = const()[name = tensor("x2_73_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_73_end_mask_0 = const()[name = tensor("x2_73_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_73_cast_fp16 = slice_by_index(begin = x2_73_begin_0, end = x2_73_end_0, end_mask = x2_73_end_mask_0, x = q_73_cast_fp16)[name = tensor("x2_73_cast_fp16")]; + tensor const_148_promoted_to_fp16 = const()[name = tensor("const_148_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2914_cast_fp16 = mul(x = x2_73_cast_fp16, y = const_148_promoted_to_fp16)[name = tensor("op_2914_cast_fp16")]; + tensor var_2916_interleave_0 = const()[name = tensor("op_2916_interleave_0"), val = tensor(false)]; + tensor var_2916_cast_fp16 = concat(axis = var_2872, interleave = var_2916_interleave_0, values = (var_2914_cast_fp16, x1_73_cast_fp16))[name = tensor("op_2916_cast_fp16")]; + tensor var_2917_cast_fp16 = mul(x = var_2916_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2917_cast_fp16")]; + tensor q_embed_37_cast_fp16 = add(x = var_2902_cast_fp16, y = var_2917_cast_fp16)[name = tensor("q_embed_37_cast_fp16")]; + tensor k_73_cast_fp16 = transpose(perm = k_73_perm_0, x = squeeze_55_cast_fp16)[name = tensor("transpose_34")]; + tensor var_2920_cast_fp16 = mul(x = k_73_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2920_cast_fp16")]; + tensor x1_75_begin_0 = const()[name = tensor("x1_75_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_75_end_0 = const()[name = tensor("x1_75_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_75_end_mask_0 = const()[name = tensor("x1_75_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_75_cast_fp16 = slice_by_index(begin = x1_75_begin_0, end = x1_75_end_0, end_mask = x1_75_end_mask_0, x = k_73_cast_fp16)[name = tensor("x1_75_cast_fp16")]; + tensor x2_75_begin_0 = const()[name = tensor("x2_75_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_75_end_0 = const()[name = tensor("x2_75_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_75_end_mask_0 = const()[name = tensor("x2_75_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_75_cast_fp16 = slice_by_index(begin = x2_75_begin_0, end = x2_75_end_0, end_mask = x2_75_end_mask_0, x = k_73_cast_fp16)[name = tensor("x2_75_cast_fp16")]; + tensor const_151_promoted_to_fp16 = const()[name = tensor("const_151_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_2932_cast_fp16 = mul(x = x2_75_cast_fp16, y = const_151_promoted_to_fp16)[name = tensor("op_2932_cast_fp16")]; + tensor var_2934_interleave_0 = const()[name = tensor("op_2934_interleave_0"), val = tensor(false)]; + tensor var_2934_cast_fp16 = concat(axis = var_2872, interleave = var_2934_interleave_0, values = (var_2932_cast_fp16, x1_75_cast_fp16))[name = tensor("op_2934_cast_fp16")]; + tensor var_2935_cast_fp16 = mul(x = var_2934_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_2935_cast_fp16")]; + tensor k_embed_37_cast_fp16 = add(x = var_2920_cast_fp16, y = var_2935_cast_fp16)[name = tensor("k_embed_37_cast_fp16")]; + tensor var_2940_transpose_x_1 = const()[name = tensor("op_2940_transpose_x_1"), val = tensor(false)]; + tensor var_2940_transpose_y_1 = const()[name = tensor("op_2940_transpose_y_1"), val = tensor(true)]; + tensor var_2940_cast_fp16 = matmul(transpose_x = var_2940_transpose_x_1, transpose_y = var_2940_transpose_y_1, x = q_embed_37_cast_fp16, y = k_embed_37_cast_fp16)[name = tensor("op_2940_cast_fp16")]; + tensor var_2941_to_fp16 = const()[name = tensor("op_2941_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_73_cast_fp16 = mul(x = var_2940_cast_fp16, y = var_2941_to_fp16)[name = tensor("attn_weights_73_cast_fp16")]; + tensor input_331_cast_fp16 = add(x = attn_weights_73_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_331_cast_fp16")]; + tensor var_2944_cast_fp16 = softmax(axis = var_2872, x = input_331_cast_fp16)[name = tensor("op_2944_cast_fp16")]; + tensor attn_output_109_transpose_x_0 = const()[name = tensor("attn_output_109_transpose_x_0"), val = tensor(false)]; + tensor attn_output_109_transpose_y_0 = const()[name = tensor("attn_output_109_transpose_y_0"), val = tensor(false)]; + tensor value_37_cast_fp16 = transpose(perm = value_37_perm_0, x = squeeze_56_cast_fp16)[name = tensor("transpose_33")]; + tensor attn_output_109_cast_fp16 = matmul(transpose_x = attn_output_109_transpose_x_0, transpose_y = attn_output_109_transpose_y_0, x = var_2944_cast_fp16, y = value_37_cast_fp16)[name = tensor("attn_output_109_cast_fp16")]; + tensor var_2948_perm_0 = const()[name = tensor("op_2948_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_2950 = const()[name = tensor("op_2950"), val = tensor([1, 512, -1])]; + tensor var_2948_cast_fp16 = transpose(perm = var_2948_perm_0, x = attn_output_109_cast_fp16)[name = tensor("transpose_32")]; + tensor var_2951_cast_fp16 = reshape(shape = var_2950, x = var_2948_cast_fp16)[name = tensor("op_2951_cast_fp16")]; + tensor model_encoder_layers_18_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_18_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(382651392)))]; + tensor linear_73_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_18_attn_Wo_weight_to_fp16, x = var_2951_cast_fp16)[name = tensor("linear_73_cast_fp16")]; + tensor input_337_cast_fp16 = add(x = input_329_cast_fp16, y = linear_73_cast_fp16)[name = tensor("input_337_cast_fp16")]; + tensor input_339_axes_0 = const()[name = tensor("input_339_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_18_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_18_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(383831104)))]; + tensor input_339_cast_fp16 = layer_norm(axes = input_339_axes_0, epsilon = var_2883_to_fp16, gamma = model_encoder_layers_18_mlp_norm_weight_to_fp16, x = input_337_cast_fp16)[name = tensor("input_339_cast_fp16")]; + tensor model_encoder_layers_18_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_18_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(383832704)))]; + tensor linear_74_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_18_mlp_Wi_weight_to_fp16, x = input_339_cast_fp16)[name = tensor("linear_74_cast_fp16")]; + tensor var_2958_split_sizes_0 = const()[name = tensor("op_2958_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_2958_axis_0 = const()[name = tensor("op_2958_axis_0"), val = tensor(-1)]; + tensor var_2958_cast_fp16_0, tensor var_2958_cast_fp16_1 = split(axis = var_2958_axis_0, split_sizes = var_2958_split_sizes_0, x = linear_74_cast_fp16)[name = tensor("op_2958_cast_fp16")]; + tensor var_2960_mode_0 = const()[name = tensor("op_2960_mode_0"), val = tensor("EXACT")]; + tensor var_2960_cast_fp16 = gelu(mode = var_2960_mode_0, x = var_2958_cast_fp16_0)[name = tensor("op_2960_cast_fp16")]; + tensor input_343_cast_fp16 = mul(x = var_2960_cast_fp16, y = var_2958_cast_fp16_1)[name = tensor("input_343_cast_fp16")]; + tensor model_encoder_layers_18_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_18_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(387371712)))]; + tensor linear_75_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_18_mlp_Wo_weight_to_fp16, x = input_343_cast_fp16)[name = tensor("linear_75_cast_fp16")]; + tensor input_347_cast_fp16 = add(x = input_337_cast_fp16, y = linear_75_cast_fp16)[name = tensor("input_347_cast_fp16")]; + tensor var_2969 = const()[name = tensor("op_2969"), val = tensor(-1)]; + tensor hidden_states_39_axes_0 = const()[name = tensor("hidden_states_39_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_19_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_19_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389141248)))]; + tensor var_2980_to_fp16 = const()[name = tensor("op_2980_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_39_cast_fp16 = layer_norm(axes = hidden_states_39_axes_0, epsilon = var_2980_to_fp16, gamma = model_encoder_layers_19_attn_norm_weight_to_fp16, x = input_347_cast_fp16)[name = tensor("hidden_states_39_cast_fp16")]; + tensor model_encoder_layers_19_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_19_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(389142848)))]; + tensor linear_76_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_19_attn_Wqkv_weight_to_fp16, x = hidden_states_39_cast_fp16)[name = tensor("linear_76_cast_fp16")]; + tensor var_2987 = const()[name = tensor("op_2987"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_79_cast_fp16 = reshape(shape = var_2987, x = linear_76_cast_fp16)[name = tensor("qkv_79_cast_fp16")]; + tensor var_2989_split_sizes_0 = const()[name = tensor("op_2989_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_2989_axis_0 = const()[name = tensor("op_2989_axis_0"), val = tensor(-3)]; + tensor var_2989_cast_fp16_0, tensor var_2989_cast_fp16_1, tensor var_2989_cast_fp16_2 = split(axis = var_2989_axis_0, split_sizes = var_2989_split_sizes_0, x = qkv_79_cast_fp16)[name = tensor("op_2989_cast_fp16")]; + tensor squeeze_57_axes_0 = const()[name = tensor("squeeze_57_axes_0"), val = tensor([-3])]; + tensor squeeze_57_cast_fp16 = squeeze(axes = squeeze_57_axes_0, x = var_2989_cast_fp16_0)[name = tensor("squeeze_57_cast_fp16")]; + tensor squeeze_58_axes_0 = const()[name = tensor("squeeze_58_axes_0"), val = tensor([-3])]; + tensor squeeze_58_cast_fp16 = squeeze(axes = squeeze_58_axes_0, x = var_2989_cast_fp16_1)[name = tensor("squeeze_58_cast_fp16")]; + tensor squeeze_59_axes_0 = const()[name = tensor("squeeze_59_axes_0"), val = tensor([-3])]; + tensor squeeze_59_cast_fp16 = squeeze(axes = squeeze_59_axes_0, x = var_2989_cast_fp16_2)[name = tensor("squeeze_59_cast_fp16")]; + tensor q_77_perm_0 = const()[name = tensor("q_77_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_77_perm_0 = const()[name = tensor("k_77_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_39_perm_0 = const()[name = tensor("value_39_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_77_cast_fp16 = transpose(perm = q_77_perm_0, x = squeeze_57_cast_fp16)[name = tensor("transpose_31")]; + tensor var_2999_cast_fp16 = mul(x = q_77_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_2999_cast_fp16")]; + tensor x1_77_begin_0 = const()[name = tensor("x1_77_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_77_end_0 = const()[name = tensor("x1_77_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_77_end_mask_0 = const()[name = tensor("x1_77_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_77_cast_fp16 = slice_by_index(begin = x1_77_begin_0, end = x1_77_end_0, end_mask = x1_77_end_mask_0, x = q_77_cast_fp16)[name = tensor("x1_77_cast_fp16")]; + tensor x2_77_begin_0 = const()[name = tensor("x2_77_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_77_end_0 = const()[name = tensor("x2_77_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_77_end_mask_0 = const()[name = tensor("x2_77_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_77_cast_fp16 = slice_by_index(begin = x2_77_begin_0, end = x2_77_end_0, end_mask = x2_77_end_mask_0, x = q_77_cast_fp16)[name = tensor("x2_77_cast_fp16")]; + tensor const_156_promoted_to_fp16 = const()[name = tensor("const_156_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_3011_cast_fp16 = mul(x = x2_77_cast_fp16, y = const_156_promoted_to_fp16)[name = tensor("op_3011_cast_fp16")]; + tensor var_3013_interleave_0 = const()[name = tensor("op_3013_interleave_0"), val = tensor(false)]; + tensor var_3013_cast_fp16 = concat(axis = var_2969, interleave = var_3013_interleave_0, values = (var_3011_cast_fp16, x1_77_cast_fp16))[name = tensor("op_3013_cast_fp16")]; + tensor var_3014_cast_fp16 = mul(x = var_3013_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_3014_cast_fp16")]; + tensor q_embed_39_cast_fp16 = add(x = var_2999_cast_fp16, y = var_3014_cast_fp16)[name = tensor("q_embed_39_cast_fp16")]; + tensor k_77_cast_fp16 = transpose(perm = k_77_perm_0, x = squeeze_58_cast_fp16)[name = tensor("transpose_30")]; + tensor var_3017_cast_fp16 = mul(x = k_77_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_3017_cast_fp16")]; + tensor x1_79_begin_0 = const()[name = tensor("x1_79_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_79_end_0 = const()[name = tensor("x1_79_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_79_end_mask_0 = const()[name = tensor("x1_79_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_79_cast_fp16 = slice_by_index(begin = x1_79_begin_0, end = x1_79_end_0, end_mask = x1_79_end_mask_0, x = k_77_cast_fp16)[name = tensor("x1_79_cast_fp16")]; + tensor x2_79_begin_0 = const()[name = tensor("x2_79_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_79_end_0 = const()[name = tensor("x2_79_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_79_end_mask_0 = const()[name = tensor("x2_79_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_79_cast_fp16 = slice_by_index(begin = x2_79_begin_0, end = x2_79_end_0, end_mask = x2_79_end_mask_0, x = k_77_cast_fp16)[name = tensor("x2_79_cast_fp16")]; + tensor const_159_promoted_to_fp16 = const()[name = tensor("const_159_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_3029_cast_fp16 = mul(x = x2_79_cast_fp16, y = const_159_promoted_to_fp16)[name = tensor("op_3029_cast_fp16")]; + tensor var_3031_interleave_0 = const()[name = tensor("op_3031_interleave_0"), val = tensor(false)]; + tensor var_3031_cast_fp16 = concat(axis = var_2969, interleave = var_3031_interleave_0, values = (var_3029_cast_fp16, x1_79_cast_fp16))[name = tensor("op_3031_cast_fp16")]; + tensor var_3032_cast_fp16 = mul(x = var_3031_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_3032_cast_fp16")]; + tensor k_embed_39_cast_fp16 = add(x = var_3017_cast_fp16, y = var_3032_cast_fp16)[name = tensor("k_embed_39_cast_fp16")]; + tensor var_3037_transpose_x_1 = const()[name = tensor("op_3037_transpose_x_1"), val = tensor(false)]; + tensor var_3037_transpose_y_1 = const()[name = tensor("op_3037_transpose_y_1"), val = tensor(true)]; + tensor var_3037_cast_fp16 = matmul(transpose_x = var_3037_transpose_x_1, transpose_y = var_3037_transpose_y_1, x = q_embed_39_cast_fp16, y = k_embed_39_cast_fp16)[name = tensor("op_3037_cast_fp16")]; + tensor var_3038_to_fp16 = const()[name = tensor("op_3038_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_77_cast_fp16 = mul(x = var_3037_cast_fp16, y = var_3038_to_fp16)[name = tensor("attn_weights_77_cast_fp16")]; + tensor input_349_cast_fp16 = add(x = attn_weights_77_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_349_cast_fp16")]; + tensor var_3041_cast_fp16 = softmax(axis = var_2969, x = input_349_cast_fp16)[name = tensor("op_3041_cast_fp16")]; + tensor attn_output_115_transpose_x_0 = const()[name = tensor("attn_output_115_transpose_x_0"), val = tensor(false)]; + tensor attn_output_115_transpose_y_0 = const()[name = tensor("attn_output_115_transpose_y_0"), val = tensor(false)]; + tensor value_39_cast_fp16 = transpose(perm = value_39_perm_0, x = squeeze_59_cast_fp16)[name = tensor("transpose_29")]; + tensor attn_output_115_cast_fp16 = matmul(transpose_x = attn_output_115_transpose_x_0, transpose_y = attn_output_115_transpose_y_0, x = var_3041_cast_fp16, y = value_39_cast_fp16)[name = tensor("attn_output_115_cast_fp16")]; + tensor var_3045_perm_0 = const()[name = tensor("op_3045_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_3047 = const()[name = tensor("op_3047"), val = tensor([1, 512, -1])]; + tensor var_3045_cast_fp16 = transpose(perm = var_3045_perm_0, x = attn_output_115_cast_fp16)[name = tensor("transpose_28")]; + tensor var_3048_cast_fp16 = reshape(shape = var_3047, x = var_3045_cast_fp16)[name = tensor("op_3048_cast_fp16")]; + tensor model_encoder_layers_19_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_19_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(392681856)))]; + tensor linear_77_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_19_attn_Wo_weight_to_fp16, x = var_3048_cast_fp16)[name = tensor("linear_77_cast_fp16")]; + tensor input_355_cast_fp16 = add(x = input_347_cast_fp16, y = linear_77_cast_fp16)[name = tensor("input_355_cast_fp16")]; + tensor input_357_axes_0 = const()[name = tensor("input_357_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_19_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_19_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(393861568)))]; + tensor input_357_cast_fp16 = layer_norm(axes = input_357_axes_0, epsilon = var_2980_to_fp16, gamma = model_encoder_layers_19_mlp_norm_weight_to_fp16, x = input_355_cast_fp16)[name = tensor("input_357_cast_fp16")]; + tensor model_encoder_layers_19_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_19_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(393863168)))]; + tensor linear_78_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_19_mlp_Wi_weight_to_fp16, x = input_357_cast_fp16)[name = tensor("linear_78_cast_fp16")]; + tensor var_3055_split_sizes_0 = const()[name = tensor("op_3055_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_3055_axis_0 = const()[name = tensor("op_3055_axis_0"), val = tensor(-1)]; + tensor var_3055_cast_fp16_0, tensor var_3055_cast_fp16_1 = split(axis = var_3055_axis_0, split_sizes = var_3055_split_sizes_0, x = linear_78_cast_fp16)[name = tensor("op_3055_cast_fp16")]; + tensor var_3057_mode_0 = const()[name = tensor("op_3057_mode_0"), val = tensor("EXACT")]; + tensor var_3057_cast_fp16 = gelu(mode = var_3057_mode_0, x = var_3055_cast_fp16_0)[name = tensor("op_3057_cast_fp16")]; + tensor input_361_cast_fp16 = mul(x = var_3057_cast_fp16, y = var_3055_cast_fp16_1)[name = tensor("input_361_cast_fp16")]; + tensor model_encoder_layers_19_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_19_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(397402176)))]; + tensor linear_79_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_19_mlp_Wo_weight_to_fp16, x = input_361_cast_fp16)[name = tensor("linear_79_cast_fp16")]; + tensor input_365_cast_fp16 = add(x = input_355_cast_fp16, y = linear_79_cast_fp16)[name = tensor("input_365_cast_fp16")]; + tensor var_3066 = const()[name = tensor("op_3066"), val = tensor(-1)]; + tensor hidden_states_41_axes_0 = const()[name = tensor("hidden_states_41_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_20_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_20_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(399171712)))]; + tensor var_3077_to_fp16 = const()[name = tensor("op_3077_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_41_cast_fp16 = layer_norm(axes = hidden_states_41_axes_0, epsilon = var_3077_to_fp16, gamma = model_encoder_layers_20_attn_norm_weight_to_fp16, x = input_365_cast_fp16)[name = tensor("hidden_states_41_cast_fp16")]; + tensor model_encoder_layers_20_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_20_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(399173312)))]; + tensor linear_80_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_20_attn_Wqkv_weight_to_fp16, x = hidden_states_41_cast_fp16)[name = tensor("linear_80_cast_fp16")]; + tensor var_3084 = const()[name = tensor("op_3084"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_83_cast_fp16 = reshape(shape = var_3084, x = linear_80_cast_fp16)[name = tensor("qkv_83_cast_fp16")]; + tensor var_3086_split_sizes_0 = const()[name = tensor("op_3086_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_3086_axis_0 = const()[name = tensor("op_3086_axis_0"), val = tensor(-3)]; + tensor var_3086_cast_fp16_0, tensor var_3086_cast_fp16_1, tensor var_3086_cast_fp16_2 = split(axis = var_3086_axis_0, split_sizes = var_3086_split_sizes_0, x = qkv_83_cast_fp16)[name = tensor("op_3086_cast_fp16")]; + tensor squeeze_60_axes_0 = const()[name = tensor("squeeze_60_axes_0"), val = tensor([-3])]; + tensor squeeze_60_cast_fp16 = squeeze(axes = squeeze_60_axes_0, x = var_3086_cast_fp16_0)[name = tensor("squeeze_60_cast_fp16")]; + tensor squeeze_61_axes_0 = const()[name = tensor("squeeze_61_axes_0"), val = tensor([-3])]; + tensor squeeze_61_cast_fp16 = squeeze(axes = squeeze_61_axes_0, x = var_3086_cast_fp16_1)[name = tensor("squeeze_61_cast_fp16")]; + tensor squeeze_62_axes_0 = const()[name = tensor("squeeze_62_axes_0"), val = tensor([-3])]; + tensor squeeze_62_cast_fp16 = squeeze(axes = squeeze_62_axes_0, x = var_3086_cast_fp16_2)[name = tensor("squeeze_62_cast_fp16")]; + tensor q_81_perm_0 = const()[name = tensor("q_81_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_81_perm_0 = const()[name = tensor("k_81_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_41_perm_0 = const()[name = tensor("value_41_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_81_cast_fp16 = transpose(perm = q_81_perm_0, x = squeeze_60_cast_fp16)[name = tensor("transpose_27")]; + tensor var_3096_cast_fp16 = mul(x = q_81_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_3096_cast_fp16")]; + tensor x1_81_begin_0 = const()[name = tensor("x1_81_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_81_end_0 = const()[name = tensor("x1_81_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_81_end_mask_0 = const()[name = tensor("x1_81_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_81_cast_fp16 = slice_by_index(begin = x1_81_begin_0, end = x1_81_end_0, end_mask = x1_81_end_mask_0, x = q_81_cast_fp16)[name = tensor("x1_81_cast_fp16")]; + tensor x2_81_begin_0 = const()[name = tensor("x2_81_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_81_end_0 = const()[name = tensor("x2_81_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_81_end_mask_0 = const()[name = tensor("x2_81_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_81_cast_fp16 = slice_by_index(begin = x2_81_begin_0, end = x2_81_end_0, end_mask = x2_81_end_mask_0, x = q_81_cast_fp16)[name = tensor("x2_81_cast_fp16")]; + tensor const_164_promoted_to_fp16 = const()[name = tensor("const_164_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_3108_cast_fp16 = mul(x = x2_81_cast_fp16, y = const_164_promoted_to_fp16)[name = tensor("op_3108_cast_fp16")]; + tensor var_3110_interleave_0 = const()[name = tensor("op_3110_interleave_0"), val = tensor(false)]; + tensor var_3110_cast_fp16 = concat(axis = var_3066, interleave = var_3110_interleave_0, values = (var_3108_cast_fp16, x1_81_cast_fp16))[name = tensor("op_3110_cast_fp16")]; + tensor var_3111_cast_fp16 = mul(x = var_3110_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_3111_cast_fp16")]; + tensor q_embed_41_cast_fp16 = add(x = var_3096_cast_fp16, y = var_3111_cast_fp16)[name = tensor("q_embed_41_cast_fp16")]; + tensor k_81_cast_fp16 = transpose(perm = k_81_perm_0, x = squeeze_61_cast_fp16)[name = tensor("transpose_26")]; + tensor var_3114_cast_fp16 = mul(x = k_81_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_3114_cast_fp16")]; + tensor x1_83_begin_0 = const()[name = tensor("x1_83_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_83_end_0 = const()[name = tensor("x1_83_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_83_end_mask_0 = const()[name = tensor("x1_83_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_83_cast_fp16 = slice_by_index(begin = x1_83_begin_0, end = x1_83_end_0, end_mask = x1_83_end_mask_0, x = k_81_cast_fp16)[name = tensor("x1_83_cast_fp16")]; + tensor x2_83_begin_0 = const()[name = tensor("x2_83_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_83_end_0 = const()[name = tensor("x2_83_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_83_end_mask_0 = const()[name = tensor("x2_83_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_83_cast_fp16 = slice_by_index(begin = x2_83_begin_0, end = x2_83_end_0, end_mask = x2_83_end_mask_0, x = k_81_cast_fp16)[name = tensor("x2_83_cast_fp16")]; + tensor const_167_promoted_to_fp16 = const()[name = tensor("const_167_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_3126_cast_fp16 = mul(x = x2_83_cast_fp16, y = const_167_promoted_to_fp16)[name = tensor("op_3126_cast_fp16")]; + tensor var_3128_interleave_0 = const()[name = tensor("op_3128_interleave_0"), val = tensor(false)]; + tensor var_3128_cast_fp16 = concat(axis = var_3066, interleave = var_3128_interleave_0, values = (var_3126_cast_fp16, x1_83_cast_fp16))[name = tensor("op_3128_cast_fp16")]; + tensor var_3129_cast_fp16 = mul(x = var_3128_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_3129_cast_fp16")]; + tensor k_embed_41_cast_fp16 = add(x = var_3114_cast_fp16, y = var_3129_cast_fp16)[name = tensor("k_embed_41_cast_fp16")]; + tensor var_3134_transpose_x_1 = const()[name = tensor("op_3134_transpose_x_1"), val = tensor(false)]; + tensor var_3134_transpose_y_1 = const()[name = tensor("op_3134_transpose_y_1"), val = tensor(true)]; + tensor var_3134_cast_fp16 = matmul(transpose_x = var_3134_transpose_x_1, transpose_y = var_3134_transpose_y_1, x = q_embed_41_cast_fp16, y = k_embed_41_cast_fp16)[name = tensor("op_3134_cast_fp16")]; + tensor var_3135_to_fp16 = const()[name = tensor("op_3135_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_81_cast_fp16 = mul(x = var_3134_cast_fp16, y = var_3135_to_fp16)[name = tensor("attn_weights_81_cast_fp16")]; + tensor input_367_cast_fp16 = add(x = attn_weights_81_cast_fp16, y = attention_mask_cast_fp16)[name = tensor("input_367_cast_fp16")]; + tensor var_3138_cast_fp16 = softmax(axis = var_3066, x = input_367_cast_fp16)[name = tensor("op_3138_cast_fp16")]; + tensor attn_output_121_transpose_x_0 = const()[name = tensor("attn_output_121_transpose_x_0"), val = tensor(false)]; + tensor attn_output_121_transpose_y_0 = const()[name = tensor("attn_output_121_transpose_y_0"), val = tensor(false)]; + tensor value_41_cast_fp16 = transpose(perm = value_41_perm_0, x = squeeze_62_cast_fp16)[name = tensor("transpose_25")]; + tensor attn_output_121_cast_fp16 = matmul(transpose_x = attn_output_121_transpose_x_0, transpose_y = attn_output_121_transpose_y_0, x = var_3138_cast_fp16, y = value_41_cast_fp16)[name = tensor("attn_output_121_cast_fp16")]; + tensor var_3142_perm_0 = const()[name = tensor("op_3142_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_3144 = const()[name = tensor("op_3144"), val = tensor([1, 512, -1])]; + tensor var_3142_cast_fp16 = transpose(perm = var_3142_perm_0, x = attn_output_121_cast_fp16)[name = tensor("transpose_24")]; + tensor var_3145_cast_fp16 = reshape(shape = var_3144, x = var_3142_cast_fp16)[name = tensor("op_3145_cast_fp16")]; + tensor model_encoder_layers_20_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_20_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(402712320)))]; + tensor linear_81_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_20_attn_Wo_weight_to_fp16, x = var_3145_cast_fp16)[name = tensor("linear_81_cast_fp16")]; + tensor input_373_cast_fp16 = add(x = input_365_cast_fp16, y = linear_81_cast_fp16)[name = tensor("input_373_cast_fp16")]; + tensor input_375_axes_0 = const()[name = tensor("input_375_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_20_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_20_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(403892032)))]; + tensor input_375_cast_fp16 = layer_norm(axes = input_375_axes_0, epsilon = var_3077_to_fp16, gamma = model_encoder_layers_20_mlp_norm_weight_to_fp16, x = input_373_cast_fp16)[name = tensor("input_375_cast_fp16")]; + tensor model_encoder_layers_20_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_20_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(403893632)))]; + tensor linear_82_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_20_mlp_Wi_weight_to_fp16, x = input_375_cast_fp16)[name = tensor("linear_82_cast_fp16")]; + tensor var_3152_split_sizes_0 = const()[name = tensor("op_3152_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_3152_axis_0 = const()[name = tensor("op_3152_axis_0"), val = tensor(-1)]; + tensor var_3152_cast_fp16_0, tensor var_3152_cast_fp16_1 = split(axis = var_3152_axis_0, split_sizes = var_3152_split_sizes_0, x = linear_82_cast_fp16)[name = tensor("op_3152_cast_fp16")]; + tensor var_3154_mode_0 = const()[name = tensor("op_3154_mode_0"), val = tensor("EXACT")]; + tensor var_3154_cast_fp16 = gelu(mode = var_3154_mode_0, x = var_3152_cast_fp16_0)[name = tensor("op_3154_cast_fp16")]; + tensor input_379_cast_fp16 = mul(x = var_3154_cast_fp16, y = var_3152_cast_fp16_1)[name = tensor("input_379_cast_fp16")]; + tensor model_encoder_layers_20_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_20_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(407432640)))]; + tensor linear_83_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_20_mlp_Wo_weight_to_fp16, x = input_379_cast_fp16)[name = tensor("linear_83_cast_fp16")]; + tensor input_383_cast_fp16 = add(x = input_373_cast_fp16, y = linear_83_cast_fp16)[name = tensor("input_383_cast_fp16")]; + tensor var_3163 = const()[name = tensor("op_3163"), val = tensor(-1)]; + tensor hidden_states_axes_0 = const()[name = tensor("hidden_states_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_21_attn_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_21_attn_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(409202176)))]; + tensor var_3174_to_fp16 = const()[name = tensor("op_3174_to_fp16"), val = tensor(0x1.5p-17)]; + tensor hidden_states_cast_fp16 = layer_norm(axes = hidden_states_axes_0, epsilon = var_3174_to_fp16, gamma = model_encoder_layers_21_attn_norm_weight_to_fp16, x = input_383_cast_fp16)[name = tensor("hidden_states_cast_fp16")]; + tensor model_encoder_layers_21_attn_Wqkv_weight_to_fp16 = const()[name = tensor("model_encoder_layers_21_attn_Wqkv_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(409203776)))]; + tensor linear_84_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_21_attn_Wqkv_weight_to_fp16, x = hidden_states_cast_fp16)[name = tensor("linear_84_cast_fp16")]; + tensor var_3181 = const()[name = tensor("op_3181"), val = tensor([1, 512, 3, -1, 64])]; + tensor qkv_87_cast_fp16 = reshape(shape = var_3181, x = linear_84_cast_fp16)[name = tensor("qkv_87_cast_fp16")]; + tensor var_3183_split_sizes_0 = const()[name = tensor("op_3183_split_sizes_0"), val = tensor([1, 1, 1])]; + tensor var_3183_axis_0 = const()[name = tensor("op_3183_axis_0"), val = tensor(-3)]; + tensor var_3183_cast_fp16_0, tensor var_3183_cast_fp16_1, tensor var_3183_cast_fp16_2 = split(axis = var_3183_axis_0, split_sizes = var_3183_split_sizes_0, x = qkv_87_cast_fp16)[name = tensor("op_3183_cast_fp16")]; + tensor squeeze_63_axes_0 = const()[name = tensor("squeeze_63_axes_0"), val = tensor([-3])]; + tensor squeeze_63_cast_fp16 = squeeze(axes = squeeze_63_axes_0, x = var_3183_cast_fp16_0)[name = tensor("squeeze_63_cast_fp16")]; + tensor squeeze_64_axes_0 = const()[name = tensor("squeeze_64_axes_0"), val = tensor([-3])]; + tensor squeeze_64_cast_fp16 = squeeze(axes = squeeze_64_axes_0, x = var_3183_cast_fp16_1)[name = tensor("squeeze_64_cast_fp16")]; + tensor squeeze_65_axes_0 = const()[name = tensor("squeeze_65_axes_0"), val = tensor([-3])]; + tensor squeeze_65_cast_fp16 = squeeze(axes = squeeze_65_axes_0, x = var_3183_cast_fp16_2)[name = tensor("squeeze_65_cast_fp16")]; + tensor q_85_perm_0 = const()[name = tensor("q_85_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor k_85_perm_0 = const()[name = tensor("k_85_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor value_perm_0 = const()[name = tensor("value_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor q_85_cast_fp16 = transpose(perm = q_85_perm_0, x = squeeze_63_cast_fp16)[name = tensor("transpose_23")]; + tensor var_3193_cast_fp16 = mul(x = q_85_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_3193_cast_fp16")]; + tensor x1_85_begin_0 = const()[name = tensor("x1_85_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_85_end_0 = const()[name = tensor("x1_85_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_85_end_mask_0 = const()[name = tensor("x1_85_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_85_cast_fp16 = slice_by_index(begin = x1_85_begin_0, end = x1_85_end_0, end_mask = x1_85_end_mask_0, x = q_85_cast_fp16)[name = tensor("x1_85_cast_fp16")]; + tensor x2_85_begin_0 = const()[name = tensor("x2_85_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_85_end_0 = const()[name = tensor("x2_85_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_85_end_mask_0 = const()[name = tensor("x2_85_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_85_cast_fp16 = slice_by_index(begin = x2_85_begin_0, end = x2_85_end_0, end_mask = x2_85_end_mask_0, x = q_85_cast_fp16)[name = tensor("x2_85_cast_fp16")]; + tensor const_172_promoted_to_fp16 = const()[name = tensor("const_172_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_3205_cast_fp16 = mul(x = x2_85_cast_fp16, y = const_172_promoted_to_fp16)[name = tensor("op_3205_cast_fp16")]; + tensor var_3207_interleave_0 = const()[name = tensor("op_3207_interleave_0"), val = tensor(false)]; + tensor var_3207_cast_fp16 = concat(axis = var_3163, interleave = var_3207_interleave_0, values = (var_3205_cast_fp16, x1_85_cast_fp16))[name = tensor("op_3207_cast_fp16")]; + tensor var_3208_cast_fp16 = mul(x = var_3207_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_3208_cast_fp16")]; + tensor q_embed_cast_fp16 = add(x = var_3193_cast_fp16, y = var_3208_cast_fp16)[name = tensor("q_embed_cast_fp16")]; + tensor k_85_cast_fp16 = transpose(perm = k_85_perm_0, x = squeeze_64_cast_fp16)[name = tensor("transpose_22")]; + tensor var_3211_cast_fp16 = mul(x = k_85_cast_fp16, y = cos_3_to_fp16)[name = tensor("op_3211_cast_fp16")]; + tensor x1_begin_0 = const()[name = tensor("x1_begin_0"), val = tensor([0, 0, 0, 0])]; + tensor x1_end_0 = const()[name = tensor("x1_end_0"), val = tensor([1, 12, 512, 32])]; + tensor x1_end_mask_0 = const()[name = tensor("x1_end_mask_0"), val = tensor([true, true, true, false])]; + tensor x1_cast_fp16 = slice_by_index(begin = x1_begin_0, end = x1_end_0, end_mask = x1_end_mask_0, x = k_85_cast_fp16)[name = tensor("x1_cast_fp16")]; + tensor x2_begin_0 = const()[name = tensor("x2_begin_0"), val = tensor([0, 0, 0, 32])]; + tensor x2_end_0 = const()[name = tensor("x2_end_0"), val = tensor([1, 12, 512, 64])]; + tensor x2_end_mask_0 = const()[name = tensor("x2_end_mask_0"), val = tensor([true, true, true, true])]; + tensor x2_cast_fp16 = slice_by_index(begin = x2_begin_0, end = x2_end_0, end_mask = x2_end_mask_0, x = k_85_cast_fp16)[name = tensor("x2_cast_fp16")]; + tensor const_175_promoted_to_fp16 = const()[name = tensor("const_175_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_3223_cast_fp16 = mul(x = x2_cast_fp16, y = const_175_promoted_to_fp16)[name = tensor("op_3223_cast_fp16")]; + tensor var_3225_interleave_0 = const()[name = tensor("op_3225_interleave_0"), val = tensor(false)]; + tensor var_3225_cast_fp16 = concat(axis = var_3163, interleave = var_3225_interleave_0, values = (var_3223_cast_fp16, x1_cast_fp16))[name = tensor("op_3225_cast_fp16")]; + tensor var_3226_cast_fp16 = mul(x = var_3225_cast_fp16, y = sin_3_to_fp16)[name = tensor("op_3226_cast_fp16")]; + tensor k_embed_cast_fp16 = add(x = var_3211_cast_fp16, y = var_3226_cast_fp16)[name = tensor("k_embed_cast_fp16")]; + tensor var_3231_transpose_x_1 = const()[name = tensor("op_3231_transpose_x_1"), val = tensor(false)]; + tensor var_3231_transpose_y_1 = const()[name = tensor("op_3231_transpose_y_1"), val = tensor(true)]; + tensor var_3231_cast_fp16 = matmul(transpose_x = var_3231_transpose_x_1, transpose_y = var_3231_transpose_y_1, x = q_embed_cast_fp16, y = k_embed_cast_fp16)[name = tensor("op_3231_cast_fp16")]; + tensor var_3232_to_fp16 = const()[name = tensor("op_3232_to_fp16"), val = tensor(0x1p-3)]; + tensor attn_weights_85_cast_fp16 = mul(x = var_3231_cast_fp16, y = var_3232_to_fp16)[name = tensor("attn_weights_85_cast_fp16")]; + tensor input_385_cast_fp16 = add(x = attn_weights_85_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("input_385_cast_fp16")]; + tensor var_3235_cast_fp16 = softmax(axis = var_3163, x = input_385_cast_fp16)[name = tensor("op_3235_cast_fp16")]; + tensor attn_output_127_transpose_x_0 = const()[name = tensor("attn_output_127_transpose_x_0"), val = tensor(false)]; + tensor attn_output_127_transpose_y_0 = const()[name = tensor("attn_output_127_transpose_y_0"), val = tensor(false)]; + tensor value_cast_fp16 = transpose(perm = value_perm_0, x = squeeze_65_cast_fp16)[name = tensor("transpose_21")]; + tensor attn_output_127_cast_fp16 = matmul(transpose_x = attn_output_127_transpose_x_0, transpose_y = attn_output_127_transpose_y_0, x = var_3235_cast_fp16, y = value_cast_fp16)[name = tensor("attn_output_127_cast_fp16")]; + tensor var_3239_perm_0 = const()[name = tensor("op_3239_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_3241 = const()[name = tensor("op_3241"), val = tensor([1, 512, -1])]; + tensor var_3239_cast_fp16 = transpose(perm = var_3239_perm_0, x = attn_output_127_cast_fp16)[name = tensor("transpose_20")]; + tensor var_3242_cast_fp16 = reshape(shape = var_3241, x = var_3239_cast_fp16)[name = tensor("op_3242_cast_fp16")]; + tensor model_encoder_layers_21_attn_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_21_attn_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(412742784)))]; + tensor linear_85_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_21_attn_Wo_weight_to_fp16, x = var_3242_cast_fp16)[name = tensor("linear_85_cast_fp16")]; + tensor input_391_cast_fp16 = add(x = input_383_cast_fp16, y = linear_85_cast_fp16)[name = tensor("input_391_cast_fp16")]; + tensor input_393_axes_0 = const()[name = tensor("input_393_axes_0"), val = tensor([-1])]; + tensor model_encoder_layers_21_mlp_norm_weight_to_fp16 = const()[name = tensor("model_encoder_layers_21_mlp_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(413922496)))]; + tensor input_393_cast_fp16 = layer_norm(axes = input_393_axes_0, epsilon = var_3174_to_fp16, gamma = model_encoder_layers_21_mlp_norm_weight_to_fp16, x = input_391_cast_fp16)[name = tensor("input_393_cast_fp16")]; + tensor model_encoder_layers_21_mlp_Wi_weight_to_fp16 = const()[name = tensor("model_encoder_layers_21_mlp_Wi_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(413924096)))]; + tensor linear_86_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = model_encoder_layers_21_mlp_Wi_weight_to_fp16, x = input_393_cast_fp16)[name = tensor("linear_86_cast_fp16")]; + tensor var_3249_split_sizes_0 = const()[name = tensor("op_3249_split_sizes_0"), val = tensor([1152, 1152])]; + tensor var_3249_axis_0 = const()[name = tensor("op_3249_axis_0"), val = tensor(-1)]; + tensor var_3249_cast_fp16_0, tensor var_3249_cast_fp16_1 = split(axis = var_3249_axis_0, split_sizes = var_3249_split_sizes_0, x = linear_86_cast_fp16)[name = tensor("op_3249_cast_fp16")]; + tensor var_3251_mode_0 = const()[name = tensor("op_3251_mode_0"), val = tensor("EXACT")]; + tensor var_3251_cast_fp16 = gelu(mode = var_3251_mode_0, x = var_3249_cast_fp16_0)[name = tensor("op_3251_cast_fp16")]; + tensor input_397_cast_fp16 = mul(x = var_3251_cast_fp16, y = var_3249_cast_fp16_1)[name = tensor("input_397_cast_fp16")]; + tensor model_encoder_layers_21_mlp_Wo_weight_to_fp16 = const()[name = tensor("model_encoder_layers_21_mlp_Wo_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(417463104)))]; + tensor linear_87_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = model_encoder_layers_21_mlp_Wo_weight_to_fp16, x = input_397_cast_fp16)[name = tensor("linear_87_cast_fp16")]; + tensor input_401_cast_fp16 = add(x = input_391_cast_fp16, y = linear_87_cast_fp16)[name = tensor("input_401_cast_fp16")]; + tensor x_89_axes_0 = const()[name = tensor("x_89_axes_0"), val = tensor([-1])]; + tensor model_encoder_final_norm_weight_to_fp16 = const()[name = tensor("model_encoder_final_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(419232640)))]; + tensor var_3257_to_fp16 = const()[name = tensor("op_3257_to_fp16"), val = tensor(0x1.5p-17)]; + tensor x_89_cast_fp16 = layer_norm(axes = x_89_axes_0, epsilon = var_3257_to_fp16, gamma = model_encoder_final_norm_weight_to_fp16, x = input_401_cast_fp16)[name = tensor("x_89_cast_fp16")]; + tensor question_type_to_fp16_dtype_0 = const()[name = tensor("question_type_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor transpose_0_to_fp16 = const()[name = tensor("transpose_0_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(419234240)))]; + tensor question_type_to_fp16 = cast(dtype = question_type_to_fp16_dtype_0, x = question_type)[name = tensor("cast_4")]; + tensor var_3262_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = transpose_0_to_fp16, x = question_type_to_fp16)[name = tensor("op_3262_cast_fp16")]; + tensor var_3264_axes_0 = const()[name = tensor("op_3264_axes_0"), val = tensor([1])]; + tensor var_3264_cast_fp16 = expand_dims(axes = var_3264_axes_0, x = var_3262_cast_fp16)[name = tensor("op_3264_cast_fp16")]; + tensor input_403_cast_fp16 = add(x = x_89_cast_fp16, y = var_3264_cast_fp16)[name = tensor("input_403_cast_fp16")]; + tensor h_1_axes_0 = const()[name = tensor("h_1_axes_0"), val = tensor([-1])]; + tensor model_head_layers_0_norm1_weight_to_fp16 = const()[name = tensor("model_head_layers_0_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(419238912)))]; + tensor model_head_layers_0_norm1_bias_to_fp16 = const()[name = tensor("model_head_layers_0_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(419240512)))]; + tensor var_3268_to_fp16 = const()[name = tensor("op_3268_to_fp16"), val = tensor(0x1.5p-17)]; + tensor h_1_cast_fp16 = layer_norm(axes = h_1_axes_0, beta = model_head_layers_0_norm1_bias_to_fp16, epsilon = var_3268_to_fp16, gamma = model_head_layers_0_norm1_weight_to_fp16, x = input_403_cast_fp16)[name = tensor("h_1_cast_fp16")]; + tensor model_head_layers_0_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_head_layers_0_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(419242112)))]; + tensor model_head_layers_0_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_head_layers_0_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(422781120)))]; + tensor linear_88_cast_fp16 = linear(bias = model_head_layers_0_self_attn_in_proj_bias_to_fp16, weight = model_head_layers_0_self_attn_in_proj_weight_to_fp16, x = h_1_cast_fp16)[name = tensor("linear_88_cast_fp16")]; + tensor var_3291_split_sizes_0 = const()[name = tensor("op_3291_split_sizes_0"), val = tensor([768, 768, 768])]; + tensor var_3291_axis_0 = const()[name = tensor("op_3291_axis_0"), val = tensor(-1)]; + tensor var_3291_cast_fp16_0, tensor var_3291_cast_fp16_1, tensor var_3291_cast_fp16_2 = split(axis = var_3291_axis_0, split_sizes = var_3291_split_sizes_0, x = linear_88_cast_fp16)[name = tensor("op_3291_cast_fp16")]; + tensor var_3300 = const()[name = tensor("op_3300"), val = tensor([1, 512, 12, 64])]; + tensor var_3301_cast_fp16 = reshape(shape = var_3300, x = var_3291_cast_fp16_0)[name = tensor("op_3301_cast_fp16")]; + tensor var_3306 = const()[name = tensor("op_3306"), val = tensor([1, 512, 12, 64])]; + tensor var_3307_cast_fp16 = reshape(shape = var_3306, x = var_3291_cast_fp16_1)[name = tensor("op_3307_cast_fp16")]; + tensor var_3312 = const()[name = tensor("op_3312"), val = tensor([1, 512, 12, 64])]; + tensor var_3313_cast_fp16 = reshape(shape = var_3312, x = var_3291_cast_fp16_2)[name = tensor("op_3313_cast_fp16")]; + tensor v_3_perm_0 = const()[name = tensor("v_3_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor var_3320_transpose_x_0 = const()[name = tensor("op_3320_transpose_x_0"), val = tensor(false)]; + tensor var_3320_transpose_y_0 = const()[name = tensor("op_3320_transpose_y_0"), val = tensor(false)]; + tensor transpose_8_perm_0 = const()[name = tensor("transpose_8_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_9_perm_0 = const()[name = tensor("transpose_9_perm_0"), val = tensor([0, 2, -1, -3])]; + tensor transpose_9 = transpose(perm = transpose_9_perm_0, x = var_3307_cast_fp16)[name = tensor("transpose_18")]; + tensor transpose_8 = transpose(perm = transpose_8_perm_0, x = var_3301_cast_fp16)[name = tensor("transpose_19")]; + tensor var_3320_cast_fp16 = matmul(transpose_x = var_3320_transpose_x_0, transpose_y = var_3320_transpose_y_0, x = transpose_8, y = transpose_9)[name = tensor("op_3320_cast_fp16")]; + tensor var_3322_to_fp16 = const()[name = tensor("op_3322_to_fp16"), val = tensor([0x1p-3])]; + tensor var_3323_cast_fp16 = mul(x = var_3320_cast_fp16, y = var_3322_to_fp16)[name = tensor("op_3323_cast_fp16")]; + tensor scores_1_cast_fp16 = add(x = var_3323_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("scores_1_cast_fp16")]; + tensor var_3326 = const()[name = tensor("op_3326"), val = tensor(-1)]; + tensor weights_1_cast_fp16 = softmax(axis = var_3326, x = scores_1_cast_fp16)[name = tensor("weights_1_cast_fp16")]; + tensor var_3329_transpose_x_0 = const()[name = tensor("op_3329_transpose_x_0"), val = tensor(false)]; + tensor var_3329_transpose_y_0 = const()[name = tensor("op_3329_transpose_y_0"), val = tensor(false)]; + tensor v_3_cast_fp16 = transpose(perm = v_3_perm_0, x = var_3313_cast_fp16)[name = tensor("transpose_17")]; + tensor var_3329_cast_fp16 = matmul(transpose_x = var_3329_transpose_x_0, transpose_y = var_3329_transpose_y_0, x = weights_1_cast_fp16, y = v_3_cast_fp16)[name = tensor("op_3329_cast_fp16")]; + tensor var_3332_perm_0 = const()[name = tensor("op_3332_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_3333 = const()[name = tensor("op_3333"), val = tensor([1, 512, 768])]; + tensor var_3332_cast_fp16 = transpose(perm = var_3332_perm_0, x = var_3329_cast_fp16)[name = tensor("transpose_16")]; + tensor input_405_cast_fp16 = reshape(shape = var_3333, x = var_3332_cast_fp16)[name = tensor("input_405_cast_fp16")]; + tensor model_head_layers_0_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_head_layers_0_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(422785792)))]; + tensor model_head_layers_0_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_head_layers_0_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(423965504)))]; + tensor linear_89_cast_fp16 = linear(bias = model_head_layers_0_self_attn_out_proj_bias_to_fp16, weight = model_head_layers_0_self_attn_out_proj_weight_to_fp16, x = input_405_cast_fp16)[name = tensor("linear_89_cast_fp16")]; + tensor input_407_cast_fp16 = add(x = input_403_cast_fp16, y = linear_89_cast_fp16)[name = tensor("input_407_cast_fp16")]; + tensor input_409_axes_0 = const()[name = tensor("input_409_axes_0"), val = tensor([-1])]; + tensor model_head_layers_0_norm2_weight_to_fp16 = const()[name = tensor("model_head_layers_0_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(423967104)))]; + tensor model_head_layers_0_norm2_bias_to_fp16 = const()[name = tensor("model_head_layers_0_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(423968704)))]; + tensor var_3339_to_fp16 = const()[name = tensor("op_3339_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_409_cast_fp16 = layer_norm(axes = input_409_axes_0, beta = model_head_layers_0_norm2_bias_to_fp16, epsilon = var_3339_to_fp16, gamma = model_head_layers_0_norm2_weight_to_fp16, x = input_407_cast_fp16)[name = tensor("input_409_cast_fp16")]; + tensor model_head_layers_0_linear1_weight_to_fp16 = const()[name = tensor("model_head_layers_0_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(423970304)))]; + tensor model_head_layers_0_linear1_bias_to_fp16 = const()[name = tensor("model_head_layers_0_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(428688960)))]; + tensor linear_90_cast_fp16 = linear(bias = model_head_layers_0_linear1_bias_to_fp16, weight = model_head_layers_0_linear1_weight_to_fp16, x = input_409_cast_fp16)[name = tensor("linear_90_cast_fp16")]; + tensor input_411_cast_fp16 = relu(x = linear_90_cast_fp16)[name = tensor("input_411_cast_fp16")]; + tensor model_head_layers_0_linear2_weight_to_fp16 = const()[name = tensor("model_head_layers_0_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(428695168)))]; + tensor model_head_layers_0_linear2_bias_to_fp16 = const()[name = tensor("model_head_layers_0_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(433413824)))]; + tensor linear_91_cast_fp16 = linear(bias = model_head_layers_0_linear2_bias_to_fp16, weight = model_head_layers_0_linear2_weight_to_fp16, x = input_411_cast_fp16)[name = tensor("linear_91_cast_fp16")]; + tensor input_413_cast_fp16 = add(x = input_407_cast_fp16, y = linear_91_cast_fp16)[name = tensor("input_413_cast_fp16")]; + tensor h_axes_0 = const()[name = tensor("h_axes_0"), val = tensor([-1])]; + tensor model_head_layers_1_norm1_weight_to_fp16 = const()[name = tensor("model_head_layers_1_norm1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(433415424)))]; + tensor model_head_layers_1_norm1_bias_to_fp16 = const()[name = tensor("model_head_layers_1_norm1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(433417024)))]; + tensor var_3349_to_fp16 = const()[name = tensor("op_3349_to_fp16"), val = tensor(0x1.5p-17)]; + tensor h_cast_fp16 = layer_norm(axes = h_axes_0, beta = model_head_layers_1_norm1_bias_to_fp16, epsilon = var_3349_to_fp16, gamma = model_head_layers_1_norm1_weight_to_fp16, x = input_413_cast_fp16)[name = tensor("h_cast_fp16")]; + tensor model_head_layers_1_self_attn_in_proj_weight_to_fp16 = const()[name = tensor("model_head_layers_1_self_attn_in_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(433418624)))]; + tensor model_head_layers_1_self_attn_in_proj_bias_to_fp16 = const()[name = tensor("model_head_layers_1_self_attn_in_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(436957632)))]; + tensor linear_92_cast_fp16 = linear(bias = model_head_layers_1_self_attn_in_proj_bias_to_fp16, weight = model_head_layers_1_self_attn_in_proj_weight_to_fp16, x = h_cast_fp16)[name = tensor("linear_92_cast_fp16")]; + tensor var_3372_split_sizes_0 = const()[name = tensor("op_3372_split_sizes_0"), val = tensor([768, 768, 768])]; + tensor var_3372_axis_0 = const()[name = tensor("op_3372_axis_0"), val = tensor(-1)]; + tensor var_3372_cast_fp16_0, tensor var_3372_cast_fp16_1, tensor var_3372_cast_fp16_2 = split(axis = var_3372_axis_0, split_sizes = var_3372_split_sizes_0, x = linear_92_cast_fp16)[name = tensor("op_3372_cast_fp16")]; + tensor var_3381 = const()[name = tensor("op_3381"), val = tensor([1, 512, 12, 64])]; + tensor var_3382_cast_fp16 = reshape(shape = var_3381, x = var_3372_cast_fp16_0)[name = tensor("op_3382_cast_fp16")]; + tensor var_3387 = const()[name = tensor("op_3387"), val = tensor([1, 512, 12, 64])]; + tensor var_3388_cast_fp16 = reshape(shape = var_3387, x = var_3372_cast_fp16_1)[name = tensor("op_3388_cast_fp16")]; + tensor var_3393 = const()[name = tensor("op_3393"), val = tensor([1, 512, 12, 64])]; + tensor var_3394_cast_fp16 = reshape(shape = var_3393, x = var_3372_cast_fp16_2)[name = tensor("op_3394_cast_fp16")]; + tensor v_perm_0 = const()[name = tensor("v_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor var_3401_transpose_x_0 = const()[name = tensor("op_3401_transpose_x_0"), val = tensor(false)]; + tensor var_3401_transpose_y_0 = const()[name = tensor("op_3401_transpose_y_0"), val = tensor(false)]; + tensor transpose_10_perm_0 = const()[name = tensor("transpose_10_perm_0"), val = tensor([0, 2, -3, -1])]; + tensor transpose_11_perm_0 = const()[name = tensor("transpose_11_perm_0"), val = tensor([0, 2, -1, -3])]; + tensor transpose_11 = transpose(perm = transpose_11_perm_0, x = var_3388_cast_fp16)[name = tensor("transpose_14")]; + tensor transpose_10 = transpose(perm = transpose_10_perm_0, x = var_3382_cast_fp16)[name = tensor("transpose_15")]; + tensor var_3401_cast_fp16 = matmul(transpose_x = var_3401_transpose_x_0, transpose_y = var_3401_transpose_y_0, x = transpose_10, y = transpose_11)[name = tensor("op_3401_cast_fp16")]; + tensor var_3403_to_fp16 = const()[name = tensor("op_3403_to_fp16"), val = tensor([0x1p-3])]; + tensor var_3404_cast_fp16 = mul(x = var_3401_cast_fp16, y = var_3403_to_fp16)[name = tensor("op_3404_cast_fp16")]; + tensor scores_cast_fp16 = add(x = var_3404_cast_fp16, y = attention_mask_3_cast_fp16)[name = tensor("scores_cast_fp16")]; + tensor var_3407 = const()[name = tensor("op_3407"), val = tensor(-1)]; + tensor weights_cast_fp16 = softmax(axis = var_3407, x = scores_cast_fp16)[name = tensor("weights_cast_fp16")]; + tensor var_3410_transpose_x_0 = const()[name = tensor("op_3410_transpose_x_0"), val = tensor(false)]; + tensor var_3410_transpose_y_0 = const()[name = tensor("op_3410_transpose_y_0"), val = tensor(false)]; + tensor v_cast_fp16 = transpose(perm = v_perm_0, x = var_3394_cast_fp16)[name = tensor("transpose_13")]; + tensor var_3410_cast_fp16 = matmul(transpose_x = var_3410_transpose_x_0, transpose_y = var_3410_transpose_y_0, x = weights_cast_fp16, y = v_cast_fp16)[name = tensor("op_3410_cast_fp16")]; + tensor var_3413_perm_0 = const()[name = tensor("op_3413_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_3414 = const()[name = tensor("op_3414"), val = tensor([1, 512, 768])]; + tensor var_3413_cast_fp16 = transpose(perm = var_3413_perm_0, x = var_3410_cast_fp16)[name = tensor("transpose_12")]; + tensor input_415_cast_fp16 = reshape(shape = var_3414, x = var_3413_cast_fp16)[name = tensor("input_415_cast_fp16")]; + tensor model_head_layers_1_self_attn_out_proj_weight_to_fp16 = const()[name = tensor("model_head_layers_1_self_attn_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(436962304)))]; + tensor model_head_layers_1_self_attn_out_proj_bias_to_fp16 = const()[name = tensor("model_head_layers_1_self_attn_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(438142016)))]; + tensor linear_93_cast_fp16 = linear(bias = model_head_layers_1_self_attn_out_proj_bias_to_fp16, weight = model_head_layers_1_self_attn_out_proj_weight_to_fp16, x = input_415_cast_fp16)[name = tensor("linear_93_cast_fp16")]; + tensor input_417_cast_fp16 = add(x = input_413_cast_fp16, y = linear_93_cast_fp16)[name = tensor("input_417_cast_fp16")]; + tensor input_419_axes_0 = const()[name = tensor("input_419_axes_0"), val = tensor([-1])]; + tensor model_head_layers_1_norm2_weight_to_fp16 = const()[name = tensor("model_head_layers_1_norm2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(438143616)))]; + tensor model_head_layers_1_norm2_bias_to_fp16 = const()[name = tensor("model_head_layers_1_norm2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(438145216)))]; + tensor var_3420_to_fp16 = const()[name = tensor("op_3420_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_419_cast_fp16 = layer_norm(axes = input_419_axes_0, beta = model_head_layers_1_norm2_bias_to_fp16, epsilon = var_3420_to_fp16, gamma = model_head_layers_1_norm2_weight_to_fp16, x = input_417_cast_fp16)[name = tensor("input_419_cast_fp16")]; + tensor model_head_layers_1_linear1_weight_to_fp16 = const()[name = tensor("model_head_layers_1_linear1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(438146816)))]; + tensor model_head_layers_1_linear1_bias_to_fp16 = const()[name = tensor("model_head_layers_1_linear1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(442865472)))]; + tensor linear_94_cast_fp16 = linear(bias = model_head_layers_1_linear1_bias_to_fp16, weight = model_head_layers_1_linear1_weight_to_fp16, x = input_419_cast_fp16)[name = tensor("linear_94_cast_fp16")]; + tensor input_421_cast_fp16 = relu(x = linear_94_cast_fp16)[name = tensor("input_421_cast_fp16")]; + tensor model_head_layers_1_linear2_weight_to_fp16 = const()[name = tensor("model_head_layers_1_linear2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(442871680)))]; + tensor model_head_layers_1_linear2_bias_to_fp16 = const()[name = tensor("model_head_layers_1_linear2_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(447590336)))]; + tensor linear_95_cast_fp16 = linear(bias = model_head_layers_1_linear2_bias_to_fp16, weight = model_head_layers_1_linear2_weight_to_fp16, x = input_421_cast_fp16)[name = tensor("linear_95_cast_fp16")]; + tensor x_cast_fp16 = add(x = input_417_cast_fp16, y = linear_95_cast_fp16)[name = tensor("x_cast_fp16")]; + tensor input_423_transpose_x_0 = const()[name = tensor("input_423_transpose_x_0"), val = tensor(false)]; + tensor input_423_transpose_y_0 = const()[name = tensor("input_423_transpose_y_0"), val = tensor(false)]; + tensor marker_map_to_fp16_dtype_0 = const()[name = tensor("marker_map_to_fp16_dtype_0"), val = tensor("fp16")]; + tensor marker_map_to_fp16 = cast(dtype = marker_map_to_fp16_dtype_0, x = marker_map)[name = tensor("cast_3")]; + tensor input_423_cast_fp16 = matmul(transpose_x = input_423_transpose_x_0, transpose_y = input_423_transpose_y_0, x = marker_map_to_fp16, y = x_cast_fp16)[name = tensor("input_423_cast_fp16")]; + tensor input_425_axes_0 = const()[name = tensor("input_425_axes_0"), val = tensor([-1])]; + tensor model_scorer_0_weight_to_fp16 = const()[name = tensor("model_scorer_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(447591936)))]; + tensor model_scorer_0_bias_to_fp16 = const()[name = tensor("model_scorer_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(447593536)))]; + tensor var_3432_to_fp16 = const()[name = tensor("op_3432_to_fp16"), val = tensor(0x1.5p-17)]; + tensor input_425_cast_fp16 = layer_norm(axes = input_425_axes_0, beta = model_scorer_0_bias_to_fp16, epsilon = var_3432_to_fp16, gamma = model_scorer_0_weight_to_fp16, x = input_423_cast_fp16)[name = tensor("input_425_cast_fp16")]; + tensor model_scorer_1_weight_to_fp16 = const()[name = tensor("model_scorer_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(447595136)))]; + tensor model_scorer_1_bias_to_fp16 = const()[name = tensor("model_scorer_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(448774848)))]; + tensor linear_96_cast_fp16 = linear(bias = model_scorer_1_bias_to_fp16, weight = model_scorer_1_weight_to_fp16, x = input_425_cast_fp16)[name = tensor("linear_96_cast_fp16")]; + tensor input_429_mode_0 = const()[name = tensor("input_429_mode_0"), val = tensor("EXACT")]; + tensor input_429_cast_fp16 = gelu(mode = input_429_mode_0, x = linear_96_cast_fp16)[name = tensor("input_429_cast_fp16")]; + tensor model_scorer_3_weight_to_fp16 = const()[name = tensor("model_scorer_3_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(448776448)))]; + tensor model_scorer_3_bias_to_fp16 = const()[name = tensor("model_scorer_3_bias_to_fp16"), val = tensor([0x1.438p-6])]; + tensor linear_97_cast_fp16 = linear(bias = model_scorer_3_bias_to_fp16, weight = model_scorer_3_weight_to_fp16, x = input_429_cast_fp16)[name = tensor("linear_97_cast_fp16")]; + tensor logits_1_axes_0 = const()[name = tensor("logits_1_axes_0"), val = tensor([-1])]; + tensor logits_1_cast_fp16 = squeeze(axes = logits_1_axes_0, x = linear_97_cast_fp16)[name = tensor("logits_1_cast_fp16")]; + tensor option_mask_axes_0 = const()[name = tensor("option_mask_axes_0"), val = tensor([-1])]; + tensor option_mask_keep_dims_0 = const()[name = tensor("option_mask_keep_dims_0"), val = tensor(false)]; + tensor option_mask_cast_fp16 = reduce_sum(axes = option_mask_axes_0, keep_dims = option_mask_keep_dims_0, x = marker_map_to_fp16)[name = tensor("option_mask_cast_fp16")]; + tensor var_3446_cast_fp16 = mul(x = logits_1_cast_fp16, y = option_mask_cast_fp16)[name = tensor("op_3446_cast_fp16")]; + tensor var_3447_to_fp16 = const()[name = tensor("op_3447_to_fp16"), val = tensor(0x1p+0)]; + tensor var_3449_cast_fp16 = sub(x = var_3447_to_fp16, y = option_mask_cast_fp16)[name = tensor("op_3449_cast_fp16")]; + tensor var_3450_to_fp16 = const()[name = tensor("op_3450_to_fp16"), val = tensor(-0x1.388p+13)]; + tensor var_3451_cast_fp16 = mul(x = var_3449_cast_fp16, y = var_3450_to_fp16)[name = tensor("op_3451_cast_fp16")]; + tensor logits_cast_fp16 = add(x = var_3446_cast_fp16, y = var_3451_cast_fp16)[name = tensor("logits_cast_fp16")]; + tensor logits_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("logits_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor var_3454 = const()[name = tensor("op_3454"), val = tensor(-1)]; + tensor probabilities_cast_fp16 = softmax(axis = var_3454, x = logits_cast_fp16)[name = tensor("probabilities_cast_fp16")]; + tensor probabilities_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("probabilities_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor var_3461_axes_0 = const()[name = tensor("op_3461_axes_0"), val = tensor([-1])]; + tensor var_3461_keep_dims_0 = const()[name = tensor("op_3461_keep_dims_0"), val = tensor(true)]; + tensor var_3461_cast_fp16 = reduce_sum(axes = var_3461_axes_0, keep_dims = var_3461_keep_dims_0, x = option_mask_cast_fp16)[name = tensor("op_3461_cast_fp16")]; + tensor var_3462_to_fp16 = const()[name = tensor("op_3462_to_fp16"), val = tensor(0x1p+1)]; + tensor const_182_to_fp16 = const()[name = tensor("const_182_to_fp16"), val = tensor(inf)]; + tensor clip_0_cast_fp16 = clip(alpha = var_3462_to_fp16, beta = const_182_to_fp16, x = var_3461_cast_fp16)[name = tensor("clip_0_cast_fp16")]; + tensor var_3465_to_fp16 = const()[name = tensor("op_3465_to_fp16"), val = tensor(0x1p-24)]; + tensor const_183_to_fp16 = const()[name = tensor("const_183_to_fp16"), val = tensor(inf)]; + tensor clip_1_cast_fp16 = clip(alpha = var_3465_to_fp16, beta = const_183_to_fp16, x = probabilities_cast_fp16)[name = tensor("clip_1_cast_fp16")]; + tensor var_3468_epsilon_0 = const()[name = tensor("op_3468_epsilon_0"), val = tensor(0x1p-149)]; + tensor var_3468_cast_fp16 = log(epsilon = var_3468_epsilon_0, x = clip_1_cast_fp16)[name = tensor("op_3468_cast_fp16")]; + tensor var_3469_cast_fp16 = mul(x = probabilities_cast_fp16, y = var_3468_cast_fp16)[name = tensor("op_3469_cast_fp16")]; + tensor var_3474_axes_0 = const()[name = tensor("op_3474_axes_0"), val = tensor([-1])]; + tensor var_3474_keep_dims_0 = const()[name = tensor("op_3474_keep_dims_0"), val = tensor(true)]; + tensor var_3474_cast_fp16 = reduce_sum(axes = var_3474_axes_0, keep_dims = var_3474_keep_dims_0, x = var_3469_cast_fp16)[name = tensor("op_3474_cast_fp16")]; + tensor const_184_promoted_to_fp16 = const()[name = tensor("const_184_promoted_to_fp16"), val = tensor(-0x1p+0)]; + tensor var_3475_cast_fp16 = mul(x = var_3474_cast_fp16, y = const_184_promoted_to_fp16)[name = tensor("op_3475_cast_fp16")]; + tensor var_3476_epsilon_0 = const()[name = tensor("op_3476_epsilon_0"), val = tensor(0x1p-149)]; + tensor var_3476_cast_fp16 = log(epsilon = var_3476_epsilon_0, x = clip_0_cast_fp16)[name = tensor("op_3476_cast_fp16")]; + tensor entropy_cast_fp16 = real_div(x = var_3475_cast_fp16, y = var_3476_cast_fp16)[name = tensor("entropy_cast_fp16")]; + tensor var_3478 = const()[name = tensor("op_3478"), val = tensor(2)]; + tensor top2_axis_0 = const()[name = tensor("top2_axis_0"), val = tensor(-1)]; + tensor top2_ascending_0 = const()[name = tensor("top2_ascending_0"), val = tensor(false)]; + tensor top2_sort_0 = const()[name = tensor("top2_sort_0"), val = tensor(true)]; + tensor top2_return_indices_0 = const()[name = tensor("top2_return_indices_0"), val = tensor(true)]; + tensor top2_cast_fp16_cast_int16_output_indices_dtype_0 = const()[name = tensor("top2_cast_fp16_cast_int16_output_indices_dtype_0"), val = tensor("uint16")]; + tensor top2_cast_fp16_cast_int16_0, tensor top2_cast_fp16_cast_int16_1 = topk(ascending = top2_ascending_0, axis = top2_axis_0, k = var_3478, output_indices_dtype = top2_cast_fp16_cast_int16_output_indices_dtype_0, return_indices = top2_return_indices_0, sort = top2_sort_0, x = probabilities_cast_fp16)[name = tensor("top2_cast_fp16_cast_int16")]; + tensor var_3493_begin_0 = const()[name = tensor("op_3493_begin_0"), val = tensor([0, 0])]; + tensor var_3493_end_0 = const()[name = tensor("op_3493_end_0"), val = tensor([1, 1])]; + tensor var_3493_end_mask_0 = const()[name = tensor("op_3493_end_mask_0"), val = tensor([true, false])]; + tensor var_3493_cast_fp16 = slice_by_index(begin = var_3493_begin_0, end = var_3493_end_0, end_mask = var_3493_end_mask_0, x = top2_cast_fp16_cast_int16_0)[name = tensor("op_3493_cast_fp16")]; + tensor var_3513_begin_0 = const()[name = tensor("op_3513_begin_0"), val = tensor([0, 1])]; + tensor var_3513_end_0 = const()[name = tensor("op_3513_end_0"), val = tensor([1, 1])]; + tensor var_3513_end_mask_0 = const()[name = tensor("op_3513_end_mask_0"), val = tensor([true, true])]; + tensor var_3513_cast_fp16 = slice_by_index(begin = var_3513_begin_0, end = var_3513_end_0, end_mask = var_3513_end_mask_0, x = top2_cast_fp16_cast_int16_0)[name = tensor("op_3513_cast_fp16")]; + tensor var_3515_cast_fp16 = sub(x = var_3493_cast_fp16, y = var_3513_cast_fp16)[name = tensor("op_3515_cast_fp16")]; + tensor _inversed_3517_y_0_to_fp16 = const()[name = tensor("_inversed_3517_y_0_to_fp16"), val = tensor(0x1.01p-8)]; + tensor _inversed_3517_cast_fp16 = mul(x = clip_0_cast_fp16, y = _inversed_3517_y_0_to_fp16)[name = tensor("_inversed_3517_cast_fp16")]; + tensor var_3519 = const()[name = tensor("op_3519"), val = tensor(-1)]; + tensor feats_interleave_0 = const()[name = tensor("feats_interleave_0"), val = tensor(false)]; + tensor feats_cast_fp16 = concat(axis = var_3519, interleave = feats_interleave_0, values = (var_3493_cast_fp16, var_3515_cast_fp16, entropy_cast_fp16, _inversed_3517_cast_fp16))[name = tensor("feats_cast_fp16")]; + tensor pooled_begin_0 = const()[name = tensor("pooled_begin_0"), val = tensor([0, 0, 0])]; + tensor pooled_end_0 = const()[name = tensor("pooled_end_0"), val = tensor([1, 1, 768])]; + tensor pooled_end_mask_0 = const()[name = tensor("pooled_end_mask_0"), val = tensor([true, false, true])]; + tensor pooled_squeeze_mask_0 = const()[name = tensor("pooled_squeeze_mask_0"), val = tensor([false, true, false])]; + tensor pooled_cast_fp16 = slice_by_index(begin = pooled_begin_0, end = pooled_end_0, end_mask = pooled_end_mask_0, squeeze_mask = pooled_squeeze_mask_0, x = x_cast_fp16)[name = tensor("pooled_cast_fp16")]; + tensor var_3530 = const()[name = tensor("op_3530"), val = tensor(-1)]; + tensor input_431_interleave_0 = const()[name = tensor("input_431_interleave_0"), val = tensor(false)]; + tensor input_431_cast_fp16 = concat(axis = var_3530, interleave = input_431_interleave_0, values = (pooled_cast_fp16, feats_cast_fp16))[name = tensor("input_431_cast_fp16")]; + tensor model_act_head_0_weight_to_fp16 = const()[name = tensor("model_act_head_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(448778048)))]; + tensor model_act_head_0_bias_to_fp16 = const()[name = tensor("model_act_head_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(449173376)))]; + tensor linear_98_cast_fp16 = linear(bias = model_act_head_0_bias_to_fp16, weight = model_act_head_0_weight_to_fp16, x = input_431_cast_fp16)[name = tensor("linear_98_cast_fp16")]; + tensor input_mode_0 = const()[name = tensor("input_mode_0"), val = tensor("EXACT")]; + tensor input_cast_fp16 = gelu(mode = input_mode_0, x = linear_98_cast_fp16)[name = tensor("input_cast_fp16")]; + tensor model_act_head_2_weight_to_fp16 = const()[name = tensor("model_act_head_2_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(449173952)))]; + tensor model_act_head_2_bias_to_fp16 = const()[name = tensor("model_act_head_2_bias_to_fp16"), val = tensor([0x1.818p-6, -0x1.4fcp-6])]; + tensor linear_99_cast_fp16 = linear(bias = model_act_head_2_bias_to_fp16, weight = model_act_head_2_weight_to_fp16, x = input_cast_fp16)[name = tensor("linear_99_cast_fp16")]; + tensor var_3536 = const()[name = tensor("op_3536"), val = tensor(-1)]; + tensor var_3538_cast_fp16 = softmax(axis = var_3536, x = linear_99_cast_fp16)[name = tensor("op_3538_cast_fp16")]; + tensor var_3538_cast_fp16_to_fp32_dtype_0 = const()[name = tensor("op_3538_cast_fp16_to_fp32_dtype_0"), val = tensor("fp32")]; + tensor logits = cast(dtype = logits_cast_fp16_to_fp32_dtype_0, x = logits_cast_fp16)[name = tensor("cast_0")]; + tensor probabilities = cast(dtype = probabilities_cast_fp16_to_fp32_dtype_0, x = probabilities_cast_fp16)[name = tensor("cast_1")]; + tensor action_probabilities = cast(dtype = var_3538_cast_fp16_to_fp32_dtype_0, x = var_3538_cast_fp16)[name = tensor("cast_2")]; + } -> (logits, probabilities, action_probabilities); +} \ No newline at end of file