diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 27f6a964..f2266d96 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -48,42 +48,19 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: ['ubuntu-20.04']
- python-version: ['3.8', '3.9', '3.10', '3.11']
- pytorch-version: ['2.0.1', '2.1.2', '2.2.2', '2.3.0']
- cuda-version: ['11.8.0', '12.2.2']
+ os: [ubuntu-22.04]
+ python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+ pytorch-version: ['2.4.0', '2.5.1', '2.6.0']
+ cuda-version: ['12.4.0']
exclude:
# see https://github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix
- # Pytorch < 2.2 does not support Python 3.12
- - pytorch-version: '1.12.1'
- python-version: '3.12'
- - pytorch-version: '1.13.1'
- python-version: '3.12'
- - pytorch-version: '2.0.1'
- python-version: '3.12'
- - pytorch-version: '2.1.2'
- python-version: '3.12'
- # Pytorch <= 1.12 does not support Python 3.11
- - pytorch-version: '1.12.1'
- python-version: '3.11'
- # Pytorch >= 2.0 only supports Python >= 3.8
- - pytorch-version: '2.0.1'
- python-version: '3.7'
- - pytorch-version: '2.1.2'
- python-version: '3.7'
+ # Pytorch < 2.5 does not support Python 3.13
- pytorch-version: '2.2.2'
- python-version: '3.7'
- - pytorch-version: '2.3.0'
- python-version: '3.7'
- - pytorch-version: '2.4.0.dev20240407'
- python-version: '3.7'
- # Pytorch <= 2.0 only supports CUDA <= 11.8
- - pytorch-version: '1.12.1'
- cuda-version: '12.2.2'
- - pytorch-version: '1.13.1'
- cuda-version: '12.2.2'
- - pytorch-version: '2.0.1'
- cuda-version: '12.2.2'
+ python-version: '3.13'
+ - pytorch-version: '2.3.1'
+ python-version: '3.13'
+ - pytorch-version: '2.4.0'
+ python-version: '3.13'
steps:
- name: Checkout
@@ -101,7 +78,7 @@ jobs:
bash -x .github/workflows/scripts/env.sh
- name: Set up Python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
@@ -113,7 +90,7 @@ jobs:
- name: Install CUDA ${{ matrix.cuda-version }}
if: ${{ matrix.cuda-version != 'cpu' }}
- uses: Jimver/cuda-toolkit@v0.2.14
+ uses: Jimver/cuda-toolkit@v0.2.19
id: cuda-toolkit
with:
cuda: ${{ matrix.cuda-version }}
@@ -121,9 +98,7 @@ jobs:
# default method is "local", and we're hitting some error with caching for CUDA 11.8 and 12.1
# method: ${{ (matrix.cuda-version == '11.8.0' || matrix.cuda-version == '12.1.0') && 'network' || 'local' }}
method: 'network'
- # We need the cuda libraries (e.g. cuSparse, cuSolver) for compiling PyTorch extensions,
- # not just nvcc
- # sub-packages: '["nvcc"]'
+ sub-packages: '["nvcc"]'
- name: Install PyTorch ${{ matrix.pytorch-version }} with CUDA ${{ matrix.cuda-version }}
run: |
@@ -167,10 +142,10 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: ['ubuntu-20.04']
+ os: [ubuntu-22.04]
python-version: ['3.10']
- pytorch-version: ['2.3.0'] # Must be the most recent version that meets requirements-cuda.txt.
- cuda-version: ['12.2.2']
+ pytorch-version: ['2.4.0']
+ cuda-version: ['12.4.0']
steps:
- name: Checkout
@@ -200,7 +175,7 @@ jobs:
- name: Install CUDA ${{ matrix.cuda-version }}
if: ${{ matrix.cuda-version != 'cpu' }}
- uses: Jimver/cuda-toolkit@v0.2.14
+ uses: Jimver/cuda-toolkit@v0.2.19
id: cuda-toolkit
with:
cuda: ${{ matrix.cuda-version }}
@@ -208,9 +183,7 @@ jobs:
# default method is "local", and we're hitting some error with caching for CUDA 11.8 and 12.1
# method: ${{ (matrix.cuda-version == '11.8.0' || matrix.cuda-version == '12.1.0') && 'network' || 'local' }}
method: 'network'
- # We need the cuda libraries (e.g. cuSparse, cuSolver) for compiling PyTorch extensions,
- # not just nvcc
- # sub-packages: '["nvcc"]'
+ sub-packages: '["nvcc"]'
- name: Install PyTorch ${{ matrix.pytorch-version }} with CUDA ${{ matrix.cuda-version }}
run: |
diff --git a/.github/workflows/scripts/pytorch-install.sh b/.github/workflows/scripts/pytorch-install.sh
index 6e61ba88..d7b89991 100644
--- a/.github/workflows/scripts/pytorch-install.sh
+++ b/.github/workflows/scripts/pytorch-install.sh
@@ -1,15 +1,18 @@
#!/bin/bash
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
python_executable=python$1
pytorch_version=$2
cuda_version=$3
pip install --upgrade pip
-# If we don't install before installing Pytorch, we get error for torch 2.0.1
-# ERROR: Could not find a version that satisfies the requirement setuptools>=40.8.0 (from versions: none)
-pip install lit
# For some reason torch 2.2.0 on python 3.12 errors saying no setuptools
-pip install setuptools
+pip install setuptools==75.8.0
+# With python 3.13 and torch 2.5.1, unless we update typing-extensions, we get error
+# AttributeError: attribute '__default__' of 'typing.ParamSpec' objects is not writable
+pip install typing-extensions==4.12.2
# We want to figure out the CUDA version to download pytorch
# e.g. we can have system CUDA version being 11.7 but if torch==1.12 then we need to download the wheel from cu116
# see https://github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix
@@ -17,8 +20,8 @@ pip install setuptools
echo $MATRIX_CUDA_VERSION
echo $MATRIX_TORCH_VERSION
export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; \
-minv = {'1.12': 113, '1.13': 116, '2.0': 117, '2.1': 118, '2.2': 118, '2.3': 118, '2.4': 118}[env['MATRIX_TORCH_VERSION']]; \
-maxv = {'1.12': 116, '1.13': 117, '2.0': 118, '2.1': 121, '2.2': 121, '2.3': 121, '2.4': 121}[env['MATRIX_TORCH_VERSION']]; \
+minv = {'2.2': 118, '2.3': 118, '2.4': 118, '2.5': 118, '2.6': 118, '2.7': 118}[env['MATRIX_TORCH_VERSION']]; \
+maxv = {'2.2': 121, '2.3': 121, '2.4': 124, '2.5': 124, '2.6': 126, '2.7': 128}[env['MATRIX_TORCH_VERSION']]; \
print(max(min(int(env['MATRIX_CUDA_VERSION']), maxv), minv))" \
)
if [[ ${pytorch_version} == *"dev"* ]]; then
diff --git a/.gitignore b/.gitignore
index 16033cd5..781c84ff 100644
--- a/.gitignore
+++ b/.gitignore
@@ -404,6 +404,7 @@ needle
results
*.json
*.jsonl
+
.vscode/
*.pt
*.pkl
@@ -415,3 +416,10 @@ build/
*.egg-info/
*.so
dist
+*.eggs/
+
+
+
+!mtraining/model_configs/**/*.json
+!mtraining/utils/comm_prof/**
+mtraining/experiments/processed_datasets/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4b92db74..6d9e73ee 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,37 +1,61 @@
+exclude: 'build'
+
default_language_version:
- python: python3
-exclude: 'dotnet'
+ python: python3
ci:
autofix_prs: true
autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
autoupdate_schedule: 'quarterly'
repos:
- - repo: https://github.com/pre-commit/pre-commit-hooks
+- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- - id: check-added-large-files
- - id: check-ast
- - id: check-yaml
- - id: check-toml
- - id: check-json
- - id: check-byte-order-marker
- exclude: .gitignore
- - id: check-merge-conflict
- - id: detect-private-key
- - id: trailing-whitespace
- - id: end-of-file-fixer
- - id: no-commit-to-branch
- - repo: https://github.com/pycqa/isort
+ - id: trailing-whitespace
+ args: [--markdown-linebreak-ext=md]
+ - id: check-ast
+ - id: check-merge-conflict
+ - id: no-commit-to-branch
+ args: ['--branch=main']
+ - id: check-added-large-files
+ args: ['--maxkb=500']
+ - id: end-of-file-fixer
+ exclude: '^(.*\.svg)$'
+ - id: check-symlinks
+ - id: check-yaml
+ - id: check-toml
+ - id: detect-private-key
+ - id: debug-statements
+ - id: check-ast
+ - id: check-json
+ - id: check-byte-order-marker
+ exclude: .gitignore
+ - id: trailing-whitespace
+
+- repo: https://github.com/Lucas-C/pre-commit-hooks
+ rev: v1.5.5
+ hooks:
+ - id: insert-license
+ files: \.(py|sh|toml)$
+ stages: [pre-commit]
+ args:
+ - --license-filepath
+ - assets/license_header.txt
+ - --use-current-year
+
+- repo: https://github.com/pycqa/isort
rev: 5.13.2
hooks:
- - id: isort
- - repo: https://github.com/psf/black
+ - id: isort
+ stages: [pre-commit]
+- repo: https://github.com/psf/black
rev: 23.3.0
hooks:
- - id: black
- exclude: minference/ops|minference/modules/minference\_forward\.py
- - repo: https://github.com/nbQA-dev/nbQA
+ - id: black
+ stages: [pre-commit]
+ exclude: minference/ops|minference/modules/minference\_forward\.py
+- repo: https://github.com/nbQA-dev/nbQA
rev: 1.7.1
hooks:
- - id: nbqa-black
+ - id: nbqa-black
+ stages: [pre-commit]
diff --git a/LeanK/README.md b/LeanK/README.md
new file mode 100644
index 00000000..2d1e191e
--- /dev/null
+++ b/LeanK/README.md
@@ -0,0 +1,159 @@
+# LeanK: Learnable K Cache Channel Pruning for Efficient Decoding
+
+Building upon the observation that QK vectors are **statically sparse** in the **channel (head_dim)** dimension, LeanK uses a learning-based approach to obtain channel-wise static mask used for KV Cache pruning.
+
+Experiments demonstrate up to **70%** K cache and **16%–18%** V cache memory reduction, reduces GPU memory and accelerates decoding without sacrificing accuracy.
+
+
+ 
+
+
+
+### Quick Start: Use LeanK with pretrained static channel patterns
+
+Requirement: install the latest version of minference and tilelang
+
+Add the following patch to enable LeanK decoding patched on Huggingface transformers:
+
+```diff
+from transformers import AutoModelForCausalLM, AutoTokenizer
++ from minference import MInference
+
+model_name = "meta-llama/Llama-3.1-8B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map="cuda",
+ _attn_implementation="flash_attention_2",
+)
+
++minference_patch = MInference(
++ attn_type="dense", model_name=model_name, kv_type="leank"
++)
++model = minference_patch(model)
+
+prompt = open("narrativeqa_example.txt").read()
+batch_inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+
+outputs = model.generate(**batch_inputs, max_new_tokens=512)
+generated_text = tokenizer.decode(
+ outputs[0][batch_inputs["input_ids"].shape[1] :], skip_special_tokens=True
+)
+print(f"Generated text: {generated_text!r}")
+```
+
+Note: for unseen sequence length, Tilelang kernel compilation may take considerable longer time. It will be much faster once the kernel is compiled and kept in cache.
+
+### Environment Setup
+
+```bash
+conda create -yn leank python=3.10
+conda activate leank
+
+conda install -y git
+conda install -y nvidia/label/cuda-12.4.0::cuda-toolkit
+conda install -y nvidia::cuda-cudart-dev
+conda install -y pytorch torchvision torchaudio pytorch-cuda=12.4 -c pytorch -c nvidia
+
+# Training environment
+pip install transformers==4.48.0 accelerate sentencepiece datasets wandb zstandard matplotlib huggingface_hub==0.25.2
+pip install ninja packaging
+pip install flash-attn==2.7.1.post4 --no-build-isolation
+pip install cloudpickle wonderwords bs4 nltk html2text pyyaml
+
+# Deployment and evaluation
+pip install tilelang==0.1.5
+pip install minference
+
+pip install jsonlines pandas seaborn tqdm
+pip install jieba mysql-connector-python fuzzywuzzy rouge SentencePiece
+pip install git+https://github.com/NVIDIA/NeMo.git
+
+pip install ipython hydra-core lightning lhotse jiwer librosa pyannote.core pyannote-core webdataset editdistance pyannote.metrics tenacity
+```
+
+
+### Data Preparation
+Download PaulGraham Essays dataset for training:
+
+```bash
+# for training
+cd leank/data
+python download_paulgraham_essay.py
+python -c "import nltk; nltk.download('punkt_tab')"
+wget https://huggingface.co/datasets/togethercomputer/Long-Data-Collections/resolve/main/fine-tune/booksum.jsonl.zst
+
+# for evaluation
+cd eval/RULER/scripts/data/synthetic/json
+python download_paulgraham_essay.py
+bash download_qa_dataset.sh
+```
+
+### Training
+
+```bash
+# run double-stage training on Llama-3.1-8B-Instruct
+torchrun --nnodes 1 --nproc_per_node 1 train.py --config scripts/llama3.1-8b.yaml
+torchrun --nnodes 1 --nproc_per_node 1 train.py --config scripts/llama3.1-8b.yaml --stage2
+
+# run double-stage training on Qwen2.5-7B-Instruct w/o Yarn
+torchrun --nnodes 1 --nproc_per_node 1 train.py --config scripts/qwen2.5-7b.yaml
+torchrun --nnodes 1 --nproc_per_node 1 train.py --config scripts/qwen2.5-7b.yaml --stage2
+
+# run double-stage training on Qwen2.5-7B-Instruct w/ Yarn
+torchrun --nnodes 1 --nproc_per_node 1 train.py --config scripts/qwen2.5-7b-yarn.yaml
+torchrun --nnodes 1 --nproc_per_node 1 train.py --config scripts/qwen2.5-7b-yarn.yaml --stage2
+```
+
+For Qwen w/ Yarn extrapolation, add the following to config.json for training and testing:
+
+```json
+"rope_scaling": {
+ "factor": 4.0,
+ "original_max_position_embeddings": 32768,
+ "type": "yarn"
+}
+```
+
+### Evaluation
+
+#### RULER
+
+```bash
+bash eval/RULER/run.sh llama3.1-8b-chat synthetic
+```
+
+#### LongBench
+
+```bash
+python eval/LongBench/pred.py --enable_leank --model_name_or_path meta-llama/Meta-Llama-3.1-8B-Instruct
+python eval/LongBench/eval.py
+```
+
+#### Kernel benchmarking
+
+```bash
+python kernel_benchmark.py
+```
+
+Dense Attention (Tile-lang): 5.31 ms
+
+LeanK Decoding (Average): 4.13 ms
+
+#### End-to-end memory and latency test
+
+```bash
+python test_memory.py
+```
+
+| Method | Seq_len | Batch_size | Gen_time | Max_memory |
+| --- | --- | --- | --- | --- |
+| HF | 4096 | 52 | 47.27 | 64.64 |
+| LeanK | 4096 | 52 | 38.32 | 57.27 |
+| LeanK | 4096 | 64 (1.2x larger) | 47.62 | 67.03 |
+
+### Acknowledgements
+
+This project incorporates code from [Tile-Lang](https://github.com/tile-ai/tilelang), [DuoAttention](https://github.com/mit-han-lab/duo-attention), [RULER](https://github.com/NVIDIA/RULER) and [LongBench](https://github.com/THUDM/LongBench). We deeply appreciate the contributions of the original authors.
diff --git a/LeanK/assets/leank_method.png b/LeanK/assets/leank_method.png
new file mode 100644
index 00000000..0cd0df79
Binary files /dev/null and b/LeanK/assets/leank_method.png differ
diff --git a/LeanK/eval/LongBench/eval_long_bench.py b/LeanK/eval/LongBench/eval_long_bench.py
new file mode 100644
index 00000000..17c17672
--- /dev/null
+++ b/LeanK/eval/LongBench/eval_long_bench.py
@@ -0,0 +1,127 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+import json
+import os
+
+import numpy as np
+from utils.metrics import (
+ classification_score,
+ code_sim_score,
+ count_score,
+ qa_f1_score,
+ qa_f1_zh_score,
+ retrieval_score,
+ retrieval_zh_score,
+ rouge_score,
+ rouge_zh_score,
+)
+
+dataset2metric = {
+ "narrativeqa": qa_f1_score,
+ "qasper": qa_f1_score,
+ "multifieldqa_en": qa_f1_score,
+ "multifieldqa_zh": qa_f1_zh_score,
+ "hotpotqa": qa_f1_score,
+ "2wikimqa": qa_f1_score,
+ "musique": qa_f1_score,
+ "dureader": rouge_zh_score,
+ "gov_report": rouge_score,
+ "qmsum": rouge_score,
+ "multi_news": rouge_score,
+ "vcsum": rouge_zh_score,
+ "trec": classification_score,
+ "triviaqa": qa_f1_score,
+ "samsum": rouge_score,
+ "lsht": classification_score,
+ "passage_retrieval_en": retrieval_score,
+ "passage_count": count_score,
+ "passage_retrieval_zh": retrieval_zh_score,
+ "lcc": code_sim_score,
+ "repobench-p": code_sim_score,
+}
+
+
+def parse_args(args=None):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model", type=str, default=None)
+ parser.add_argument("--e", action="store_true", help="Evaluate on LongBench-E")
+ return parser.parse_args(args)
+
+
+def scorer_e(dataset, predictions, answers, lengths, all_classes):
+ scores = {"0-4k": [], "4-8k": [], "8k+": []}
+ for prediction, ground_truths, length in zip(predictions, answers, lengths):
+ score = 0.0
+ if dataset in ["trec", "triviaqa", "samsum", "lsht"]:
+ prediction = prediction.lstrip("\n").split("\n")[0]
+ for ground_truth in ground_truths:
+ score = max(
+ score,
+ dataset2metric[dataset](
+ prediction, ground_truth, all_classes=all_classes
+ ),
+ )
+ if length < 4000:
+ scores["0-4k"].append(score)
+ elif length < 8000:
+ scores["4-8k"].append(score)
+ else:
+ scores["8k+"].append(score)
+ for key in scores.keys():
+ scores[key] = round(100 * np.mean(scores[key]), 2)
+ return scores
+
+
+def scorer(dataset, predictions, answers, all_classes):
+ total_score = 0.0
+ for prediction, ground_truths in zip(predictions, answers):
+ score = 0.0
+ if dataset in ["trec", "triviaqa", "samsum", "lsht"]:
+ prediction = prediction.lstrip("\n").split("\n")[0]
+ for ground_truth in ground_truths:
+ score = max(
+ score,
+ dataset2metric[dataset](
+ prediction, ground_truth, all_classes=all_classes
+ ),
+ )
+ total_score += score
+ return round(100 * total_score / len(predictions), 2)
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ scores = dict()
+ if args.e:
+ path = f"longbench_rst_e/{args.model}/"
+ else:
+ path = f"longbench_rst/{args.model}/"
+ all_files = os.listdir(path)
+ print("Evaluating on:", all_files)
+ for filename in all_files:
+ if not filename.endswith("jsonl"):
+ continue
+ predictions, answers, lengths = [], [], []
+ dataset = filename.split(".")[0]
+ with open(f"{path}{filename}", "r", encoding="utf-8") as f:
+ for line in f:
+ data = json.loads(line)
+ predictions.append(data["pred"])
+ answers.append(data["answers"])
+ all_classes = data["all_classes"]
+ if "length" in data:
+ lengths.append(data["length"])
+ if args.e:
+ score = scorer_e(dataset, predictions, answers, lengths, all_classes)
+ else:
+ score = scorer(dataset, predictions, answers, all_classes)
+ scores[dataset] = score
+ if args.e:
+ out_path = f"longbench_rst_e/{args.model}/result.json"
+ else:
+ out_path = f"longbench_rst/{args.model}/result.json"
+ with open(out_path, "w") as f:
+ print(scores)
+ json.dump(scores, f, ensure_ascii=False, indent=4)
diff --git a/LeanK/eval/LongBench/pred_long_bench.py b/LeanK/eval/LongBench/pred_long_bench.py
new file mode 100644
index 00000000..4d4f0795
--- /dev/null
+++ b/LeanK/eval/LongBench/pred_long_bench.py
@@ -0,0 +1,257 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import json
+import os
+import random
+
+import numpy as np
+import torch
+from datasets import load_dataset
+from tqdm import tqdm
+
+os.environ["WANDB_DISABLED"] = "true"
+
+from transformers import AutoTokenizer, LlamaConfig, Qwen2Config
+from utils.process_args import process_args
+
+from minference import MInference
+
+
+# build prompt for chat models
+def build_chat(tokenizer, prompt, model_name):
+ if "llama-3" in model_name.lower() and "instruct" in model_name.lower():
+ prompt = f"<|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+ else:
+ messages = [
+ {"role": "user", "content": prompt},
+ ]
+ prompt = tokenizer.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=True
+ )
+ return prompt
+
+
+def post_process(response, model_name):
+ if "xgen" in model_name:
+ response = response.strip().replace("Assistant:", "")
+ elif "internlm" in model_name:
+ response = response.split("")[0]
+ return response
+
+
+def get_pred(
+ model,
+ tokenizer,
+ data,
+ max_length,
+ max_gen,
+ prompt_format,
+ dataset,
+ device,
+ model_name,
+):
+ preds = []
+ for json_obj in tqdm(data):
+ prompt = prompt_format.format(**json_obj)
+ # truncate to fit max_length
+ tokenized_prompt = tokenizer(
+ prompt, truncation=False, return_tensors="pt"
+ ).input_ids[0]
+ if len(tokenized_prompt) > max_length:
+ half = int(max_length / 2)
+ prompt = tokenizer.decode(
+ tokenized_prompt[:half], skip_special_tokens=True
+ ) + tokenizer.decode(tokenized_prompt[-half:], skip_special_tokens=True)
+ if dataset not in [
+ "trec",
+ "triviaqa",
+ "samsum",
+ "lsht",
+ "lcc",
+ "repobench-p",
+ ]: # chat models are better off without build prompts on these tasks
+ prompt = build_chat(tokenizer, prompt, model_name)
+ input = tokenizer(prompt, truncation=False, return_tensors="pt").to(device)
+ context_length = input.input_ids.shape[-1]
+ eos_token_ids = [tokenizer.eos_token_id]
+ if "llama-3" in model_name.lower():
+ eos_token_ids.append(
+ tokenizer.encode("<|eot_id|>", add_special_tokens=False)[0]
+ )
+ eos_token_ids.append(
+ tokenizer.encode("<|eom_id|>", add_special_tokens=False)[0]
+ )
+ eos_token_ids.append(
+ tokenizer.encode("<|end_of_text|>", add_special_tokens=False)[0]
+ )
+ if dataset == "samsum":
+ eos_token_ids.append(tokenizer.encode("\n", add_special_tokens=False)[-1])
+ if (
+ dataset == "samsum"
+ ): # prevent illegal output on samsum (model endlessly repeat "\nDialogue"), might be a prompting issue
+ output = model.generate(
+ **input,
+ max_new_tokens=max_gen,
+ num_beams=1,
+ do_sample=False,
+ temperature=1.0,
+ min_length=context_length + 1,
+ eos_token_id=eos_token_ids,
+ )[0]
+ else:
+ output = model.generate(
+ **input,
+ max_new_tokens=max_gen,
+ num_beams=1,
+ do_sample=False,
+ temperature=1.0,
+ eos_token_id=eos_token_ids,
+ )[0]
+ pred = tokenizer.decode(output[context_length:], skip_special_tokens=True)
+ pred = post_process(pred, model_name)
+ print("pred is:", pred)
+ preds.append(
+ {
+ "pred": pred,
+ "answers": json_obj["answers"],
+ "all_classes": json_obj["all_classes"],
+ "length": json_obj["length"],
+ }
+ )
+ return preds
+
+
+def seed_everything(seed):
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed(seed)
+ np.random.seed(seed)
+ random.seed(seed)
+ torch.backends.cudnn.benchmark = False
+ torch.backends.cudnn.deterministic = True
+ torch.cuda.manual_seed_all(seed)
+
+
+if __name__ == "__main__":
+ seed_everything(42)
+ model2path = json.load(open("config/model2path.json", "r"))
+ model2maxlen = json.load(open("config/model2maxlen.json", "r"))
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+ # define your model
+ model_args, data_args = process_args()
+ model_name = model_args.model_name_or_path.split("/")[-1]
+ dtype = torch.bfloat16
+
+ if (
+ "llama" in model_args.model_name_or_path.lower()
+ or "longchat" in model_args.model_name_or_path.lower()
+ ):
+ config = LlamaConfig.from_pretrained(model_args.model_name_or_path)
+ tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path)
+
+ elif "qwen2" in model_args.model_name_or_path.lower():
+ config = Qwen2Config.from_pretrained(model_args.model_name_or_path)
+ tokenizer = AutoTokenizer.from_pretrained(
+ model_args.model_name_or_path, use_fast=False, trust_remote_code=True
+ )
+
+ if (
+ "llama" in model_args.model_name_or_path.lower()
+ or "longchat" in model_args.model_name_or_path.lower()
+ ):
+ from transformers import LlamaForCausalLM
+
+ model = LlamaForCausalLM.from_pretrained(
+ pretrained_model_name_or_path=model_args.model_name_or_path,
+ config=config,
+ torch_dtype=dtype,
+ low_cpu_mem_usage=True,
+ use_flash_attention_2=True,
+ device_map="auto",
+ )
+
+ elif "qwen2" in model_args.model_name_or_path.lower():
+ from transformers import Qwen2ForCausalLM
+
+ model = Qwen2ForCausalLM.from_pretrained(
+ pretrained_model_name_or_path=model_args.model_name_or_path,
+ config=config,
+ torch_dtype=dtype,
+ low_cpu_mem_usage=True,
+ use_flash_attention_2=True,
+ device_map="auto",
+ )
+
+ if model_args.enable_leank:
+ minference_patch = MInference(
+ attn_type="dense", model_name=model_name, kv_type="leank"
+ )
+ model = minference_patch(model)
+
+ model.eval()
+ max_length = model2maxlen[model_name]
+ if data_args.e:
+ datasets = [
+ "qasper",
+ "multifieldqa_en",
+ "hotpotqa",
+ "2wikimqa",
+ "gov_report",
+ "multi_news",
+ "trec",
+ "triviaqa",
+ "samsum",
+ "passage_count",
+ "passage_retrieval_en",
+ "lcc",
+ "repobench-p",
+ ]
+ else:
+ datasets = ["triviaqa"]
+ # ["qmsum", "qasper", "multifieldqa_en", "hotpotqa", "2wikimqa", "gov_report", "multi_news",
+ # "trec", "triviaqa", "samsum", "passage_count", "passage_retrieval_en", "lcc", "repobench-p"]
+ # we design specific prompt format and max generation length for each task, feel free to modify them to optimize model output
+ dataset2prompt = json.load(open("config/dataset2prompt.json", "r"))
+ dataset2maxlen = json.load(open("config/dataset2maxlen.json", "r"))
+ # predict on each dataset
+ if not os.path.exists("longbench_rst_e"):
+ os.makedirs("longbench_rst_e")
+ if not os.path.exists("longbench_rst"):
+ os.makedirs("longbench_rst")
+ for dataset in datasets:
+ if data_args.e:
+ data = load_dataset("THUDM/LongBench", f"{dataset}_e", split="test")
+ if not os.path.exists(
+ f"longbench_rst/{model_name}_{max_length}_{model_args.k_bits}bits_group{model_args.group_size}_residual{model_args.residual_length}"
+ ):
+ os.makedirs(
+ f"longbench_rst/{model_name}_{max_length}_{model_args.k_bits}bits_group{model_args.group_size}_residual{model_args.residual_length}"
+ )
+ out_path = f"longbench_rst/{model_name}_{max_length}_{model_args.k_bits}bits_group{model_args.group_size}_residual{model_args.residual_length}/{dataset}.jsonl"
+ else:
+ data = load_dataset("THUDM/LongBench", dataset, split="test")
+ if not os.path.exists(
+ f"longbench_rst_e/{model_name}_{max_length}_{model_args.k_bits}bits_group{model_args.group_size}_residual{model_args.residual_length}"
+ ):
+ os.makedirs(
+ f"longbench_rst_e/{model_name}_{max_length}_{model_args.k_bits}bits_group{model_args.group_size}_residual{model_args.residual_length}"
+ )
+ out_path = f"longbench_rst_e/{model_name}_{max_length}_{model_args.k_bits}bits_group{model_args.group_size}_residual{model_args.residual_length}/{dataset}.jsonl"
+ prompt_format = dataset2prompt[dataset]
+ max_gen = dataset2maxlen[dataset]
+ preds = get_pred(
+ model,
+ tokenizer,
+ data,
+ max_length,
+ max_gen,
+ prompt_format,
+ dataset,
+ device,
+ model_name,
+ )
+ with open(out_path, "w", encoding="utf-8") as f:
+ for pred in preds:
+ json.dump(pred, f, ensure_ascii=False)
+ f.write("\n")
diff --git a/LeanK/eval/LongBench/utils/data.py b/LeanK/eval/LongBench/utils/data.py
new file mode 100644
index 00000000..8dffcaf1
--- /dev/null
+++ b/LeanK/eval/LongBench/utils/data.py
@@ -0,0 +1,107 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import random
+
+import numpy as np
+import torch
+from datasets import load_dataset
+from torch.utils.data import DataLoader, Dataset
+
+
+class TextDataset(torch.utils.data.IterableDataset):
+ def __init__(self, data, tokenizer, seqlen, col_key, cutoff=1000):
+ self.tokenizer = tokenizer
+ self.col_key = col_key
+ self.cutoff = cutoff
+ self.block_size = seqlen
+ if cutoff is None:
+ cutoff = len(data)
+ tokenized_datasets = [self.tokenizer(data[i][col_key]) for i in range(cutoff)]
+ grouped_dataset = self.group_texts(tokenized_datasets)
+ self.input_ids = grouped_dataset["input_ids"]
+ self.labels = grouped_dataset["labels"]
+ self.data = [
+ dict(input_ids=self.input_ids[i], labels=self.labels[i])
+ for i in range(len(self.input_ids))
+ ]
+
+ def __len__(self):
+ return len(self.input_ids)
+
+ def __getitem__(self, i):
+ return dict(input_ids=self.input_ids[i], labels=self.labels[i])
+
+ def __iter__(self):
+ return iter(self.data)
+
+ def group_texts(self, examples):
+ # Concatenate all texts.
+ # Initialize an empty dictionary
+ concatenated_examples = {}
+
+ # Loop through the list of dictionaries
+ for d in examples:
+ # Loop through the keys in each dictionary
+ for key in d.keys():
+ # If the key is not already a key in the dict_of_lists, create a new list
+ if key not in concatenated_examples:
+ concatenated_examples[key] = []
+ # Append the value to the list associated with the key in dict_of_lists
+ concatenated_examples[key].extend(d[key])
+ total_length = len(concatenated_examples["input_ids"])
+ # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
+ # customize this part to your needs.
+ if total_length >= self.block_size:
+ total_length = (total_length // self.block_size) * self.block_size
+ # Split by chunks of max_len.
+ result = {
+ k: [
+ t[i : i + self.block_size]
+ for i in range(0, total_length, self.block_size)
+ ]
+ for k, t in concatenated_examples.items()
+ }
+ result["labels"] = result["input_ids"].copy()
+ return result
+
+
+def set_seed(seed):
+ np.random.seed(seed)
+ torch.random.manual_seed(seed)
+ random.seed(seed)
+
+
+def get_c4(n_train_samples, n_eval_samples, seqlen, tokenizer):
+ # raw_tra_data = load_dataset("c4", split="train")
+ raw_tra_data = load_dataset(
+ "allenai/c4",
+ "allenai--c4",
+ data_files={"train": "en/c4-train.00000-of-01024.json.gz"},
+ split="train",
+ )
+ # raw_val_data = load_dataset("c4", split="validation")
+ raw_val_data = load_dataset(
+ "allenai/c4",
+ "allenai--c4",
+ data_files={"validation": "en/c4-validation.00000-of-00008.json.gz"},
+ split="validation",
+ )
+ train_dataset = TextDataset(
+ raw_tra_data, tokenizer, col_key="text", cutoff=n_train_samples, seqlen=seqlen
+ )
+ val_dataset = TextDataset(
+ raw_val_data,
+ tokenizer,
+ col_key="text",
+ cutoff=n_eval_samples, # todo: change to 1100
+ seqlen=seqlen,
+ )
+ return train_dataset, val_dataset
+
+
+def get_loaders(name, enc, n_train_samples=128, n_eval_samples=1024, seqlen=2048):
+ if "c4" in name:
+ return get_c4(n_train_samples, n_eval_samples, seqlen, enc)
+ else:
+ raise NotImplementedError
diff --git a/LeanK/eval/LongBench/utils/metrics.py b/LeanK/eval/LongBench/utils/metrics.py
new file mode 100644
index 00000000..927de4e1
--- /dev/null
+++ b/LeanK/eval/LongBench/utils/metrics.py
@@ -0,0 +1,166 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import difflib
+import re
+import string
+from collections import Counter
+from typing import List
+
+import jieba
+from fuzzywuzzy import fuzz
+from rouge import Rouge
+
+
+def normalize_answer(s):
+ """Lower text and remove punctuation, articles and extra whitespace."""
+
+ def remove_articles(text):
+ return re.sub(r"\b(a|an|the)\b", " ", text)
+
+ def white_space_fix(text):
+ return " ".join(text.split())
+
+ def remove_punc(text):
+ exclude = set(string.punctuation)
+ return "".join(ch for ch in text if ch not in exclude)
+
+ def lower(text):
+ return text.lower()
+
+ return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def normalize_zh_answer(s):
+ """Lower text and remove punctuation, extra whitespace."""
+
+ def white_space_fix(text):
+ return "".join(text.split())
+
+ def remove_punc(text):
+ cn_punctuation = "！？｡。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏."
+ all_punctuation = set(string.punctuation + cn_punctuation)
+ return "".join(ch for ch in text if ch not in all_punctuation)
+
+ def lower(text):
+ return text.lower()
+
+ return white_space_fix(remove_punc(lower(s)))
+
+
+def count_score(prediction, ground_truth, **kwargs):
+ numbers = re.findall(r"\d+", prediction)
+ right_num = 0
+ for number in numbers:
+ if str(number) == str(ground_truth):
+ right_num += 1
+ final_score = 0.0 if len(numbers) == 0 else right_num / len(numbers)
+ return float(final_score)
+
+
+def retrieval_score(prediction, ground_truth, **kwargs):
+ pattern = r"Paragraph (\d+)"
+ matches = re.findall(pattern, ground_truth)
+ ground_truth_id = matches[0]
+ numbers = re.findall(r"\d+", prediction)
+ right_num = 0
+ for number in numbers:
+ if str(number) == str(ground_truth_id):
+ right_num += 1
+ final_score = 0.0 if len(numbers) == 0 else right_num / len(numbers)
+ return float(final_score)
+
+
+def retrieval_zh_score(prediction, ground_truth, **kwargs):
+ pattern = r"段落(\d+)"
+ matches = re.findall(pattern, ground_truth)
+ ground_truth_id = matches[0]
+ numbers = re.findall(r"\d+", prediction)
+ right_num = 0
+ for number in numbers:
+ if str(number) == str(ground_truth_id):
+ right_num += 1
+ final_score = 0.0 if len(numbers) == 0 else right_num / len(numbers)
+ return float(final_score)
+
+
+def code_sim_score(prediction, ground_truth, **kwargs):
+ all_lines = prediction.lstrip("\n").split("\n")
+ prediction = ""
+ for line in all_lines:
+ if ("`" not in line) and ("#" not in line) and ("//" not in line):
+ prediction = line
+ break
+ return fuzz.ratio(prediction, ground_truth) / 100
+
+
+def classification_score(prediction, ground_truth, **kwargs):
+ em_match_list = []
+ all_classes = kwargs["all_classes"]
+ for class_name in all_classes:
+ if class_name in prediction:
+ em_match_list.append(class_name)
+ for match_term in em_match_list:
+ if match_term in ground_truth and match_term != ground_truth:
+ em_match_list.remove(match_term)
+ if em_match_list != 0:
+ if ground_truth in em_match_list:
+ score = 1.0 / len(em_match_list)
+ else:
+ score = 0.0
+ else:
+ best_match = None
+ highest_similarity = 0
+ for string in all_classes:
+ similarity = difflib.SequenceMatcher(None, string, prediction).ratio()
+ if similarity > highest_similarity:
+ highest_similarity = similarity
+ best_match = string
+ score = float(best_match == ground_truth)
+ return score
+
+
+def rouge_score(prediction, ground_truth, **kwargs):
+ rouge = Rouge()
+ try:
+ scores = rouge.get_scores([prediction], [ground_truth], avg=True)
+ except:
+ return 0.0
+ return scores["rouge-l"]["f"]
+
+
+def rouge_zh_score(prediction, ground_truth, **kwargs):
+ prediction = " ".join(list(jieba.cut(prediction, cut_all=False)))
+ ground_truth = " ".join(list(jieba.cut(ground_truth, cut_all=False)))
+ score = rouge_score(prediction, ground_truth)
+ return score
+
+
+def f1_score(prediction, ground_truth, **kwargs):
+ common = Counter(prediction) & Counter(ground_truth)
+ num_same = sum(common.values())
+ if num_same == 0:
+ return 0
+ precision = 1.0 * num_same / len(prediction)
+ recall = 1.0 * num_same / len(ground_truth)
+ f1 = (2 * precision * recall) / (precision + recall)
+ return f1
+
+
+def qa_f1_score(prediction, ground_truth, **kwargs):
+ normalized_prediction = normalize_answer(prediction)
+ normalized_ground_truth = normalize_answer(ground_truth)
+
+ prediction_tokens = normalized_prediction.split()
+ ground_truth_tokens = normalized_ground_truth.split()
+ return f1_score(prediction_tokens, ground_truth_tokens)
+
+
+def qa_f1_zh_score(prediction, ground_truth, **kwargs):
+ prediction_tokens = list(jieba.cut(prediction, cut_all=False))
+ ground_truth_tokens = list(jieba.cut(ground_truth, cut_all=False))
+ prediction_tokens = [normalize_zh_answer(token) for token in prediction_tokens]
+ ground_truth_tokens = [normalize_zh_answer(token) for token in ground_truth_tokens]
+ prediction_tokens = [token for token in prediction_tokens if len(token) > 0]
+ ground_truth_tokens = [token for token in ground_truth_tokens if len(token) > 0]
+ return f1_score(prediction_tokens, ground_truth_tokens)
diff --git a/LeanK/eval/LongBench/utils/process_args.py b/LeanK/eval/LongBench/utils/process_args.py
new file mode 100644
index 00000000..033ff799
--- /dev/null
+++ b/LeanK/eval/LongBench/utils/process_args.py
@@ -0,0 +1,124 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# coding=utf-8
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+from dataclasses import dataclass, field
+from typing import Optional
+
+import transformers
+
+
+@dataclass
+class ModelArguments:
+ model_name_or_path: str = field(
+ default=None, metadata={"help": "Output model local path, do not set manually"}
+ )
+ k_bits: Optional[int] = field(
+ default=2,
+ metadata={"help": "KV_cache quantization bits."},
+ )
+ v_bits: Optional[int] = field(
+ default=2,
+ metadata={"help": "KV_cache quantization bits."},
+ )
+ k_quant_dim: Optional[str] = field(
+ default="token",
+ metadata={"help": "KV_cache quantization bits."},
+ )
+ v_quant_dim: Optional[str] = field(
+ default="token",
+ metadata={"help": "KV_cache quantization bits."},
+ )
+ group_size: Optional[int] = field(
+ default=32,
+ metadata={"help": "KV_cache quantization group size."},
+ )
+ residual_length: Optional[int] = field(
+ default=128,
+ metadata={"help": "KV_cache residual length."},
+ )
+ output_model_filename: Optional[str] = field(
+ default="test-output", metadata={"help": "Output model relative manifold path"}
+ )
+ load_quant: Optional[str] = field(
+ default=None,
+ metadata={"help": "The path to a quantized model"},
+ )
+ w_bit: Optional[int] = field(
+ default=4,
+ metadata={"help": "The model weight bit width."},
+ )
+ lora: Optional[bool] = field(
+ default=False,
+ metadata={"help": "Whether to use LoRA"},
+ )
+ lora_mode: Optional[str] = field(
+ default="q",
+ metadata={"help": "LoRA mode"},
+ )
+ lora_r: Optional[int] = field(
+ default=1,
+ metadata={"help": "LoRA r"},
+ )
+ lora_alpha: Optional[float] = field(
+ default=1.0,
+ metadata={"help": "LoRA alpha"},
+ )
+ lora_dropout: Optional[float] = field(
+ default=0.0,
+ metadata={"help": "LoRA dropout"},
+ )
+ enable_leank: Optional[str] = field(
+ default=True,
+ metadata={"help": "enable LeanK decoding"},
+ )
+ enable_think: Optional[str] = field(
+ default=False,
+ metadata={"help": "enable ThinK decoding"},
+ )
+
+
+@dataclass
+class DataArguments:
+ dataset: Optional[str] = field(
+ default="c4",
+ metadata={"help": "The dataset used for fine-tuning the model."},
+ )
+ eval_tasks: Optional[str] = field(
+ default="wikitext",
+ metadata={"help": "The dataset used for evaluation."},
+ )
+ tasks: Optional[str] = field(
+ default="wikitext",
+ metadata={"help": "The dataset used for evaluation."},
+ )
+ batch_size: Optional[int] = field(
+ default=1,
+ metadata={"help": "The batch size."},
+ )
+ num_fewshot: Optional[int] = field(
+ default=0,
+ metadata={"help": "The number of fewshot examples."},
+ )
+ output_path: Optional[str] = field(
+ default="./outputs",
+ metadata={"help": "The output path."},
+ )
+ e: Optional[bool] = field(
+ default=False,
+ metadata={"help": "Evaluate on LongBench-E."},
+ )
+
+
+def process_args():
+ parser = transformers.HfArgumentParser((ModelArguments, DataArguments))
+ model_args, data_args = parser.parse_args_into_dataclasses()
+
+ return model_args, data_args
diff --git a/LeanK/eval/RULER/config_models.sh b/LeanK/eval/RULER/config_models.sh
new file mode 100644
index 00000000..4650e589
--- /dev/null
+++ b/LeanK/eval/RULER/config_models.sh
@@ -0,0 +1,109 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TEMPERATURE="0.0" # greedy
+TOP_P="1.0"
+TOP_K="32"
+SEQ_LENGTHS=(
+ # 4096
+ # 8192
+ # 16384
+ # 32768
+ 65536
+ # 131072
+)
+
+MODEL_SELECT() {
+ MODEL_NAME=$1
+ MODEL_DIR=$2
+ ENGINE_DIR=$3
+
+ case $MODEL_NAME in
+ llama2-7b-chat)
+ MODEL_PATH="${MODEL_DIR}/llama2-7b-chat-hf"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="vllm"
+ ;;
+ llama3.1-8b-chat)
+ MODEL_PATH="${MODEL_DIR}/Meta-Llama-3.1-8B-Instruct"
+ MODEL_TEMPLATE_TYPE="meta-llama3"
+ MODEL_FRAMEWORK="hf"
+ ;;
+ qwen2.5-7b)
+ MODEL_PATH="${MODEL_DIR}/Qwen2.5-7B-Instruct"
+ MODEL_TEMPLATE_TYPE="meta-llama3"
+ MODEL_FRAMEWORK="hf"
+ ;;
+ jamba1.5-mini)
+ MODEL_PATH="${MODEL_DIR}/Jamba-1.5-Mini"
+ MODEL_TEMPLATE_TYPE="jamba"
+ MODEL_FRAMEWORK="vllm"
+ ;;
+ gpt-3.5-turbo)
+ MODEL_PATH="gpt-3.5-turbo-0125"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="openai"
+ TOKENIZER_PATH="cl100k_base"
+ TOKENIZER_TYPE="openai"
+ OPENAI_API_KEY=""
+ AZURE_ID=""
+ AZURE_SECRET=""
+ AZURE_ENDPOINT=""
+ ;;
+ gpt-4-turbo)
+ MODEL_PATH="gpt-4"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="openai"
+ TOKENIZER_PATH="cl100k_base"
+ TOKENIZER_TYPE="openai"
+ OPENAI_API_KEY=""
+ AZURE_ID=""
+ AZURE_SECRET=""
+ AZURE_ENDPOINT=""
+ ;;
+ gemini_1.0_pro)
+ MODEL_PATH="gemini-1.0-pro-latest"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="gemini"
+ TOKENIZER_PATH=$MODEL_PATH
+ TOKENIZER_TYPE="gemini"
+ GEMINI_API_KEY=""
+ ;;
+ gemini_1.5_pro)
+ MODEL_PATH="gemini-1.5-pro-latest"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="gemini"
+ TOKENIZER_PATH=$MODEL_PATH
+ TOKENIZER_TYPE="gemini"
+ GEMINI_API_KEY=""
+ ;;
+ esac
+
+
+ if [ -z "${TOKENIZER_PATH}" ]; then
+ if [ -f ${MODEL_PATH}/tokenizer.model ]; then
+ TOKENIZER_PATH=${MODEL_PATH}/tokenizer.model
+ TOKENIZER_TYPE="nemo"
+ else
+ TOKENIZER_PATH=${MODEL_PATH}
+ TOKENIZER_TYPE="hf"
+ fi
+ fi
+
+
+ echo "$MODEL_PATH:$MODEL_TEMPLATE_TYPE:$MODEL_FRAMEWORK:$TOKENIZER_PATH:$TOKENIZER_TYPE:$OPENAI_API_KEY:$GEMINI_API_KEY:$AZURE_ID:$AZURE_SECRET:$AZURE_ENDPOINT"
+}
diff --git a/LeanK/eval/RULER/config_tasks.sh b/LeanK/eval/RULER/config_tasks.sh
new file mode 100644
index 00000000..8d419118
--- /dev/null
+++ b/LeanK/eval/RULER/config_tasks.sh
@@ -0,0 +1,49 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+NUM_SAMPLES=5
+REMOVE_NEWLINE_TAB=false
+STOP_WORDS=""
+
+if [ -z "${STOP_WORDS}" ]; then
+ STOP_WORDS=""
+else
+ STOP_WORDS="--stop_words \"${STOP_WORDS}\""
+fi
+
+if [ "${REMOVE_NEWLINE_TAB}" = false ]; then
+ REMOVE_NEWLINE_TAB=""
+else
+ REMOVE_NEWLINE_TAB="--remove_newline_tab"
+fi
+
+# task name in `synthetic.yaml`
+synthetic=(
+ # "niah_single_1"
+ # "niah_single_2"
+ # "niah_single_3"
+ # "niah_multikey_1"
+ # "niah_multikey_2"
+ "niah_multikey_3"
+ # "niah_multivalue"
+ # "niah_multiquery"
+ # "vt"
+ # "fwe"
+ # "cwe"
+ # "qa_1"
+ # "qa_2"
+)
diff --git a/LeanK/eval/RULER/data/prepare.py b/LeanK/eval/RULER/data/prepare.py
new file mode 100644
index 00000000..49675e0c
--- /dev/null
+++ b/LeanK/eval/RULER/data/prepare.py
@@ -0,0 +1,199 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Prepare jsonl with field `input` and `outputs`.
+{
+ "index" int,
+ "input": str,
+ "outputs": [str],
+}
+
+python prepare.py \
+ --save_dir ./ \
+ --benchmark synthetic \
+ --task niah_single_1 \
+ --tokenizer_path tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --model_template_type base \
+ --num_samples 10 \
+"""
+import argparse
+import importlib
+import os
+import subprocess
+import time
+from pathlib import Path
+
+import nltk
+import yaml
+from template import Templates
+
+try:
+ nltk.data.find("tokenizers/punkt")
+except LookupError:
+ nltk.download("punkt")
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--benchmark", type=str, default="synthetic", help="Options: [synthetic]"
+)
+parser.add_argument("--task", type=str, required=True, help="tasks in benchmark")
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--num_samples",
+ type=int,
+ default=500,
+ help="maximum number of samples we want to test",
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument(
+ "--model_template_type", type=str, default="base", help="Options in `template.py`"
+)
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+parser.add_argument(
+ "--chunk_idx", type=int, default=0, help="index of current split chunk"
+)
+parser.add_argument("--chunk_amount", type=int, default=1, help="size of split chunk")
+
+args = parser.parse_args()
+
+
+def main():
+ start_time = time.time()
+ curr_folder = os.path.dirname(os.path.abspath(__file__))
+
+ try:
+ module = importlib.import_module(f"{args.benchmark}.constants")
+ except ImportError:
+ print(f"Module data.{args.benchmark}.constants not found.")
+
+ tasks_base = module.TASKS
+ with open(os.path.join(curr_folder, f"../{args.benchmark}.yaml"), "r") as f:
+ tasks_customized = yaml.safe_load(f)
+
+ if args.task not in tasks_customized:
+ raise ValueError(f"{args.task} is not found in config_tasks.yaml")
+
+ config = tasks_customized.get(args.task)
+ config.update(tasks_base[config["task"]])
+
+ # Add templates
+ assert args.model_template_type in Templates, print(
+ f"{args.model_template_type} is not found in {Templates.keys()}"
+ )
+ model_template = Templates[args.model_template_type]
+ task_template = config["template"]
+
+ # Add answer prefix for all models
+ answer_prefix = config["answer_prefix"] if "answer_prefix" in config else ""
+ config["template"] = (
+ model_template.format(task_template=task_template) + answer_prefix
+ )
+
+ # Split task into multiple chunks
+ chunks = [
+ (args.num_samples // args.chunk_amount)
+ + (1 if i < args.num_samples % args.chunk_amount else 0)
+ for i in range(args.chunk_amount)
+ ]
+ num_samples = chunks[args.chunk_idx]
+ pre_samples = sum(chunks[: args.chunk_idx])
+
+ random_seed = args.random_seed + args.chunk_idx
+
+ save_file = args.save_dir / args.task / f"{args.subset}.jsonl"
+ file_exists = False
+ if os.path.exists(save_file):
+ with open(save_file, "r") as f:
+ data = f.readlines()
+ if len(data) == args.num_samples:
+ file_exists = True
+
+ if not file_exists:
+ try:
+ script = os.path.join(curr_folder, args.benchmark, f"{config['task']}.py")
+ additional_args = " ".join(
+ [f"--{k} {v}" for k, v in config["args"].items()]
+ )
+ command = f"""python {script} \
+ --save_dir {args.save_dir} \
+ --save_name {args.task} \
+ --subset {args.subset} \
+ --tokenizer_path {args.tokenizer_path} \
+ --tokenizer_type {args.tokenizer_type} \
+ --max_seq_length {args.max_seq_length} \
+ --tokens_to_generate {config['tokens_to_generate']} \
+ --num_samples {num_samples} \
+ --random_seed {random_seed} \
+ {additional_args} \
+ {f"--remove_newline_tab" if args.remove_newline_tab else ""} \
+ {f"--pre_samples {pre_samples}" if config['task'] == 'qa' else ""} \
+ --template "{config['template']}"
+ """
+ print(command)
+ result = subprocess.run(
+ command,
+ shell=True,
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ )
+
+ if result.returncode == 0:
+ print("Output:")
+ print(result.stdout)
+ else:
+ print("Error:")
+ print(result.stderr)
+ except subprocess.CalledProcessError as e:
+ print("Error output:", e.stderr)
+
+ print(f"Prepare {args.task} with lines: {args.num_samples} to {save_file}")
+ print(f"Used time: {round((time.time() - start_time) / 60, 1)} minutes")
+ else:
+ print(
+ f"Skip preparing {args.task} with lines: {args.num_samples} to {save_file} (file exists)"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/data/synthetic/common_words_extraction.py b/LeanK/eval/RULER/data/synthetic/common_words_extraction.py
new file mode 100644
index 00000000..7dbb4d14
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/common_words_extraction.py
@@ -0,0 +1,221 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+"""
+Create a dataset jsonl file for common words extraction.
+
+python common_words_extraction.py \
+ --save_dir=./ \
+ --save_name=vt \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --tokens_to_generate 30 \
+ --num_samples 10 \
+ --random_seed 42 \
+ -freq_cw 30 --freq_ucw 3 --num_cw 10 \
+ --template "[INST] Below is a numbered list of words. In these words, some appear more often than others. Memorize the ones that appear most often.\n{context}\nQuestion: What are the 10 most common words in the above list? [/INST] Answer: The top 10 words that appear most often in the list are:"
+"""
+
+import argparse
+import os
+import random
+import sys
+from pathlib import Path
+
+import wonderwords
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate",
+ type=int,
+ required=True,
+ help="expected generated token amount.",
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+parser.add_argument("--freq_cw", type=int, default=30)
+parser.add_argument("--freq_ucw", type=int, default=3)
+parser.add_argument("--num_cw", type=int, default=10)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+nouns = wonderwords.random_word._get_words_from_text_file("nounlist.txt")
+adjs = wonderwords.random_word._get_words_from_text_file("adjectivelist.txt")
+verbs = wonderwords.random_word._get_words_from_text_file("verblist.txt")
+words = nouns + adjs + verbs
+words = sorted(list(set(words)))
+random.Random(args.random_seed).shuffle(words)
+
+
+def get_example(num_words, common_repeats=30, uncommon_repeats=3, common_nums=10):
+ word_list_full = random.sample(words, num_words)
+ common, uncommon = word_list_full[:common_nums], word_list_full[common_nums:]
+ word_list = common * int(common_repeats) + uncommon * int(uncommon_repeats)
+ random.Random(args.random_seed).shuffle(word_list)
+
+ # Formatting the word list as "1. word1 2. word2 3. word3 ..."
+ context = " ".join([f"{i + 1}. {word}" for i, word in enumerate(word_list)])
+
+ return context, common
+
+
+def generate_input_output(num_words):
+ if args.max_seq_length < 4096:
+ context_example, answer_example = get_example(20, 3, 1, args.num_cw)
+ context, answer = get_example(num_words, 6, 1, args.num_cw)
+ else:
+ context_example, answer_example = get_example(40, 10, 3, args.num_cw)
+ context, answer = get_example(
+ num_words, args.freq_cw, args.freq_ucw, args.num_cw
+ )
+
+ template = args.template
+
+ input_example = template.format(
+ context=context_example,
+ query="",
+ ) + " ".join([f"{i + 1}. {word}" for i, word in enumerate(answer_example)])
+
+ input_text = template.format(
+ context=context,
+ query="",
+ )
+
+ return input_example + "\n" + input_text, answer
+
+
+def sys_word_pair_random(
+ num_samples: int, max_seq_length: int, save_dir: str, incremental: int = 10
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ # Find the perfect num_words
+ num_words = incremental
+
+ total_tokens = 0
+ while total_tokens + tokens_to_generate < max_seq_length:
+ input_text, answer = generate_input_output(num_words)
+ # Calculate the number of tokens in the example
+ total_tokens = len(
+ TOKENIZER.text_to_tokens(
+ input_text
+ + " "
+ + " ".join([f"{i + 1}. {word}" for i, word in enumerate(answer)])
+ )
+ )
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Words: {num_words}"
+ )
+ if total_tokens + tokens_to_generate > max_seq_length:
+ num_words -= incremental
+ break
+
+ num_words += incremental
+ if num_words > len(words):
+ num_words = len(words)
+ break
+
+ print("num_words:", num_words)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_words = num_words
+ while True:
+ try:
+ input_text, answer = generate_input_output(used_words)
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_words > incremental:
+ used_words -= incremental
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ write_jsons = sys_word_pair_random(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ save_dir=args.save_dir,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/data/synthetic/constants.py b/LeanK/eval/RULER/data/synthetic/constants.py
new file mode 100644
index 00000000..fed446c1
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/constants.py
@@ -0,0 +1,53 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+"""
+Add a new task (required arguments):
+
+TASK_NAME: {
+ 'tokens_to_generate': how many tokens we want to generate.
+ 'template': the template with at least {context} and {query}.
+}
+"""
+
+TASKS = {
+ "niah": {
+ "tokens_to_generate": 128,
+ "template": """Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned in the provided text?""",
+ "answer_prefix": """ The special magic {type_needle_v} for {query} mentioned in the provided text are""",
+ },
+ "variable_tracking": {
+ "tokens_to_generate": 30,
+ "template": """Memorize and track the chain(s) of variable assignment hidden in the following text.\n\n{context}\nQuestion: Find all variables that are assigned the value {query} in the text above.""",
+ "answer_prefix": """ Answer: According to the chain(s) of variable assignment in the text above, {num_v} variables are assgined the value {query}, they are: """,
+ },
+ "common_words_extraction": {
+ "tokens_to_generate": 120,
+ "template": """Below is a numbered list of words. In these words, some appear more often than others. Memorize the ones that appear most often.\n{context}\nQuestion: What are the 10 most common words in the above list?""",
+ "answer_prefix": """ Answer: The top 10 words that appear most often in the list are:""",
+ },
+ "freq_words_extraction": {
+ "tokens_to_generate": 50,
+ "template": """Read the following coded text and track the frequency of each coded word. Find the three most frequently appeared coded words. {context}\nQuestion: Do not provide any explanation. Please ignore the dots '....'. What are the three most frequently appeared words in the above coded text?""",
+ "answer_prefix": """ Answer: According to the coded text above, the three most frequently appeared words are:""",
+ },
+ "qa": {
+ "tokens_to_generate": 32,
+ "template": """Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query}""",
+ "answer_prefix": """ Answer:""",
+ },
+}
diff --git a/LeanK/eval/RULER/data/synthetic/freq_words_extraction.py b/LeanK/eval/RULER/data/synthetic/freq_words_extraction.py
new file mode 100644
index 00000000..953ee995
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/freq_words_extraction.py
@@ -0,0 +1,204 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+"""
+Create a dataset jsonl file for frequent words extraction.
+
+python freq_words_extraction.py \
+ --save_dir=./ \
+ --save_name=vt \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --tokens_to_generate 30 \
+ --num_samples 10 \
+ --random_seed 42 \
+ --alpha 2.0 \
+ --template "[INST] Read the following coded text and track the frequency of each coded word. Find the three most frequently appeared coded words. {context}\nQuestion: Do not provide any explanation. Please ignore the dots '....'. What are the three most frequently appeared words in the above coded text? [/INST] Answer: According to the coded text above, the three most frequently appeared words are:"
+"""
+
+import argparse
+import os
+import random
+import string
+import sys
+from pathlib import Path
+
+import numpy as np
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from scipy.special import zeta
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate", type=int, default=50, help="number of tokens to generate"
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+parser.add_argument(
+ "--coded_wordlen", type=int, default=6, help="length of synthetic word"
+)
+parser.add_argument(
+ "--vocab_size", type=int, default=-1, help="synthetic vocab size to sample from"
+)
+parser.add_argument("--alpha", type=float, default=2.0, help="zeta distribution alpha")
+parser.add_argument("--add_fewshot", action="store_true", default=False)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+
+def generate_input_output(
+ max_len, num_words=-1, coded_wordlen=6, vocab_size=2000, incremental=10, alpha=2.0
+):
+ # generate vocab
+ vocab = [
+ "".join(random.choices(string.ascii_lowercase, k=coded_wordlen))
+ for _ in range(vocab_size)
+ ]
+ while len(set(vocab)) < vocab_size:
+ vocab.append("".join(random.choices(string.ascii_lowercase, k=coded_wordlen)))
+ vocab = sorted(list(set(vocab)))
+ random.Random(args.random_seed).shuffle(vocab)
+ vocab[0] = "..." # treat the top ranked as noise
+
+ # sample words
+ template = args.template
+
+ def gen_text(num_words):
+ k = np.arange(1, len(vocab) + 1)
+ sampled_cnt = num_words * (k**-alpha) / zeta(alpha)
+ sampled_words = [[w] * zi for w, zi in zip(vocab, sampled_cnt.astype(int))]
+ sampled_words = [x for wlst in sampled_words for x in wlst]
+ random.Random(args.random_seed).shuffle(sampled_words)
+ return template.format(context=" ".join(sampled_words), query=""), vocab[1:4]
+
+ if num_words > 0:
+ num_words = num_words
+ text, answer = gen_text(num_words)
+ while len(TOKENIZER.text_to_tokens(text)) > max_len:
+ num_words -= incremental
+ text, answer = gen_text(num_words)
+ else:
+ num_words = max_len // coded_wordlen # init
+ text, answer = gen_text(num_words)
+ while len(TOKENIZER.text_to_tokens(text)) < max_len:
+ num_words += incremental
+ text, answer = gen_text(num_words)
+ num_words -= incremental
+ text, answer = gen_text(num_words)
+ return text, answer, num_words
+
+
+def sys_kwext(num_samples: int, max_seq_length: int, incremental: int = 10):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ vocab_size = max_seq_length // 50 if args.vocab_size == -1 else args.vocab_size
+
+ # get number of words
+ input_max_len = max_seq_length
+ _, _, num_example_words = generate_input_output(
+ input_max_len,
+ coded_wordlen=args.coded_wordlen,
+ vocab_size=vocab_size,
+ incremental=input_max_len // 32,
+ alpha=args.alpha,
+ )
+ print("num_example_words:", num_example_words)
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ # construct input
+ input_max_len = max_seq_length
+ input_text, answer, _ = generate_input_output(
+ input_max_len,
+ num_words=num_example_words,
+ coded_wordlen=args.coded_wordlen,
+ vocab_size=vocab_size,
+ incremental=input_max_len // 32,
+ alpha=args.alpha,
+ )
+
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+ write_jsons = sys_kwext(
+ num_samples=args.num_samples, max_seq_length=args.max_seq_length, incremental=10
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/data/synthetic/json/PaulGrahamEssays_URLs.txt b/LeanK/eval/RULER/data/synthetic/json/PaulGrahamEssays_URLs.txt
new file mode 100644
index 00000000..b9a38e44
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/json/PaulGrahamEssays_URLs.txt
@@ -0,0 +1,218 @@
+http://www.paulgraham.com/13sentences.html
+http://www.paulgraham.com/5founders.html
+http://www.paulgraham.com/6631327.html
+http://www.paulgraham.com/95.html
+http://www.paulgraham.com/ace.html
+http://www.paulgraham.com/airbnb.html
+http://www.paulgraham.com/airbnbs.html
+http://www.paulgraham.com/alien.html
+http://www.paulgraham.com/altair.html
+http://www.paulgraham.com/ambitious.html
+http://www.paulgraham.com/america.html
+http://www.paulgraham.com/angelinvesting.html
+http://www.paulgraham.com/artistsship.html
+http://www.paulgraham.com/badeconomy.html
+http://www.paulgraham.com/better.html
+http://www.paulgraham.com/bronze.html
+http://www.paulgraham.com/bubble.html
+http://www.paulgraham.com/charisma.html
+http://www.paulgraham.com/cities.html
+http://www.paulgraham.com/college.html
+http://www.paulgraham.com/colleges.html
+http://www.paulgraham.com/conformism.html
+http://www.paulgraham.com/control.html
+http://www.paulgraham.com/convergence.html
+http://www.paulgraham.com/convince.html
+http://www.paulgraham.com/cred.html
+http://www.paulgraham.com/credentials.html
+http://www.paulgraham.com/determination.html
+http://www.paulgraham.com/die.html
+http://www.paulgraham.com/disagree.html
+http://www.paulgraham.com/disc.html
+http://www.paulgraham.com/discover.html
+http://www.paulgraham.com/distraction.html
+http://www.paulgraham.com/divergence.html
+http://www.paulgraham.com/donate.html
+http://www.paulgraham.com/ds.html
+http://www.paulgraham.com/early.html
+http://www.paulgraham.com/earnest.html
+http://www.paulgraham.com/equity.html
+http://www.paulgraham.com/essay.html
+http://www.paulgraham.com/ffb.html
+http://www.paulgraham.com/fh.html
+http://www.paulgraham.com/fix.html
+http://www.paulgraham.com/fn.html
+http://www.paulgraham.com/foundersatwork.html
+http://www.paulgraham.com/fp.html
+http://www.paulgraham.com/fr.html
+http://www.paulgraham.com/fundraising.html
+http://www.paulgraham.com/future.html
+http://www.paulgraham.com/genius.html
+http://www.paulgraham.com/getideas.html
+http://www.paulgraham.com/good.html
+http://www.paulgraham.com/goodart.html
+http://www.paulgraham.com/googles.html
+http://www.paulgraham.com/greatwork.html
+http://www.paulgraham.com/growth.html
+http://www.paulgraham.com/guidetoinvestors.html
+http://www.paulgraham.com/hackernews.html
+http://www.paulgraham.com/head.html
+http://www.paulgraham.com/herd.html
+http://www.paulgraham.com/heresy.html
+http://www.paulgraham.com/heroes.html
+http://www.paulgraham.com/highres.html
+http://www.paulgraham.com/hiresfund.html
+http://www.paulgraham.com/hiring.html
+http://www.paulgraham.com/hp.html
+http://www.paulgraham.com/hs.html
+http://www.paulgraham.com/hundred.html
+http://www.paulgraham.com/hw.html
+http://www.paulgraham.com/hwh.html
+http://www.paulgraham.com/icad.html
+http://www.paulgraham.com/ideas.html
+http://www.paulgraham.com/identity.html
+http://www.paulgraham.com/ineq.html
+http://www.paulgraham.com/inequality.html
+http://www.paulgraham.com/investors.html
+http://www.paulgraham.com/invtrend.html
+http://www.paulgraham.com/javacover.html
+http://www.paulgraham.com/jessica.html
+http://www.paulgraham.com/judgement.html
+http://www.paulgraham.com/kate.html
+http://www.paulgraham.com/kids.html
+http://www.paulgraham.com/ladder.html
+http://www.paulgraham.com/lesson.html
+http://www.paulgraham.com/lies.html
+http://www.paulgraham.com/lwba.html
+http://www.paulgraham.com/mac.html
+http://www.paulgraham.com/makersschedule.html
+http://www.paulgraham.com/marginal.html
+http://www.paulgraham.com/maybe.html
+http://www.paulgraham.com/mean.html
+http://www.paulgraham.com/microsoft.html
+http://www.paulgraham.com/mit.html
+http://www.paulgraham.com/name.html
+http://www.paulgraham.com/nerds.html
+http://www.paulgraham.com/newthings.html
+http://www.paulgraham.com/noob.html
+http://www.paulgraham.com/noop.html
+http://www.paulgraham.com/notnot.html
+http://www.paulgraham.com/nov.html
+http://www.paulgraham.com/nthings.html
+http://www.paulgraham.com/opensource.html
+http://www.paulgraham.com/organic.html
+http://www.paulgraham.com/orth.html
+http://www.paulgraham.com/own.html
+http://www.paulgraham.com/patentpledge.html
+http://www.paulgraham.com/pgh.html
+http://www.paulgraham.com/pinch.html
+http://www.paulgraham.com/polls.html
+http://www.paulgraham.com/power.html
+http://www.paulgraham.com/prcmc.html
+http://www.paulgraham.com/procrastination.html
+http://www.paulgraham.com/progbot.html
+http://www.paulgraham.com/prop62.html
+http://www.paulgraham.com/property.html
+http://www.paulgraham.com/publishing.html
+http://www.paulgraham.com/pypar.html
+http://www.paulgraham.com/ramenprofitable.html
+http://www.paulgraham.com/randomness.html
+http://www.paulgraham.com/re.html
+http://www.paulgraham.com/read.html
+http://www.paulgraham.com/real.html
+http://www.paulgraham.com/really.html
+http://www.paulgraham.com/relres.html
+http://www.paulgraham.com/revolution.html
+http://www.paulgraham.com/richnow.html
+http://www.paulgraham.com/road.html
+http://www.paulgraham.com/ronco.html
+http://www.paulgraham.com/safe.html
+http://www.paulgraham.com/say.html
+http://www.paulgraham.com/schlep.html
+http://www.paulgraham.com/seesv.html
+http://www.paulgraham.com/segway.html
+http://www.paulgraham.com/selfindulgence.html
+http://www.paulgraham.com/sfp.html
+http://www.paulgraham.com/simply.html
+http://www.paulgraham.com/smart.html
+http://www.paulgraham.com/softwarepatents.html
+http://www.paulgraham.com/spam.html
+http://www.paulgraham.com/speak.html
+http://www.paulgraham.com/start.html
+http://www.paulgraham.com/startupfunding.html
+http://www.paulgraham.com/startuphubs.html
+http://www.paulgraham.com/startupideas.html
+http://www.paulgraham.com/startupmistakes.html
+http://www.paulgraham.com/stuff.html
+http://www.paulgraham.com/superlinear.html
+http://www.paulgraham.com/swan.html
+http://www.paulgraham.com/tablets.html
+http://www.paulgraham.com/talk.html
+http://www.paulgraham.com/taste.html
+http://www.paulgraham.com/think.html
+http://www.paulgraham.com/top.html
+http://www.paulgraham.com/trolls.html
+http://www.paulgraham.com/twitter.html
+http://www.paulgraham.com/usa.html
+http://www.paulgraham.com/users.html
+http://www.paulgraham.com/venturecapital.html
+http://www.paulgraham.com/wealth.html
+http://www.paulgraham.com/webstartups.html
+http://www.paulgraham.com/whyyc.html
+http://www.paulgraham.com/word.html
+http://www.paulgraham.com/words.html
+http://www.paulgraham.com/work.html
+http://www.paulgraham.com/writing44.html
+http://www.paulgraham.com/wtax.html
+http://www.paulgraham.com/yahoo.html
+http://www.paulgraham.com/ycombinator.html
+http://www.paulgraham.com/ycstart.html
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/addiction.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/aord.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/apple.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/avg.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/before.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/bias.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/boss.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/copy.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/corpdev.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/desres.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/diff.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/ecw.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/founders.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/foundervisa.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gap.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gba.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gh.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/goodtaste.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/hubs.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/iflisp.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/island.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/know.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/langdes.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/laundry.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/love.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/mod.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/newideas.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/nft.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/philosophy.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/popular.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/pow.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/rootsoflisp.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/rss.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/siliconvalley.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/startuplessons.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/submarine.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/sun.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/superangels.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/todo.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/unions.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/useful.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vb.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vcsqueeze.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vw.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/want.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/web20.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/weird.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/wisdom.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/worked.txt
diff --git a/LeanK/eval/RULER/data/synthetic/json/download_paulgraham_essay.py b/LeanK/eval/RULER/data/synthetic/json/download_paulgraham_essay.py
new file mode 100755
index 00000000..d23da508
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/json/download_paulgraham_essay.py
@@ -0,0 +1,88 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import glob
+import json
+import os
+import shutil
+import urllib.request
+
+import html2text
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+
+temp_folder_repo = "essay_repo"
+temp_folder_html = "essay_html"
+os.makedirs(temp_folder_repo, exist_ok=True)
+os.makedirs(temp_folder_html, exist_ok=True)
+
+h = html2text.HTML2Text()
+h.ignore_images = True
+h.ignore_tables = True
+h.escape_all = True
+h.reference_links = False
+h.mark_code = False
+
+with open("PaulGrahamEssays_URLs.txt") as f:
+ urls = [line.strip() for line in f]
+
+for url in tqdm(urls):
+ if ".html" in url:
+ filename = url.split("/")[-1].replace(".html", ".txt")
+ try:
+ with urllib.request.urlopen(url) as website:
+ content = website.read().decode("unicode_escape", "utf-8")
+ soup = BeautifulSoup(content, "html.parser")
+ specific_tag = soup.find("font")
+ parsed = h.handle(str(specific_tag))
+
+ with open(os.path.join(temp_folder_html, filename), "w") as file:
+ file.write(parsed)
+
+ except Exception as e:
+ print(f"Fail download {filename}, ({e})")
+
+ else:
+ filename = url.split("/")[-1]
+ try:
+ with urllib.request.urlopen(url) as website:
+ content = website.read().decode("utf-8")
+
+ with open(os.path.join(temp_folder_repo, filename), "w") as file:
+ file.write(content)
+
+ except Exception as e:
+ print(f"Fail download {filename}, ({e})")
+
+files_repo = sorted(glob.glob(os.path.join(temp_folder_repo, "*.txt")))
+files_html = sorted(glob.glob(os.path.join(temp_folder_html, "*.txt")))
+print(
+ f"Download {len(files_repo)} essays from `https://github.com/gkamradt/LLMTest_NeedleInAHaystack/`"
+)
+print(f"Download {len(files_html)} essays from `http://www.paulgraham.com/`")
+
+text = ""
+for file in files_repo + files_html:
+ with open(file, "r") as f:
+ text += f.read()
+
+with open("PaulGrahamEssays.json", "w") as f:
+ json.dump({"text": text}, f)
+
+
+shutil.rmtree(temp_folder_repo)
+shutil.rmtree(temp_folder_html)
diff --git a/LeanK/eval/RULER/data/synthetic/json/download_qa_dataset.sh b/LeanK/eval/RULER/data/synthetic/json/download_qa_dataset.sh
new file mode 100644
index 00000000..1efae2a4
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/json/download_qa_dataset.sh
@@ -0,0 +1,19 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json -O squad.json
+wget http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_distractor_v1.json -O hotpotqa.json
diff --git a/LeanK/eval/RULER/data/synthetic/niah.py b/LeanK/eval/RULER/data/synthetic/niah.py
new file mode 100644
index 00000000..0220dac5
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/niah.py
@@ -0,0 +1,346 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+"""
+Create a dataset jsonl file for needle in a haystack.
+
+python niah.py \
+ --save_dir=./ \
+ --save_name=niah_single \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type=nemo \
+ --max_seq_length=4096 \
+ --tokens_to_generate=128 \
+ --num_samples=10 \
+ --template="Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned in the provided text? The special magic {type_needle_v} for {query} mentioned in the provided text are"
+"""
+import argparse
+import importlib
+import json
+import os
+import random
+import re
+import sys
+import uuid
+from pathlib import Path
+
+import numpy as np
+import wonderwords
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from nltk.tokenize import sent_tokenize
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+# Basic Configurations
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate",
+ type=int,
+ required=True,
+ help="expected generated token amount.",
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+# Complexity Configurations
+parser.add_argument("--num_needle_k", type=int, default=1)
+parser.add_argument("--num_needle_v", type=int, default=1)
+parser.add_argument("--num_needle_q", type=int, default=1)
+parser.add_argument(
+ "--type_haystack",
+ type=str,
+ default="essay",
+ help="[Options] repeat, essay, needle.",
+)
+parser.add_argument(
+ "--type_needle_k",
+ type=str,
+ default="words",
+ help="[Options] numbers, words, uuids.",
+)
+parser.add_argument(
+ "--type_needle_v",
+ type=str,
+ default="numbers",
+ help="[Options] numbers, words, uuids.",
+)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+args.num_needle_k = max(args.num_needle_k, args.num_needle_q)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+# Define Needle/Haystack Format
+needle = "One of the special magic {type_needle_v} for {key} is: {value}."
+if args.type_haystack == "essay":
+ essay = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "json/PaulGrahamEssays.json"
+ )
+ essay = json.load(open(essay))["text"]
+ haystack = re.sub(r"\s+", " ", essay).split(" ")
+elif args.type_haystack == "repeat":
+ haystack = "The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again."
+elif args.type_haystack == "needle":
+ haystack = needle
+else:
+ raise NotImplementedError(f"{args.type_haystack} is not implemented.")
+
+
+# Words
+nouns = wonderwords.random_word._get_words_from_text_file("nounlist.txt")
+adjs = wonderwords.random_word._get_words_from_text_file("adjectivelist.txt")
+# verbs = wonderwords.random_word._get_words_from_text_file("verblist.txt")
+words = [f"{adj}-{noun}" for adj in adjs for noun in nouns]
+words = sorted(list(set(words)))
+
+
+# Positions
+DEPTHS = list(np.round(np.linspace(0, 100, num=40, endpoint=True)).astype(int))
+
+
+def generate_random_number(num_digits=7):
+ lower_bound = 10 ** (num_digits - 1)
+ upper_bound = 10**num_digits - 1
+ return str(random.randint(lower_bound, upper_bound))
+
+
+def generate_random_word():
+ word = random.choice(words)
+ return word
+
+
+def generate_random_uuid():
+ return str(uuid.UUID(int=random.getrandbits(128), version=4))
+
+
+def generate_random(type_needle: str):
+ if type_needle == "numbers":
+ return generate_random_number()
+ elif type_needle == "words":
+ return generate_random_word()
+ elif type_needle == "uuids":
+ return generate_random_uuid()
+ else:
+ raise NotImplementedError(f"{args.type_needle} is not implemented.")
+
+
+def generate_input_output(num_haystack):
+ keys, values, needles = [], [], []
+ for _ in range(args.num_needle_k):
+ keys.append(generate_random(args.type_needle_k))
+ value = []
+ for _ in range(args.num_needle_v):
+ value.append(generate_random(args.type_needle_v))
+ needles.append(
+ needle.format(
+ type_needle_v=args.type_needle_v,
+ key=keys[-1],
+ value=value[-1],
+ )
+ )
+ values.append(value)
+
+ random.Random(args.random_seed).shuffle(needles)
+
+ # Context
+ if args.type_haystack == "essay":
+ text = " ".join(haystack[:num_haystack])
+ document_sents = sent_tokenize(text.strip())
+ insertion_positions = (
+ [0]
+ + sorted(
+ [
+ int(len(document_sents) * (depth / 100))
+ for depth in random.sample(DEPTHS, len(needles))
+ ]
+ )
+ + [len(document_sents)]
+ )
+ document_sents_list = []
+ for i in range(1, len(insertion_positions)):
+ last_pos = insertion_positions[i - 1]
+ next_pos = insertion_positions[i]
+ document_sents_list.append(" ".join(document_sents[last_pos:next_pos]))
+ if i - 1 < len(needles):
+ document_sents_list.append(needles[i - 1])
+ context = " ".join(document_sents_list)
+
+ else:
+ if args.type_haystack == "repeat":
+ sentences = [haystack] * num_haystack
+ elif args.type_haystack == "needle":
+ sentences = [
+ haystack.format(
+ type_needle_v=args.type_needle_v,
+ key=generate_random(args.type_needle_k),
+ value=generate_random(args.type_needle_v),
+ )
+ for _ in range(num_haystack)
+ ]
+
+ indexes = sorted(random.sample(range(num_haystack), len(needles)), reverse=True)
+ for index, element in zip(indexes, needles):
+ sentences.insert(index, element)
+ context = "\n".join(sentences)
+
+ ## Query and Answer
+ indices = random.sample(range(args.num_needle_k), args.num_needle_q)
+ queries = [keys[i] for i in indices]
+ answers = [a for i in indices for a in values[i]]
+ query = (
+ ", ".join(queries[:-1]) + ", and " + queries[-1]
+ if len(queries) > 1
+ else queries[0]
+ )
+
+ template = args.template
+ type_needle_v = args.type_needle_v
+ if args.num_needle_q * args.num_needle_v == 1:
+ template = template.replace("Some", "A")
+ template = template.replace("are all", "is")
+ template = template.replace("are", "is")
+ template = template.replace("answers", "answer")
+ type_needle_v = type_needle_v[:-1] # remove "s"
+
+ input_text = template.format(
+ type_needle_v=type_needle_v,
+ context=context,
+ query=query,
+ )
+
+ return input_text, answers
+
+
+def generate_samples(
+ num_samples: int, max_seq_length: int, save_dir: str, incremental: int = 500
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ if args.type_haystack == "essay":
+ incremental = 500
+ elif args.type_haystack == "repeat":
+ incremental = 25
+ elif args.type_haystack == "needle":
+ incremental = 25
+
+ if args.type_haystack != "essay" and args.max_seq_length < 4096:
+ incremental = 5
+
+ num_haystack = incremental
+
+ total_tokens = 0 # Track the total tokens generated for the first example
+ while total_tokens + tokens_to_generate < max_seq_length:
+ input_text, answer = generate_input_output(num_haystack)
+ # Calculate the number of tokens in the example
+ total_tokens = len(TOKENIZER.text_to_tokens(input_text + " ".join(answer)))
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Haystack: {num_haystack}"
+ )
+ if total_tokens + tokens_to_generate > max_seq_length:
+ num_haystack -= incremental
+ break
+
+ if args.type_haystack == "essay" and num_haystack > len(haystack):
+ num_haystack = len(haystack)
+ break
+
+ num_haystack += incremental
+
+ print("Num haystack:", num_haystack)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_haystack = num_haystack
+ while True:
+ try:
+ input_text, answer = generate_input_output(used_haystack)
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_haystack > incremental:
+ used_haystack -= incremental
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ write_jsons = generate_samples(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ save_dir=args.save_dir,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/data/synthetic/qa.py b/LeanK/eval/RULER/data/synthetic/qa.py
new file mode 100644
index 00000000..71a212bd
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/qa.py
@@ -0,0 +1,276 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+"""
+Create a dataset jsonl file for QA task.
+
+python qa.py \
+ --save_dir=./ \
+ --save_name=niah_single \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type=nemo \
+ --max_seq_length=4096 \
+ --tokens_to_generate=128 \
+ --num_samples=10 \
+ --template="Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query} Answer:"
+"""
+import argparse
+import json
+import os
+import random
+import re
+import sys
+from pathlib import Path
+
+import numpy as np
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+# Basic Configurations
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate",
+ type=int,
+ required=True,
+ help="expected generated token amount.",
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument(
+ "--pre_samples", type=int, default=0, help="number of samples are already generated"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, required=True, help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+# Complexity Configurations
+parser.add_argument("--dataset", type=str, required=True, help="dataset file")
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+
+# Read SQuAD QA dataset
+def read_squad(file):
+ with open(file) as f:
+ data = json.load(f)
+
+ total_docs = [p["context"] for d in data["data"] for p in d["paragraphs"]]
+ total_docs = sorted(list(set(total_docs)))
+ total_docs_dict = {c: idx for idx, c in enumerate(total_docs)}
+
+ total_qas = []
+ for d in data["data"]:
+ more_docs = [total_docs_dict[p["context"]] for p in d["paragraphs"]]
+ for p in d["paragraphs"]:
+ for qas in p["qas"]:
+ if not qas["is_impossible"]:
+ total_qas.append(
+ {
+ "query": qas["question"],
+ "outputs": [a["text"] for a in qas["answers"]],
+ "context": [total_docs_dict[p["context"]]],
+ "more_context": [
+ idx
+ for idx in more_docs
+ if idx != total_docs_dict[p["context"]]
+ ],
+ }
+ )
+
+ return total_qas, total_docs
+
+
+# Read Hotpot QA dataset
+def read_hotpotqa(file):
+ with open(file) as f:
+ data = json.load(f)
+
+ total_docs = [f"{t}\n{''.join(p)}" for d in data for t, p in d["context"]]
+ total_docs = sorted(list(set(total_docs)))
+ total_docs_dict = {c: idx for idx, c in enumerate(total_docs)}
+
+ total_qas = []
+ for d in data:
+ total_qas.append(
+ {
+ "query": d["question"],
+ "outputs": [d["answer"]],
+ "context": [
+ total_docs_dict[f"{t}\n{''.join(p)}"] for t, p in d["context"]
+ ],
+ }
+ )
+
+ return total_qas, total_docs
+
+
+DOCUMENT_PROMPT = "Document {i}:\n{document}"
+if args.dataset == "squad":
+ QAS, DOCS = read_squad(
+ os.path.join(os.path.dirname(os.path.abspath(__file__)), "json/squad.json")
+ )
+elif args.dataset == "hotpotqa":
+ QAS, DOCS = read_hotpotqa(
+ os.path.join(os.path.dirname(os.path.abspath(__file__)), "json/hotpotqa.json")
+ )
+else:
+ raise NotImplementedError(f"{args.dataset} is not implemented.")
+
+
+def generate_input_output(index, num_docs):
+ curr_q = QAS[index]["query"]
+ curr_a = QAS[index]["outputs"]
+ curr_docs = QAS[index]["context"]
+ curr_more = QAS[index].get("more_context", [])
+ if num_docs < len(DOCS):
+ if (num_docs - len(curr_docs)) > len(curr_more):
+ addition_docs = [
+ i for i, d in enumerate(DOCS) if i not in curr_docs + curr_more
+ ]
+ all_docs = (
+ curr_docs
+ + curr_more
+ + random.sample(
+ addition_docs, max(0, num_docs - len(curr_docs) - len(curr_more))
+ )
+ )
+ else:
+ all_docs = curr_docs + random.sample(curr_more, num_docs - len(curr_docs))
+
+ all_docs = [DOCS[idx] for idx in all_docs]
+ else:
+ all_docs = DOCS
+
+ random.Random(args.random_seed).shuffle(all_docs)
+
+ context = "\n\n".join(
+ [DOCUMENT_PROMPT.format(i=i + 1, document=d) for i, d in enumerate(all_docs)]
+ )
+ input_text = args.template.format(context=context, query=curr_q)
+ return input_text, curr_a
+
+
+def generate_samples(
+ num_samples: int, max_seq_length: int, save_dir: str, incremental: int = 10
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ # Find the perfect num_docs
+ num_docs = incremental
+
+ total_tokens = 0 # Track the total tokens generated for this example
+ while total_tokens + tokens_to_generate < max_seq_length:
+ input_text, answer = generate_input_output(0, num_docs)
+ # Calculate the number of tokens in the example
+ total_tokens = len(TOKENIZER.text_to_tokens(input_text + f" {answer}"))
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Docs: {num_docs}"
+ )
+ if total_tokens + tokens_to_generate > max_seq_length:
+ num_docs -= incremental
+ break
+
+ num_docs += incremental
+ if num_docs > len(DOCS):
+ num_docs = len(DOCS)
+ break
+ print("Number of documents:", num_docs)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_docs = num_docs
+ while True:
+ try:
+ input_text, answer = generate_input_output(
+ index + args.pre_samples, used_docs
+ )
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_docs > incremental:
+ used_docs -= incremental
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ write_jsons = generate_samples(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ save_dir=args.save_dir,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/data/synthetic/variable_tracking.py b/LeanK/eval/RULER/data/synthetic/variable_tracking.py
new file mode 100644
index 00000000..222346f9
--- /dev/null
+++ b/LeanK/eval/RULER/data/synthetic/variable_tracking.py
@@ -0,0 +1,292 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+"""
+Create a dataset jsonl file for variable tracking.
+
+python variable_tracking.py \
+ --save_dir=./ \
+ --save_name=vt \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --tokens_to_generate 30 \
+ --num_samples 10 \
+ --random_seed 42 \
+ --num_chains 1 --num_hops 4 \
+ --template "[INST] Memorize and track the chain(s) of variable assignment hidden in the following text.\n\n{context}\nQuestion: Find all variables that are assigned the value {query} in the text above. [/INST] Answer: According to the chain(s) of variable assignment in the text above, {num_v} variables are assgined the value {query}, they are: "
+"""
+import argparse
+import os
+import random
+import string
+import sys
+from pathlib import Path
+
+from constants import TASKS
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+import numpy as np
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate", type=int, default=120, help="number of tokens to generate"
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+parser.add_argument(
+ "--num_chains", type=int, default=1, help="number of inserted variable chains"
+)
+parser.add_argument(
+ "--num_hops", type=int, default=4, help="number of hops in each chain"
+)
+parser.add_argument("--add_fewshot", action="store_true", default=False)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+
+def generate_chains(num_chains, num_hops, is_icl=False):
+ vars_all = []
+ k = 5 if not is_icl else 3
+ num_hops = num_hops if not is_icl else min(10, num_hops)
+ vars_all = [
+ "".join(random.choices(string.ascii_uppercase, k=k)).upper()
+ for _ in range((num_hops + 1) * num_chains)
+ ]
+ while len(set(vars_all)) < num_chains * (num_hops + 1):
+ vars_all.append("".join(random.choices(string.ascii_uppercase, k=k)).upper())
+
+ vars_ret = []
+ chains_ret = []
+ for i in range(0, len(vars_all), num_hops + 1):
+ this_vars = vars_all[i : i + num_hops + 1]
+ vars_ret.append(this_vars)
+ this_chain = [f"VAR {this_vars[0]} = {np.random.randint(10000, 99999)}"]
+ for j in range(num_hops):
+ this_chain.append(f"VAR {this_vars[j+1]} = VAR {this_vars[j]} ")
+ chains_ret.append(this_chain)
+ return vars_ret, chains_ret
+
+
+def generate_input_output(num_noises, num_chains, num_hops, is_icl=False):
+ vars, chains = generate_chains(num_chains, num_hops, is_icl=is_icl)
+
+ noise = "The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again.\n"
+
+ # Create a list of the repeated noise
+ sentences = [noise] * num_noises
+ if len(sentences) <= len(chains[0]):
+ sentences = [
+ n + "." if len(n.strip()) > 0 else n
+ for n in [x for noise in sentences for x in noise.split(".")]
+ ]
+ try:
+ assert len(sentences) > len(
+ chains[0]
+ ), "Noises too short, unable to generate data"
+ except:
+ print("reduces chain length for not enough noises")
+ chains = [chain[: len(sentences) - 1] for chain in chains]
+ # sample random positions to insert variable assignment
+ for chain_i in chains:
+ # sample random positions (sorted) to insert variable assignment
+ positions = list(sorted(random.sample(range(len(sentences)), len(chain_i))))
+ for insert_pi, j in zip(positions, range(len(chain_i))):
+ sentences.insert(insert_pi + j, chain_i[j])
+
+ # Insert the passkey sentence at the random position
+ context = " ".join(sentences)
+ context = context.replace(". \n", ".\n")
+
+ template = args.template
+ if (
+ is_icl
+ and template
+ != TASKS["variable_tracking"]["template"]
+ + TASKS["variable_tracking"]["answer_prefix"]
+ ):
+ # remove model template
+ cutoff = template.index(TASKS["variable_tracking"]["template"][:20])
+ cutoff_ans = template.index(TASKS["variable_tracking"]["answer_prefix"][:10])
+ template = (
+ " ".join(template[cutoff:cutoff_ans].split()[:-1]) + template[cutoff_ans:]
+ )
+
+ value = chains[0][0].split("=")[-1].strip()
+ input_text = template.format(context=context, query=value, num_v=num_hops + 1)
+
+ return input_text, vars[0]
+
+
+def randomize_icl(icl_example):
+ icl_tgt_cut = icl_example.index(TASKS["variable_tracking"]["answer_prefix"][-10:])
+ icl_tgt = icl_example[icl_tgt_cut + 10 :].strip().split()
+ for item in icl_tgt:
+ new_item = "".join(random.choices(string.ascii_uppercase, k=len(item))).upper()
+ icl_example = icl_example.replace(item, new_item)
+ return icl_example
+
+
+def sys_vartrack_w_noise_random(
+ num_samples: int,
+ max_seq_length: int,
+ incremental: int = 10,
+ num_chains: int = 1,
+ num_hops: int = 4,
+ add_fewshot: bool = True,
+ icl_example: str = None,
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ # Find the perfect num_noises
+ num_noises = incremental
+
+ total_tokens = 0 # Track the total tokens generated for this example
+ example_tokens = 0
+ if add_fewshot and (icl_example is not None):
+ icl_example_out = " ".join(icl_example["outputs"])
+ icl_example = icl_example["input"] + " " + icl_example_out + "\n\n"
+ example_tokens = len(TOKENIZER.text_to_tokens(icl_example))
+
+ while total_tokens + tokens_to_generate + example_tokens < max_seq_length:
+ input_text, answer = generate_input_output(
+ num_noises, num_chains, num_hops, is_icl=add_fewshot & (icl_example is None)
+ )
+ # Calculate the number of tokens in the example
+ total_tokens = len(TOKENIZER.text_to_tokens(input_text + f" {answer}"))
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate + example_tokens} | Noises: {num_noises}"
+ )
+ if total_tokens + tokens_to_generate + example_tokens > max_seq_length:
+ num_noises -= incremental
+ break
+ num_noises += incremental
+ print("Num noises:", num_noises)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_noises = num_noises
+ while True:
+ try:
+ input_text, answer = generate_input_output(
+ used_noises,
+ num_chains,
+ num_hops,
+ is_icl=add_fewshot & (icl_example is None),
+ )
+ length = (
+ len(TOKENIZER.text_to_tokens(input_text))
+ + tokens_to_generate
+ + example_tokens
+ )
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_noises > incremental:
+ used_noises -= incremental
+
+ if add_fewshot and (icl_example is not None):
+ # insert icl_example between model template and input
+ cutoff = input_text.index(TASKS["variable_tracking"]["template"][:20])
+ input_text = (
+ input_text[:cutoff]
+ + randomize_icl(icl_example)
+ + "\n\n"
+ + input_text[cutoff:]
+ )
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ icl_example = sys_vartrack_w_noise_random(
+ num_samples=1,
+ max_seq_length=500,
+ incremental=5,
+ num_chains=args.num_chains,
+ num_hops=args.num_hops,
+ )[0]
+ write_jsons = sys_vartrack_w_noise_random(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ num_chains=args.num_chains,
+ num_hops=args.num_hops,
+ icl_example=icl_example,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/data/template.py b/LeanK/eval/RULER/data/template.py
new file mode 100644
index 00000000..c1490eeb
--- /dev/null
+++ b/LeanK/eval/RULER/data/template.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Templates = {
+ "base": "{task_template}",
+ "meta-chat": "[INST] {task_template} [/INST]",
+ "vicuna-chat": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {task_template} ASSISTANT:",
+ "lwm-chat": "You are a helpful assistant. USER: {task_template} ASSISTANT: ",
+ "command-r-chat": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{task_template}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
+ "chatglm-chat": "[gMASK]sop<|user|> \n {task_template}<|assistant|> \n ",
+ "RWKV": "User: hi\n\nAssistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it\n\nUser: {task_template}\n\nAssistant:",
+ "Phi3": "<|user|>\n{task_template}<|end|>\n<|assistant|>\n",
+ "meta-llama3": "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{task_template}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n",
+ "jamba": "<|startoftext|><|bom|><|system|> <|eom|><|bom|><|user|> {task_template}<|eom|><|bom|><|assistant|>",
+}
diff --git a/LeanK/eval/RULER/data/tokenizer.py b/LeanK/eval/RULER/data/tokenizer.py
new file mode 100644
index 00000000..0586321a
--- /dev/null
+++ b/LeanK/eval/RULER/data/tokenizer.py
@@ -0,0 +1,116 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from typing import List
+
+from tenacity import retry, stop_after_attempt, wait_fixed, wait_random
+
+
+def select_tokenizer(tokenizer_type, tokenizer_path):
+ if tokenizer_type == "nemo":
+ return NeMoSentencePieceTokenizer(model_path=tokenizer_path)
+ elif tokenizer_type == "hf":
+ return HFTokenizer(model_path=tokenizer_path)
+ elif tokenizer_type == "openai":
+ return OpenAITokenizer(model_path=tokenizer_path)
+ elif tokenizer_type == "gemini":
+ return GeminiTokenizer(model_path=tokenizer_path)
+ else:
+ raise ValueError(f"Unknown tokenizer_type {tokenizer_type}")
+
+
+class NeMoSentencePieceTokenizer:
+ """
+ Tokenizer from NeMo SentencePieceTokenizer
+ """
+
+ def __init__(self, model_path) -> None:
+ from nemo.collections.common.tokenizers.sentencepiece_tokenizer import (
+ SentencePieceTokenizer,
+ )
+
+ self.tokenizer = SentencePieceTokenizer(model_path=model_path)
+
+ def text_to_tokens(self, text: str) -> List[str]:
+ tokens = self.tokenizer.text_to_tokens(text)
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ text = self.tokenizer.tokens_to_text(tokens)
+ return text
+
+
+class HFTokenizer:
+ """
+ Tokenizer from HF models
+ """
+
+ def __init__(self, model_path) -> None:
+ from transformers import AutoTokenizer
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ model_path, trust_remote_code=True
+ )
+
+ def text_to_tokens(self, text: str) -> List[str]:
+ tokens = self.tokenizer.tokenize(text)
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ text = self.tokenizer.convert_tokens_to_string(tokens)
+ return text
+
+
+class OpenAITokenizer:
+ """
+ Tokenizer from tiktoken
+ """
+
+ def __init__(self, model_path="cl100k_base") -> None:
+ import tiktoken
+
+ self.tokenizer = tiktoken.get_encoding(model_path)
+
+ def text_to_tokens(self, text: str) -> List[int]:
+ tokens = self.tokenizer.encode(text)
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ text = self.tokenizer.decode(tokens)
+ return text
+
+
+class GeminiTokenizer:
+ """
+ Tokenizer from gemini
+ """
+
+ def __init__(self, model_path="gemini-1.5-pro-latest") -> None:
+ import google.generativeai as genai
+
+ genai.configure(api_key=os.environ["GEMINI_API_KEY"])
+ self.model = genai.GenerativeModel(model_path)
+
+ @retry(wait=wait_fixed(60) + wait_random(0, 10), stop=stop_after_attempt(3))
+ def text_to_tokens(self, text: str) -> List[int]:
+ tokens = list(range(self.model.count_tokens(text).total_tokens))
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ pass
diff --git a/LeanK/eval/RULER/eval/evaluate.py b/LeanK/eval/RULER/eval/evaluate.py
new file mode 100644
index 00000000..a1175ef4
--- /dev/null
+++ b/LeanK/eval/RULER/eval/evaluate.py
@@ -0,0 +1,241 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Get summary.csv with score and null predictions amount.
+
+Running
+```
+python evaluate.py \
+ --data_dir /path/to/your/prediction_jsonl_folder \
+ --benchmark synthetic
+```
+"""
+
+import argparse
+import os
+import re
+
+import nltk
+
+try:
+ nltk.data.find("tokenizers/punkt")
+except LookupError:
+ nltk.download("punkt")
+
+import importlib
+from collections import defaultdict
+from pathlib import Path
+
+import pandas as pd
+import yaml
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--data_dir", type=str, required=True, help="path to the prediction jsonl files"
+)
+parser.add_argument(
+ "--benchmark", type=str, default="synthetic", help="Options: [synthetic]"
+)
+parser.add_argument(
+ "--verbose", type=int, default=0, help="how many lines you want to display."
+)
+args = parser.parse_args()
+
+
+def postprocess_pred(predict_str: str, task_config: dict):
+ predict_str = predict_str.strip()
+
+ # Remove all non-printable characters
+ np_pattern = re.compile(r"[\x00-\x1f]")
+ predict_str = np_pattern.sub("\n", predict_str).strip()
+
+ return predict_str
+
+
+def get_pred_and_ref(
+ predictions_file: str,
+ task_config: dict,
+ input_field: str = "input",
+ references_field: str = "outputs",
+ prediction_field: str = "pred",
+ metadata_field: str = "others",
+):
+ lines = read_manifest(predictions_file)
+
+ inputs = []
+ predicts = []
+ references = []
+ indices = []
+
+ for line in tqdm(lines):
+ input = line[input_field]
+ predict = line[prediction_field]
+ predict = postprocess_pred(predict, task_config)
+ reference = line.get(references_field, [line.get("output", "")])
+ index = line[metadata_field].get("id", line["index"])
+
+ inputs.append(input)
+ predicts.append(predict)
+ references.append(reference)
+ indices.append(index)
+
+ return inputs, predicts, references, indices
+
+
+def run_evaluation_per_task(task_config: dict, predictions_file: str, verbose: int = 0):
+ inputs, predicts, references, indices = get_pred_and_ref(
+ predictions_file=predictions_file,
+ task_config=task_config,
+ )
+
+ task_nulls = f"{sum([len(x)==0 for x in predicts])}/{len(predicts)}"
+
+ if len(references) > 0 and references[0][0] is not None:
+ task_score = task_config["metric_fn"](predicts, references)
+ else:
+ task_score = 0.0
+
+ if verbose != 0:
+ print("=" * 40)
+ for i, (input, reference, predict) in enumerate(
+ zip(inputs, references, predicts)
+ ):
+ print(f"Input : {input}")
+ print(f"Reference : {reference}")
+ print(f"Prediction: {predict}")
+ print("=" * 40)
+ if i > verbose:
+ break
+
+ return task_score, task_nulls, predicts, indices
+
+
+def write_evaluation(results: dict):
+ tasks = list(results.keys())
+ score = [results[task]["score"] for task in tasks]
+ nulls = [results[task]["nulls"] for task in tasks]
+ dfs = [
+ ["Tasks"] + tasks,
+ ["Score"] + score,
+ ["Nulls"] + nulls,
+ ]
+
+ output_file = os.path.join(
+ args.data_dir, "summary.csv" if len(tasks) > 1 else f"summary-{tasks[0]}.csv"
+ )
+ df = pd.DataFrame(dfs)
+ df.to_csv(output_file, index=False)
+ print("\n=============================================\n")
+ print(df)
+ print(f"\nSaved eval results to {output_file}")
+
+
+def write_submission(results: dict):
+ COLUMNS = ["Task", "ID", "Prediction"]
+ dfs = pd.DataFrame(columns=COLUMNS, data=[])
+
+ for task, result in results.items():
+ df = pd.DataFrame(
+ {"Task": task, "ID": result["indices"], "Prediction": result["predicts"]}
+ )
+ dfs = pd.concat((dfs, df[COLUMNS]))
+
+ output_file = os.path.join(args.data_dir, "submission.csv")
+ dfs = dfs.reset_index(drop=True)
+ dfs.to_csv(output_file, index=False)
+ print(f"\nSaved submission results to {output_file}")
+
+
+def aggregate_chunk(folder):
+ jsonl_files = [file for file in os.listdir(folder) if Path(file).suffix == ".jsonl"]
+ chunk_files = sorted(
+ [file for file in jsonl_files if re.match(r".*[^_]+-\d+\.jsonl", file)]
+ )
+ chunk_files_dict = defaultdict(list)
+ for file in chunk_files:
+ task = "-".join(file.split("-")[:-1])
+ chunk_files_dict[task].append(file)
+
+ for task, files in chunk_files_dict.items():
+ lines = []
+ for file in sorted(files):
+ file = os.path.join(folder, file)
+ lines += read_manifest(file)
+ os.remove(file) # Remove chunk files
+ write_manifest(os.path.join(folder, f"{task}.jsonl"), lines)
+
+
+def main():
+ curr_folder = os.path.dirname(os.path.abspath(__file__))
+
+ try:
+ module = importlib.import_module(f"{args.benchmark}.constants")
+ except ImportError:
+ print(f"Module eval.{args.benchmark}.constants not found.")
+
+ tasks_base = module.TASKS
+ with open(os.path.join(curr_folder, f"../{args.benchmark}.yaml"), "r") as f:
+ tasks_customized = yaml.safe_load(f)
+
+ TASKS = tasks_customized
+ for _, config in TASKS.items():
+ config.update(tasks_base[config["task"]])
+
+ print(f"Total tasks: {list(TASKS.keys())}")
+
+ # Aggregate all prediction files
+ aggregate_chunk(args.data_dir)
+
+ # Get scores and nulls
+ jsonl_files = [
+ file for file in os.listdir(args.data_dir) if Path(file).suffix == ".jsonl"
+ ]
+ eval_results = {}
+ subm_results = {}
+
+ for task, config in TASKS.items():
+ if f"{task}.jsonl" not in jsonl_files:
+ print(f"Prediction file {task}.jsonl is not found.")
+ continue
+
+ print(f"Evaluate task {task}...")
+ task_score, task_nulls, predicts, indices = run_evaluation_per_task(
+ predictions_file=os.path.join(args.data_dir, f"{task}.jsonl"),
+ task_config=config,
+ )
+ eval_results[task] = {
+ "score": task_score,
+ "nulls": task_nulls,
+ }
+ subm_results[task] = {
+ "predicts": predicts,
+ "indices": indices,
+ }
+
+ # Write to csv
+ write_evaluation(eval_results)
+ write_submission(subm_results)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/eval/synthetic/constants.py b/LeanK/eval/RULER/eval/synthetic/constants.py
new file mode 100644
index 00000000..186bf4e6
--- /dev/null
+++ b/LeanK/eval/RULER/eval/synthetic/constants.py
@@ -0,0 +1,69 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Add a new task:
+
+TASK_NAME: {
+ 'metric_fn': the metric function with input (predictions: [str], references: [[str]]) to compute score.
+}
+"""
+
+
+def string_match_part(preds, refs):
+ score = (
+ sum(
+ [
+ max([1.0 if r.lower() in pred.lower() else 0.0 for r in ref])
+ for pred, ref in zip(preds, refs)
+ ]
+ )
+ / len(preds)
+ * 100
+ )
+ return round(score, 2)
+
+
+def string_match_all(preds, refs):
+ score = (
+ sum(
+ [
+ sum([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) / len(ref)
+ for pred, ref in zip(preds, refs)
+ ]
+ )
+ / len(preds)
+ * 100
+ )
+ return round(score, 2)
+
+
+TASKS = {
+ "niah": {
+ "metric_fn": string_match_all,
+ },
+ "variable_tracking": {
+ "metric_fn": string_match_all,
+ },
+ "common_words_extraction": {
+ "metric_fn": string_match_all,
+ },
+ "freq_words_extraction": {"metric_fn": string_match_all},
+ "qa": {
+ "metric_fn": string_match_part,
+ },
+}
diff --git a/LeanK/eval/RULER/pred/call_api.py b/LeanK/eval/RULER/pred/call_api.py
new file mode 100644
index 00000000..91801cde
--- /dev/null
+++ b/LeanK/eval/RULER/pred/call_api.py
@@ -0,0 +1,389 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Prepare prediction jsonl with field `pred` .
+dataset jsonl:
+{
+ "index" int,
+ "input": str,
+ "outputs": [str],
+}
+
+prediction jsonl:
+{
+ "index" int,
+ "input": str,
+ "outputs": [str],
+ "pred": str,
+}
+"""
+
+import argparse
+import importlib
+import json
+import math
+import os
+import sys
+import threading
+import time
+import traceback
+from pathlib import Path
+
+import yaml
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from tqdm import tqdm
+
+SERVER_TYPES = (
+ "trtllm",
+ "vllm",
+ "sglang",
+ "openai",
+ "gemini",
+ "hf",
+ "mamba",
+)
+
+
+class ServerAction(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ namespace.server_type = values
+
+
+parser = argparse.ArgumentParser()
+# Data
+parser.add_argument(
+ "--data_dir", type=Path, required=True, help="path to load the dataset jsonl files"
+)
+parser.add_argument(
+ "--save_dir",
+ type=Path,
+ required=True,
+ help="path to save the prediction jsonl files",
+)
+parser.add_argument(
+ "--benchmark", type=str, default="synthetic", help="Options: [synthetic]"
+)
+parser.add_argument(
+ "--task", type=str, required=True, help="Options: tasks in benchmark"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--chunk_idx", type=int, default=0, help="index of current split chunk"
+)
+parser.add_argument("--chunk_amount", type=int, default=1, help="size of split chunk")
+
+# Server
+parser.add_argument(
+ "--server_type", default="nemo", action=ServerAction, choices=SERVER_TYPES
+)
+parser.add_argument("--server_host", type=str, default="127.0.0.1")
+parser.add_argument("--server_port", type=str, default="5000")
+parser.add_argument("--ssh_server", type=str)
+parser.add_argument("--ssh_key_path", type=str)
+parser.add_argument(
+ "--model_name_or_path",
+ type=str,
+ default="gpt-3.5-turbo",
+ help="supported models from OpenAI or HF (provide a key or a local path to the checkpoint)",
+)
+
+# Inference
+parser.add_argument("--temperature", type=float, default=1.0)
+parser.add_argument("--top_k", type=int, default=32)
+parser.add_argument("--top_p", type=float, default=1.0)
+parser.add_argument("--random_seed", type=int, default=0)
+parser.add_argument("--stop_words", type=str, default="")
+parser.add_argument("--sliding_window_size", type=int)
+parser.add_argument("--threads", type=int, default=4)
+parser.add_argument("--batch_size", type=int, default=1)
+parser.add_argument("--enable_leank", type=bool, default=True)
+
+args = parser.parse_args()
+args.stop_words = list(filter(None, args.stop_words.split(",")))
+if args.server_type == "hf" or args.server_type == "gemini":
+ args.threads = 1
+
+
+def get_llm(tokens_to_generate):
+ if args.server_type == "trtllm":
+ from client_wrappers import TRTLLMClient
+
+ llm = TRTLLMClient(
+ server_host=args.server_host,
+ server_port=args.server_port,
+ ssh_server=args.ssh_server,
+ ssh_key_path=args.ssh_key_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ max_attention_window_size=args.sliding_window_size,
+ )
+
+ elif args.server_type == "vllm":
+ from client_wrappers import VLLMClient
+
+ llm = VLLMClient(
+ server_host=args.server_host,
+ server_port=args.server_port,
+ ssh_server=args.ssh_server,
+ ssh_key_path=args.ssh_key_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ )
+
+ elif args.server_type == "sglang":
+ from client_wrappers import SGLClient
+
+ llm = SGLClient(
+ server_host=args.server_host,
+ server_port=args.server_port,
+ ssh_server=args.ssh_server,
+ ssh_key_path=args.ssh_key_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ )
+
+ elif args.server_type == "openai":
+ from client_wrappers import OpenAIClient
+
+ llm = OpenAIClient(
+ model_name=args.model_name_or_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ )
+
+ elif args.server_type == "gemini":
+ from client_wrappers import GeminiClient
+
+ llm = GeminiClient(
+ model_name=args.model_name_or_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ )
+
+ elif args.server_type == "hf":
+ from model_wrappers import HuggingFaceModel
+
+ llm = HuggingFaceModel(
+ name_or_path=args.model_name_or_path,
+ do_sample=args.temperature > 0,
+ repetition_penalty=1,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ stop=args.stop_words,
+ max_new_tokens=tokens_to_generate,
+ enable_leank=args.enable_leank,
+ )
+
+ elif args.server_type == "mamba":
+ from model_wrappers import MambaModel
+
+ # mamba uses its own generation function, do not pass in do_sample
+ # https://github.com/state-spaces/mamba/blob/009bec5ee37f586844a3fc89c040a9c1a9d8badf/mamba_ssm/utils/generation.py#L121
+ llm = MambaModel(
+ name_or_path=args.model_name_or_path,
+ repetition_penalty=1,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ stop=args.stop_words,
+ max_new_tokens=tokens_to_generate,
+ )
+
+ else:
+ raise RuntimeError(f"Unsupported server type {args.server_type}")
+
+ return llm
+
+
+def main():
+ start_time = time.time()
+
+ curr_folder = os.path.dirname(os.path.abspath(__file__))
+
+ try:
+ sys.path.append(os.path.dirname(curr_folder))
+ module = importlib.import_module(f"data.{args.benchmark}.constants")
+ except ImportError:
+ print(f"Module data.{args.benchmark}.constants not found.")
+
+ tasks_base = module.TASKS
+ with open(os.path.join(curr_folder, f"../{args.benchmark}.yaml"), "r") as f:
+ tasks_customized = yaml.safe_load(f)
+
+ if args.task not in tasks_customized:
+ raise ValueError(f"{args.task} is not found in config_tasks.yaml")
+
+ config = tasks_customized.get(args.task)
+ config.update(tasks_base[config["task"]])
+
+ task_file = args.data_dir / args.task / f"{args.subset}.jsonl"
+
+ if args.chunk_amount > 1:
+ pred_file = args.save_dir / f"{args.task}-{args.chunk_idx}.jsonl"
+ else:
+ pred_file = args.save_dir / f"{args.task}.jsonl"
+
+ print(f"Predict {args.task} \nfrom {task_file}\nto {pred_file}")
+ pred_file.parent.mkdir(parents=True, exist_ok=True)
+
+ # Load data
+ if os.path.exists(pred_file):
+ pred_index = [sample["index"] for sample in read_manifest(pred_file)]
+ data = [
+ sample
+ for sample in read_manifest(task_file)
+ if sample["index"] not in pred_index
+ ]
+ else:
+ data = read_manifest(task_file)
+
+ # Load api
+ llm = get_llm(config["tokens_to_generate"])
+
+ def get_output(
+ idx_list,
+ index_list,
+ input_list,
+ outputs_list,
+ others_list,
+ truncation_list,
+ length_list,
+ ):
+ nonlocal llm
+
+ while True:
+ try:
+ pred_list = llm.process_batch(prompts=input_list)
+ break
+ except Exception as e:
+ traceback.print_exc()
+
+ zipped_iter = zip(
+ pred_list,
+ idx_list,
+ index_list,
+ input_list,
+ outputs_list,
+ others_list,
+ truncation_list,
+ length_list,
+ )
+
+ for pred, idx, index, input, outputs, others, truncation, length in zipped_iter:
+ if isinstance(pred["text"], str):
+ pred_text = pred["text"]
+ elif len(pred["text"]) > 0:
+ pred_text = pred["text"][0]
+ else:
+ pred_text = ""
+
+ outputs_parallel[idx] = {
+ "index": index,
+ "pred": pred_text,
+ "input": input,
+ "outputs": outputs,
+ "others": others,
+ "truncation": truncation,
+ "length": length,
+ }
+
+ threads = []
+ outputs_parallel = [{} for _ in range(len(data))]
+
+ batched_data = []
+ batch = []
+ for idx, data_point in enumerate(data):
+ data_point["idx"] = idx
+
+ if len(batch) >= args.batch_size:
+ batched_data.append(batch)
+ batch = []
+
+ batch.append(data_point)
+
+ if len(batch):
+ batched_data.append(batch)
+
+ # setting buffering=1 to force to dump the output after every line, so that we can see intermediate generations
+ with open(pred_file, "at", encoding="utf-8", buffering=1) as fout:
+ # the data is processed sequentially, so we can store the start and end of current processing window
+ start_idx = 0 # window: [start_idx, end_idx]
+
+ for batch_idx, batch in tqdm(enumerate(batched_data), total=len(batched_data)):
+ idx_list = [data_point["idx"] for data_point in batch]
+ end_idx = idx_list[-1] # the data in a batch is ordered
+
+ thread = threading.Thread(
+ target=get_output,
+ kwargs=dict(
+ idx_list=idx_list,
+ index_list=[data_point["index"] for data_point in batch],
+ input_list=[data_point["input"] for data_point in batch],
+ outputs_list=[data_point["outputs"] for data_point in batch],
+ others_list=[data_point.get("others", {}) for data_point in batch],
+ truncation_list=[
+ data_point.get("truncation", -1) for data_point in batch
+ ],
+ length_list=[data_point.get("length", -1) for data_point in batch],
+ ),
+ )
+ thread.start()
+ threads.append(thread)
+
+ is_last_batch = batch_idx == len(batched_data) - 1
+
+ if (len(threads) == args.threads) or is_last_batch:
+ for thread in threads:
+ thread.join()
+ threads = []
+
+ # dump the results in current processing window on disk
+ for idx in range(start_idx, end_idx + 1):
+ if len(outputs_parallel[idx]) > 0:
+ fout.write(json.dumps(outputs_parallel[idx]) + "\n")
+
+ start_idx = end_idx + 1
+
+ print(f"Used time: {round((time.time() - start_time) / 60, 1)} minutes")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/LeanK/eval/RULER/pred/client_wrappers.py b/LeanK/eval/RULER/pred/client_wrappers.py
new file mode 100644
index 00000000..a112ce55
--- /dev/null
+++ b/LeanK/eval/RULER/pred/client_wrappers.py
@@ -0,0 +1,418 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import abc
+import json
+import multiprocessing
+import os
+import re
+import sys
+import time
+import traceback
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import List, Tuple, Union
+
+import requests
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+
+class Client(abc.ABC):
+ def __init__(
+ self,
+ server_host,
+ server_port="5000",
+ ssh_server=None,
+ ssh_key_path=None,
+ **generation_kwargs,
+ ):
+ self.server_host = server_host
+ self.server_port = server_port
+ self.ssh_server = os.getenv("SSH_SERVER", ssh_server)
+ self.ssh_key_path = os.getenv("SSH_KEY_PATH", ssh_key_path)
+ self.generation_kwargs = generation_kwargs
+
+ @abc.abstractmethod
+ def _single_call(
+ self,
+ prompts,
+ ):
+ pass
+
+ def __call__(self, prompt: str, **kwargs):
+ request = self.generation_kwargs
+ # prompts are added later
+ request["prompts"] = [f"{prompt}"]
+ if "others" in kwargs:
+ request["others"] = kwargs["others"]
+
+ outputs = self._single_call(**request)
+ response = {"text": outputs}
+ return response
+
+ @retry(wait=wait_random_exponential(min=15, max=60), stop=stop_after_attempt(3))
+ def _send_request(self, request, route="generate"):
+ if self.ssh_server and self.ssh_key_path:
+ import sshtunnel_requests
+
+ sshtunnel_request = sshtunnel_requests.from_url(
+ f"ssh://{self.ssh_server}:22", self.ssh_key_path
+ )
+ outputs = sshtunnel_request.put(
+ url="http://{}:{}/{}".format(self.server_host, self.server_port, route),
+ data=json.dumps(request),
+ headers={"Content-Type": "application/json"},
+ ).json()
+ else:
+ outputs = requests.put(
+ url="http://{}:{}/{}".format(self.server_host, self.server_port, route),
+ data=json.dumps(request),
+ headers={"Content-Type": "application/json"},
+ ).json()
+ return outputs
+
+ def process_batch(self, prompts: List[str], **kwargs) -> List[dict]:
+ num_threads = max(96, multiprocessing.cpu_count() * 16)
+ with ThreadPoolExecutor(num_threads) as executor:
+ futures = []
+ for prompt in prompts:
+ futures.append(
+ executor.submit(
+ self.__call__,
+ prompt,
+ **kwargs,
+ )
+ )
+ rets = [f.result() for f in futures]
+ return rets
+
+
+class TRTLLMClient(Client):
+ def _single_call(
+ self,
+ prompts,
+ tokens_to_generate,
+ temperature,
+ top_p,
+ top_k,
+ random_seed,
+ stop: List[str],
+ max_attention_window_size=None,
+ ):
+ request = {
+ "prompts": prompts,
+ "tokens_to_generate": tokens_to_generate,
+ "temperature": temperature,
+ "top_k": top_k,
+ "top_p": top_p,
+ "random_seed": random_seed,
+ "stop_words_list": ",".join(stop),
+ }
+ if max_attention_window_size:
+ request["max_attention_window_size"] = max_attention_window_size
+
+ outputs = self._send_request(request)
+ return outputs
+
+
+class VLLMClient(Client):
+ def _single_call(
+ self,
+ prompts,
+ tokens_to_generate,
+ temperature,
+ top_p,
+ top_k,
+ random_seed,
+ stop: List[str],
+ ):
+ request = {
+ "prompt": prompts[0],
+ "max_tokens": tokens_to_generate,
+ "temperature": temperature,
+ "top_k": top_k,
+ "top_p": top_p,
+ "stop": stop,
+ }
+ # TODO: random seed is not supported?
+ outputs = self._send_request(request)
+ outputs = outputs["text"]
+ return outputs
+
+
+class SGLClient(Client):
+ def _single_call(
+ self,
+ prompts,
+ tokens_to_generate,
+ temperature,
+ top_p,
+ top_k,
+ random_seed,
+ stop: List[str],
+ ):
+ request = {
+ "text": prompts[0],
+ "sampling_params": {
+ "max_new_tokens": tokens_to_generate,
+ "temperature": temperature,
+ "top_k": top_k,
+ "top_p": top_p,
+ "stop": stop,
+ },
+ }
+ # TODO: random seed is not supported?
+ outputs = self._send_request(request)
+ outputs = outputs["text"]
+ return outputs
+
+
+class OpenAIClient:
+ def __init__(self, model_name, **generation_kwargs):
+ model2length = {
+ # OpenAI
+ "gpt-4": 8192,
+ "gpt-4-0613": 8192,
+ "gpt-4-1106-preview": 128000,
+ "gpt-4-0125-preview": 128000,
+ "gpt-4-turbo-preview": 128000,
+ "gpt-3.5-turbo-0125": 16385,
+ "gpt-3.5-turbo-1106": 16385,
+ "gpt-3.5-turbo-0613": 4096,
+ "gpt-3.5-turbo": 16385,
+ "gpt-3.5-turbo-16k": 16385,
+ "gpt-3.5-turbo-16k-0613": 16385,
+ # Azure
+ "gpt-4-32k": 32768,
+ "gpt-4": 128000,
+ "gpt-35-turbo-16k": 16384,
+ }
+ self.openai_api_key = os.environ["OPENAI_API_KEY"]
+ self.azure_api_id = os.environ["AZURE_API_ID"]
+ self.azure_api_secret = os.environ["AZURE_API_SECRET"]
+ self.azure_api_endpoint = os.environ["AZURE_API_ENDPOINT"]
+ self.model_name = model_name
+
+ # Azure
+ if self.azure_api_id and self.azure_api_secret:
+ if "gpt-3.5" in model_name:
+ self.model_name = "gpt-35-turbo-16k"
+ if "gpt-4" in model_name:
+ self.model_name = "gpt-4"
+
+ import tiktoken
+
+ self.encoding = tiktoken.get_encoding("cl100k_base")
+ self.max_length = model2length[self.model_name]
+ self.generation_kwargs = generation_kwargs
+ self._create_client()
+
+ def _create_client(
+ self,
+ ):
+ from openai import AzureOpenAI, OpenAI
+
+ # OpenAI
+ if self.openai_api_key:
+ self.client = OpenAI(api_key=self.openai_api_key)
+
+ # Azure
+ elif self.azure_api_id and self.azure_api_secret:
+ self.client = AzureOpenAI(
+ api_key=self.get_azure_api_key(
+ self.azure_api_id,
+ self.azure_api_secret,
+ self.azure_api_endpoint,
+ ),
+ api_version="2024-02-15-preview",
+ azure_endpoint=os.path.join(self.azure_api_endpoint, "llm/v1/azure"),
+ )
+
+ def _count_tokens(self, messages):
+ tokens_per_message = 3
+ tokens_per_name = 1
+ num_tokens = 0
+ for message in messages:
+ num_tokens += tokens_per_message
+ for key, value in message.items():
+ num_tokens += len(self.encoding.encode(value))
+ if key == "name":
+ num_tokens += tokens_per_name
+ num_tokens += 3 # every reply is primed with <|start|>assistant<|message|>
+ return num_tokens
+
+ @retry(wait=wait_random_exponential(min=15, max=60), stop=stop_after_attempt(3))
+ def _send_request(self, request):
+ try:
+ response = self.client.chat.completions.create(
+ model=self.model_name,
+ messages=request["msgs"],
+ max_tokens=request["tokens_to_generate"],
+ temperature=request["temperature"],
+ seed=request["random_seed"],
+ top_p=request["top_p"],
+ stop=request["stop"],
+ )
+ except Exception as e:
+ print(f"Error occurred while calling OpenAI: {e}")
+ if self.azure_api_id and self.azure_api_secret and e.status_code == 401:
+ # token expired
+ self._create_client()
+
+ return response
+
+ def __call__(
+ self,
+ prompt: str,
+ ):
+ # system_msg = [{"role": "system", "content": ""}]
+ system_msg = []
+ user_assistant_msgs = [{"role": "user", "content": prompt}]
+ msgs = system_msg + user_assistant_msgs
+ openai_length = self._count_tokens(msgs)
+ request = self.generation_kwargs
+
+ tokens_to_generate_new = self.max_length - openai_length
+ if tokens_to_generate_new < request["tokens_to_generate"]:
+ print(
+ f"Reduce generate tokens from {request['tokens_to_generate']} to {tokens_to_generate_new}"
+ )
+ request["tokens_to_generate"] = tokens_to_generate_new
+
+ request["msgs"] = msgs
+ outputs = self._send_request(request)
+ response = {"text": [outputs.choices[0].message.content]}
+ return response
+
+ def get_azure_api_key(
+ self,
+ p_client_id,
+ p_client_secret,
+ p_token_url,
+ p_scope="azureopenai-readwrite",
+ cache_file="azure_openai_key.json",
+ ):
+ base_path = Path(__file__).parent
+ file_path = Path.joinpath(base_path, cache_file)
+
+ # Check if the token is cached
+ renew = True
+ if os.path.exists(file_path):
+ with open(file_path, "r") as f:
+ token = json.load(f)
+ renew = True if time.time() > token["expires_in"] else False
+
+ if renew:
+ # Get a new token from the OAuth server
+ response = requests.post(
+ os.path.join(p_token_url, "oauth/api/v1/ssa/default/token"),
+ data={
+ "grant_type": "client_credentials",
+ "client_id": p_client_id,
+ "client_secret": p_client_secret,
+ "scope": p_scope,
+ },
+ )
+ response.raise_for_status()
+ token = response.json()
+ token["expires_in"] += time.time()
+ with open(file_path, "w") as f:
+ json.dump(token, f)
+
+ authToken = token["access_token"]
+ return authToken
+
+
+class GeminiClient:
+ def __init__(self, model_name, **generation_kwargs):
+ model2length = {
+ "gemini-1.0-pro-latest": (30720, 2048),
+ "gemini-1.5-pro-latest": (1048576, 8192),
+ }
+
+ self.model_name = model_name
+ self.model = self._initialize_model()
+ self.max_input_length = model2length[model_name][0]
+ self.max_output_length = model2length[model_name][1]
+ assert generation_kwargs["tokens_to_generate"] < self.max_output_length, print(
+ f"tokens_to_generate exceeds {self.max_output_length}"
+ )
+
+ import google.generativeai as genai
+
+ self.config = genai.GenerationConfig(
+ candidate_count=1,
+ stop_sequences=generation_kwargs["stop"],
+ max_output_tokens=generation_kwargs["tokens_to_generate"],
+ temperature=generation_kwargs["temperature"],
+ top_p=generation_kwargs["top_p"],
+ top_k=generation_kwargs["top_k"],
+ )
+
+ from google.generativeai.types import HarmBlockThreshold, HarmCategory
+
+ self.safety_settings = {
+ HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
+ HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
+ HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
+ HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
+ }
+
+ @retry(wait=wait_random_exponential(min=60, max=60), stop=stop_after_attempt(3))
+ def _send_request(self, request):
+ try:
+ response = self.model.generate_content(
+ request["prompt"],
+ generation_config=request["config"],
+ safety_settings=self.safety_settings,
+ )
+ except Exception as e:
+ traceback.print_exc()
+ return None
+ return response
+
+ def __call__(
+ self,
+ prompt: str,
+ ):
+ assert (
+ self.model.count_tokens(prompt).total_tokens < self.max_input_length
+ ), print(f"input length exceeds {self.max_input_length}")
+
+ request = {
+ "prompt": prompt,
+ "config": self.config,
+ }
+
+ outputs = self._send_request(request)
+
+ try:
+ response = {"text": [outputs.candidates[0].content.parts[0].text]}
+ except Exception as e:
+ response = {"text": []}
+ print(outputs)
+ traceback.print_exc()
+
+ return response
+
+ def _initialize_model(self):
+ import google.generativeai as genai
+
+ genai.configure(api_key=os.environ["GEMINI_API_KEY"])
+ return genai.GenerativeModel(self.model_name)
diff --git a/LeanK/eval/RULER/pred/model_wrappers.py b/LeanK/eval/RULER/pred/model_wrappers.py
new file mode 100644
index 00000000..4ff706a1
--- /dev/null
+++ b/LeanK/eval/RULER/pred/model_wrappers.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+from typing import Dict, List, Optional
+
+import requests
+import torch
+
+from minference import MInference
+
+
+class HuggingFaceModel:
+ def __init__(self, name_or_path: str, **generation_kwargs) -> None:
+ from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ name_or_path, trust_remote_code=True
+ )
+
+ if "Yarn-Llama" in name_or_path:
+ model_kwargs = None
+ else:
+ model_kwargs = {"attn_implementation": "flash_attention_2"}
+
+ self.pipeline = None
+ self.model = AutoModelForCausalLM.from_pretrained(
+ name_or_path,
+ trust_remote_code=True,
+ device_map="auto",
+ torch_dtype=torch.bfloat16,
+ )
+
+ self.generation_kwargs = generation_kwargs
+ self.stop = self.generation_kwargs.pop("stop")
+
+ model_name = name_or_path.split("/")[-1]
+
+ if self.generation_kwargs.pop("enable_leank"):
+ minference_patch = MInference(
+ attn_type="dense", model_name=model_name, kv_type="leank"
+ )
+ self.model = minference_patch(self.model)
+
+ if self.tokenizer.pad_token is None:
+ # add pad token to allow batching (known issue for llama2)
+ self.tokenizer.padding_side = "left"
+ self.tokenizer.pad_token = self.tokenizer.eos_token
+ self.tokenizer.pad_token_id = self.tokenizer.eos_token_id
+
+ def __call__(self, prompt: str, **kwargs) -> dict:
+ return self.process_batch([prompt], **kwargs)[0]
+
+ def process_batch(self, prompts: List[str], **kwargs) -> List[dict]:
+ if self.pipeline is None:
+ inputs = self.tokenizer(prompts, return_tensors="pt", padding=True).to(
+ self.model.device
+ )
+ generated_ids = self.model.generate(**inputs, **self.generation_kwargs)
+ generated_texts = self.tokenizer.batch_decode(
+ generated_ids, skip_special_tokens=True
+ )
+ else:
+ output = self.pipeline(
+ text_inputs=prompts,
+ **self.generation_kwargs,
+ )
+ assert len(output) == len(prompts)
+ # output in the form of a list of list of dictionaries
+ # outer list len = batch size
+ # inner list len = 1
+ generated_texts = [llm_result[0]["generated_text"] for llm_result in output]
+
+ results = []
+
+ for text, prompt in zip(generated_texts, prompts):
+ # remove the input form the generated text
+ # This is a workaround for the llama3 tokenizer not being able to reproduce the same prompt after tokenization
+ # see Issue https://github.com/NVIDIA/RULER/issues/54 for explaination
+ if self.pipeline is None:
+ tokenized_prompt = self.tokenizer(
+ prompt, return_tensors="pt", padding=True
+ )
+ prompt = self.tokenizer.decode(
+ tokenized_prompt.input_ids[0], skip_special_tokens=True
+ )
+ if text.startswith(prompt):
+ text = text[len(prompt) :]
+
+ if self.stop is not None:
+ for s in self.stop:
+ text = text.split(s)[0]
+
+ results.append({"text": [text]})
+
+ return results
+
+
+class MambaModel:
+ def __init__(self, name_or_path: str, **generation_kwargs) -> None:
+ from mamba_ssm.models.mixer_seq_simple import MambaLMHeadModel
+ from transformers import AutoTokenizer
+
+ self.tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+ self.device = "cuda"
+ self.model = MambaLMHeadModel.from_pretrained(
+ name_or_path, device=self.device, dtype=torch.bfloat16
+ )
+ self.generation_kwargs = generation_kwargs
+ self.stop = self.generation_kwargs.pop("stop")
+ self.max_genlen = self.generation_kwargs.pop("max_new_tokens")
+ self.minp = 0.0
+
+ def __call__(self, prompt: str, **kwargs) -> Dict[str, List[str]]:
+ # tokenize
+ tokens = self.tokenizer(prompt, return_tensors="pt")
+ input_ids = tokens.input_ids.to(self.device)
+ max_length = input_ids.shape[1] + self.max_genlen
+
+ # generate
+ out = self.model.generate(
+ input_ids=input_ids,
+ max_length=max_length,
+ cg=True,
+ return_dict_in_generate=True,
+ output_scores=True,
+ enable_timing=False,
+ **self.generation_kwargs,
+ )
+ assert len(out.sequences) == 1
+ # detok
+ return {"text": [self.tokenizer.decode(out.sequences[0][input_ids.shape[1] :])]}
+
+ def process_batch(self, prompts: List[str], **kwargs) -> List[dict]:
+ # FIXME: naive implementation
+ return [self.__call__(prompt, **kwargs) for prompt in prompts]
diff --git a/LeanK/eval/RULER/pred/serve_trt.py b/LeanK/eval/RULER/pred/serve_trt.py
new file mode 100644
index 00000000..68546cee
--- /dev/null
+++ b/LeanK/eval/RULER/pred/serve_trt.py
@@ -0,0 +1,279 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# adapted from https://github.com/Kipok/NeMo-Skills/blob/v0.1/nemo_skills/inference/server/serve_trt.py
+
+import json
+import logging
+import sys
+from argparse import ArgumentParser
+from pathlib import Path
+
+import numpy as np
+import tensorrt_llm
+import torch
+from flask import Flask, jsonify, request
+from flask_restful import Api, Resource
+from mpi4py import MPI
+from tensorrt_llm.runtime import ModelRunnerCpp
+from transformers import AutoTokenizer
+
+
+class TritonServerGenerate(Resource):
+ def __init__(self, model):
+ self.model = model
+ self.comm = MPI.COMM_WORLD
+
+ def generate(
+ self,
+ prompts,
+ max_new_tokens,
+ temperature,
+ top_k,
+ top_p,
+ repetition_penalty,
+ random_seed,
+ stop_words_list,
+ max_attention_window_size=None,
+ ):
+ output = self.model.forward(
+ prompts,
+ max_output_token=max_new_tokens,
+ top_k=top_k,
+ top_p=top_p,
+ temperature=temperature,
+ repetition_penalty=repetition_penalty,
+ random_seed=random_seed,
+ stop_words_list=stop_words_list,
+ max_attention_window_size=max_attention_window_size,
+ )
+ return output
+
+ def put(self):
+ logging.info("request IP: " + str(request.remote_addr))
+ logging.info(json.dumps(request.get_json()))
+
+ input_request = request.get_json()
+
+ tokens_to_generate = input_request.get("tokens_to_generate", 64)
+ temperature = input_request.get("temperature", 1.0)
+ top_k = input_request.get("top_k", 0)
+ top_p = input_request.get("top_p", 1.0)
+ repetition_penalty = input_request.get("repetition_penalty", 1.2)
+ stop_words_list = input_request.get("stop_words_list")
+ max_attention_window_size = input_request.get("max_attention_window_size")
+ random_seed = input_request.get("random_seed", 0)
+ prompts = input_request["prompts"]
+
+ data = dict(
+ prompts=prompts,
+ max_new_tokens=tokens_to_generate,
+ temperature=temperature,
+ top_k=top_k,
+ top_p=top_p,
+ repetition_penalty=repetition_penalty,
+ random_seed=random_seed,
+ stop_words_list=stop_words_list,
+ max_attention_window_size=max_attention_window_size,
+ )
+ self.comm.Barrier()
+ data = self.comm.bcast(data, root=0)
+
+ out = self.generate(**data)
+ return jsonify(out)
+
+
+def parse_input(input_texts: str, tokenizer):
+ batch_input_ids = [
+ tokenizer.encode(
+ input_text,
+ add_special_tokens=False, # TODO: does this need to be true?
+ )
+ for input_text in input_texts
+ ]
+ batch_input_ids = [
+ torch.tensor(x, dtype=torch.int32, device="cuda") for x in batch_input_ids
+ ]
+ input_lengths = [x.size(0) for x in batch_input_ids]
+
+ return batch_input_ids, input_lengths
+
+
+def get_output(output_ids, input_lengths, max_output_len, tokenizer, eos_token):
+ num_beams = output_ids.size(1)
+ assert num_beams == 1
+ output_texts = []
+ for idx, input_len in enumerate(input_lengths):
+ output_begin = input_len
+ output_end = input_len + max_output_len
+ outputs = output_ids[idx][0][output_begin:output_end]
+ eos_ids = (outputs == eos_token).nonzero(as_tuple=True)[-1]
+ if len(eos_ids) > 0:
+ outputs = outputs[: eos_ids[0]]
+ outputs = outputs.tolist()
+ output_texts.append(tokenizer.decode(outputs))
+ return output_texts
+
+
+def prepare_stop_words(stop_words_list, tokenizer):
+ # adapted from https://github.com/NVIDIA/TensorRT-LLM/blob/b310ec675145c9ee7668592549f733df4abf1e94/tensorrt_llm/runtime/generation.py#L46
+ flat_ids = []
+ offsets = []
+ for batch_stop_words in stop_words_list:
+ item_flat_ids = []
+ item_offsets = []
+
+ for word in batch_stop_words:
+ # there is a known issue in TensorRT-LLM that word ids are not unique and might change depending on
+ # where in the text it appears. In our case we mainly need to stop on ids as they appear in the middle
+ # of the text. The following is a workaround to get such ids that works for both kind of stop
+ # words as well as newlines that we commonly use. But note that it's not a universal fix, so this might
+ # require refactoring if different stop words are used in the future.
+ # Eventually, this needs to be fixed inside TensorRT-LLM itself.
+ ids = tokenizer.encode("magic" + word)
+ ids = ids[2:] # skipping "magic"
+
+ if len(ids) == 0:
+ continue
+
+ item_flat_ids += ids
+ item_offsets.append(len(ids))
+
+ flat_ids.append(np.array(item_flat_ids))
+ offsets.append(np.cumsum(np.array(item_offsets)))
+
+ pad_to = max(1, max(len(ids) for ids in flat_ids))
+
+ for i, (ids, offs) in enumerate(zip(flat_ids, offsets)):
+ flat_ids[i] = np.pad(ids, (0, pad_to - len(ids)), constant_values=0)
+ offsets[i] = np.pad(offs, (0, pad_to - len(offs)), constant_values=-1)
+
+ stop_words = np.array([flat_ids, offsets], dtype="int32").transpose((1, 0, 2))
+ return torch.Tensor(stop_words).to(torch.int32).to("cuda").contiguous()
+
+
+def load_tokenizer(tokenizer_dir: str):
+ tokenizer = AutoTokenizer.from_pretrained(
+ tokenizer_dir,
+ legacy=False,
+ trust_remote_code=True,
+ )
+
+ if tokenizer.pad_token_id is None:
+ tokenizer.pad_token_id = tokenizer.eos_token_id
+ pad_id = tokenizer.pad_token_id
+ end_id = tokenizer.eos_token_id
+
+ return tokenizer, pad_id, end_id
+
+
+class TensorRTLLM:
+ def __init__(self, model_path: str):
+ self.tokenizer, self.pad_id, self.end_id = load_tokenizer(
+ tokenizer_dir=model_path
+ )
+ self.runner = ModelRunnerCpp.from_dir(
+ engine_dir=model_path, rank=tensorrt_llm.mpi_rank()
+ )
+
+ @torch.no_grad()
+ def forward(
+ self,
+ input_texts,
+ max_output_token,
+ top_k,
+ top_p,
+ temperature,
+ repetition_penalty,
+ random_seed,
+ stop_words_list,
+ max_attention_window_size,
+ ):
+ batch_input_ids, input_lengths = parse_input(input_texts, self.tokenizer)
+
+ stop_words_list = [stop_words_list for _ in range(len(input_texts))]
+ stop_words_list = prepare_stop_words(stop_words_list, self.tokenizer)
+
+ # TODO: return dictionary with a proper error reporting
+ try:
+ output_ids = self.runner.generate(
+ batch_input_ids,
+ max_new_tokens=max_output_token,
+ end_id=self.end_id,
+ pad_id=self.pad_id,
+ temperature=temperature,
+ top_k=top_k,
+ top_p=top_p,
+ repetition_penalty=repetition_penalty,
+ random_seed=random_seed,
+ stop_words_list=stop_words_list,
+ max_attention_window_size=max_attention_window_size,
+ return_dict=False,
+ )
+ torch.cuda.synchronize()
+
+ output = get_output(
+ output_ids, input_lengths, max_output_token, self.tokenizer, self.end_id
+ )
+ except RuntimeError as e:
+ logging.error("RuntimeError: %s", e)
+ output = [f"RuntimeError: {e}"] * len(input_texts)
+
+ return output
+
+
+class WrapperServer:
+ def __init__(self, model_path: str):
+ self.comm = MPI.COMM_WORLD
+ self.rank = self.comm.Get_rank()
+
+ self.model = TensorRTLLM(model_path=model_path)
+
+ if self.rank == 0:
+ self.app = Flask(__file__, static_url_path="")
+ api = Api(self.app)
+ api.add_resource(
+ TritonServerGenerate, "/generate", resource_class_args=[self.model]
+ )
+
+ def run(self, url, port=5000):
+ if self.rank == 0:
+ self.app.run(url, threaded=True, port=port, debug=False)
+ else:
+ self.worker_loop()
+
+ def worker_loop(self):
+ triton = TritonServerGenerate(self.model)
+ while True:
+ self.comm.Barrier()
+ data = None
+ data = self.comm.bcast(data, root=0)
+ triton.generate(**data)
+
+
+if __name__ == "__main__":
+ # TODO: can we reuse normal logger here?
+ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
+
+ parser = ArgumentParser()
+ parser.add_argument("--model_path", required=True)
+ parser.add_argument("--host", type=str, default="0.0.0.0")
+ parser.add_argument("--port", type=int, default=5000)
+ args = parser.parse_args()
+
+ server = WrapperServer(model_path=args.model_path)
+ server.run(args.host, args.port)
diff --git a/LeanK/eval/RULER/pred/serve_vllm.py b/LeanK/eval/RULER/pred/serve_vllm.py
new file mode 100644
index 00000000..e6d72589
--- /dev/null
+++ b/LeanK/eval/RULER/pred/serve_vllm.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# adapted from https://github.com/vllm-project/vllm/blob/v0.4.0/vllm/entrypoints/api_server.py
+
+import argparse
+import json
+from typing import AsyncGenerator
+
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, Response, StreamingResponse
+from vllm.engine.arg_utils import AsyncEngineArgs
+from vllm.engine.async_llm_engine import AsyncLLMEngine
+from vllm.sampling_params import SamplingParams
+from vllm.utils import random_uuid
+
+TIMEOUT_KEEP_ALIVE = 5 # seconds.
+app = FastAPI()
+engine = None
+
+
+@app.get("/health")
+async def health() -> Response:
+ """Health check."""
+ return Response(status_code=200)
+
+
+@app.put("/generate")
+async def generate(request: Request) -> Response:
+ """Generate completion for the request.
+
+ The request should be a JSON object with the following fields:
+ - prompt: the prompt to use for the generation.
+ - stream: whether to stream the results or not.
+ - other fields: the sampling parameters (See `SamplingParams` for details).
+ """
+ request_dict = await request.json()
+ prompt = request_dict.pop("prompt")
+ stream = request_dict.pop("stream", False)
+ sampling_params = SamplingParams(**request_dict)
+ request_id = random_uuid()
+
+ results_generator = engine.generate(prompt, sampling_params, request_id)
+
+ # Streaming case
+ async def stream_results() -> AsyncGenerator[bytes, None]:
+ async for request_output in results_generator:
+ prompt = request_output.prompt
+ text_outputs = [prompt + output.text for output in request_output.outputs]
+ ret = {"text": text_outputs}
+ yield (json.dumps(ret) + "\0").encode("utf-8")
+
+ if stream:
+ return StreamingResponse(stream_results())
+
+ # Non-streaming case
+ final_output = None
+ async for request_output in results_generator:
+ if await request.is_disconnected():
+ # Abort the request if the client disconnects.
+ await engine.abort(request_id)
+ return Response(status_code=499)
+ final_output = request_output
+ assert final_output is not None
+ text_outputs = [output.text for output in final_output.outputs]
+ ret = {"text": text_outputs}
+ return JSONResponse(ret)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--host", type=str, default="0.0.0.0")
+ parser.add_argument("--port", type=int, default=5000)
+ parser.add_argument("--ssl-keyfile", type=str, default=None)
+ parser.add_argument("--ssl-certfile", type=str, default=None)
+ parser.add_argument(
+ "--root-path",
+ type=str,
+ default=None,
+ help="FastAPI root_path when app is behind a path based routing proxy",
+ )
+ parser = AsyncEngineArgs.add_cli_args(parser)
+ args = parser.parse_args()
+
+ engine_args = AsyncEngineArgs.from_cli_args(args)
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
+
+ app.root_path = args.root_path
+ uvicorn.run(
+ app,
+ host=args.host,
+ port=args.port,
+ log_level="debug",
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
+ ssl_keyfile=args.ssl_keyfile,
+ ssl_certfile=args.ssl_certfile,
+ )
diff --git a/LeanK/eval/RULER/run.sh b/LeanK/eval/RULER/run.sh
new file mode 100644
index 00000000..c89aca9f
--- /dev/null
+++ b/LeanK/eval/RULER/run.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# container: docker.io/cphsieh/ruler:0.1.0
+# bash run.sh MODEL_NAME BENCHMARK_NAME
+
+if [ $# -ne 2 ]; then
+ echo "Usage: $0 $1 "
+ exit 1
+fi
+
+
+# Root Directories
+GPUS="1" # GPU size for tensor_parallel.
+ROOT_DIR="leank_eval" # the path that stores generated task samples and model predictions.
+MODEL_DIR="../../../.." # the path that contains individual model folders from HUggingface.
+ENGINE_DIR="." # the path that contains individual engine folders from TensorRT-LLM.
+BATCH_SIZE=1 # increase to improve GPU utilization
+
+
+# Model and Tokenizer
+source config_models.sh
+MODEL_NAME=${1}
+MODEL_CONFIG=$(MODEL_SELECT ${MODEL_NAME} ${MODEL_DIR} ${ENGINE_DIR})
+IFS=":" read MODEL_PATH MODEL_TEMPLATE_TYPE MODEL_FRAMEWORK TOKENIZER_PATH TOKENIZER_TYPE OPENAI_API_KEY GEMINI_API_KEY AZURE_ID AZURE_SECRET AZURE_ENDPOINT <<< "$MODEL_CONFIG"
+if [ -z "${MODEL_PATH}" ]; then
+ echo "Model: ${MODEL_NAME} is not supported"
+ exit 1
+fi
+
+
+export OPENAI_API_KEY=${OPENAI_API_KEY}
+export GEMINI_API_KEY=${GEMINI_API_KEY}
+export AZURE_API_ID=${AZURE_ID}
+export AZURE_API_SECRET=${AZURE_SECRET}
+export AZURE_API_ENDPOINT=${AZURE_ENDPOINT}
+
+
+# Benchmark and Tasks
+source config_tasks.sh
+BENCHMARK=${2}
+declare -n TASKS=$BENCHMARK
+if [ -z "${TASKS}" ]; then
+ echo "Benchmark: ${BENCHMARK} is not supported"
+ exit 1
+fi
+
+
+# Start server (you may want to run in other container.)
+if [ "$MODEL_FRAMEWORK" == "vllm" ]; then
+ python pred/serve_vllm.py \
+ --model=${MODEL_PATH} \
+ --tensor-parallel-size=${GPUS} \
+ --dtype bfloat16 \
+ --disable-custom-all-reduce \
+ &
+
+elif [ "$MODEL_FRAMEWORK" == "trtllm" ]; then
+ python pred/serve_trt.py \
+ --model_path=${MODEL_PATH} \
+ &
+
+elif [ "$MODEL_FRAMEWORK" == "sglang" ]; then
+ python -m sglang.launch_server \
+ --model-path ${MODEL_PATH} \
+ --tp ${GPUS} \
+ --port 5000 \
+ --enable-flashinfer \
+ &
+ # use sglang/test/killall_sglang.sh to kill sglang server if it hangs
+
+fi
+
+
+# Start client (prepare data / call model API / obtain final metrics)
+total_time=0
+for MAX_SEQ_LENGTH in "${SEQ_LENGTHS[@]}"; do
+
+ RESULTS_DIR="${ROOT_DIR}/${MODEL_NAME}/${BENCHMARK}/${MAX_SEQ_LENGTH}"
+ DATA_DIR="${RESULTS_DIR}/data"
+ PRED_DIR="${RESULTS_DIR}/pred"
+ mkdir -p ${DATA_DIR}
+ mkdir -p ${PRED_DIR}
+
+ for TASK in "${TASKS[@]}"; do
+ python data/prepare.py \
+ --save_dir ${DATA_DIR} \
+ --benchmark ${BENCHMARK} \
+ --task ${TASK} \
+ --tokenizer_path ${TOKENIZER_PATH} \
+ --tokenizer_type ${TOKENIZER_TYPE} \
+ --max_seq_length ${MAX_SEQ_LENGTH} \
+ --model_template_type ${MODEL_TEMPLATE_TYPE} \
+ --num_samples ${NUM_SAMPLES} \
+ ${REMOVE_NEWLINE_TAB}
+
+ start_time=$(date +%s)
+ python pred/call_api.py \
+ --data_dir ${DATA_DIR} \
+ --save_dir ${PRED_DIR} \
+ --benchmark ${BENCHMARK} \
+ --task ${TASK} \
+ --server_type ${MODEL_FRAMEWORK} \
+ --model_name_or_path ${MODEL_PATH} \
+ --temperature ${TEMPERATURE} \
+ --top_k ${TOP_K} \
+ --top_p ${TOP_P} \
+ --batch_size ${BATCH_SIZE} \
+ ${STOP_WORDS}
+ end_time=$(date +%s)
+ time_diff=$((end_time - start_time))
+ total_time=$((total_time + time_diff))
+ done
+
+ python eval/evaluate.py \
+ --data_dir ${PRED_DIR} \
+ --benchmark ${BENCHMARK}
+done
+
+echo "Total time spent on call_api: $total_time seconds"
diff --git a/LeanK/eval/RULER/synthetic.yaml b/LeanK/eval/RULER/synthetic.yaml
new file mode 100644
index 00000000..823ed733
--- /dev/null
+++ b/LeanK/eval/RULER/synthetic.yaml
@@ -0,0 +1,121 @@
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+niah_single_1:
+ task: niah
+ args:
+ type_haystack: repeat
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_single_2:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_single_3:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: uuids
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multikey_1:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 4
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multikey_2:
+ task: niah
+ args:
+ type_haystack: needle
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multikey_3:
+ task: niah
+ args:
+ type_haystack: needle
+ type_needle_k: uuids
+ type_needle_v: uuids
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multivalue:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 4
+ num_needle_q: 1
+
+niah_multiquery:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 4
+
+vt:
+ task: variable_tracking
+ args:
+ num_chains: 1
+ num_hops: 4
+
+cwe:
+ task: common_words_extraction
+ args:
+ freq_cw: 30
+ freq_ucw: 3
+ num_cw: 10
+
+fwe:
+ task: freq_words_extraction
+ args:
+ alpha: 2.0
+
+qa_1:
+ task: qa
+ args:
+ dataset: squad
+
+qa_2:
+ task: qa
+ args:
+ dataset: hotpotqa
diff --git a/LeanK/eval/kernel_benchmark.py b/LeanK/eval/kernel_benchmark.py
new file mode 100644
index 00000000..d759077f
--- /dev/null
+++ b/LeanK/eval/kernel_benchmark.py
@@ -0,0 +1,183 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+
+import numpy as np
+import tilelang
+import tilelang.language as T
+import torch
+from tilelang.autotuner import *
+from tilelang_flash_decoding import flashattn
+
+from minference.modules.leank import reorder_channel_mask
+from minference.ops.leank_flash_decoding import leank_flashattn
+
+llama_pattern = torch.load("../../minference/configs/leank/llama3.1-8b-instruct.pth")
+
+supported_dims = [0, 32, 64, 96, 128]
+layer_boundaries = []
+layer_counts = []
+for i in range(llama_pattern.shape[0]):
+ layer_mask = llama_pattern[i]
+ channel_mask = layer_mask.sum(dim=-1)
+ layer_full_attn_channels = reorder_channel_mask(
+ channel_mask,
+ layer_mask,
+ supported_dims,
+ )
+ dim_cnt = (layer_mask.sum(dim=-1) == supported_dims[0]).sum().item()
+ boundaries = [dim_cnt]
+ counts = [supported_dims[0]]
+ for d in supported_dims[1:]:
+ nheads = (layer_mask.sum(dim=-1) == d).sum().item()
+ if nheads > 0:
+ dim_cnt += nheads
+ boundaries.append(dim_cnt)
+ counts.append(d)
+ layer_boundaries.append(boundaries)
+ layer_counts.append(counts)
+
+
+def get_heuristic_config() -> dict:
+ # Get CUDA device properties
+ if not torch.cuda.is_available():
+ raise RuntimeError("CUDA is not available")
+ device = torch.cuda.current_device()
+ sm_major, sm_minor = torch.cuda.get_device_capability(device)
+ sm_version = sm_major * 10 + sm_minor
+ print(f"CUDA device capability: {sm_version}")
+ if sm_version == 89:
+ return {
+ "block_N": 128,
+ "block_H": 64,
+ "num_split": 8,
+ "num_stages": 0,
+ "threads": 128,
+ }
+ else:
+ return {
+ "block_N": 64,
+ "block_H": 64,
+ "num_split": 2,
+ "num_stages": 1,
+ "threads": 128,
+ }
+
+
+def main(
+ batch: int = 1,
+ heads: int = 32,
+ groups: int = 8,
+ kv_seqlen: int = 8192,
+ dim: int = 128,
+ tune: bool = False,
+):
+ batch, heads, groups, kv_seqlen, dim = batch, heads, groups, kv_seqlen, dim
+ qk_flops = 2 * batch * heads * kv_seqlen * dim
+ kv_fulllen = 1024
+ kv_seqlen -= 1024
+ pv_flops = 2 * batch * heads * kv_seqlen * dim
+ total_flops = qk_flops + pv_flops
+ heads_per_group = heads // groups
+ layer_idx = 0
+
+ config = get_heuristic_config()
+
+ if not tune:
+ kernel = flashattn(batch, heads, groups, kv_seqlen, kv_seqlen, dim, **config)
+ profiler = kernel.get_profiler(
+ tensor_supply_type=tilelang.TensorSupplyType.Auto
+ )
+ base_latency = profiler.do_bench(warmup=500)
+ print("Dense Attention (Tile-lang): {:.2f} ms".format(base_latency))
+
+ times = []
+
+ for boundaries, counts in zip(layer_boundaries, layer_counts):
+ l, r = boundaries[0], -1
+ number_groups = []
+ kernel_kwargs = []
+ for boundary, count in zip(boundaries[1:], counts[1:]):
+ r = boundary
+ number_groups.append(r - l)
+ l = r
+
+ n_groups = len(number_groups)
+ if n_groups < 4:
+ number_groups += [0] * (4 - n_groups)
+
+ kernel_kwargs = [batch, heads]
+ kernel_kwargs += [i * heads_per_group for i in number_groups]
+ kernel_kwargs += [groups]
+ kernel_kwargs += number_groups
+ kernel_kwargs += [kv_seqlen, kv_seqlen, kv_fulllen, kv_fulllen, dim]
+ kernel_kwargs += counts[1:]
+ if n_groups < 4:
+ kernel_kwargs += [0] * (4 - n_groups)
+ kernel_kwargs += [n_groups]
+ kernel_kwargs += ["bfloat16"]
+
+ layer_idx += 1
+ if not tune:
+ program = leank_flashattn(*kernel_kwargs)(**config)
+ kernel = tilelang.compile(
+ program, out_idx=[5 * (n_groups + 1) + 1 + (n_groups > 0)]
+ )
+ profiler = kernel.get_profiler(
+ tensor_supply_type=tilelang.TensorSupplyType.Auto
+ )
+ latency = profiler.do_bench(warmup=500)
+ print(f"Layer {layer_idx}", "LeanK Attention: {:.2f} ms".format(latency))
+ times.append(latency)
+ else:
+ best_result = leank_flashattn(**kernel_kwargs, tune=tune)
+ best_latency = best_result.latency
+ best_config = best_result.config
+ print(f"Best latency: {best_latency}")
+ print(f"Best TFlops: {total_flops / best_latency * 1e-9}")
+ print(f"Best config: {best_config}")
+
+ print("---------------------")
+ print("Dense Attention (Tile-lang): {:.2f} ms".format(base_latency))
+ print("LeanK Decoding (Average): {:.2f} ms".format(np.mean(times)))
+
+
+def get_heuristic_config() -> dict:
+ # Get CUDA device properties
+ if not torch.cuda.is_available():
+ raise RuntimeError("CUDA is not available")
+ device = torch.cuda.current_device()
+ sm_major, sm_minor = torch.cuda.get_device_capability(device)
+ sm_version = sm_major * 10 + sm_minor
+ print(f"CUDA device capability: {sm_version}")
+ if sm_version == 89:
+ return {
+ "block_N": 128,
+ "block_H": 64,
+ "num_split": 8,
+ "num_stages": 0,
+ "threads": 128,
+ }
+ else:
+ return {
+ "block_N": 64,
+ "block_H": 64,
+ "num_split": 8,
+ "num_stages": 1,
+ "threads": 128,
+ }
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--batch", type=int, default=64, help="batch size")
+ parser.add_argument("--heads", type=int, default=32, help="heads")
+ parser.add_argument("--groups", type=int, default=8, help="groups")
+ parser.add_argument(
+ "--kv_seqlen", type=int, default=32768, help="kv sequence length"
+ )
+ parser.add_argument("--dim", type=int, default=128, help="dim")
+ parser.add_argument("--tune", action="store_true", help="tune configs")
+ args = parser.parse_args()
+ main(args.batch, args.heads, args.groups, args.kv_seqlen, args.dim, args.tune)
diff --git a/LeanK/eval/test_memory.py b/LeanK/eval/test_memory.py
new file mode 100644
index 00000000..634cc38b
--- /dev/null
+++ b/LeanK/eval/test_memory.py
@@ -0,0 +1,144 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+import time
+
+import numpy as np
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from minference import MInference
+
+
+def main(model_name):
+ n_times = 6
+ n_warmup = 1
+
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype=torch.bfloat16,
+ device_map="cuda:0",
+ attn_implementation="flash_attention_2",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+ # Test HF Baseline
+ for bs in [52]:
+ for t in range(n_times):
+ seq_len = 4096
+ dur_list = []
+ mem_list = []
+ torch.cuda.reset_peak_memory_stats()
+ torch.cuda.empty_cache()
+ input_ids = torch.randint(
+ 0,
+ model.vocab_size,
+ (
+ bs,
+ seq_len,
+ ),
+ ).to(model.device)
+ attention_mask = torch.randint(0, 2, (bs, seq_len)).to(model.device)
+ start_time = time.time()
+
+ with torch.no_grad():
+ out = model.generate(
+ input_ids,
+ attention_mask=attention_mask,
+ pad_token_id=tokenizer.eos_token_id,
+ max_new_tokens=127,
+ do_sample=False,
+ return_legacy_cache=True,
+ )
+ torch.cuda.synchronize()
+
+ end_time = time.time()
+ print("run", t, "takes", (end_time - start_time), "seconds")
+
+ peak_memory = torch.cuda.max_memory_allocated()
+ if t > n_warmup:
+ dur_list.append((end_time - start_time))
+ mem_list.append(peak_memory / 1024 / 1024 / 1024)
+ print(
+ "------------------------------------------------------------------------------------------------------------"
+ )
+ print(
+ "HF seq_len: {:<20} batch_size: {:<20} time: {:<10.2f} memory: {:<10.2f} ".format(
+ seq_len, bs, np.mean(dur_list), np.mean(mem_list)
+ )
+ )
+ print(
+ "------------------------------------------------------------------------------------------------------------"
+ )
+
+ minference_patch = MInference(
+ attn_type="dense",
+ model_name=model_name,
+ kv_type="leank",
+ attn_kwargs={"recent_size": 250, "sink_size": 6, "accumu_size": 128},
+ )
+ model = minference_patch(model)
+
+ # test LeanK
+ for bs in [52, 64]:
+ for t in range(n_times):
+ seq_len = 4096
+ dur_list = []
+ mem_list = []
+ torch.cuda.reset_peak_memory_stats()
+ torch.cuda.empty_cache()
+ input_ids = torch.randint(
+ 0,
+ model.vocab_size,
+ (
+ bs,
+ seq_len,
+ ),
+ ).to(model.device)
+ attention_mask = torch.randint(0, 2, (bs, seq_len)).to(model.device)
+
+ start_time = time.time()
+
+ with torch.no_grad():
+ model.generate(
+ input_ids,
+ attention_mask=attention_mask,
+ pad_token_id=tokenizer.eos_token_id,
+ max_new_tokens=127,
+ do_sample=False,
+ temperature=1.0,
+ )
+ torch.cuda.synchronize()
+
+ end_time = time.time()
+ print("run", t, "takes", (end_time - start_time), "seconds")
+
+ peak_memory = torch.cuda.max_memory_allocated()
+ if t > n_warmup:
+ dur_list.append((end_time - start_time))
+ mem_list.append(peak_memory / 1024 / 1024 / 1024)
+ torch.cuda.empty_cache()
+ print(
+ "------------------------------------------------------------------------------------------------------------"
+ )
+ print(
+ "LeanK seq_len: {:<20} batch_size: {:<20} time: {:<10.2f} memory: {:<10.2f} ".format(
+ seq_len, bs, np.mean(dur_list), np.mean(mem_list)
+ )
+ )
+ print(
+ "------------------------------------------------------------------------------------------------------------"
+ )
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--model_name",
+ type=str,
+ default="meta-llama/Meta-Llama-3.1-8B-Instruct",
+ help="model path",
+ )
+ args = parser.parse_args()
+ main(args.model_name)
diff --git a/LeanK/eval/tilelang_flash_decoding.py b/LeanK/eval/tilelang_flash_decoding.py
new file mode 100644
index 00000000..02105760
--- /dev/null
+++ b/LeanK/eval/tilelang_flash_decoding.py
@@ -0,0 +1,336 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import tilelang
+import tilelang.language as T
+import torch
+import torch.nn.functional as F
+from tilelang.autotuner import *
+
+torch.random.manual_seed(0)
+
+
+@tilelang.jit(out_idx=[6])
+def flashattn(
+ batch,
+ heads,
+ groups,
+ seqlen_kv,
+ config_len,
+ dim,
+ block_N,
+ block_H,
+ num_split,
+ num_stages,
+ threads,
+):
+ scale = (1.0 / dim) ** 0.5 * 1.44269504 # log2(e)
+ shape_q = [batch, heads, dim]
+ shape_k = [batch, seqlen_kv, groups, dim]
+ shape_v = [batch, seqlen_kv, groups, dim]
+ shape_o = [batch, heads, dim]
+ dtype = "float16"
+ accum_dtype = "float"
+ kv_group_num = heads // groups
+
+ part_shape = [batch, heads, num_split, dim]
+ valid_block_H = min(block_H, kv_group_num)
+ valid_block_N = min(block_N, seqlen_kv // num_split)
+
+ @T.macro
+ def flash_attn(
+ Q: T.Tensor(shape_q, dtype),
+ K: T.Tensor(shape_k, dtype),
+ V: T.Tensor(shape_v, dtype),
+ mask: T.Tensor([batch, seqlen_kv, groups], "uint8"),
+ Output: T.Tensor([batch, heads, dim], dtype),
+ ):
+ with T.Kernel(batch, heads // valid_block_H, num_split, threads=threads) as (
+ bx,
+ by,
+ bz,
+ ):
+ Q_shared = T.alloc_shared([block_H, dim], dtype)
+ K_shared = T.alloc_shared([block_N, dim], dtype)
+ V_shared = T.alloc_shared([block_N, dim], dtype)
+ O_shared = T.alloc_shared([valid_block_H, dim], dtype)
+ acc_s = T.alloc_fragment([block_H, block_N], accum_dtype)
+ acc_s_cast = T.alloc_fragment([block_H, block_N], dtype)
+ mask_local = T.alloc_fragment([block_N], "uint8")
+ acc_o = T.alloc_fragment([block_H, dim], accum_dtype)
+ scores_max = T.alloc_fragment([block_H], accum_dtype)
+ scores_max_prev = T.alloc_fragment([block_H], accum_dtype)
+ scores_scale = T.alloc_fragment([block_H], accum_dtype)
+ scores_sum = T.alloc_fragment([block_H], accum_dtype)
+ logsum = T.alloc_fragment([block_H], accum_dtype)
+
+ bid = bx
+ hid = by
+ cur_kv_head = hid // (kv_group_num // valid_block_H)
+
+ T.copy(
+ Q[bid, hid * valid_block_H : hid * valid_block_H + block_H, :], Q_shared
+ )
+ T.fill(acc_o, 0)
+ T.fill(logsum, 0)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+
+ loop_range = T.ceildiv((seqlen_kv // num_split), block_N)
+ for k in T.Pipelined(loop_range, num_stages=num_stages):
+ T.copy(
+ K[bid, k * block_N : (k + 1) * block_N, cur_kv_head, :], K_shared
+ )
+ T.copy(
+ mask[bid, k * block_N : (k + 1) * block_N, cur_kv_head], mask_local
+ )
+ T.clear(acc_s)
+ T.gemm(
+ Q_shared,
+ K_shared,
+ acc_s,
+ transpose_B=True,
+ policy=T.GemmWarpPolicy.FullRow,
+ )
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.if_then_else(
+ mask_local[j] != 0, acc_s[i, j], -T.infinity(accum_dtype)
+ )
+ T.copy(scores_max, scores_max_prev)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ T.reduce_max(acc_s, scores_max, dim=1, clear=False)
+ for i in T.Parallel(block_H):
+ scores_scale[i] = T.exp2(
+ scores_max_prev[i] * scale - scores_max[i] * scale
+ )
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.exp2(acc_s[i, j] * scale - scores_max[i] * scale)
+ T.reduce_sum(acc_s, scores_sum, dim=1)
+ for i in T.Parallel(block_H):
+ logsum[i] = logsum[i] * scores_scale[i] + scores_sum[i]
+ T.copy(acc_s, acc_s_cast)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] *= scores_scale[i]
+ T.copy(
+ V[bid, k * block_N : (k + 1) * block_N, cur_kv_head, :], V_shared
+ )
+ T.gemm(acc_s_cast, V_shared, acc_o, policy=T.GemmWarpPolicy.FullRow)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] /= logsum[i]
+ for i in T.Parallel(block_H):
+ logsum[i] = T.log2(logsum[i]) + scores_max[i] * scale
+ T.copy(acc_o[:valid_block_H, :], O_shared)
+ T.copy(
+ O_shared,
+ Output[bid, hid * valid_block_H : (hid + 1) * valid_block_H, :],
+ )
+
+ @T.macro
+ def flash_attn_split(
+ Q: T.Tensor(shape_q, dtype),
+ K: T.Tensor(shape_k, dtype),
+ V: T.Tensor(shape_v, dtype),
+ mask: T.Tensor([batch, seqlen_kv, groups], "uint8"),
+ glse: T.Tensor([batch, heads, num_split], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ ):
+ with T.Kernel(batch, heads // valid_block_H, num_split, threads=threads) as (
+ bx,
+ by,
+ bz,
+ ):
+ Q_shared = T.alloc_shared([block_H, dim], dtype)
+ K_shared = T.alloc_shared([block_N, dim], dtype)
+ V_shared = T.alloc_shared([block_N, dim], dtype)
+ O_shared = T.alloc_shared([valid_block_H, dim], dtype)
+ acc_s = T.alloc_fragment([block_H, block_N], accum_dtype)
+ acc_s_cast = T.alloc_fragment([block_H, block_N], dtype)
+ mask_local = T.alloc_fragment([block_N], "uint8")
+ acc_o = T.alloc_fragment([block_H, dim], accum_dtype)
+ scores_max = T.alloc_fragment([block_H], accum_dtype)
+ scores_max_prev = T.alloc_fragment([block_H], accum_dtype)
+ scores_scale = T.alloc_fragment([block_H], accum_dtype)
+ scores_sum = T.alloc_fragment([block_H], accum_dtype)
+ logsum = T.alloc_fragment([block_H], accum_dtype)
+
+ bid = bx
+ hid = by
+ sid = bz
+ cur_kv_head = hid // (kv_group_num // valid_block_H)
+
+ T.copy(
+ Q[bid, hid * valid_block_H : hid * valid_block_H + block_H, :], Q_shared
+ )
+ T.fill(acc_o, 0)
+ T.fill(logsum, 0)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+
+ loop_range = T.ceildiv((seqlen_kv // num_split), block_N)
+
+ per_block_len = T.ceildiv(config_len, num_split)
+ this_block_end = T.min(per_block_len * (sid + 1), seqlen_kv)
+ this_block_begin = per_block_len * sid
+
+ if this_block_begin < seqlen_kv:
+ loop_range = T.ceildiv(per_block_len, block_N)
+ for k in T.Pipelined(loop_range, num_stages=num_stages):
+ if per_block_len * sid + k * block_N < seqlen_kv:
+ T.copy(
+ K[
+ bid,
+ per_block_len * sid
+ + k * block_N : per_block_len * sid
+ + (k + 1) * block_N,
+ cur_kv_head,
+ :,
+ ],
+ K_shared,
+ )
+ T.copy(
+ mask[
+ bid,
+ per_block_len * sid
+ + k * block_N : per_block_len * sid
+ + (k + 1) * block_N,
+ cur_kv_head,
+ ],
+ mask_local,
+ )
+ T.clear(acc_s)
+ T.gemm(
+ Q_shared,
+ K_shared,
+ acc_s,
+ transpose_B=True,
+ policy=T.GemmWarpPolicy.FullRow,
+ )
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.if_then_else(
+ (mask_local[j] != 0) & (j < seqlen_kv // num_split),
+ acc_s[i, j],
+ -T.infinity(accum_dtype),
+ )
+ T.copy(scores_max, scores_max_prev)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ T.reduce_max(acc_s, scores_max, dim=1, clear=False)
+ for i in T.Parallel(block_H):
+ scores_scale[i] = T.exp2(
+ scores_max_prev[i] * scale - scores_max[i] * scale
+ )
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.exp2(
+ acc_s[i, j] * scale - scores_max[i] * scale
+ )
+ T.reduce_sum(acc_s, scores_sum, dim=1)
+ for i in T.Parallel(block_H):
+ logsum[i] = logsum[i] * scores_scale[i] + scores_sum[i]
+ T.copy(acc_s, acc_s_cast)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] *= scores_scale[i]
+ T.copy(
+ V[
+ bid,
+ per_block_len * sid
+ + k * block_N : per_block_len * sid
+ + (k + 1) * block_N,
+ cur_kv_head,
+ :,
+ ],
+ V_shared,
+ )
+ T.gemm(
+ acc_s_cast, V_shared, acc_o, policy=T.GemmWarpPolicy.FullRow
+ )
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] /= logsum[i]
+ for i in T.Parallel(block_H):
+ logsum[i] = T.log2(logsum[i]) + scores_max[i] * scale
+
+ for i in T.Parallel(block_H):
+ if i < valid_block_H:
+ glse[bid, hid * valid_block_H + i, sid] = logsum[i]
+ T.copy(acc_o[:valid_block_H, :], O_shared)
+ T.copy(
+ O_shared,
+ Output_partial[
+ bid, hid * valid_block_H : (hid + 1) * valid_block_H, sid, :
+ ],
+ )
+
+ @T.macro
+ def combine(
+ glse: T.Tensor([batch, heads, num_split], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ with T.Kernel(heads, batch, threads=128) as (by, bz):
+ po_local = T.alloc_fragment([dim], dtype)
+ o_accum_local = T.alloc_fragment([dim], accum_dtype)
+ lse_local = T.alloc_fragment([num_split, 128], dtype)
+ lse_local_split = T.alloc_local([1], accum_dtype)
+ lse_logsum_local = T.alloc_local([1], accum_dtype)
+ lse_max_local = T.alloc_fragment([128], accum_dtype)
+ scale_local = T.alloc_local([1], accum_dtype)
+
+ T.annotate_layout(
+ {
+ lse_logsum_local: T.Fragment(
+ lse_logsum_local.shape, forward_thread_fn=lambda i: i
+ ),
+ lse_max_local: T.Fragment(
+ lse_max_local.shape, forward_thread_fn=lambda i: i
+ ),
+ # lse_local: (local_id, thread_id)
+ lse_local: T.Fragment(
+ lse_local.shape, forward_fn=lambda i, j: (j, i)
+ ),
+ }
+ )
+
+ T.clear(lse_logsum_local)
+ T.clear(o_accum_local)
+ for k, j in T.Parallel(num_split, 128):
+ lse_local[k, j] = glse[bz, by, k]
+ T.reduce_max(lse_local, lse_max_local, dim=0, clear=True)
+ for k in T.Pipelined(num_split, num_stages=1):
+ lse_local_split[0] = glse[bz, by, k]
+ lse_logsum_local[0] += T.exp2(lse_local_split[0] - lse_max_local[0])
+ lse_logsum_local[0] = T.log2(lse_logsum_local[0]) + lse_max_local[0]
+ for k in T.serial(num_split):
+ for i in T.Parallel(dim):
+ po_local[i] = Output_partial[bz, by, k, i]
+ lse_local_split[0] = glse[bz, by, k]
+ scale_local[0] = T.exp2(lse_local_split[0] - lse_logsum_local[0])
+ for i in T.Parallel(dim):
+ o_accum_local[i] += po_local[i] * scale_local[0]
+ for i in T.Parallel(dim):
+ Output[bz, by, i] = o_accum_local[i]
+
+ @T.prim_func
+ def flashattn_gqa_decode_split(
+ Q: T.Tensor(shape_q, dtype),
+ K: T.Tensor(shape_k, dtype),
+ V: T.Tensor(shape_v, dtype),
+ mask: T.Tensor([batch, seqlen_kv, groups], "uint8"),
+ glse: T.Tensor([batch, heads, num_split], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ flash_attn_split(Q, K, V, mask, glse, Output_partial)
+ combine(glse, Output_partial, Output)
+
+ @T.prim_func
+ def flashattn_gqa_decode_no_split(
+ Q: T.Tensor(shape_q, dtype),
+ K: T.Tensor(shape_k, dtype),
+ V: T.Tensor(shape_v, dtype),
+ mask: T.Tensor([batch, seqlen_kv, groups], "uint8"),
+ glse: T.Tensor([batch, heads, num_split], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ flash_attn(Q, K, V, mask, Output)
+
+ if num_split > 1:
+ return flashattn_gqa_decode_split
+ else:
+ return flashattn_gqa_decode_no_split
diff --git a/LeanK/leank/__init__.py b/LeanK/leank/__init__.py
new file mode 100644
index 00000000..9df7121d
--- /dev/null
+++ b/LeanK/leank/__init__.py
@@ -0,0 +1,2 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
diff --git a/LeanK/leank/data.py b/LeanK/leank/data.py
new file mode 100644
index 00000000..fdab195c
--- /dev/null
+++ b/LeanK/leank/data.py
@@ -0,0 +1,583 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+import re
+from dataclasses import dataclass
+from typing import Dict, Sequence
+
+import torch
+import transformers
+from datasets import load_dataset
+from torch.utils.data import Dataset, IterableDataset
+
+
+def get_dataset(dataset_name, split="train", size=None):
+ dataset = load_dataset("json", data_files=dataset_name, split=split)
+ if size is not None:
+ dataset = dataset.select(range(size))
+ return dataset
+
+
+class MultiplePasskeyRetrievalDataset(Dataset):
+ PASSKEY_ALPHABET = [
+ "alpha",
+ "bravo",
+ "charlie",
+ "delta",
+ "echo",
+ "foxtrot",
+ "golf",
+ "hotel",
+ "india",
+ "juliett",
+ "kilo",
+ "lima",
+ "mike",
+ "november",
+ "oscar",
+ "papa",
+ "quebec",
+ "romeo",
+ "sierra",
+ "tango",
+ "uniform",
+ "victor",
+ "whiskey",
+ "xray",
+ "yankee",
+ "zulu",
+ ]
+
+ ORDINAL_NUMBERS = [
+ "first",
+ "second",
+ "third",
+ "fourth",
+ "fifth",
+ "sixth",
+ "seventh",
+ "eighth",
+ "ninth",
+ "tenth",
+ "eleventh",
+ "twelfth",
+ "thirteenth",
+ "fourteenth",
+ "fifteenth",
+ "sixteenth",
+ "seventeenth",
+ "eighteenth",
+ "nineteenth",
+ "twentieth",
+ ]
+
+ def __init__(
+ self,
+ haystack_dataset,
+ tokenizer: transformers.PreTrainedTokenizer,
+ max_length=None,
+ passkey_length=32,
+ num_passkeys=10,
+ needle="Remeber this sequence of words, it's the {ordinal_number} passkey to the vault: ",
+ retrieval_question="Based on the content of the book, what is the {ordinal_number} passkey to the vault?\nPasskey: ",
+ prompt1="<|im_start|> This is a very long story book: ",
+ prompt2=" .\n\n",
+ buffer_size=300,
+ seperator="\n\n",
+ min_depth_ratio=0.1,
+ max_depth_ratio=0.9,
+ context_lengths_num_intervals=20,
+ depth_ratio_num_intervals=20,
+ context_length_min=None,
+ context_length_max=None,
+ pad_to_multiple_of=16,
+ ):
+ super(MultiplePasskeyRetrievalDataset, self).__init__()
+
+ self.tokenizer = tokenizer
+
+ self.max_length = (
+ max_length if max_length is not None else tokenizer.model_max_length
+ )
+ self.max_depth_ratio = max_depth_ratio
+ self.min_depth_ratio = min_depth_ratio
+ self.context_lengths_num_intervals = context_lengths_num_intervals
+ self.depth_ratio_num_intervals = depth_ratio_num_intervals
+
+ if context_length_min is None or context_length_max is None:
+ self.context_length_min = self.context_length_max = self.max_length
+ else:
+ self.context_length_min = context_length_min
+ self.context_length_max = context_length_max
+
+ self.context_length_intervals = torch.linspace(
+ self.context_length_min,
+ self.context_length_max,
+ context_lengths_num_intervals,
+ dtype=torch.int,
+ )
+
+ self.depth_ratio_intervals = torch.linspace(
+ min_depth_ratio, max_depth_ratio, depth_ratio_num_intervals
+ )
+
+ self.passkey_length = passkey_length
+
+ self.num_passkeys = num_passkeys
+
+ self.haystack = ""
+
+ for sample in haystack_dataset["text"]:
+ if self._get_token_nums(self.haystack) >= self.context_length_max:
+ break
+ self.haystack += sample
+
+ self.haystack = self._trim(self.haystack, self.context_length_max)
+
+ self.needle = needle
+ self.needle_tokens_list = [
+ self.tokenizer.encode(
+ self.needle.format(ordinal_number=ordinal_number),
+ add_special_tokens=False,
+ )
+ for ordinal_number in self.ORDINAL_NUMBERS[: self.num_passkeys]
+ ]
+ self.retrieval_question_tokens_list = [
+ self.tokenizer.encode(
+ retrieval_question.format(ordinal_number=ordinal_number),
+ add_special_tokens=False,
+ )
+ for ordinal_number in self.ORDINAL_NUMBERS[: self.num_passkeys]
+ ]
+
+ self.haystack_tokens = self.tokenizer.encode(
+ self.haystack, add_special_tokens=False
+ )
+ self.seperator_tokens = self.tokenizer.encode(
+ seperator, add_special_tokens=False
+ )
+ self.prompt1_tokens = self.tokenizer.encode(prompt1, add_special_tokens=True)
+ self.prompt2_tokens = self.tokenizer.encode(prompt2, add_special_tokens=False)
+
+ passkey = self._generate_passkey()
+ passkey_tokens = self.tokenizer.encode(passkey, add_special_tokens=False)
+ needle_tokens = self.needle_tokens_list[0] + passkey_tokens
+
+ other_input_len = (
+ len(self.prompt1_tokens)
+ + len(self.prompt2_tokens)
+ + (
+ len(self.seperator_tokens)
+ + len(needle_tokens)
+ + len(self.seperator_tokens)
+ + len(self.retrieval_question_tokens_list[0])
+ + len(passkey_tokens)
+ )
+ * self.num_passkeys
+ )
+ if (
+ len(self.haystack_tokens) + other_input_len
+ > self.context_length_max - buffer_size
+ ):
+ self.haystack_tokens = self.haystack_tokens[
+ : self.context_length_max - buffer_size - other_input_len
+ ]
+
+ def _generate_passkey(self):
+ random_seq = torch.randint(
+ 0, len(self.PASSKEY_ALPHABET), (self.passkey_length,)
+ )
+ passkey = " ".join([self.PASSKEY_ALPHABET[i] for i in random_seq])
+ return passkey
+
+ def __len__(self):
+ return len(self.context_length_intervals)
+
+ def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+ context_length = self.context_length_intervals[i]
+ # randomly sample self.num_passkeys depth ratios in self.depth_ratio_intervals
+ depth_ratios = (
+ self.depth_ratio_intervals[
+ torch.randperm(self.depth_ratio_num_intervals)[: self.num_passkeys]
+ ]
+ .sort()
+ .values
+ )
+ passkey_tokens_list = [
+ self.tokenizer.encode(self._generate_passkey(), add_special_tokens=False)
+ for _ in range(self.num_passkeys)
+ ]
+ context = self._insert_needle(context_length, depth_ratios, passkey_tokens_list)
+ return self._construct_input(context, passkey_tokens_list)
+
+ def _trim(self, context, context_length):
+ tokens = self.tokenizer.encode(context, add_special_tokens=False)
+ if len(tokens) > context_length:
+ context = self.tokenizer.decode(tokens[:context_length])
+ return context
+
+ def _get_token_nums(self, context):
+ return len(self.tokenizer.encode(context))
+
+ def _insert_needle(self, context_length, depth_ratios, passkey_tokens_list):
+ haystack_tokens = self.haystack_tokens[:context_length]
+
+ context = []
+ last_insertion_point = 0
+
+ for i, (depth_ratio, passkey_tokens) in enumerate(
+ zip(depth_ratios, passkey_tokens_list)
+ ):
+ insertion_point = int(len(haystack_tokens) * depth_ratio)
+
+ needle_tokens = self.needle_tokens_list[i] + passkey_tokens
+
+ context += (
+ haystack_tokens[last_insertion_point:insertion_point]
+ + self.seperator_tokens
+ + needle_tokens
+ + self.seperator_tokens
+ )
+ last_insertion_point = insertion_point
+
+ context += haystack_tokens[last_insertion_point:]
+
+ return context
+
+ def _construct_input(self, context_tokens, passkey_tokens_list):
+ qa_tokens = []
+ for i, (passkey_tokens, retrieval_question_tokens) in enumerate(
+ zip(passkey_tokens_list, self.retrieval_question_tokens_list)
+ ):
+ qa_tokens += (
+ retrieval_question_tokens + passkey_tokens + self.seperator_tokens
+ )
+
+ context_tokens = self.prompt1_tokens + context_tokens
+
+ # pad to multiple of 16
+ if len(context_tokens) + len(qa_tokens) + len(self.prompt2_tokens) % 16 != 0:
+ pad_len = (
+ 16
+ - (len(context_tokens) + len(qa_tokens) + len(self.prompt2_tokens)) % 16
+ )
+ context_tokens += self.haystack_tokens[-pad_len:]
+
+ context_tokens += self.prompt2_tokens
+
+ input_ids = torch.tensor(context_tokens + qa_tokens)
+
+ assert input_ids.size(0) % 16 == 0
+
+ labels = torch.tensor([-100] * len(context_tokens) + qa_tokens)
+ length_context = len(context_tokens)
+ # self.tokenizer.encode(input_text, return_tensors="pt").shape[-1]
+ # labels = torch.tensor([-100] * length_context + [0] * (input_ids.shape[-1] - length_context))
+
+ return dict(input_ids=input_ids, labels=labels, length_context=length_context)
+ # return dict(input_ids=input_ids, labels=labels)
+
+
+@dataclass
+class DataCollator(object):
+ """Collate examples for supervised fine-tuning."""
+
+ tokenizer: transformers.PreTrainedTokenizer
+
+ def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
+ input_ids, labels, length_context = tuple(
+ [instance[key] for instance in instances]
+ for key in ("input_ids", "labels", "length_context")
+ )
+ input_ids = torch.nn.utils.rnn.pad_sequence(
+ input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
+ )
+ labels = torch.nn.utils.rnn.pad_sequence(
+ labels, batch_first=True, padding_value=-100
+ )
+
+ ret_dict = dict(
+ input_ids=input_ids,
+ labels=labels,
+ attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
+ length_context=length_context,
+ )
+ for key in instances[0].keys():
+ if key not in ret_dict:
+ ret_dict[key] = torch.stack([instance[key] for instance in instances])
+ return ret_dict
+
+
+def get_supervised_dataloader(
+ dataset, tokenizer, batch_size, num_workers=4, shuffle=True, sampler=None
+):
+ collator = DataCollator(tokenizer)
+ dataloader = torch.utils.data.DataLoader(
+ dataset,
+ batch_size=batch_size,
+ num_workers=num_workers,
+ collate_fn=collator,
+ shuffle=None if sampler is not None else shuffle,
+ sampler=sampler,
+ )
+ return dataloader
+
+
+import json
+import random
+import uuid
+
+import numpy as np
+import wonderwords
+from nltk.tokenize import sent_tokenize
+from tqdm import tqdm
+
+nouns = wonderwords.random_word._get_words_from_text_file("nounlist.txt")
+adjs = wonderwords.random_word._get_words_from_text_file("adjectivelist.txt")
+words = [f"{adj}-{noun}" for adj in adjs for noun in nouns]
+words = sorted(list(set(words)))
+
+DEPTHS = list(np.round(np.linspace(0, 100, num=40, endpoint=True)).astype(int))
+
+
+class PasskeyRetrievalDataset(Dataset):
+ def __init__(
+ self,
+ tokenizer: transformers.PreTrainedTokenizer,
+ uuid_length=128,
+ context_lengths_num_intervals=200,
+ context_length_min=None,
+ context_length_max=None,
+ num_needle_k={"essay": 1, "needle": 1},
+ num_needle_v={"essay": [4], "needle": [1]},
+ num_needle_q={"essay": 1, "needle": 1},
+ random_seed=42,
+ template="""<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n
+ Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n
+ {context}\n
+ What are all the special magic {type_needle_v} for {query} mentioned in the provided text?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n
+ The special magic {type_needle_v} for {query} mentioned in the provided text are """,
+ tokens_to_generate=128,
+ type_needle_v={"essay": "numbers", "needle": "uuids"},
+ type_needle_k={"essay": "words", "needle": "uuids"},
+ type_haystack=["needle", "essay"],
+ ):
+ super(PasskeyRetrievalDataset, self).__init__()
+
+ self.tokenizer = tokenizer
+ self.uuid_length = uuid_length
+ self.num_needle_k = num_needle_k
+ self.num_needle_v = num_needle_v
+ self.num_needle_q = num_needle_q
+ self.random_seed = random_seed
+ self.template = template
+ self.type_needle_v = type_needle_v
+ self.type_needle_k = type_needle_k
+ self.type_haystack = type_haystack
+
+ self.context_length_min = context_length_min
+ self.context_length_max = context_length_max
+
+ self.context_length_intervals = torch.linspace(
+ self.context_length_min,
+ self.context_length_max,
+ context_lengths_num_intervals,
+ dtype=torch.int,
+ )
+
+ self.uuid_length = uuid_length
+ self.haystack = {}
+ needle = "One of the special magic {type_needle_v} for {key} is: {value}."
+ if "essay" in type_haystack:
+ essay = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "data/PaulGrahamEssays.json"
+ )
+ essay = json.load(open(essay))["text"]
+ self.haystack["essay"] = re.sub(r"\s+", " ", essay).split(" ")
+ if "repeat" in type_haystack:
+ self.haystack[
+ "repeat"
+ ] = "The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again."
+ if "needle" in type_haystack:
+ self.haystack["needle"] = needle
+ self.needle = needle
+
+ self.tokens_to_generate = tokens_to_generate
+
+ self.incremental = {}
+
+ if "essay" in type_haystack:
+ self.incremental["essay"] = 500
+ if "repeat" in type_haystack:
+ self.incremental["repeat"] = 25
+ if "needle" in type_haystack:
+ self.incremental["needle"] = 25
+
+ self.num_haystack = {}
+
+ for type in type_haystack:
+ num_haystack = self.incremental[type]
+ total_tokens = 0
+ while total_tokens + tokens_to_generate < context_length_max:
+ input_text, answer = self.generate_input_output(
+ num_haystack, template, type
+ )
+ total_tokens = self.tokenizer.encode(
+ input_text + " ".join(answer), return_tensors="pt"
+ ).shape[-1]
+ print(
+ f"Max length {context_length_max} | Current length {total_tokens + tokens_to_generate} | Haystack: {num_haystack}"
+ )
+ if total_tokens + tokens_to_generate > context_length_max:
+ num_haystack -= self.incremental[type]
+ break
+
+ num_haystack += self.incremental[type]
+
+ self.num_haystack[type] = num_haystack
+ print("Num haystack of", type, ":", num_haystack)
+
+ def __len__(self):
+ return len(self.context_length_intervals)
+
+ def generate_random(self):
+ return str(uuid.UUID(int=random.getrandbits(self.uuid_length), version=4))
+
+ def generate_random_number(self, num_digits=7):
+ lower_bound = 10 ** (num_digits - 1)
+ upper_bound = 10**num_digits - 1
+ return str(random.randint(lower_bound, upper_bound))
+
+ def generate_random_word(self):
+ word = random.choice(words)
+ return word
+
+ def generate_random_uuid(self):
+ return str(uuid.UUID(int=random.getrandbits(128), version=4))
+
+ def generate_random(self, type_needle: str):
+ if type_needle == "numbers":
+ return self.generate_random_number()
+ elif type_needle == "words":
+ return self.generate_random_word()
+ elif type_needle == "uuids":
+ return self.generate_random_uuid()
+ else:
+ raise NotImplementedError(f"{type_needle} is not implemented.")
+
+ def generate_input_output(self, num_haystack, template, type):
+ keys, values, needles = [], [], []
+ for _ in range(self.num_needle_k[type]):
+ keys.append(self.generate_random(self.type_needle_k[type]))
+ value = []
+ for _ in range(random.choice(self.num_needle_v[type])):
+ value.append(self.generate_random(self.type_needle_v[type]))
+ needles.append(
+ self.needle.format(
+ type_needle_v=self.type_needle_v[type],
+ key=keys[-1],
+ value=value[-1],
+ )
+ )
+ values.append(value)
+
+ random.Random(self.random_seed).shuffle(needles)
+
+ if type == "essay":
+ text = " ".join(self.haystack[type][:num_haystack])
+ document_sents = sent_tokenize(text.strip())
+ insertion_positions = (
+ [0]
+ + sorted(
+ [
+ int(len(document_sents) * (depth / 100))
+ for depth in random.sample(DEPTHS, len(needles))
+ ]
+ )
+ + [len(document_sents)]
+ )
+ document_sents_list = []
+ for i in range(1, len(insertion_positions)):
+ last_pos = insertion_positions[i - 1]
+ next_pos = insertion_positions[i]
+ document_sents_list.append(" ".join(document_sents[last_pos:next_pos]))
+ if i - 1 < len(needles):
+ document_sents_list.append(needles[i - 1])
+ context = " ".join(document_sents_list)
+
+ else:
+ if type == "repeat":
+ sentences = [self.haystack[type]] * num_haystack
+ elif type == "needle":
+ sentences = [
+ self.haystack[type].format(
+ type_needle_v=self.type_needle_v[type],
+ key=self.generate_random(self.type_needle_k[type]),
+ value=self.generate_random(self.type_needle_v[type]),
+ )
+ for _ in range(num_haystack)
+ ]
+
+ indexes = sorted(
+ random.sample(range(num_haystack), len(needles)), reverse=True
+ )
+ for index, element in zip(indexes, needles):
+ sentences.insert(index, element)
+ context = "\n".join(sentences)
+
+ indices = random.sample(range(self.num_needle_k[type]), self.num_needle_q[type])
+ queries = [keys[i] for i in indices]
+ answers = [a for i in indices for a in values[i]]
+ query = (
+ ", ".join(queries[:-1]) + ", and " + queries[-1]
+ if len(queries) > 1
+ else queries[0]
+ )
+
+ type_needle_v = self.type_needle_v[type]
+ if self.num_needle_q[type] * self.num_needle_v[type] == 1:
+ template = template.replace("Some", "A")
+ template = template.replace("are all", "is")
+ template = template.replace("are", "is")
+ template = template.replace("answers", "answer")
+ type_needle_v = type_needle_v[:-1] # remove "s"
+
+ input_text = template.format(
+ type_needle_v=type_needle_v,
+ context=context,
+ query=query,
+ )
+
+ return input_text, answers
+
+ def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+ context_length = self.context_length_intervals[i]
+ type = self.type_haystack[i % len(self.type_haystack)]
+ used_haystack = self.num_haystack[type]
+ while True:
+ try:
+ input_text, answer = self.generate_input_output(
+ used_haystack, self.template, type
+ )
+ length = (
+ self.tokenizer.encode(input_text, return_tensors="pt").shape[-1]
+ + self.tokens_to_generate
+ )
+ assert length <= context_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_haystack > self.incremental[type]:
+ used_haystack -= self.incremental[type]
+
+ input_ids = self.tokenizer.encode(input_text + answer[0], return_tensors="pt")
+ length_context = self.tokenizer.encode(input_text, return_tensors="pt").shape[
+ -1
+ ]
+ labels = torch.tensor(
+ [-100] * length_context + [0] * (input_ids.shape[-1] - length_context)
+ )
+
+ return dict(
+ input_ids=input_ids[0], labels=labels, length_context=length_context
+ )
diff --git a/LeanK/leank/data/PaulGrahamEssays_URLs.txt b/LeanK/leank/data/PaulGrahamEssays_URLs.txt
new file mode 100644
index 00000000..b9a38e44
--- /dev/null
+++ b/LeanK/leank/data/PaulGrahamEssays_URLs.txt
@@ -0,0 +1,218 @@
+http://www.paulgraham.com/13sentences.html
+http://www.paulgraham.com/5founders.html
+http://www.paulgraham.com/6631327.html
+http://www.paulgraham.com/95.html
+http://www.paulgraham.com/ace.html
+http://www.paulgraham.com/airbnb.html
+http://www.paulgraham.com/airbnbs.html
+http://www.paulgraham.com/alien.html
+http://www.paulgraham.com/altair.html
+http://www.paulgraham.com/ambitious.html
+http://www.paulgraham.com/america.html
+http://www.paulgraham.com/angelinvesting.html
+http://www.paulgraham.com/artistsship.html
+http://www.paulgraham.com/badeconomy.html
+http://www.paulgraham.com/better.html
+http://www.paulgraham.com/bronze.html
+http://www.paulgraham.com/bubble.html
+http://www.paulgraham.com/charisma.html
+http://www.paulgraham.com/cities.html
+http://www.paulgraham.com/college.html
+http://www.paulgraham.com/colleges.html
+http://www.paulgraham.com/conformism.html
+http://www.paulgraham.com/control.html
+http://www.paulgraham.com/convergence.html
+http://www.paulgraham.com/convince.html
+http://www.paulgraham.com/cred.html
+http://www.paulgraham.com/credentials.html
+http://www.paulgraham.com/determination.html
+http://www.paulgraham.com/die.html
+http://www.paulgraham.com/disagree.html
+http://www.paulgraham.com/disc.html
+http://www.paulgraham.com/discover.html
+http://www.paulgraham.com/distraction.html
+http://www.paulgraham.com/divergence.html
+http://www.paulgraham.com/donate.html
+http://www.paulgraham.com/ds.html
+http://www.paulgraham.com/early.html
+http://www.paulgraham.com/earnest.html
+http://www.paulgraham.com/equity.html
+http://www.paulgraham.com/essay.html
+http://www.paulgraham.com/ffb.html
+http://www.paulgraham.com/fh.html
+http://www.paulgraham.com/fix.html
+http://www.paulgraham.com/fn.html
+http://www.paulgraham.com/foundersatwork.html
+http://www.paulgraham.com/fp.html
+http://www.paulgraham.com/fr.html
+http://www.paulgraham.com/fundraising.html
+http://www.paulgraham.com/future.html
+http://www.paulgraham.com/genius.html
+http://www.paulgraham.com/getideas.html
+http://www.paulgraham.com/good.html
+http://www.paulgraham.com/goodart.html
+http://www.paulgraham.com/googles.html
+http://www.paulgraham.com/greatwork.html
+http://www.paulgraham.com/growth.html
+http://www.paulgraham.com/guidetoinvestors.html
+http://www.paulgraham.com/hackernews.html
+http://www.paulgraham.com/head.html
+http://www.paulgraham.com/herd.html
+http://www.paulgraham.com/heresy.html
+http://www.paulgraham.com/heroes.html
+http://www.paulgraham.com/highres.html
+http://www.paulgraham.com/hiresfund.html
+http://www.paulgraham.com/hiring.html
+http://www.paulgraham.com/hp.html
+http://www.paulgraham.com/hs.html
+http://www.paulgraham.com/hundred.html
+http://www.paulgraham.com/hw.html
+http://www.paulgraham.com/hwh.html
+http://www.paulgraham.com/icad.html
+http://www.paulgraham.com/ideas.html
+http://www.paulgraham.com/identity.html
+http://www.paulgraham.com/ineq.html
+http://www.paulgraham.com/inequality.html
+http://www.paulgraham.com/investors.html
+http://www.paulgraham.com/invtrend.html
+http://www.paulgraham.com/javacover.html
+http://www.paulgraham.com/jessica.html
+http://www.paulgraham.com/judgement.html
+http://www.paulgraham.com/kate.html
+http://www.paulgraham.com/kids.html
+http://www.paulgraham.com/ladder.html
+http://www.paulgraham.com/lesson.html
+http://www.paulgraham.com/lies.html
+http://www.paulgraham.com/lwba.html
+http://www.paulgraham.com/mac.html
+http://www.paulgraham.com/makersschedule.html
+http://www.paulgraham.com/marginal.html
+http://www.paulgraham.com/maybe.html
+http://www.paulgraham.com/mean.html
+http://www.paulgraham.com/microsoft.html
+http://www.paulgraham.com/mit.html
+http://www.paulgraham.com/name.html
+http://www.paulgraham.com/nerds.html
+http://www.paulgraham.com/newthings.html
+http://www.paulgraham.com/noob.html
+http://www.paulgraham.com/noop.html
+http://www.paulgraham.com/notnot.html
+http://www.paulgraham.com/nov.html
+http://www.paulgraham.com/nthings.html
+http://www.paulgraham.com/opensource.html
+http://www.paulgraham.com/organic.html
+http://www.paulgraham.com/orth.html
+http://www.paulgraham.com/own.html
+http://www.paulgraham.com/patentpledge.html
+http://www.paulgraham.com/pgh.html
+http://www.paulgraham.com/pinch.html
+http://www.paulgraham.com/polls.html
+http://www.paulgraham.com/power.html
+http://www.paulgraham.com/prcmc.html
+http://www.paulgraham.com/procrastination.html
+http://www.paulgraham.com/progbot.html
+http://www.paulgraham.com/prop62.html
+http://www.paulgraham.com/property.html
+http://www.paulgraham.com/publishing.html
+http://www.paulgraham.com/pypar.html
+http://www.paulgraham.com/ramenprofitable.html
+http://www.paulgraham.com/randomness.html
+http://www.paulgraham.com/re.html
+http://www.paulgraham.com/read.html
+http://www.paulgraham.com/real.html
+http://www.paulgraham.com/really.html
+http://www.paulgraham.com/relres.html
+http://www.paulgraham.com/revolution.html
+http://www.paulgraham.com/richnow.html
+http://www.paulgraham.com/road.html
+http://www.paulgraham.com/ronco.html
+http://www.paulgraham.com/safe.html
+http://www.paulgraham.com/say.html
+http://www.paulgraham.com/schlep.html
+http://www.paulgraham.com/seesv.html
+http://www.paulgraham.com/segway.html
+http://www.paulgraham.com/selfindulgence.html
+http://www.paulgraham.com/sfp.html
+http://www.paulgraham.com/simply.html
+http://www.paulgraham.com/smart.html
+http://www.paulgraham.com/softwarepatents.html
+http://www.paulgraham.com/spam.html
+http://www.paulgraham.com/speak.html
+http://www.paulgraham.com/start.html
+http://www.paulgraham.com/startupfunding.html
+http://www.paulgraham.com/startuphubs.html
+http://www.paulgraham.com/startupideas.html
+http://www.paulgraham.com/startupmistakes.html
+http://www.paulgraham.com/stuff.html
+http://www.paulgraham.com/superlinear.html
+http://www.paulgraham.com/swan.html
+http://www.paulgraham.com/tablets.html
+http://www.paulgraham.com/talk.html
+http://www.paulgraham.com/taste.html
+http://www.paulgraham.com/think.html
+http://www.paulgraham.com/top.html
+http://www.paulgraham.com/trolls.html
+http://www.paulgraham.com/twitter.html
+http://www.paulgraham.com/usa.html
+http://www.paulgraham.com/users.html
+http://www.paulgraham.com/venturecapital.html
+http://www.paulgraham.com/wealth.html
+http://www.paulgraham.com/webstartups.html
+http://www.paulgraham.com/whyyc.html
+http://www.paulgraham.com/word.html
+http://www.paulgraham.com/words.html
+http://www.paulgraham.com/work.html
+http://www.paulgraham.com/writing44.html
+http://www.paulgraham.com/wtax.html
+http://www.paulgraham.com/yahoo.html
+http://www.paulgraham.com/ycombinator.html
+http://www.paulgraham.com/ycstart.html
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/addiction.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/aord.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/apple.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/avg.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/before.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/bias.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/boss.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/copy.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/corpdev.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/desres.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/diff.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/ecw.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/founders.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/foundervisa.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gap.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gba.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gh.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/goodtaste.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/hubs.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/iflisp.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/island.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/know.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/langdes.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/laundry.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/love.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/mod.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/newideas.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/nft.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/philosophy.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/popular.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/pow.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/rootsoflisp.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/rss.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/siliconvalley.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/startuplessons.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/submarine.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/sun.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/superangels.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/todo.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/unions.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/useful.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vb.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vcsqueeze.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vw.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/want.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/web20.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/weird.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/wisdom.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/worked.txt
diff --git a/LeanK/leank/data/download_paulgraham_essay.py b/LeanK/leank/data/download_paulgraham_essay.py
new file mode 100755
index 00000000..d23da508
--- /dev/null
+++ b/LeanK/leank/data/download_paulgraham_essay.py
@@ -0,0 +1,88 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import glob
+import json
+import os
+import shutil
+import urllib.request
+
+import html2text
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+
+temp_folder_repo = "essay_repo"
+temp_folder_html = "essay_html"
+os.makedirs(temp_folder_repo, exist_ok=True)
+os.makedirs(temp_folder_html, exist_ok=True)
+
+h = html2text.HTML2Text()
+h.ignore_images = True
+h.ignore_tables = True
+h.escape_all = True
+h.reference_links = False
+h.mark_code = False
+
+with open("PaulGrahamEssays_URLs.txt") as f:
+ urls = [line.strip() for line in f]
+
+for url in tqdm(urls):
+ if ".html" in url:
+ filename = url.split("/")[-1].replace(".html", ".txt")
+ try:
+ with urllib.request.urlopen(url) as website:
+ content = website.read().decode("unicode_escape", "utf-8")
+ soup = BeautifulSoup(content, "html.parser")
+ specific_tag = soup.find("font")
+ parsed = h.handle(str(specific_tag))
+
+ with open(os.path.join(temp_folder_html, filename), "w") as file:
+ file.write(parsed)
+
+ except Exception as e:
+ print(f"Fail download {filename}, ({e})")
+
+ else:
+ filename = url.split("/")[-1]
+ try:
+ with urllib.request.urlopen(url) as website:
+ content = website.read().decode("utf-8")
+
+ with open(os.path.join(temp_folder_repo, filename), "w") as file:
+ file.write(content)
+
+ except Exception as e:
+ print(f"Fail download {filename}, ({e})")
+
+files_repo = sorted(glob.glob(os.path.join(temp_folder_repo, "*.txt")))
+files_html = sorted(glob.glob(os.path.join(temp_folder_html, "*.txt")))
+print(
+ f"Download {len(files_repo)} essays from `https://github.com/gkamradt/LLMTest_NeedleInAHaystack/`"
+)
+print(f"Download {len(files_html)} essays from `http://www.paulgraham.com/`")
+
+text = ""
+for file in files_repo + files_html:
+ with open(file, "r") as f:
+ text += f.read()
+
+with open("PaulGrahamEssays.json", "w") as f:
+ json.dump({"text": text}, f)
+
+
+shutil.rmtree(temp_folder_repo)
+shutil.rmtree(temp_folder_html)
diff --git a/LeanK/leank/loss.py b/LeanK/leank/loss.py
new file mode 100644
index 00000000..5d72320d
--- /dev/null
+++ b/LeanK/leank/loss.py
@@ -0,0 +1,8 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+
+def l1_loss(x):
+ numel = x.numel()
+ l1 = x.abs().sum()
+ return l1 / numel
diff --git a/LeanK/leank/patch/__init__.py b/LeanK/leank/patch/__init__.py
new file mode 100644
index 00000000..600ebb7e
--- /dev/null
+++ b/LeanK/leank/patch/__init__.py
@@ -0,0 +1,92 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+
+import numpy as np
+import torch
+
+from .llama import (
+ enable_llama_training,
+ full_attn_forward_llama,
+ get_scaling_factors_llama,
+ map_scaling_factors_llama,
+ scaled_attn_forward_llama,
+ set_scaling_factors_llama,
+)
+from .qwen import (
+ enable_qwen_training,
+ full_attn_forward_qwen,
+ map_scaling_factors_qwen,
+ scaled_attn_forward_qwen,
+ set_scaling_factors_qwen,
+)
+
+
+def enable_training(
+ model,
+ sink_size,
+ recent_size,
+ initial_value=1.0,
+ enable_ulysses_attention=False,
+ scaling_factors=None,
+):
+ if "llama" in model.config.model_type:
+ enable_llama_training(
+ model,
+ sink_size,
+ recent_size,
+ initial_value=initial_value,
+ enable_ulysses_attention=enable_ulysses_attention,
+ scaling_factors=scaling_factors,
+ )
+ elif "qwen" in model.config.model_type:
+ enable_qwen_training(
+ model,
+ sink_size,
+ recent_size,
+ initial_value=initial_value,
+ enable_ulysses_attention=enable_ulysses_attention,
+ scaling_factors=scaling_factors,
+ )
+ else:
+ raise ValueError(f"Model type {model.config.model_type} not supported")
+
+
+def get_scaling_factors(model):
+ if "llama" in model.config.model_type:
+ return get_scaling_factors_llama(model)
+ elif "qwen" in model.config.model_type:
+ return set_scaling_factors_qwen(model)
+ else:
+ raise ValueError(f"Model type {model.config.model_type} not supported")
+
+
+def set_scaling_factors(model, scaling_factors):
+ if "llama" in model.config.model_type:
+ model = set_scaling_factors_llama(model, scaling_factors)
+ elif "qwen" in model.config.model_type:
+ model = set_scaling_factors_qwen(model, scaling_factors)
+ else:
+ raise ValueError(f"Model type {model.config.model_type} not supported")
+ return model
+
+
+def map_scaling_factors(model, func):
+ if "llama" in model.config.model_type:
+ return map_scaling_factors_llama(model, func)
+ elif "qwen" in model.config.model_type:
+ return map_scaling_factors_qwen(model, func)
+ else:
+ raise ValueError(f"Model type {model.config.model_type} not supported")
+
+
+def load_scaling_factors(load_dir, filename="scaling_factors.tsv"):
+ scaling_factors = np.loadtxt(
+ os.path.join(load_dir, filename),
+ dtype=float,
+ delimiter="\t",
+ )
+ scaling_factors = np.clip(scaling_factors, 0, 1)
+ scaling_factors = torch.tensor(scaling_factors, dtype=torch.float32)
+ return scaling_factors
diff --git a/LeanK/leank/patch/llama.py b/LeanK/leank/patch/llama.py
new file mode 100644
index 00000000..b6bfc119
--- /dev/null
+++ b/LeanK/leank/patch/llama.py
@@ -0,0 +1,360 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Part of the code is adapted from DuoAttention (https://github.com/mit-han-lab/duo-attention). We thank the authors for their work.
+
+import math
+import os
+from typing import Optional, Tuple
+
+import torch
+from torch import nn
+from transformers.cache_utils import Cache
+from transformers.modeling_flash_attention_utils import (
+ FlashAttentionKwargs,
+ _flash_attention_forward,
+)
+from transformers.models.llama.modeling_llama import (
+ LlamaForCausalLM,
+ LlamaModel,
+ apply_rotary_pos_emb,
+ repeat_kv,
+)
+from transformers.processing_utils import Unpack
+
+from ..ulysses import UlyssesAttention, UlyssesAttentionDecode
+from .tuple_kv_cache import enable_tuple_kv_cache_for_llama
+
+
+class BinaryMask(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, bar):
+ return torch.where(bar.to(x.device) == 1.0, x, 0)
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ return grad_output, None # Straight-through estimator
+
+
+def scaled_attn(
+ full_query_states: torch.Tensor,
+ full_key_states: torch.Tensor,
+ full_value_states: torch.Tensor,
+ scaling_factors: torch.Tensor,
+ length_context: int,
+ num_key_value_groups: int,
+ sink_size: int = 128,
+ recent_size: int = 1024,
+):
+ dtype = full_query_states.dtype
+ q_len = full_query_states.shape[1]
+ hdim = full_query_states.shape[-1]
+ prefill_query_states = full_query_states[:, :length_context, :, :]
+ prefill_key_states = full_key_states[:, :length_context, :, :]
+ prefill_value_states = full_value_states[:, :length_context, :, :]
+
+ with torch.no_grad():
+ prefill_attn_output = _flash_attention_forward(
+ prefill_query_states,
+ prefill_key_states,
+ prefill_value_states,
+ None,
+ q_len,
+ dropout=0.0,
+ is_causal=True,
+ )
+
+ decode_query_states = full_query_states[:, length_context:, :, :]
+
+ full_key_states_masked = full_key_states.clone()
+ full_key_states = full_key_states.transpose(1, 2).to(torch.float32)
+
+ full_key_states_masked = (
+ (full_key_states_masked * scaling_factors[:, :1, :, :])
+ .transpose(1, 2)
+ .to(torch.float32)
+ )
+
+ decode_query_states = decode_query_states.transpose(1, 2).to(torch.float32)
+ full_value_states = full_value_states.transpose(1, 2).to(torch.float32)
+
+ full_key_states = repeat_kv(full_key_states, num_key_value_groups)
+ full_key_states_masked = repeat_kv(full_key_states_masked, num_key_value_groups)
+ full_value_states = repeat_kv(full_value_states, num_key_value_groups)
+
+ attn_weights = torch.matmul(
+ decode_query_states, full_key_states.transpose(2, 3)
+ ) / math.sqrt(hdim)
+ attn_weights_masked = torch.matmul(
+ decode_query_states, full_key_states_masked.transpose(2, 3)
+ ) / math.sqrt(hdim)
+
+ query_len = q_len - length_context
+
+ causal_mask = (
+ torch.full(
+ (query_len, query_len), fill_value=torch.finfo(attn_weights.dtype).min
+ )
+ .to(attn_weights.dtype)
+ .to(attn_weights.dtype)
+ )
+ causal_mask = torch.triu(causal_mask, diagonal=1)[None, None, ...].to(
+ attn_weights.device
+ )
+
+ attn_weights[:, :, -query_len:, -query_len:] += causal_mask
+ attn_weights_masked[:, :, -query_len:, -query_len:] += causal_mask
+
+ full_mask = torch.zeros(query_len, q_len).to(attn_weights.device)
+ full_mask[:, -recent_size:] = 1
+ full_mask[:, -query_len - recent_size : -recent_size] = torch.triu(
+ torch.full((query_len, query_len), fill_value=1), diagonal=1
+ )
+ full_mask[:, :sink_size] = 1
+ full_mask = full_mask[None, None, ...]
+
+ attn_weights = attn_weights * full_mask + attn_weights_masked * (1 - full_mask)
+
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ decode_query_states.dtype
+ )
+ attn_output = torch.matmul(attn_weights, full_value_states)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ streaming_attn_output = torch.cat((prefill_attn_output, attn_output), dim=1)
+
+ return streaming_attn_output.to(dtype)
+
+
+def scaled_attn_forward_llama(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+ attention_mask: Optional[torch.Tensor],
+ length_context: Optional[int] = None,
+ past_key_value: Optional[Cache] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, _ = hidden_states.size()
+
+ scaling_factors = self.scaling_factors.clamp(0, 1).view(
+ 1, 1, self.num_key_value_heads, self.head_dim
+ )
+
+ if self.mask_round is not None:
+ scaling_factors = BinaryMask.apply(
+ scaling_factors,
+ torch.Tensor(self.mask_round).reshape(scaling_factors.shape),
+ )
+
+ full_query_states = self.q_proj(hidden_states)
+ full_key_states = self.k_proj(hidden_states)
+ full_value_states = self.v_proj(hidden_states)
+ full_query_states = full_query_states.view(
+ bsz, q_len, self.num_heads, self.head_dim
+ )
+ full_key_states = full_key_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ )
+ full_value_states = full_value_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ )
+
+ cos, sin = position_embeddings
+
+ full_query_states, full_key_states = apply_rotary_pos_emb(
+ full_query_states,
+ full_key_states,
+ cos,
+ sin,
+ unsqueeze_dim=2, # unsqueeze_dim=2 for the flash attention
+ )
+
+ decode_attn_output = self.scaled_attn_func(
+ full_query_states,
+ full_key_states,
+ full_value_states,
+ scaling_factors,
+ length_context=length_context,
+ num_key_value_groups=self.num_key_value_groups,
+ )
+
+ decode_attn_output = decode_attn_output.reshape(bsz, q_len, -1).contiguous()
+ decode_attn_output = self.o_proj(decode_attn_output)
+
+ return decode_attn_output, None
+
+
+def full_attn_forward_llama(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+ attention_mask: Optional[torch.Tensor],
+ length_context: Optional[int] = None,
+ past_key_value: Optional[Cache] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, _ = hidden_states.size()
+
+ with torch.no_grad():
+ input_shape = hidden_states.shape[:-1]
+ hidden_shape = (*input_shape, -1, self.head_dim)
+
+ full_query_states = self.q_proj(hidden_states)
+ full_key_states = self.k_proj(hidden_states)
+ full_value_states = self.v_proj(hidden_states)
+ full_query_states = full_query_states.view(hidden_shape)
+ full_key_states = full_key_states.view(hidden_shape)
+ full_value_states = full_value_states.view(hidden_shape)
+
+ cos, sin = position_embeddings
+
+ full_query_states, full_key_states = apply_rotary_pos_emb(
+ full_query_states,
+ full_key_states,
+ cos,
+ sin,
+ unsqueeze_dim=2, # unsqueeze_dim=2 for the flash attention
+ )
+
+ full_attn_output = self.full_attn_func(
+ full_query_states,
+ full_key_states,
+ full_value_states,
+ attention_mask,
+ q_len,
+ dropout=0.0,
+ is_causal=self.is_causal,
+ )
+ full_attn_output = full_attn_output.reshape(bsz, q_len, -1).contiguous()
+ full_attn_output = self.o_proj(full_attn_output)
+
+ return full_attn_output, None
+
+
+def enable_llama_training(
+ model: LlamaForCausalLM,
+ sink_size,
+ recent_size,
+ scaling_factors=None,
+ initial_value=1.0,
+ enable_ulysses_attention=False,
+):
+ enable_tuple_kv_cache_for_llama(model)
+ device = next(model.parameters()).device
+ dtype = next(model.parameters()).dtype
+
+ for idx, layer in enumerate(model.model.layers):
+ module = layer.self_attn
+ module.sink_size = sink_size
+ module.recent_size = recent_size
+ module.register_parameter(
+ "scaling_factors",
+ nn.Parameter(
+ (
+ scaling_factors[idx]
+ .reshape(model.config.num_key_value_heads, module.head_dim)
+ .to(device, dtype)
+ )
+ if scaling_factors is not None
+ else (
+ torch.ones(
+ (model.config.num_key_value_heads, module.head_dim),
+ device=device,
+ dtype=dtype,
+ requires_grad=True,
+ )
+ * initial_value
+ )
+ ),
+ )
+ module.num_heads = model.config.num_attention_heads
+ module.num_key_value_heads = model.config.num_key_value_heads
+ module.scaling_factors.requires_grad = True
+ module.sink_size = sink_size
+ module.recent_size = recent_size
+
+ if not enable_ulysses_attention:
+ module.scaled_attn_func = scaled_attn
+ module.full_attn_func = _flash_attention_forward
+ else:
+ module.scaled_attn_func = UlyssesAttentionDecode(
+ attn_func=scaled_attn,
+ )
+ module.full_attn_func = UlyssesAttention(
+ attn_func=_flash_attention_forward,
+ )
+
+
+def get_scaling_factors_llama(model):
+ scaling_factors = []
+ if isinstance(model, LlamaForCausalLM):
+ for layer in model.model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ scaling_factors.append(module.scaling_factors)
+ elif isinstance(model, LlamaModel):
+ for layer in model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ scaling_factors.append(module.scaling_factors)
+ else:
+ raise ValueError("Model type not supported")
+
+ return scaling_factors
+
+
+def set_scaling_factors_llama(model, scaling_factors):
+ print(scaling_factors)
+ print("set_scaling_factors_llama")
+ if isinstance(model, LlamaForCausalLM):
+ print("Resume from LlamaForCausalLM")
+ for layer_idx, layer in enumerate(model.model.layers):
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ module.scaling_factors.data = scaling_factors[layer_idx].to(
+ module.scaling_factors.device, module.scaling_factors.dtype
+ )
+ elif isinstance(model, LlamaModel):
+ print("Resume from LlamaModel")
+ for layer_idx, layer in enumerate(model.layers):
+ module = layer.self_attn
+ import torch.distributed as dist
+ from torch.distributed._tensor import DeviceMesh
+ from torch.distributed.tensor import DTensor
+
+ mesh = DeviceMesh(
+ device_type="cuda",
+ mesh=[i for i in range(int(os.environ["WORLD_SIZE"]))],
+ )
+
+ # Convert the tensor to a DTensor
+ dtensor = DTensor.from_local(
+ scaling_factors[layer_idx],
+ )
+
+ module.scaling_factors.data = dtensor.to(
+ module.scaling_factors.device, module.scaling_factors.dtype
+ )
+ else:
+ raise ValueError("Model type not supported")
+
+
+def map_scaling_factors_llama(model, func):
+ if isinstance(model, LlamaForCausalLM):
+ for layer in model.model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ func(module.scaling_factors)
+ elif isinstance(model, LlamaModel):
+ for layer in model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ func(module.scaling_factors)
+ else:
+ raise ValueError("Model type not supported")
diff --git a/LeanK/leank/patch/qwen.py b/LeanK/leank/patch/qwen.py
new file mode 100644
index 00000000..dac4d969
--- /dev/null
+++ b/LeanK/leank/patch/qwen.py
@@ -0,0 +1,341 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Part of the code is adapted from DuoAttention (https://github.com/mit-han-lab/duo-attention). We thank the authors for their work.
+
+import math
+import os
+import types
+from typing import Optional, Tuple
+
+import torch
+from torch import nn
+from transformers.cache_utils import Cache
+from transformers.modeling_flash_attention_utils import (
+ FlashAttentionKwargs,
+ _flash_attention_forward,
+)
+from transformers.models.qwen2.modeling_qwen2 import (
+ Qwen2ForCausalLM,
+ Qwen2Model,
+ apply_rotary_pos_emb,
+ repeat_kv,
+)
+from transformers.processing_utils import Unpack
+
+from ..ulysses import UlyssesAttention, UlyssesAttentionDecode
+from .tuple_kv_cache import enable_tuple_kv_cache_for_qwen
+
+
+class BinaryMask(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, bar):
+ return torch.where(bar.to(x.device) == 1.0, x, 0)
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ return grad_output, None # Straight-through estimator
+
+
+def scaled_attn(
+ full_query_states: torch.Tensor,
+ full_key_states: torch.Tensor,
+ full_value_states: torch.Tensor,
+ scaling_factors: torch.Tensor,
+ length_context: int,
+ num_key_value_groups: int,
+ sink_size: int = 128,
+ recent_size: int = 1024,
+):
+ dtype = full_query_states.dtype
+ q_len = full_query_states.shape[1]
+ hdim = full_query_states.shape[-1]
+ prefill_query_states = full_query_states[:, :length_context, :, :]
+ prefill_key_states = full_key_states[:, :length_context, :, :]
+ prefill_value_states = full_value_states[:, :length_context, :, :]
+
+ with torch.no_grad():
+ prefill_attn_output = _flash_attention_forward(
+ prefill_query_states,
+ prefill_key_states,
+ prefill_value_states,
+ None,
+ q_len,
+ dropout=0.0,
+ is_causal=True,
+ )
+
+ decode_query_states = full_query_states[:, length_context:, :, :]
+
+ full_key_states_masked = full_key_states.clone()
+ full_key_states = full_key_states.transpose(1, 2).to(torch.float32)
+
+ full_key_states_masked = (
+ (full_key_states_masked * scaling_factors[:, :1, :, :])
+ .transpose(1, 2)
+ .to(torch.float32)
+ )
+
+ decode_query_states = decode_query_states.transpose(1, 2).to(torch.float32)
+ full_value_states = full_value_states.transpose(1, 2).to(torch.float32)
+
+ full_key_states = repeat_kv(full_key_states, num_key_value_groups)
+ full_key_states_masked = repeat_kv(full_key_states_masked, num_key_value_groups)
+ full_value_states = repeat_kv(full_value_states, num_key_value_groups)
+
+ attn_weights = torch.matmul(
+ decode_query_states, full_key_states.transpose(2, 3)
+ ) / math.sqrt(hdim)
+ attn_weights_masked = torch.matmul(
+ decode_query_states, full_key_states_masked.transpose(2, 3)
+ ) / math.sqrt(hdim)
+
+ query_len = q_len - length_context
+
+ causal_mask = (
+ torch.full(
+ (query_len, query_len), fill_value=torch.finfo(attn_weights.dtype).min
+ )
+ .to(attn_weights.dtype)
+ .to(attn_weights.dtype)
+ )
+ causal_mask = torch.triu(causal_mask, diagonal=1)[None, None, ...].to(
+ attn_weights.device
+ )
+
+ attn_weights[:, :, -query_len:, -query_len:] += causal_mask
+ attn_weights_masked[:, :, -query_len:, -query_len:] += causal_mask
+
+ full_mask = torch.zeros(query_len, q_len).to(attn_weights.device)
+ full_mask[:, -recent_size:] = 1
+ full_mask[:, -query_len - recent_size : -recent_size] = torch.triu(
+ torch.full((query_len, query_len), fill_value=1), diagonal=1
+ )
+ full_mask[:, :sink_size] = 1
+ full_mask = full_mask[None, None, ...]
+
+ attn_weights = attn_weights * full_mask + attn_weights_masked * (1 - full_mask)
+
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ decode_query_states.dtype
+ )
+ attn_output = torch.matmul(attn_weights, full_value_states)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ streaming_attn_output = torch.cat((prefill_attn_output, attn_output), dim=1)
+
+ return streaming_attn_output.to(dtype)
+
+
+def scaled_attn_forward_qwen(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+ attention_mask: Optional[torch.Tensor],
+ length_context: Optional[int] = None,
+ past_key_value: Optional[Cache] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, _ = hidden_states.size()
+
+ scaling_factors = self.scaling_factors.clamp(0, 1).view(
+ 1, 1, self.num_key_value_heads, self.head_dim
+ )
+ if self.mask_round is not None:
+ scaling_factors = BinaryMask.apply(
+ scaling_factors,
+ torch.Tensor(self.mask_round).reshape(scaling_factors.shape),
+ )
+
+ full_query_states = self.q_proj(hidden_states)
+ full_key_states = self.k_proj(hidden_states)
+ full_value_states = self.v_proj(hidden_states)
+ full_query_states = full_query_states.view(
+ bsz, q_len, self.num_heads, self.head_dim
+ )
+ full_key_states = full_key_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ )
+ full_value_states = full_value_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ )
+
+ cos, sin = position_embeddings
+
+ full_query_states, full_key_states = apply_rotary_pos_emb(
+ full_query_states,
+ full_key_states,
+ cos,
+ sin,
+ unsqueeze_dim=2, # unsqueeze_dim=2 for the flash attention
+ )
+
+ decode_attn_output = self.scaled_attn_func(
+ full_query_states,
+ full_key_states,
+ full_value_states,
+ scaling_factors,
+ length_context=length_context,
+ num_key_value_groups=self.num_key_value_groups,
+ sink_size=self.sink_size,
+ recent_size=self.recent_size,
+ )
+
+ decode_attn_output = decode_attn_output.reshape(bsz, q_len, -1).contiguous()
+ decode_attn_output = self.o_proj(decode_attn_output)
+
+ return decode_attn_output, None
+
+
+def full_attn_forward_qwen(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+ attention_mask: Optional[torch.Tensor],
+ length_context: Optional[int] = None,
+ past_key_value: Optional[Cache] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, _ = hidden_states.size()
+
+ with torch.no_grad():
+ input_shape = hidden_states.shape[:-1]
+ hidden_shape = (*input_shape, -1, self.head_dim)
+
+ full_query_states = self.q_proj(hidden_states)
+ full_key_states = self.k_proj(hidden_states)
+ full_value_states = self.v_proj(hidden_states)
+ full_query_states = full_query_states.view(hidden_shape)
+ full_key_states = full_key_states.view(hidden_shape)
+ full_value_states = full_value_states.view(hidden_shape)
+
+ cos, sin = position_embeddings
+
+ full_query_states, full_key_states = apply_rotary_pos_emb(
+ full_query_states,
+ full_key_states,
+ cos,
+ sin,
+ unsqueeze_dim=2, # unsqueeze_dim=2 for the flash attention
+ )
+
+ full_attn_output = self.full_attn_func(
+ full_query_states,
+ full_key_states,
+ full_value_states,
+ attention_mask,
+ q_len,
+ dropout=0.0,
+ is_causal=self.is_causal,
+ )
+ full_attn_output = full_attn_output.reshape(bsz, q_len, -1).contiguous()
+ full_attn_output = self.o_proj(full_attn_output)
+
+ return full_attn_output, None
+
+
+def enable_qwen_training(
+ model: Qwen2ForCausalLM,
+ sink_size,
+ recent_size,
+ scaling_factors=None,
+ initial_value=1.0,
+ enable_ulysses_attention=False,
+):
+ enable_tuple_kv_cache_for_qwen(model)
+ device = next(model.parameters()).device
+ dtype = next(model.parameters()).dtype
+
+ for idx, layer in enumerate(model.model.layers):
+ module = layer.self_attn
+ module.sink_size = sink_size
+ module.recent_size = recent_size
+ module.register_parameter(
+ "scaling_factors",
+ nn.Parameter(
+ (scaling_factors[idx].to(device, dtype))
+ if scaling_factors is not None
+ else (
+ torch.ones(
+ model.config.num_key_value_heads * module.head_dim,
+ device=device,
+ dtype=dtype,
+ requires_grad=True,
+ )
+ * initial_value
+ )
+ ),
+ )
+ module.scaling_factors.requires_grad = True
+ module.num_heads = model.config.num_attention_heads
+ module.num_key_value_heads = model.config.num_key_value_heads
+ module.sink_size = sink_size
+ module.recent_size = recent_size
+
+ if not enable_ulysses_attention:
+ module.scaled_attn_func = scaled_attn
+ module.full_attn_func = _flash_attention_forward
+ else:
+ module.scaled_attn_func = UlyssesAttentionDecode(
+ attn_func=scaled_attn,
+ )
+ module.full_attn_func = UlyssesAttention(
+ attn_func=_flash_attention_forward,
+ )
+
+
+def set_scaling_factors_qwen(model):
+ scaling_factors = []
+ if isinstance(model, Qwen2ForCausalLM):
+ for layer in model.model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ scaling_factors.append(module.scaling_factors)
+ elif isinstance(model, Qwen2Model):
+ for layer in model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ scaling_factors.append(module.scaling_factors)
+ else:
+ raise ValueError("Model type not supported")
+
+ return scaling_factors
+
+
+def set_scaling_factors_qwen(model):
+ scaling_factors = []
+ if isinstance(model, Qwen2ForCausalLM):
+ for layer in model.model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ scaling_factors.append(module.scaling_factors)
+ elif isinstance(model, Qwen2Model):
+ for layer in model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ scaling_factors.append(module.scaling_factors)
+ else:
+ raise ValueError("Model type not supported")
+
+ return scaling_factors
+
+
+def map_scaling_factors_qwen(model, func):
+ if isinstance(model, Qwen2ForCausalLM):
+ for layer in model.model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ func(module.scaling_factors)
+ elif isinstance(model, Qwen2Model):
+ for layer in model.layers:
+ module = layer.self_attn
+ if not hasattr(module, "scaling_factors"):
+ continue
+ func(module.scaling_factors)
+ else:
+ raise ValueError("Model type not supported")
diff --git a/LeanK/leank/patch/tuple_kv_cache.py b/LeanK/leank/patch/tuple_kv_cache.py
new file mode 100644
index 00000000..11dfed05
--- /dev/null
+++ b/LeanK/leank/patch/tuple_kv_cache.py
@@ -0,0 +1,618 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Part of the code is adapted from DuoAttention (https://github.com/mit-han-lab/duo-attention). We thank the authors for their work.
+
+import types
+from typing import Optional, Tuple
+
+import torch
+import torch.functional as F
+from flash_attn import flash_attn_func, flash_attn_varlen_func
+from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.models.llama.modeling_llama import (
+ BaseModelOutputWithPast,
+ List,
+ LlamaForCausalLM,
+ Union,
+ apply_rotary_pos_emb,
+)
+from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM
+from transformers.processing_utils import Unpack
+
+
+def _get_unpad_data(padding_mask):
+ seqlens_in_batch = padding_mask.sum(dim=-1, dtype=torch.int32)
+ indices = torch.nonzero(padding_mask.flatten(), as_tuple=False).flatten()
+ max_seqlen_in_batch = seqlens_in_batch.max().item()
+ cu_seqlens = F.pad(
+ torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0)
+ )
+ return (
+ indices,
+ cu_seqlens,
+ max_seqlen_in_batch,
+ )
+
+
+def old_flash_attention_2_forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ padding_mask: Optional[torch.LongTensor] = None,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ # LlamaFlashAttention2 attention does not support output_attentions
+ output_attentions = False
+
+ bsz, q_len, _ = hidden_states.size()
+
+ query_states = self.q_proj(hidden_states)
+ key_states = self.k_proj(hidden_states)
+ value_states = self.v_proj(hidden_states)
+
+ # Flash attention requires the input to have the shape
+ # batch_size x seq_length x head_dime x hidden_dim
+ # therefore we just need to keep the original shape
+ query_states = query_states.view(
+ bsz, q_len, self.num_heads, self.head_dim
+ ).transpose(1, 2)
+ key_states = key_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+ value_states = value_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+
+ kv_seq_len = key_states.shape[-2]
+ if past_key_value is not None:
+ kv_seq_len += past_key_value[0].shape[-2]
+
+ cos, sin = self.rotary_emb(value_states, position_ids)
+
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+ if past_key_value is not None:
+ # reuse k, v, self_attention
+ key_states = torch.cat([past_key_value[0], key_states], dim=2)
+ value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+ past_key_value = (key_states, value_states) if use_cache else None
+
+ query_states = query_states.transpose(1, 2)
+ key_states = key_states.transpose(1, 2)
+ value_states = value_states.transpose(1, 2)
+
+ # TODO: llama does not have dropout in the config??
+ # It is recommended to use dropout with FA according to the docs
+ # when training.
+ dropout_rate = 0.0 # if not self.training else self.attn_dropout
+
+ # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+ # therefore the input hidden states gets silently casted in float32. Hence, we need
+ # cast them back in float16 just to be sure everything works as expected.
+ # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+ # in fp32. (LlamaRMSNorm handles it correctly)
+ input_dtype = query_states.dtype
+ if input_dtype == torch.float32:
+ query_states = query_states.to(torch.float16)
+ key_states = key_states.to(torch.float16)
+ value_states = value_states.to(torch.float16)
+
+ attn_output = self._flash_attention_forward(
+ query_states,
+ key_states,
+ value_states,
+ padding_mask,
+ q_len,
+ dropout=dropout_rate,
+ )
+
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+ attn_output = self.o_proj(attn_output)
+
+ if not output_attentions:
+ attn_weights = None
+
+ return attn_output, attn_weights, past_key_value
+
+
+def _flash_attention_forward(
+ self,
+ query_states,
+ key_states,
+ value_states,
+ padding_mask,
+ query_length,
+ dropout=0.0,
+ softmax_scale=None,
+):
+ """
+ Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+ first unpad the input, then computes the attention scores and pad the final attention scores.
+
+ Args:
+ query_states (`torch.Tensor`):
+ Input query states to be passed to Flash Attention API
+ key_states (`torch.Tensor`):
+ Input key states to be passed to Flash Attention API
+ value_states (`torch.Tensor`):
+ Input value states to be passed to Flash Attention API
+ padding_mask (`torch.Tensor`):
+ The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+ position of padding tokens and 1 for the position of non-padding tokens.
+ dropout (`int`, *optional*):
+ Attention dropout
+ softmax_scale (`float`, *optional*):
+ The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+ """
+ # Contains at least one padding token in the sequence
+ if padding_mask is not None:
+ batch_size = query_states.shape[0]
+ (
+ query_states,
+ key_states,
+ value_states,
+ indices_q,
+ cu_seq_lens,
+ max_seq_lens,
+ ) = self._upad_input(
+ query_states, key_states, value_states, padding_mask, query_length
+ )
+
+ cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+ max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+ attn_output_unpad = flash_attn_varlen_func(
+ query_states,
+ key_states,
+ value_states,
+ cu_seqlens_q=cu_seqlens_q,
+ cu_seqlens_k=cu_seqlens_k,
+ max_seqlen_q=max_seqlen_in_batch_q,
+ max_seqlen_k=max_seqlen_in_batch_k,
+ dropout_p=dropout,
+ softmax_scale=softmax_scale,
+ causal=True,
+ )
+
+ attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+ else:
+ attn_output = flash_attn_func(
+ query_states,
+ key_states,
+ value_states,
+ dropout,
+ softmax_scale=softmax_scale,
+ causal=True,
+ )
+
+ return attn_output
+
+
+def _upad_input(self, query_layer, key_layer, value_layer, padding_mask, query_length):
+ indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(padding_mask)
+ batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+ key_layer = index_first_axis(
+ key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+ indices_k,
+ )
+ value_layer = index_first_axis(
+ value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+ indices_k,
+ )
+ if query_length == kv_seq_len:
+ query_layer = index_first_axis(
+ query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim),
+ indices_k,
+ )
+ cu_seqlens_q = cu_seqlens_k
+ max_seqlen_in_batch_q = max_seqlen_in_batch_k
+ indices_q = indices_k
+ elif query_length == 1:
+ max_seqlen_in_batch_q = 1
+ cu_seqlens_q = torch.arange(
+ batch_size + 1, dtype=torch.int32, device=query_layer.device
+ ) # There is a memcpy here, that is very bad.
+ indices_q = cu_seqlens_q[:-1]
+ query_layer = query_layer.squeeze(1)
+ else:
+ # The -q_len: slice assumes left padding.
+ padding_mask = padding_mask[:, -query_length:]
+ query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(
+ query_layer, padding_mask
+ )
+
+ return (
+ query_layer,
+ key_layer,
+ value_layer,
+ indices_q,
+ (cu_seqlens_q, cu_seqlens_k),
+ (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+ )
+
+
+def old_llama_model_forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Cache] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ length_context: Optional[int] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **flash_attn_kwargs: Unpack[FlashAttentionKwargs],
+) -> Union[Tuple, BaseModelOutputWithPast]:
+ output_attentions = (
+ output_attentions
+ if output_attentions is not None
+ else self.config.output_attentions
+ )
+ output_hidden_states = (
+ output_hidden_states
+ if output_hidden_states is not None
+ else self.config.output_hidden_states
+ )
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ # retrieve input_ids and inputs_embeds
+ if input_ids is not None and inputs_embeds is not None:
+ raise ValueError(
+ "You cannot specify both input_ids and inputs_embeds at the same time"
+ )
+
+ if inputs_embeds is None:
+ inputs_embeds = self.embed_tokens(input_ids)
+
+ if use_cache and past_key_values is None:
+ past_key_values = DynamicCache()
+
+ if cache_position is None:
+ past_seen_tokens = (
+ past_key_values.get_seq_length() if past_key_values is not None else 0
+ )
+ cache_position = torch.arange(
+ past_seen_tokens,
+ past_seen_tokens + inputs_embeds.shape[1],
+ device=inputs_embeds.device,
+ )
+
+ if position_ids is None:
+ position_ids = cache_position.unsqueeze(0)
+
+ causal_mask = self._update_causal_mask(
+ attention_mask,
+ inputs_embeds,
+ cache_position,
+ past_key_values,
+ output_attentions,
+ )
+
+ hidden_states = inputs_embeds
+
+ # create position embeddings to be shared across the decoder layers
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+ # decoder layers
+ all_hidden_states = () if output_hidden_states else None
+ all_self_attns = () if output_attentions else None
+
+ for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ if self.gradient_checkpointing and self.training:
+ layer_outputs = self._gradient_checkpointing_func(
+ decoder_layer.__call__,
+ hidden_states,
+ causal_mask,
+ position_ids,
+ past_key_values,
+ output_attentions,
+ use_cache,
+ cache_position,
+ position_embeddings,
+ )
+ else:
+ layer_outputs = decoder_layer(
+ hidden_states,
+ attention_mask=causal_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_values,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ length_context=length_context,
+ **flash_attn_kwargs,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_self_attns += (layer_outputs[1],)
+
+ hidden_states = self.norm(hidden_states)
+
+ # add hidden states from the last decoder layer
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ output = BaseModelOutputWithPast(
+ last_hidden_state=hidden_states,
+ past_key_values=past_key_values if use_cache else None,
+ hidden_states=all_hidden_states,
+ attentions=all_self_attns,
+ )
+
+ return output if return_dict else output.to_tuple()
+
+
+def old_llama_decoder_layer_forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ output_attentions: Optional[bool] = False,
+ use_cache: Optional[bool] = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[
+ Tuple[torch.Tensor, torch.Tensor]
+ ] = None, # necessary, but kept here for BC
+ length_context: Optional[int] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+ residual = hidden_states
+
+ hidden_states = self.input_layernorm(hidden_states)
+
+ # Self Attention
+ hidden_states, self_attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ length_context=length_context,
+ **kwargs,
+ )
+ hidden_states = residual + hidden_states
+
+ # Fully Connected
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (self_attn_weights,)
+
+ return outputs
+
+
+def enable_tuple_kv_cache_for_llama(model: LlamaForCausalLM):
+ model.model._prepare_decoder_attention_mask = lambda *args, **kwargs: None
+ model.model.forward = types.MethodType(old_llama_model_forward, model.model)
+ for idx in range(len(model.model.layers)):
+ model.model.layers[idx].forward = types.MethodType(
+ old_llama_decoder_layer_forward, model.model.layers[idx]
+ )
+ model.model.layers[idx].self_attn.forward = types.MethodType(
+ old_flash_attention_2_forward, model.model.layers[idx].self_attn
+ )
+ model.model.layers[idx].self_attn._upad_input = types.MethodType(
+ _upad_input, model.model.layers[idx].self_attn
+ )
+ model.model.layers[idx].self_attn._flash_attention_forward = types.MethodType(
+ _flash_attention_forward, model.model.layers[idx].self_attn
+ )
+
+
+def old_qwen_model_forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ length_context: Optional[int] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **flash_attn_kwargs: Unpack[FlashAttentionKwargs],
+) -> Union[Tuple, BaseModelOutputWithPast]:
+ output_attentions = (
+ output_attentions
+ if output_attentions is not None
+ else self.config.output_attentions
+ )
+ output_hidden_states = (
+ output_hidden_states
+ if output_hidden_states is not None
+ else self.config.output_hidden_states
+ )
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ # retrieve input_ids and inputs_embeds
+ if input_ids is not None and inputs_embeds is not None:
+ raise ValueError(
+ "You cannot specify both input_ids and inputs_embeds at the same time"
+ )
+
+ if inputs_embeds is None:
+ inputs_embeds = self.embed_tokens(input_ids)
+
+ if use_cache and past_key_values is None:
+ past_key_values = DynamicCache()
+
+ if cache_position is None:
+ past_seen_tokens = (
+ past_key_values.get_seq_length() if past_key_values is not None else 0
+ )
+ cache_position = torch.arange(
+ past_seen_tokens,
+ past_seen_tokens + inputs_embeds.shape[1],
+ device=inputs_embeds.device,
+ )
+
+ if position_ids is None:
+ position_ids = cache_position.unsqueeze(0)
+
+ causal_mask = self._update_causal_mask(
+ attention_mask,
+ inputs_embeds,
+ cache_position,
+ past_key_values,
+ output_attentions,
+ )
+
+ hidden_states = inputs_embeds
+
+ # create position embeddings to be shared across the decoder layers
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+ # decoder layers
+ all_hidden_states = () if output_hidden_states else None
+ all_self_attns = () if output_attentions else None
+
+ for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ if self.gradient_checkpointing and self.training:
+ layer_outputs = self._gradient_checkpointing_func(
+ decoder_layer.__call__,
+ hidden_states,
+ causal_mask,
+ position_ids,
+ past_key_values,
+ output_attentions,
+ use_cache,
+ cache_position,
+ position_embeddings,
+ )
+ else:
+ layer_outputs = decoder_layer(
+ hidden_states,
+ attention_mask=causal_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_values,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ length_context=length_context,
+ **flash_attn_kwargs,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_self_attns += (layer_outputs[1],)
+
+ hidden_states = self.norm(hidden_states)
+
+ # add hidden states from the last decoder layer
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ output = BaseModelOutputWithPast(
+ last_hidden_state=hidden_states,
+ past_key_values=past_key_values if use_cache else None,
+ hidden_states=all_hidden_states,
+ attentions=all_self_attns,
+ )
+
+ return output if return_dict else output.to_tuple()
+
+
+def old_qwen_decoder_layer_forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ output_attentions: Optional[bool] = False,
+ use_cache: Optional[bool] = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[
+ Tuple[torch.Tensor, torch.Tensor]
+ ] = None, # necessary, but kept here for BC
+ length_context: Optional[int] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+ residual = hidden_states
+
+ hidden_states = self.input_layernorm(hidden_states)
+
+ # Self Attention
+ hidden_states, self_attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ length_context=length_context,
+ **kwargs,
+ )
+ hidden_states = residual + hidden_states
+
+ # Fully Connected
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (self_attn_weights,)
+
+ return outputs
+
+
+def enable_tuple_kv_cache_for_qwen(model: Qwen2ForCausalLM):
+ model.model._prepare_decoder_attention_mask = lambda *args, **kwargs: None
+ model.model.forward = types.MethodType(old_qwen_model_forward, model.model)
+ for idx in range(len(model.model.layers)):
+ model.model.layers[idx].forward = types.MethodType(
+ old_qwen_decoder_layer_forward, model.model.layers[idx]
+ )
+ model.model.layers[idx].self_attn.forward = types.MethodType(
+ old_flash_attention_2_forward, model.model.layers[idx].self_attn
+ )
+ model.model.layers[idx].self_attn._upad_input = types.MethodType(
+ _upad_input, model.model.layers[idx].self_attn
+ )
+ model.model.layers[idx].self_attn._flash_attention_forward = types.MethodType(
+ _flash_attention_forward, model.model.layers[idx].self_attn
+ )
diff --git a/LeanK/leank/ulysses/__init__.py b/LeanK/leank/ulysses/__init__.py
new file mode 100644
index 00000000..d3c6df92
--- /dev/null
+++ b/LeanK/leank/ulysses/__init__.py
@@ -0,0 +1,4 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from .attn_layer import UlyssesAttention, UlyssesAttentionDecode
diff --git a/LeanK/leank/ulysses/attn_layer.py b/LeanK/leank/ulysses/attn_layer.py
new file mode 100644
index 00000000..d447996b
--- /dev/null
+++ b/LeanK/leank/ulysses/attn_layer.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+from typing import Any
+
+import torch
+import torch.distributed as dist
+from torch import Tensor
+
+from .utils import SeqAllToAll4D
+
+
+class UlyssesAttention(torch.nn.Module):
+ """Initialization.
+
+ Arguments:
+ local_attention (Module): local attention with q,k,v
+ sequence_process_group (ProcessGroup): sequence parallel process group
+ scatter_idx (int): scatter_idx for all2all comm
+ gather_idx (int): gather_idx for all2all comm
+ """
+
+ def __init__(
+ self,
+ attn_func,
+ sequence_process_group: dist.ProcessGroup = None,
+ scatter_idx: int = 2,
+ gather_idx: int = 1,
+ ) -> None:
+ super(UlyssesAttention, self).__init__()
+ self.spg = sequence_process_group
+ self.scatter_idx = scatter_idx
+ self.gather_idx = gather_idx
+ self.attn_func = attn_func
+
+ def forward(
+ self,
+ query: Tensor,
+ key: Tensor,
+ value: Tensor,
+ *args: Any,
+ **kwargs: Any,
+ ) -> Tensor:
+ """forward
+
+ Arguments:
+ query (Tensor): query input to the layer
+ key (Tensor): key input to the layer
+ value (Tensor): value input to the layer
+ args: other args
+
+ Returns:
+ * output (Tensor): context output
+ """
+ q = SeqAllToAll4D.apply(self.spg, query, self.scatter_idx, self.gather_idx)
+ k = SeqAllToAll4D.apply(self.spg, key, self.scatter_idx, self.gather_idx)
+ v = SeqAllToAll4D.apply(self.spg, value, self.scatter_idx, self.gather_idx)
+
+ context_layer = self.attn_func(
+ q,
+ k,
+ v,
+ *args,
+ **kwargs,
+ )
+
+ # (bs, seq_len, head_cnt/N, head_size) -> (bs, seq_len/N, head_cnt, head_size)
+ # scatter 1, gather 2
+ output = SeqAllToAll4D.apply(
+ self.spg, context_layer, self.gather_idx, self.scatter_idx
+ )
+
+ # out e.g., [s/p::h]
+ return output
+
+
+class UlyssesAttentionDecode(torch.nn.Module):
+ """Initialization.
+
+ Arguments:
+ local_attention (Module): local attention with q,k,v
+ sequence_process_group (ProcessGroup): sequence parallel process group
+ scatter_idx (int): scatter_idx for all2all comm
+ gather_idx (int): gather_idx for all2all comm
+ """
+
+ def __init__(
+ self,
+ attn_func,
+ sequence_process_group: dist.ProcessGroup = None,
+ scatter_idx: int = 2,
+ gather_idx: int = 1,
+ ) -> None:
+ super(UlyssesAttentionDecode, self).__init__()
+ self.spg = sequence_process_group
+ self.scatter_idx = scatter_idx
+ self.gather_idx = gather_idx
+ self.attn_func = attn_func
+
+ def forward(
+ self,
+ query: Tensor,
+ key: Tensor,
+ value: Tensor,
+ key2: Tensor,
+ *args: Any,
+ **kwargs: Any,
+ ) -> Tensor:
+ """forward
+
+ Arguments:
+ query (Tensor): query input to the layer
+ key (Tensor): key input to the layer
+ value (Tensor): value input to the layer
+ args: other args
+
+ Returns:
+ * output (Tensor): context output
+ """
+
+ # import IPython; IPython.embed()
+
+ q = SeqAllToAll4D.apply(self.spg, query, self.scatter_idx, self.gather_idx)
+ k = SeqAllToAll4D.apply(self.spg, key, self.scatter_idx, self.gather_idx)
+ v = SeqAllToAll4D.apply(self.spg, value, self.scatter_idx, self.gather_idx)
+ k2 = SeqAllToAll4D.apply(self.spg, key2, self.scatter_idx, self.gather_idx)
+
+ # import IPython; IPython.embed()
+
+ context_layer = self.attn_func(
+ q,
+ k,
+ v,
+ k2,
+ *args,
+ **kwargs,
+ )
+
+ # (bs, seq_len, head_cnt/N, head_size) -> (bs, seq_len/N, head_cnt, head_size)
+ # scatter 1, gather 2
+ output = SeqAllToAll4D.apply(
+ self.spg, context_layer, self.gather_idx, self.scatter_idx
+ )
+
+ # out e.g., [s/p::h]
+ return output
diff --git a/LeanK/leank/ulysses/utils.py b/LeanK/leank/ulysses/utils.py
new file mode 100644
index 00000000..9e3d16f0
--- /dev/null
+++ b/LeanK/leank/ulysses/utils.py
@@ -0,0 +1,231 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+from typing import Any, Tuple
+
+import torch
+import torch.distributed as dist
+from torch import Tensor
+
+
+def all_to_all_4D(
+ input: torch.tensor, scatter_idx: int = 2, gather_idx: int = 1, group=None
+) -> torch.tensor:
+ """
+ all-to-all for QKV
+
+ Args:
+ input (torch.tensor): a tensor sharded along dim scatter dim
+ scatter_idx (int): default 1
+ gather_idx (int): default 2
+ group : torch process group
+
+ Returns:
+ torch.tensor: resharded tensor (bs, seqlen/P, hc, hs)
+ """
+ assert (
+ input.dim() == 4
+ ), f"input must be 4D tensor, got {input.dim()} and shape {input.shape}"
+
+ seq_world_size = dist.get_world_size(group)
+
+ if scatter_idx == 2 and gather_idx == 1:
+ # input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen/P, hc, hs) output: (bs, seqlen, hc/P, hs)
+ # P: number of workers, hc: head count, hs: head size
+ bs, shard_seqlen, hc, hs = input.shape
+ seqlen = shard_seqlen * seq_world_size
+ shard_hc = hc // seq_world_size
+
+ # transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
+ # (bs, seqlen/P, hc, hs) -reshape-> (bs, seq_len/P, P, hc/P, hs) -transpose(0,2)-> (P, seq_len/P, bs, hc/P, hs)
+ input_t = (
+ input.reshape(bs, shard_seqlen, seq_world_size, shard_hc, hs)
+ .transpose(0, 2)
+ .contiguous()
+ )
+
+ output = torch.empty_like(input_t)
+ # https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
+ # (P, seq_len/P, bs, hc/P, hs) scatter seqlen -all2all-> (P, seq_len/P, bs, hc/P, hs) scatter head
+ dist.all_to_all_single(output, input_t, group=group)
+
+ # if scattering the seq-dim, transpose the heads back to the original dimension
+ output = output.reshape(seqlen, bs, shard_hc, hs)
+
+ # (seq_len, bs, hc/P, hs) -reshape-> (bs, seq_len, hc/P, hs)
+ output = output.transpose(0, 1).contiguous()
+
+ return output.reshape(bs, seqlen, shard_hc, hs).contiguous()
+ elif scatter_idx == 1 and gather_idx == 2:
+ # input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen, hc/P, hs) output: (bs, seqlen/P, hc, hs)
+ bs, seqlen, shard_hc, hs = input.shape
+ hc = shard_hc * seq_world_size
+ shard_seqlen = seqlen // seq_world_size
+ seq_world_size = dist.get_world_size(group)
+
+ # transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
+ # (bs, seqlen, hc/P, hs) -reshape-> (bs, P, seq_len/P, hc/P, hs) -transpose(0, 3)-> (hc/P, P, seqlen/P, bs, hs) -transpose(0, 1) -> (P, hc/P, seqlen/P, bs, hs)
+ input_t = (
+ input.reshape(bs, seq_world_size, shard_seqlen, shard_hc, hs)
+ .transpose(0, 3)
+ .transpose(0, 1)
+ .contiguous()
+ .reshape(seq_world_size, shard_hc, shard_seqlen, bs, hs)
+ )
+
+ output = torch.empty_like(input_t)
+ # https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
+ # (P, bs x hc/P, seqlen/P, hs) scatter seqlen -all2all-> (P, bs x seq_len/P, hc/P, hs) scatter head
+ dist.all_to_all_single(output, input_t, group=group)
+
+ # if scattering the seq-dim, transpose the heads back to the original dimension
+ output = output.reshape(hc, shard_seqlen, bs, hs)
+
+ # (hc, seqlen/N, bs, hs) -tranpose(0,2)-> (bs, seqlen/N, hc, hs)
+ output = output.transpose(0, 2).contiguous()
+
+ return output.reshape(bs, shard_seqlen, hc, hs).contiguous()
+ else:
+ raise RuntimeError("scatter_idx must be 1 or 2 and gather_idx must be 1 or 2")
+
+
+class SeqAllToAll4D(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx: Any,
+ group: dist.ProcessGroup,
+ input: Tensor,
+ scatter_idx: int,
+ gather_idx: int,
+ ) -> Tensor:
+ ctx.group = group
+ ctx.scatter_idx = scatter_idx
+ ctx.gather_idx = gather_idx
+
+ return all_to_all_4D(input, scatter_idx, gather_idx, group=group)
+
+ @staticmethod
+ def backward(ctx: Any, *grad_output: Tensor) -> Tuple[None, Tensor, None, None]:
+ return (
+ None,
+ SeqAllToAll4D.apply(
+ ctx.group, *grad_output, ctx.gather_idx, ctx.scatter_idx
+ ),
+ None,
+ None,
+ )
+
+
+def all_to_all_5D(
+ input: torch.tensor, scatter_idx: int = 3, gather_idx: int = 1, group=None
+) -> torch.tensor:
+ """
+ all-to-all for QKV
+ forward (bs, seqlen/N, 3, hc, hs) -> (bs, seqlen, 3, hc/N, hs)
+
+ Args:
+ input (torch.tensor): a tensor sharded along dim scatter dim
+ scatter_idx (int): default 1
+ gather_idx (int): default 2
+ group : torch process group
+
+ Returns:
+ torch.tensor: resharded tensor (bs, seqlen/P, 3, hc, hs)
+ """
+ assert (
+ input.dim() == 5
+ ), f"input must be 5D tensor, got {input.dim()} and shape {input.shape}"
+
+ seq_world_size = dist.get_world_size(group)
+
+ if scatter_idx == 3 and gather_idx == 1:
+ # input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen/P, 3, hc, hs) output: (bs, seqlen, 3, hc/P, hs)
+ bs, shard_seqlen, t_cnt, hc, hs = input.shape
+
+ assert t_cnt == 3
+ seqlen = shard_seqlen * seq_world_size
+ shard_hc = hc // seq_world_size
+
+ # transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
+ # (bs, seqlen/P, 3, hc, hs) -reshape-> (bs, seq_len/P, 3, P, hc/P, hs) -transpose(0,3)-> (P, seq_len/P, 3, bs, hc/P, hs)
+ input_t = (
+ input.reshape(bs, shard_seqlen, 3, seq_world_size, shard_hc, hs)
+ .transpose(0, 3)
+ .contiguous()
+ )
+
+ output = torch.empty_like(input_t)
+ # https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
+ # (P, seq_len/P, 3, bs, hc/P, hs) scatter seqlen -all2all-> (P, seq_len/P, 3, bs, hc/P, hs) scatter head
+ dist.all_to_all_single(output, input_t, group=group)
+
+ # if scattering the seq-dim, transpose the heads back to the original dimension
+ output = output.reshape(seqlen, 3, bs, shard_hc, hs)
+
+ # (seq_len, 3, bs, hc/P, hs) -trans-> (bs, seq_len, 3, hc/P, hs)
+ output = output.transpose(0, 2).transpose(1, 2).contiguous()
+
+ return output.reshape(bs, seqlen, 3, shard_hc, hs).contiguous()
+ elif scatter_idx == 1 and gather_idx == 3:
+ # input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen, hc/P, hs) output: (bs, seqlen/P, hc, hs)
+ bs, seqlen, _, shard_hc, hs = input.shape
+ hc = shard_hc * seq_world_size
+ shard_seqlen = seqlen // seq_world_size
+ seq_world_size = dist.get_world_size(group)
+
+ # transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
+ # (bs, seqlen, 3, hc/P, hs) -reshape-> (bs, P, seq_len/P, 3, hc/P, hs) -transpose(0, 4)-> (hc/P, P, seqlen/P, 3, bs, hs) -transpose(0, 1) -> (P, hc/P, seqlen/P, 3, bs, hs)
+ input_t = (
+ input.reshape(bs, seq_world_size, shard_seqlen, 3, shard_hc, hs)
+ .transpose(0, 4)
+ .transpose(0, 1)
+ .contiguous()
+ .reshape(seq_world_size, shard_hc, shard_seqlen, 3, bs, hs)
+ )
+
+ output = torch.empty_like(input_t)
+ # https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
+ # (P, bs x hc/P, seqlen/P, hs) scatter seqlen -all2all-> (P, bs x seq_len/P, hc/P, hs) scatter head
+ dist.all_to_all_single(output, input_t, group=group)
+
+ # if scattering the seq-dim, transpose the heads back to the original dimension
+ output = output.reshape(hc, shard_seqlen, 3, bs, hs)
+
+ # (hc, seqlen/N, bs, hs) -tranpose(0,2)-> (bs, seqlen/N, hc, hs)
+ output = output.transpose(0, 3).contiguous()
+
+ return output.reshape(bs, shard_seqlen, 3, hc, hs).contiguous()
+ else:
+ raise RuntimeError("scatter_idx must be 1 or 3 and gather_idx must be 1 or 3")
+
+
+class SeqAllToAll5D(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx: Any,
+ group: dist.ProcessGroup,
+ input: Tensor,
+ scatter_idx: int = 3,
+ gather_idx: int = 1,
+ ) -> Tensor:
+ ctx.group = group
+ ctx.scatter_idx = scatter_idx
+ ctx.gather_idx = gather_idx
+
+ return all_to_all_5D(input, scatter_idx, gather_idx, group=group)
+
+ @staticmethod
+ def backward(ctx: Any, *grad_output: Tensor) -> Tuple[None, Tensor, None, None]:
+ return (
+ None,
+ SeqAllToAll5D.apply(
+ ctx.group, *grad_output, ctx.gather_idx, ctx.scatter_idx
+ ),
+ None,
+ None,
+ )
diff --git a/LeanK/leank/utils.py b/LeanK/leank/utils.py
new file mode 100644
index 00000000..e46d8ee0
--- /dev/null
+++ b/LeanK/leank/utils.py
@@ -0,0 +1,181 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import transformers
+from accelerate import dispatch_model, infer_auto_device_map
+from accelerate.utils import get_balanced_memory
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="kv_reduction")
+
+ parser.add_argument(
+ "--model_name", type=str, default="Meta-Llama/Llama-3.1-8B-Instruct"
+ )
+ parser.add_argument("--config_name", type=str, default=None)
+
+ parser.add_argument("--dataset_format", type=str, default="multiple_passkey")
+ parser.add_argument("--split", type=str, default="train")
+ parser.add_argument("--lr", type=float, default=1e-1)
+ parser.add_argument("--num_steps", type=int, default=1000)
+ parser.add_argument("--batch_size", type=int, default=1)
+ parser.add_argument("--max_length", type=int, default=4096)
+ parser.add_argument("--context_length_min", type=int, default=1024)
+ parser.add_argument("--context_length_max", type=int, default=4096)
+ parser.add_argument("--context_lengths_num_intervals", type=int, default=20)
+ parser.add_argument("--depth_ratio_num_intervals", type=int, default=10)
+ parser.add_argument("--num_passkeys", type=int, default=10)
+ parser.add_argument("--output_dir", type=str, default="outputs")
+ parser.add_argument("--sink_size", type=int, default=64)
+ parser.add_argument("--recent_size", type=int, default=256)
+ parser.add_argument("--deploy_sink_size", type=int, default=None)
+ parser.add_argument("--deploy_recent_size", type=int, default=None)
+ parser.add_argument("--reg_weight", type=float, default=0.05)
+ parser.add_argument("--initial_value", type=float, default=1.0)
+ parser.add_argument("--exp_name", type=str, default=None)
+ parser.add_argument("--enable_pp", action="store_true")
+ parser.add_argument("--enable_tp", action="store_true")
+ parser.add_argument("--disable_wandb", action="store_true")
+ parser.add_argument("--min_needle_depth_ratio", type=float, default=0)
+ parser.add_argument("--max_needle_depth_ratio", type=float, default=1.0)
+ parser.add_argument("--save_steps", type=int, default=50)
+ parser.add_argument("--gradient_accumulation_steps", type=int, default=1)
+ parser.add_argument("--resume", action="store_true")
+ parser.add_argument("--rope_theta", type=float, default=None)
+ parser.add_argument("--device", type=str, default="0")
+ parser.add_argument("--ratio", type=float, default=0.7)
+ parser.add_argument("--align", type=int, default=32)
+ parser.add_argument("--stage2", action="store_true")
+ parser.add_argument("--stage1_rst_path", type=str, default=None)
+
+ parser.add_argument(
+ "--supervision",
+ type=str,
+ default="distill",
+ choices=["classify", "distill"],
+ )
+
+ # Eval params
+ parser.add_argument("--n_samples", type=int, default=None)
+ parser.add_argument("--task", type=str, default="default")
+ parser.add_argument("--attn_load_dir", type=str, default=None)
+ parser.add_argument("--threshold", type=float, default=0.5)
+ parser.add_argument("--sparsity", type=float, default=None)
+ parser.add_argument("--passkey_length", type=int, default=32)
+ parser.add_argument("--context_length", type=int, default=16384)
+ parser.add_argument("--generation_length", type=int, default=256)
+ parser.add_argument("--stride_length", type=int, default=256)
+ parser.add_argument("--prefilling_chunk_size", type=int, default=4096)
+
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--config", type=str, required=True, help="Path to YAML config")
+
+ args = parser.parse_args()
+
+ args.device = parse_device(args.device)
+ return args
+
+
+def parse_device(device: str):
+ if "," in device:
+ return [int(d) for d in device.split(",")]
+ elif device in ["auto", "cpu"]:
+ return device
+ return f"cuda:{device}"
+
+
+def get_model(model_name):
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype=torch.bfloat16,
+ low_cpu_mem_usage=True,
+ attn_implementation="eager",
+ )
+
+ if hasattr(model.config, "sliding_window") and model.config.sliding_window is None:
+ model.config.sliding_window = model.config.max_position_embeddings
+
+ return model
+
+
+def get_tokenizer(tokenizer_name):
+ tokenizer = transformers.AutoTokenizer.from_pretrained(
+ tokenizer_name, use_fast=False, trust_remote_code=True
+ )
+
+ if tokenizer.pad_token_id is None:
+ if tokenizer.eos_token_id is not None:
+ tokenizer.pad_token_id = tokenizer.eos_token_id
+ else:
+ tokenizer.pad_token_id = 0
+
+ return tokenizer
+
+
+def convert_to_list(scaling_factors):
+ num_pruned_layers = len(scaling_factors)
+ for idx in range(num_pruned_layers):
+ scaling_factors[idx] = (
+ scaling_factors[idx].detach().flatten().float().cpu().tolist()
+ )
+ return scaling_factors
+
+
+def visualize_patterns(scaling_factors):
+ img = np.array(scaling_factors)
+ fig = plt.figure(figsize=(10, 10))
+ plt.imshow(img, cmap="coolwarm", interpolation="nearest", aspect="auto")
+ plt.xlabel("Attention Heads")
+ plt.ylabel("Layers")
+ plt.colorbar(fraction=0.046, pad=0.04)
+ # scale the color to 0-1
+ plt.clim(0, 1)
+ plt.tight_layout()
+ plt.title("Ratio of Full Attention Computations")
+ return fig
+
+
+def seed_everything(seed):
+ import os
+ import random
+
+ import numpy as np
+ import torch
+
+ random.seed(seed)
+ os.environ["PYTHONHASHSEED"] = str(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed(seed)
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = True
+
+
+def save_scaling_factors(scaling_factors, output_filename):
+ np.savetxt(
+ output_filename,
+ np.array(scaling_factors),
+ delimiter="\t",
+ )
+
+
+def sparsify_scaling_factors(scaling_factors, ratio, round_to):
+ l = len(scaling_factors)
+ h, d = scaling_factors[0].shape
+
+ scaling_factors = torch.stack(scaling_factors).flatten().float().cpu().numpy()
+
+ threshold = np.quantile(scaling_factors, ratio)
+ mask = (scaling_factors >= threshold).astype(float)
+
+ head_k = (mask.reshape(l, h, d).sum(-1) + round_to // 2) // round_to * round_to
+
+ ind = np.argsort(scaling_factors.reshape(l, h, d))[:, :, ::-1]
+ reverse_ind = np.argsort(ind)
+ mask_round = reverse_ind < head_k[:, :, None]
+ return mask_round
diff --git a/LeanK/narrativeqa_example.txt b/LeanK/narrativeqa_example.txt
new file mode 100644
index 00000000..7acefd1b
--- /dev/null
+++ b/LeanK/narrativeqa_example.txt
@@ -0,0 +1,2347 @@
+<|start_header_id|>user<|end_header_id|>
+
+You are given a story, which can be either a novel or a movie script, and a question. Answer the question asconcisely as you can, using a single phrase if possible. Do not provide any explanation.
+
+Story: Transcribed from the 1915 Martin Secker edition by David Price, email
+ccx074@pglaf.org
+
+ [Picture: Book cover]
+
+
+
+
+
+ THE
+ COXON FUND
+
+
+ BY HENRY JAMES
+
+ [Picture: Decorative graphic]
+
+ * * * * *
+
+ LONDON: MARTIN SECKER
+ NUMBER FIVE JOHN STREET ADELPHI
+
+ * * * * *
+
+ This edition first published 1915
+
+ The text follows that of the
+ Definitive Edition
+
+ * * * * *
+
+
+
+
+I
+
+
+“THEY’VE got him for life!” I said to myself that evening on my way back
+to the station; but later on, alone in the compartment (from Wimbledon to
+Waterloo, before the glory of the District Railway) I amended this
+declaration in the light of the sense that my friends would probably
+after all not enjoy a monopoly of Mr. Saltram. I won’t pretend to have
+taken his vast measure on that first occasion, but I think I had achieved
+a glimpse of what the privilege of his acquaintance might mean for many
+persons in the way of charges accepted. He had been a great experience,
+and it was this perhaps that had put me into the frame of foreseeing how
+we should all, sooner or later, have the honour of dealing with him as a
+whole. Whatever impression I then received of the amount of this total,
+I had a full enough vision of the patience of the Mulvilles. He was to
+stay all the winter: Adelaide dropped it in a tone that drew the sting
+from the inevitable emphasis. These excellent people might indeed have
+been content to give the circle of hospitality a diameter of six months;
+but if they didn’t say he was to stay all summer as well it was only
+because this was more than they ventured to hope. I remember that at
+dinner that evening he wore slippers, new and predominantly purple, of
+some queer carpet-stuff; but the Mulvilles were still in the stage of
+supposing that he might be snatched from them by higher bidders. At a
+later time they grew, poor dears, to fear no snatching; but theirs was a
+fidelity which needed no help from competition to make them proud.
+Wonderful indeed as, when all was said, you inevitably pronounced Frank
+Saltram, it was not to be overlooked that the Kent Mulvilles were in
+their way still more extraordinary: as striking an instance as could
+easily be encountered of the familiar truth that remarkable men find
+remarkable conveniences.
+
+They had sent for me from Wimbledon to come out and dine, and there had
+been an implication in Adelaide’s note—judged by her notes alone she
+might have been thought silly—that it was a case in which something
+momentous was to be determined or done. I had never known them not be in
+a “state” about somebody, and I dare say I tried to be droll on this
+point in accepting their invitation. On finding myself in the presence
+of their latest discovery I had not at first felt irreverence droop—and,
+thank heaven, I have never been absolutely deprived of that alternative
+in Mr. Saltram’s company. I saw, however—I hasten to declare it—that
+compared to this specimen their other phoenixes had been birds of
+inconsiderable feather, and I afterwards took credit to myself for not
+having even in primal bewilderments made a mistake about the essence of
+the man. He had an incomparable gift; I never was blind to it—it dazzles
+me still. It dazzles me perhaps even more in remembrance than in fact,
+for I’m not unaware that for so rare a subject the imagination goes to
+some expense, inserting a jewel here and there or giving a twist to a
+plume. How the art of portraiture would rejoice in this figure if the
+art of portraiture had only the canvas! Nature, in truth, had largely
+rounded it, and if memory, hovering about it, sometimes holds her breath,
+this is because the voice that comes back was really golden.
+
+Though the great man was an inmate and didn’t dress, he kept dinner on
+this occasion waiting, and the first words he uttered on coming into the
+room were an elated announcement to Mulville that he had found out
+something. Not catching the allusion and gaping doubtless a little at
+his face, I privately asked Adelaide what he had found out. I shall
+never forget the look she gave me as she replied: “Everything!” She
+really believed it. At that moment, at any rate, he had found out that
+the mercy of the Mulvilles was infinite. He had previously of course
+discovered, as I had myself for that matter, that their dinners were
+soignés. Let me not indeed, in saying this, neglect to declare that I
+shall falsify my counterfeit if I seem to hint that there was in his
+nature any ounce of calculation. He took whatever came, but he never
+plotted for it, and no man who was so much of an absorbent can ever have
+been so little of a parasite. He had a system of the universe, but he
+had no system of sponging—that was quite hand-to-mouth. He had fine
+gross easy senses, but it was not his good-natured appetite that wrought
+confusion. If he had loved us for our dinners we could have paid with
+our dinners, and it would have been a great economy of finer matter. I
+make free in these connexions with the plural possessive because if I was
+never able to do what the Mulvilles did, and people with still bigger
+houses and simpler charities, I met, first and last, every demand of
+reflexion, of emotion—particularly perhaps those of gratitude and of
+resentment. No one, I think, paid the tribute of giving him up so often,
+and if it’s rendering honour to borrow wisdom I’ve a right to talk of my
+sacrifices. He yielded lessons as the sea yields fish—I lived for a
+while on this diet. Sometimes it almost appeared to me that his massive
+monstrous failure—if failure after all it was—had been designed for my
+private recreation. He fairly pampered my curiosity; but the history of
+that experience would take me too far. This is not the large canvas I
+just now spoke of, and I wouldn’t have approached him with my present
+hand had it been a question of all the features. Frank Saltram’s
+features, for artistic purposes, are verily the anecdotes that are to be
+gathered. Their name is legion, and this is only one, of which the
+interest is that it concerns even more closely several other persons.
+Such episodes, as one looks back, are the little dramas that made up the
+innumerable facets of the big drama—which is yet to be reported.
+
+
+
+
+II
+
+
+IT is furthermore remarkable that though the two stories are distinct—my
+own, as it were, and this other—they equally began, in a manner, the
+first night of my acquaintance with Frank Saltram, the night I came back
+from Wimbledon so agitated with a new sense of life that, in London, for
+the very thrill of it, I could only walk home. Walking and swinging my
+stick, I overtook, at Buckingham Gate, George Gravener, and George
+Gravener’s story may be said to have begun with my making him, as our
+paths lay together, come home with me for a talk. I duly remember, let
+me parenthesise, that it was still more that of another person, and also
+that several years were to elapse before it was to extend to a second
+chapter. I had much to say to him, none the less, about my visit to the
+Mulvilles, whom he more indifferently knew, and I was at any rate so
+amusing that for long afterwards he never encountered me without asking
+for news of the old man of the sea. I hadn’t said Mr. Saltram was old,
+and it was to be seen that he was of an age to outweather George
+Gravener. I had at that time a lodging in Ebury Street, and Gravener was
+staying at his brother’s empty house in Eaton Square. At Cambridge, five
+years before, even in our devastating set, his intellectual power had
+seemed to me almost awful. Some one had once asked me privately, with
+blanched cheeks, what it was then that after all such a mind as that left
+standing. “It leaves itself!” I could recollect devoutly replying. I
+could smile at present for this remembrance, since before we got to Ebury
+Street I was struck with the fact that, save in the sense of being well
+set up on his legs, George Gravener had actually ceased to tower. The
+universe he laid low had somehow bloomed again—the usual eminences were
+visible. I wondered whether he had lost his humour, or only, dreadful
+thought, had never had any—not even when I had fancied him most
+Aristophanesque. What was the need of appealing to laughter, however, I
+could enviously enquire, where you might appeal so confidently to
+measurement? Mr. Saltram’s queer figure, his thick nose and hanging lip,
+were fresh to me: in the light of my old friend’s fine cold symmetry they
+presented mere success in amusing as the refuge of conscious ugliness.
+Already, at hungry twenty-six, Gravener looked as blank and parliamentary
+as if he were fifty and popular. In my scrap of a residence—he had a
+worldling’s eye for its futile conveniences, but never a comrade’s joke—I
+sounded Frank Saltram in his ears; a circumstance I mention in order to
+note that even then I was surprised at his impatience of my enlivenment.
+As he had never before heard of the personage it took indeed the form of
+impatience of the preposterous Mulvilles, his relation to whom, like
+mine, had had its origin in an early, a childish intimacy with the young
+Adelaide, the fruit of multiplied ties in the previous generation. When
+she married Kent Mulville, who was older than Gravener and I and much
+more amiable, I gained a friend, but Gravener practically lost one. We
+reacted in different ways from the form taken by what he called their
+deplorable social action—the form (the term was also his) of nasty
+second-rate gush. I may have held in my ‘for intérieur’ that the good
+people at Wimbledon were beautiful fools, but when he sniffed at them I
+couldn’t help taking the opposite line, for I already felt that even
+should we happen to agree it would always be for reasons that differed.
+It came home to me that he was admirably British as, without so much as a
+sociable sneer at my bookbinder, he turned away from the serried rows of
+my little French library.
+
+“Of course I’ve never seen the fellow, but it’s clear enough he’s a
+humbug.”
+
+“Clear ‘enough’ is just what it isn’t,” I replied; “if it only were!”
+That ejaculation on my part must have been the beginning of what was to
+be later a long ache for final frivolous rest. Gravener was profound
+enough to remark after a moment that in the first place he couldn’t be
+anything but a Dissenter, and when I answered that the very note of his
+fascination was his extraordinary speculative breadth my friend retorted
+that there was no cad like your cultivated cad, and that I might depend
+upon discovering—since I had had the levity not already to have
+enquired—that my shining light proceeded, a generation back, from a
+Methodist cheesemonger. I confess I was struck with his insistence, and
+I said, after reflexion: “It may be—I admit it may be; but why on earth
+are you so sure?”—asking the question mainly to lay him the trap of
+saying that it was because the poor man didn’t dress for dinner. He took
+an instant to circumvent my trap and come blandly out the other side.
+
+“Because the Kent Mulvilles have invented him. They’ve an infallible
+hand for frauds. All their geese are swans. They were born to be duped,
+they like it, they cry for it, they don’t know anything from anything,
+and they disgust one—luckily perhaps!—with Christian charity.” His
+vehemence was doubtless an accident, but it might have been a strange
+foreknowledge. I forget what protest I dropped; it was at any rate
+something that led him to go on after a moment: “I only ask one
+thing—it’s perfectly simple. Is a man, in a given case, a real
+gentleman?”
+
+“A real gentleman, my dear fellow—that’s so soon said!”
+
+“Not so soon when he isn’t! If they’ve got hold of one this time he must
+be a great rascal!”
+
+“I might feel injured,” I answered, “if I didn’t reflect that they don’t
+rave about me.”
+
+“Don’t be too sure! I’ll grant that he’s a gentleman,” Gravener
+presently added, “if you’ll admit that he’s a scamp.”
+
+“I don’t know which to admire most, your logic or your benevolence.”
+
+My friend coloured at this, but he didn’t change the subject. “Where did
+they pick him up?”
+
+“I think they were struck with something he had published.”
+
+“I can fancy the dreary thing!”
+
+“I believe they found out he had all sorts of worries and difficulties.”
+
+“That of course wasn’t to be endured, so they jumped at the privilege of
+paying his debts!” I professed that I knew nothing about his debts, and
+I reminded my visitor that though the dear Mulvilles were angels they
+were neither idiots nor millionaires. What they mainly aimed at was
+reuniting Mr. Saltram to his wife. “I was expecting to hear he has
+basely abandoned her,” Gravener went on, at this, “and I’m too glad you
+don’t disappoint me.”
+
+I tried to recall exactly what Mrs. Mulville had told me. “He didn’t
+leave her—no. It’s she who has left him.”
+
+“Left him to us?” Gravener asked. “The monster—many thanks! I decline
+to take him.”
+
+“You’ll hear more about him in spite of yourself. I can’t, no, I really
+can’t resist the impression that he’s a big man.” I was already
+mastering—to my shame perhaps be it said—just the tone my old friend
+least liked.
+
+“It’s doubtless only a trifle,” he returned, “but you haven’t happened to
+mention what his reputation’s to rest on.”
+
+“Why on what I began by boring you with—his extraordinary mind.”
+
+“As exhibited in his writings?”
+
+“Possibly in his writings, but certainly in his talk, which is far and
+away the richest I ever listened to.”
+
+“And what’s it all about?”
+
+“My dear fellow, don’t ask me! About everything!” I pursued, reminding
+myself of poor Adelaide. “About his ideas of things,” I then more
+charitably added. “You must have heard him to know what I mean—it’s
+unlike anything that ever was heard.” I coloured, I admit, I overcharged
+a little, for such a picture was an anticipation of Saltram’s later
+development and still more of my fuller acquaintance with him. However,
+I really expressed, a little lyrically perhaps, my actual imagination of
+him when I proceeded to declare that, in a cloud of tradition, of legend,
+he might very well go down to posterity as the greatest of all great
+talkers. Before we parted George Gravener had wondered why such a row
+should be made about a chatterbox the more and why he should be pampered
+and pensioned. The greater the wind-bag the greater the calamity. Out
+of proportion to everything else on earth had come to be this wagging of
+the tongue. We were drenched with talk—our wretched age was dying of it.
+I differed from him here sincerely, only going so far as to concede, and
+gladly, that we were drenched with sound. It was not however the mere
+speakers who were killing us—it was the mere stammerers. Fine talk was
+as rare as it was refreshing—the gift of the gods themselves, the one
+starry spangle on the ragged cloak of humanity. How many men were there
+who rose to this privilege, of how many masters of conversation could he
+boast the acquaintance? Dying of talk?—why we were dying of the lack of
+it! Bad writing wasn’t talk, as many people seemed to think, and even
+good wasn’t always to be compared to it. From the best talk indeed the
+best writing had something to learn. I fancifully added that we too
+should peradventure be gilded by the legend, should be pointed at for
+having listened, for having actually heard. Gravener, who had glanced at
+his watch and discovered it was midnight, found to all this a retort
+beautifully characteristic of him.
+
+“There’s one little fact to be borne in mind in the presence equally of
+the best talk and of the worst.” He looked, in saying this, as if he
+meant great things, and I was sure he could only mean once more that
+neither of them mattered if a man wasn’t a real gentleman. Perhaps it
+was what he did mean; he deprived me however of the exultation of being
+right by putting the truth in a slightly different way. “The only thing
+that really counts for one’s estimate of a person is his conduct.” He
+had his watch still in his palm, and I reproached him with unfair play in
+having ascertained beforehand that it was now the hour at which I always
+gave in. My pleasantry so far failed to mollify him that he promptly
+added that to the rule he had just enunciated there was absolutely no
+exception.
+
+“None whatever?”
+
+“None whatever.”
+
+“Trust me then to try to be good at any price!” I laughed as I went with
+him to the door. “I declare I will be, if I have to be horrible!”
+
+
+
+
+III
+
+
+IF that first night was one of the liveliest, or at any rate was the
+freshest, of my exaltations, there was another, four years later, that
+was one of my great discomposures. Repetition, I well knew by this time,
+was the secret of Saltram’s power to alienate, and of course one would
+never have seen him at his finest if one hadn’t seen him in his remorses.
+They set in mainly at this season and were magnificent, elemental,
+orchestral. I was quite aware that one of these atmospheric disturbances
+was now due; but none the less, in our arduous attempt to set him on his
+feet as a lecturer, it was impossible not to feel that two failures were
+a large order, as we said, for a short course of five. This was the
+second time, and it was past nine o’clock; the audience, a muster
+unprecedented and really encouraging, had fortunately the attitude of
+blandness that might have been looked for in persons whom the promise of
+(if I’m not mistaken) An Analysis of Primary Ideas had drawn to the
+neighbourhood of Upper Baker Street. There was in those days in that
+region a petty lecture-hall to be secured on terms as moderate as the
+funds left at our disposal by the irrepressible question of the
+maintenance of five small Saltrams—I include the mother—and one large
+one. By the time the Saltrams, of different sizes, were all maintained
+we had pretty well poured out the oil that might have lubricated the
+machinery for enabling the most original of men to appear to maintain
+them.
+
+It was I, the other time, who had been forced into the breach, standing
+up there for an odious lamplit moment to explain to half a dozen thin
+benches, where earnest brows were virtuously void of anything so cynical
+as a suspicion, that we couldn’t so much as put a finger on Mr. Saltram.
+There was nothing to plead but that our scouts had been out from the
+early hours and that we were afraid that on one of his walks abroad—he
+took one, for meditation, whenever he was to address such a company—some
+accident had disabled or delayed him. The meditative walks were a
+fiction, for he never, that any one could discover, prepared anything but
+a magnificent prospectus; hence his circulars and programmes, of which I
+possess an almost complete collection, are the solemn ghosts of
+generations never born. I put the case, as it seemed to me, at the best;
+but I admit I had been angry, and Kent Mulville was shocked at my want of
+public optimism. This time therefore I left the excuses to his more
+practised patience, only relieving myself in response to a direct appeal
+from a young lady next whom, in the hall, I found myself sitting. My
+position was an accident, but if it had been calculated the reason would
+scarce have eluded an observer of the fact that no one else in the room
+had an approach to an appearance. Our philosopher’s “tail” was
+deplorably limp. This visitor was the only person who looked at her
+ease, who had come a little in the spirit of adventure. She seemed to
+carry amusement in her handsome young head, and her presence spoke, a
+little mystifyingly, of a sudden extension of Saltram’s sphere of
+influence. He was doing better than we hoped, and he had chosen such an
+occasion, of all occasions, to succumb to heaven knew which of his fond
+infirmities. The young lady produced an impression of auburn hair and
+black velvet, and had on her other hand a companion of obscurer type,
+presumably a waiting-maid. She herself might perhaps have been a foreign
+countess, and before she addressed me I had beguiled our sorry interval
+by finding in her a vague recall of the opening of some novel of Madame
+Sand. It didn’t make her more fathomable to pass in a few minutes from
+this to the certitude that she was American; it simply engendered
+depressing reflexions as to the possible check to contributions from
+Boston. She asked me if, as a person apparently more initiated, I would
+recommend further waiting, and I answered that if she considered I was on
+my honour I would privately deprecate it. Perhaps she didn’t; at any
+rate our talk took a turn that prolonged it till she became aware we were
+left almost alone. I presently ascertained she knew Mrs. Saltram, and
+this explained in a manner the miracle. The brotherhood of the friends
+of the husband was as nothing to the brotherhood, or perhaps I should say
+the sisterhood, of the friends of the wife. Like the Kent Mulvilles I
+belonged to both fraternities, and even better than they I think I had
+sounded the abyss of Mrs. Saltram’s wrongs. She bored me to extinction,
+and I knew but too well how she had bored her husband; but there were
+those who stood by her, the most efficient of whom were indeed the
+handful of poor Saltram’s backers. They did her liberal justice, whereas
+her mere patrons and partisans had nothing but hatred for our
+philosopher. I’m bound to say it was we, however—we of both camps, as it
+were—who had always done most for her.
+
+I thought my young lady looked rich—I scarcely knew why; and I hoped she
+had put her hand in her pocket. I soon made her out, however, not at all
+a fine fanatic—she was but a generous, irresponsible enquirer. She had
+come to England to see her aunt, and it was at her aunt’s she had met the
+dreary lady we had all so much on our mind. I saw she’d help to pass the
+time when she observed that it was a pity this lady wasn’t intrinsically
+more interesting. That was refreshing, for it was an article of faith in
+Mrs. Saltram’s circle—at least among those who scorned to know her horrid
+husband—that she was attractive on her merits. She was in truth a most
+ordinary person, as Saltram himself would have been if he hadn’t been a
+prodigy. The question of vulgarity had no application to him, but it was
+a measure his wife kept challenging you to apply. I hasten to add that
+the consequences of your doing so were no sufficient reason for his
+having left her to starve. “He doesn’t seem to have much force of
+character,” said my young lady; at which I laughed out so loud that my
+departing friends looked back at me over their shoulders as if I were
+making a joke of their discomfiture. My joke probably cost Saltram a
+subscription or two, but it helped me on with my interlocutress. “She
+says he drinks like a fish,” she sociably continued, “and yet she allows
+that his mind’s wonderfully clear.” It was amusing to converse with a
+pretty girl who could talk of the clearness of Saltram’s mind. I
+expected next to hear she had been assured he was awfully clever. I
+tried to tell her—I had it almost on my conscience—what was the proper
+way to regard him; an effort attended perhaps more than ever on this
+occasion with the usual effect of my feeling that I wasn’t after all very
+sure of it. She had come to-night out of high curiosity—she had wanted
+to learn this proper way for herself. She had read some of his papers
+and hadn’t understood them; but it was at home, at her aunt’s, that her
+curiosity had been kindled—kindled mainly by his wife’s remarkable
+stories of his want of virtue. “I suppose they ought to have kept me
+away,” my companion dropped, “and I suppose they’d have done so if I
+hadn’t somehow got an idea that he’s fascinating. In fact Mrs. Saltram
+herself says he is.”
+
+“So you came to see where the fascination resides? Well, you’ve seen!”
+
+My young lady raised fine eyebrows. “Do you mean in his bad faith?”
+
+“In the extraordinary effects of it; his possession, that is, of some
+quality or other that condemns us in advance to forgive him the
+humiliation, as I may call it, to which he has subjected us.”
+
+“The humiliation?”
+
+“Why mine, for instance, as one of his guarantors, before you as the
+purchaser of a ticket.”
+
+She let her charming gay eyes rest on me. “You don’t look humiliated a
+bit, and if you did I should let you off, disappointed as I am; for the
+mysterious quality you speak of is just the quality I came to see.”
+
+“Oh, you can’t ‘see’ it!” I cried.
+
+“How then do you get at it?”
+
+“You don’t! You mustn’t suppose he’s good-looking,” I added.
+
+“Why his wife says he’s lovely!”
+
+My hilarity may have struck her as excessive, but I confess it broke out
+afresh. Had she acted only in obedience to this singular plea, so
+characteristic, on Mrs. Saltram’s part, of what was irritating in the
+narrowness of that lady’s point of view? “Mrs. Saltram,” I explained,
+“undervalues him where he’s strongest, so that, to make up for it
+perhaps, she overpraises him where he’s weak. He’s not, assuredly,
+superficially attractive; he’s middle-aged, fat, featureless save for his
+great eyes.”
+
+“Yes, his great eyes,” said my young lady attentively. She had evidently
+heard all about his great eyes—the beaux yeux for which alone we had
+really done it all.
+
+“They’re tragic and splendid—lights on a dangerous coast. But he moves
+badly and dresses worse, and altogether he’s anything but smart.”
+
+My companion, who appeared to reflect on this, after a moment appealed.
+“Do you call him a real gentleman?”
+
+I started slightly at the question, for I had a sense of recognising it:
+George Gravener, years before, that first flushed night, had put me face
+to face with it. It had embarrassed me then, but it didn’t embarrass me
+now, for I had lived with it and overcome it and disposed of it. “A real
+gentleman? Emphatically not!”
+
+My promptitude surprised her a little, but I quickly felt how little it
+was to Gravener I was now talking. “Do you say that because he’s—what do
+you call it in England?—of humble extraction?”
+
+“Not a bit. His father was a country school-master and his mother the
+widow of a sexton, but that has nothing to do with it. I say it simply
+because I know him well.”
+
+“But isn’t it an awful drawback?”
+
+“Awful—quite awful.”
+
+“I mean isn’t it positively fatal?”
+
+“Fatal to what? Not to his magnificent vitality.”
+
+Again she had a meditative moment. “And is his magnificent vitality the
+cause of his vices?”
+
+“Your questions are formidable, but I’m glad you put them. I was
+thinking of his noble intellect. His vices, as you say, have been much
+exaggerated: they consist mainly after all in one comprehensive defect.”
+
+“A want of will?”
+
+“A want of dignity.”
+
+“He doesn’t recognise his obligations?”
+
+“On the contrary, he recognises them with effusion, especially in public:
+he smiles and bows and beckons across the street to them. But when they
+pass over he turns away, and he speedily loses them in the crowd. The
+recognition’s purely spiritual—it isn’t in the least social. So he
+leaves all his belongings to other people to take care of. He accepts
+favours, loans, sacrifices—all with nothing more deterrent than an agony
+of shame. Fortunately we’re a little faithful band, and we do what we
+can.” I held my tongue about the natural children, engendered, to the
+number of three, in the wantonness of his youth. I only remarked that he
+did make efforts—often tremendous ones. “But the efforts,” I said,
+“never come to much: the only things that come to much are the
+abandonments, the surrenders.”
+
+“And how much do they come to?”
+
+“You’re right to put it as if we had a big bill to pay, but, as I’ve told
+you before, your questions are rather terrible. They come, these mere
+exercises of genius, to a great sum total of poetry, of philosophy, a
+mighty mass of speculation, notation, quotation. The genius is there,
+you see, to meet the surrender; but there’s no genius to support the
+defence.”
+
+“But what is there, after all, at his age, to show?”
+
+“In the way of achievement recognised and reputation established?” I
+asked. “To ‘show’ if you will, there isn’t much, since his writing,
+mostly, isn’t as fine, isn’t certainly as showy, as his talk. Moreover
+two-thirds of his work are merely colossal projects and announcements.
+‘Showing’ Frank Saltram is often a poor business,” I went on: “we
+endeavoured, you’ll have observed, to show him to-night! However, if he
+had lectured he’d have lectured divinely. It would just have been his
+talk.”
+
+“And what would his talk just have been?”
+
+I was conscious of some ineffectiveness, as well perhaps as of a little
+impatience, as I replied: “The exhibition of a splendid intellect.” My
+young lady looked not quite satisfied at this, but as I wasn’t prepared
+for another question I hastily pursued: “The sight of a great suspended
+swinging crystal—huge lucid lustrous, a block of light—flashing back
+every impression of life and every possibility of thought!”
+
+This gave her something to turn over till we had passed out to the dusky
+porch of the hall, in front of which the lamps of a quiet brougham were
+almost the only thing Saltram’s treachery hadn’t extinguished. I went
+with her to the door of her carriage, out of which she leaned a moment
+after she had thanked me and taken her seat. Her smile even in the
+darkness was pretty. “I do want to see that crystal!”
+
+“You’ve only to come to the next lecture.”
+
+“I go abroad in a day or two with my aunt.”
+
+“Wait over till next week,” I suggested. “It’s quite worth it.”
+
+She became grave. “Not unless he really comes!” At which the brougham
+started off, carrying her away too fast, fortunately for my manners, to
+allow me to exclaim “Ingratitude!”
+
+
+
+
+IV
+
+
+MRS. SALTRAM made a great affair of her right to be informed where her
+husband had been the second evening he failed to meet his audience. She
+came to me to ascertain, but I couldn’t satisfy her, for in spite of my
+ingenuity I remained in ignorance. It wasn’t till much later that I
+found this had not been the case with Kent Mulville, whose hope for the
+best never twirled the thumbs of him more placidly than when he happened
+to know the worst. He had known it on the occasion I speak of—that is
+immediately after. He was impenetrable then, but ultimately confessed.
+What he confessed was more than I shall now venture to make public. It
+was of course familiar to me that Saltram was incapable of keeping the
+engagements which, after their separation, he had entered into with
+regard to his wife, a deeply wronged, justly resentful, quite
+irreproachable and insufferable person. She often appeared at my
+chambers to talk over his lapses; for if, as she declared, she had washed
+her hands of him, she had carefully preserved the water of this ablution,
+which she handed about for analysis. She had arts of her own of exciting
+one’s impatience, the most infallible of which was perhaps her assumption
+that we were kind to her because we liked her. In reality her personal
+fall had been a sort of social rise—since I had seen the moment when, in
+our little conscientious circle, her desolation almost made her the
+fashion. Her voice was grating and her children ugly; moreover she hated
+the good Mulvilles, whom I more and more loved. They were the people who
+by doing most for her husband had in the long run done most for herself;
+and the warm confidence with which he had laid his length upon them was a
+pressure gentle compared with her stiffer persuadability. I’m bound to
+say he didn’t criticise his benefactors, though practically he got tired
+of them; she, however, had the highest standards about eleemosynary
+forms. She offered the odd spectacle of a spirit puffed up by
+dependence, and indeed it had introduced her to some excellent society.
+She pitied me for not knowing certain people who aided her and whom she
+doubtless patronised in turn for their luck in not knowing me. I dare
+say I should have got on with her better if she had had a ray of
+imagination—if it had occasionally seemed to occur to her to regard
+Saltram’s expressions of his nature in any other manner than as separate
+subjects of woe. They were all flowers of his character, pearls strung
+on an endless thread; but she had a stubborn little way of challenging
+them one after the other, as if she never suspected that he had a
+character, such as it was, or that deficiencies might be organic; the
+irritating effect of a mind incapable of a generalisation. One might
+doubtless have overdone the idea that there was a general licence for
+such a man; but if this had happened it would have been through one’s
+feeling that there could be none for such a woman.
+
+I recognised her superiority when I asked her about the aunt of the
+disappointed young lady: it sounded like a sentence from an
+English-French or other phrase-book. She triumphed in what she told me
+and she may have triumphed still more in what she withheld. My friend of
+the other evening, Miss Anvoy, had but lately come to England; Lady
+Coxon, the aunt, had been established here for years in consequence of
+her marriage with the late Sir Gregory of that name. She had a house in
+the Regent’s Park, a Bath-chair and a fernery; and above all she had
+sympathy. Mrs. Saltram had made her acquaintance through mutual friends.
+This vagueness caused me to feel how much I was out of it and how large
+an independent circle Mrs. Saltram had at her command. I should have
+been glad to know more about the disappointed young lady, but I felt I
+should know most by not depriving her of her advantage, as she might have
+mysterious means of depriving me of my knowledge. For the present,
+moreover, this experience was stayed, Lady Coxon having in fact gone
+abroad accompanied by her niece. The niece, besides being immensely
+clever, was an heiress, Mrs. Saltram said; the only daughter and the
+light of the eyes of some great American merchant, a man, over there, of
+endless indulgences and dollars. She had pretty clothes and pretty
+manners, and she had, what was prettier still, the great thing of all.
+The great thing of all for Mrs. Saltram was always sympathy, and she
+spoke as if during the absence of these ladies she mightn’t know where to
+turn for it. A few months later indeed, when they had come back, her
+tone perceptibly changed: she alluded to them, on my leading her up to
+it, rather as to persons in her debt for favours received. What had
+happened I didn’t know, but I saw it would take only a little more or a
+little less to make her speak of them as thankless subjects of social
+countenance—people for whom she had vainly tried to do something. I
+confess I saw how it wouldn’t be in a mere week or two that I should rid
+myself of the image of Ruth Anvoy, in whose very name, when I learnt it,
+I found something secretly to like. I should probably neither see her
+nor hear of her again: the knight’s widow (he had been mayor of
+Clockborough) would pass away and the heiress would return to her
+inheritance. I gathered with surprise that she had not communicated to
+his wife the story of her attempt to hear Mr..Saltram, and I founded this
+reticence on the easy supposition that Mrs. Saltram had fatigued by
+overpressure the spring of the sympathy of which she boasted. The girl
+at any rate would forget the small adventure, be distracted, take a
+husband; besides which she would lack occasion to repeat her experiment.
+
+We clung to the idea of the brilliant course, delivered without an
+accident, that, as a lecturer, would still make the paying public aware
+of our great man, but the fact remained that in the case of an
+inspiration so unequal there was treachery, there was fallacy at least,
+in the very conception of a series. In our scrutiny of ways and means we
+were inevitably subject to the old convention of the synopsis, the
+syllabus, partly of course not to lose the advantage of his grand free
+hand in drawing up such things; but for myself I laughed at our playbills
+even while I stickled for them. It was indeed amusing work to be
+scrupulous for Frank Saltram, who also at moments laughed about it, so
+far as the comfort of a sigh so unstudied as to be cheerful might pass
+for such a sound. He admitted with a candour all his own that he was in
+truth only to be depended on in the Mulvilles’ drawing-room. “Yes,” he
+suggestively allowed, “it’s there, I think, that I’m at my best; quite
+late, when it gets toward eleven—and if I’ve not been too much worried.”
+We all knew what too much worry meant; it meant too enslaved for the hour
+to the superstition of sobriety. On the Saturdays I used to bring my
+portmanteau, so as not to have to think of eleven o’clock trains. I had
+a bold theory that as regards this temple of talk and its altars of
+cushioned chintz, its pictures and its flowers, its large fireside and
+clear lamplight, we might really arrive at something if the Mulvilles
+would but charge for admission. Here it was, however, that they
+shamelessly broke down; as there’s a flaw in every perfection this was
+the inexpugnable refuge of their egotism. They declined to make their
+saloon a market, so that Saltram’s golden words continued the sole coin
+that rang there. It can have happened to no man, however, to be paid a
+greater price than such an enchanted hush as surrounded him on his
+greatest nights. The most profane, on these occasions, felt a presence;
+all minor eloquence grew dumb. Adelaide Mulville, for the pride of her
+hospitality, anxiously watched the door or stealthily poked the fire. I
+used to call it the music-room, for we had anticipated Bayreuth. The
+very gates of the kingdom of light seemed to open and the horizon of
+thought to flash with the beauty of a sunrise at sea.
+
+In the consideration of ways and means, the sittings of our little board,
+we were always conscious of the creak of Mrs. Saltram’s shoes. She
+hovered, she interrupted, she almost presided, the state of affairs being
+mostly such as to supply her with every incentive for enquiring what was
+to be done next. It was the pressing pursuit of this knowledge that, in
+concatenations of omnibuses and usually in very wet weather, led her so
+often to my door. She thought us spiritless creatures with editors and
+publishers; but she carried matters to no great effect when she
+personally pushed into back-shops. She wanted all moneys to be paid to
+herself: they were otherwise liable to such strange adventures. They
+trickled away into the desert—they were mainly at best, alas, a slender
+stream. The editors and the publishers were the last people to take this
+remarkable thinker at the valuation that has now pretty well come to be
+established. The former were half-distraught between the desire to “cut”
+him and the difficulty of finding a crevice for their shears; and when a
+volume on this or that portentous subject was proposed to the latter they
+suggested alternative titles which, as reported to our friend, brought
+into his face the noble blank melancholy that sometimes made it handsome.
+The title of an unwritten book didn’t after all much matter, but some
+masterpiece of Saltram’s may have died in his bosom of the shudder with
+which it was then convulsed. The ideal solution, failing the fee at Kent
+Mulville’s door, would have been some system of subscription to projected
+treatises with their non-appearance provided for—provided for, I mean, by
+the indulgence of subscribers. The author’s real misfortune was that
+subscribers were so wretchedly literal. When they tastelessly enquired
+why publication hadn’t ensued I was tempted to ask who in the world had
+ever been so published. Nature herself had brought him out in voluminous
+form, and the money was simply a deposit on borrowing the work.
+
+
+
+
+V
+
+
+I WAS doubtless often a nuisance to my friends in those years; but there
+were sacrifices I declined to make, and I never passed the hat to George
+Gravener. I never forgot our little discussion in Ebury Street, and I
+think it stuck in my throat to have to treat him to the avowal I had
+found so easy to Mss Anvoy. It had cost me nothing to confide to this
+charming girl, but it would have cost me much to confide to the friend of
+my youth, that the character of the “real gentleman” wasn’t an attribute
+of the man I took such pains for. Was this because I had already
+generalised to the point of perceiving that women are really the
+unfastidious sex? I knew at any rate that Gravener, already quite in
+view but still hungry and frugal, had naturally enough more ambition than
+charity. He had sharp aims for stray sovereigns, being in view most from
+the tall steeple of Clockborough. His immediate ambition was to occupy à
+lui seul the field of vision of that smokily-seeing city, and all his
+movements and postures were calculated for the favouring angle. The
+movement of the hand as to the pocket had thus to alternate gracefully
+with the posture of the hand on the heart. He talked to Clockborough in
+short only less beguilingly than Frank Saltram talked to his electors;
+with the difference to our credit, however, that we had already voted and
+that our candidate had no antagonist but himself. He had more than once
+been at Wimbledon—it was Mrs. Mulville’s work not mine—and by the time
+the claret was served had seen the god descend. He took more pains to
+swing his censer than I had expected, but on our way back to town he
+forestalled any little triumph I might have been so artless as to express
+by the observation that such a man was—a hundred times!—a man to use and
+never a man to be used by. I remember that this neat remark humiliated
+me almost as much as if virtually, in the fever of broken slumbers, I
+hadn’t often made it myself. The difference was that on Gravener’s part
+a force attached to it that could never attach to it on mine. He was
+able to use people—he had the machinery; and the irony of Saltram’s being
+made showy at Clockborough came out to me when he said, as if he had no
+memory of our original talk and the idea were quite fresh to him: “I hate
+his type, you know, but I’ll be hanged if I don’t put some of those
+things in. I can find a place for them: we might even find a place for
+the fellow himself.” I myself should have had some fear—not, I need
+scarcely say, for the “things” themselves, but for some other things very
+near them; in fine for the rest of my eloquence.
+
+Later on I could see that the oracle of Wimbledon was not in this case so
+appropriate as he would have been had the polities of the gods only
+coincided more exactly with those of the party. There was a distinct
+moment when, without saying anything more definite to me, Gravener
+entertained the idea of annexing Mr. Saltram. Such a project was
+delusive, for the discovery of analogies between his body of doctrine and
+that pressed from headquarters upon Clockborough—the bottling, in a word,
+of the air of those lungs for convenient public uncorking in
+corn-exchanges—was an experiment for which no one had the leisure. The
+only thing would have been to carry him massively about, paid, caged,
+clipped; to turn him on for a particular occasion in a particular
+channel. Frank Saltram’s channel, however, was essentially not
+calculable, and there was no knowing what disastrous floods might have
+ensued. For what there would have been to do The Empire, the great
+newspaper, was there to look to; but it was no new misfortune that there
+were delicate situations in which The Empire broke down. In fine there
+was an instinctive apprehension that a clever young journalist
+commissioned to report on Mr. Saltram might never come back from the
+errand. No one knew better than George Gravener that that was a time
+when prompt returns counted double. If he therefore found our friend an
+exasperating waste of orthodoxy it was because of his being, as he said,
+poor Gravener, up in the clouds, not because he was down in the dust.
+The man would have been, just as he was, a real enough gentleman if he
+could have helped to put in a real gentleman. Gravener’s great objection
+to the actual member was that he was not one.
+
+Lady Coxon had a fine old house, a house with “grounds,” at Clockborough,
+which she had let; but after she returned from abroad I learned from Mrs.
+Saltram that the lease had fallen in and that she had gone down to resume
+possession. I could see the faded red livery, the big square shoulders,
+the high-walled garden of this decent abode. As the rumble of
+dissolution grew louder the suitor would have pressed his suit, and I
+found myself hoping the politics of the late Mayor’s widow wouldn’t be
+such as to admonish her to ask him to dinner; perhaps indeed I went so
+far as to pray, they would naturally form a bar to any contact. I tried
+to focus the many-buttoned page, in the daily airing, as he perhaps even
+pushed the Bath-chair over somebody’s toes. I was destined to hear, none
+the less, through Mrs. Saltram—who, I afterwards learned, was in
+correspondence with Lady Coxon’s housekeeper—that Gravener was known to
+have spoken of the habitation I had in my eye as the pleasantest thing at
+Clockborough. On his part, I was sure, this was the voice not of envy
+but of experience. The vivid scene was now peopled, and I could see him
+in the old-time garden with Miss Anvoy, who would be certain, and very
+justly, to think him good-looking. It would be too much to describe
+myself as troubled by this play of surmise; but I occur to remember the
+relief, singular enough, of feeling it suddenly brushed away by an
+annoyance really much greater; an annoyance the result of its happening
+to come over me about that time with a rush that I was simply ashamed of
+Frank Saltram. There were limits after all, and my mark at last had been
+reached.
+
+I had had my disgusts, if I may allow myself to-day such an expression;
+but this was a supreme revolt. Certain things cleared up in my mind,
+certain values stood out. It was all very well to have an unfortunate
+temperament; there was nothing so unfortunate as to have, for practical
+purposes, nothing else. I avoided George Gravener at this moment and
+reflected that at such a time I should do so most effectually by leaving
+England. I wanted to forget Frank Saltram—that was all. I didn’t want
+to do anything in the world to him but that. Indignation had withered on
+the stalk, and I felt that one could pity him as much as one ought only
+by never thinking of him again. It wasn’t for anything he had done to
+me; it was for what he had done to the Mulvilles. Adelaide cried about
+it for a week, and her husband, profiting by the example so signally
+given him of the fatal effect of a want of character, left the letter,
+the drop too much, unanswered. The letter, an incredible one, addressed
+by Saltram to Wimbledon during a stay with the Pudneys at Ramsgate, was
+the central feature of the incident, which, however, had many features,
+each more painful than whichever other we compared it with. The Pudneys
+had behaved shockingly, but that was no excuse. Base ingratitude, gross
+indecency—one had one’s choice only of such formulas as that the more
+they fitted the less they gave one rest. These are dead aches now, and I
+am under no obligation, thank heaven, to be definite about the business.
+There are things which if I had had to tell them—well, would have stopped
+me off here altogether.
+
+I went abroad for the general election, and if I don’t know how much, on
+the Continent, I forgot, I at least know how much I missed, him. At a
+distance, in a foreign land, ignoring, abjuring, unlearning him, I
+discovered what he had done for me. I owed him, oh unmistakeably,
+certain noble conceptions; I had lighted my little taper at his smoky
+lamp, and lo it continued to twinkle. But the light it gave me just
+showed me how much more I wanted. I was pursued of course by letters
+from Mrs. Saltram which I didn’t scruple not to read, though quite aware
+her embarrassments couldn’t but be now of the gravest. I sacrificed to
+propriety by simply putting them away, and this is how, one day as my
+absence drew to an end, my eye, while I rummaged in my desk for another
+paper, was caught by a name on a leaf that had detached itself from the
+packet. The allusion was to Miss Anvoy, who, it appeared, was engaged to
+be married to Mr. George Gravener; and the news was two months old. A
+direct question of Mrs. Saltram’s had thus remained unanswered—she had
+enquired of me in a postscript what sort of man this aspirant to such a
+hand might be. The great other fact about him just then was that he had
+been triumphantly returned for Clockborough in the interest of the party
+that had swept the country—so that I might easily have referred Mrs.
+Saltram to the journals of the day. Yet when I at last wrote her that I
+was coming home and would discharge my accumulated burden by seeing her,
+I but remarked in regard to her question that she must really put it to
+Miss Anvoy.
+
+
+
+
+VI
+
+
+I HAD almost avoided the general election, but some of its consequences,
+on my return, had smartly to be faced. The season, in London, began to
+breathe again and to flap its folded wings. Confidence, under the new
+Ministry, was understood to be reviving, and one of the symptoms, in a
+social body, was a recovery of appetite. People once more fed together,
+and it happened that, one Saturday night, at somebody’s house, I fed with
+George Gravener. When the ladies left the room I moved up to where he
+sat and begged to congratulate him. “On my election?” he asked after a
+moment; so that I could feign, jocosely, not to have heard of that
+triumph and to be alluding to the rumour of a victory still more
+personal. I dare say I coloured however, for his political success had
+momentarily passed out of my mind. What was present to it was that he
+was to marry that beautiful girl; and yet his question made me conscious
+of some discomposure—I hadn’t intended to put this before everything. He
+himself indeed ought gracefully to have done so, and I remember thinking
+the whole man was in this assumption that in expressing my sense of what
+he had won I had fixed my thoughts on his “seat.” We straightened the
+matter out, and he was so much lighter in hand than I had lately seen him
+that his spirits might well have been fed from a twofold source. He was
+so good as to say that he hoped I should soon make the acquaintance of
+Miss Anvoy, who, with her aunt, was presently coming up to town. Lady
+Coxon, in the country, had been seriously unwell, and this had delayed
+their arrival. I told him I had heard the marriage would be a splendid
+one; on which, brightened and humanised by his luck, he laughed and said
+“Do you mean for her?” When I had again explained what I meant he went
+on: “Oh she’s an American, but you’d scarcely know it; unless, perhaps,”
+he added, “by her being used to more money than most girls in England,
+even the daughters of rich men. That wouldn’t in the least do for a
+fellow like me, you know, if it wasn’t for the great liberality of her
+father. He really has been most kind, and everything’s quite
+satisfactory.” He added that his eldest brother had taken a tremendous
+fancy to her and that during a recent visit at Coldfield she had nearly
+won over Lady Maddock. I gathered from something he dropped later on
+that the free-handed gentleman beyond the seas had not made a settlement,
+but had given a handsome present and was apparently to be looked to,
+across the water, for other favours. People are simplified alike by
+great contentments and great yearnings, and, whether or no it was
+Gravener’s directness that begot my own, I seem to recall that in some
+turn taken by our talk he almost imposed it on me as an act of decorum to
+ask if Miss Anvoy had also by chance expectations from her aunt. My
+enquiry drew out that Lady Coxon, who was the oddest of women, would have
+in any contingency to act under her late husband’s will, which was odder
+still, saddling her with a mass of queer obligations complicated with
+queer loopholes. There were several dreary people, Coxon cousins, old
+maids, to whom she would have more or less to minister. Gravener
+laughed, without saying no, when I suggested that the young lady might
+come in through a loophole; then suddenly, as if he suspected my turning
+a lantern on him, he declared quite dryly: “That’s all rot—one’s moved by
+other springs!”
+
+A fortnight later, at Lady Coxon’s own house, I understood well enough
+the springs one was moved by. Gravener had spoken of me there as an old
+friend, and I received a gracious invitation to dine. The Knight’s widow
+was again indisposed—she had succumbed at the eleventh hour; so that I
+found Miss Anvoy bravely playing hostess without even Gravener’s help,
+since, to make matters worse, he had just sent up word that the House,
+the insatiable House, with which he supposed he had contracted for easier
+terms, positively declined to release him. I was struck with the
+courage, the grace and gaiety of the young lady left thus to handle the
+fauna and flora of the Regent’s Park. I did what I could to help her to
+classify them, after I had recovered from the confusion of seeing her
+slightly disconcerted at perceiving in the guest introduced by her
+intended the gentleman with whom she had had that talk about Frank
+Saltram. I had at this moment my first glimpse of the fact that she was
+a person who could carry a responsibility; but I leave the reader to
+judge of my sense of the aggravation, for either of us, of such a burden,
+when I heard the servant announce Mrs. Saltram. From what immediately
+passed between the two ladies I gathered that the latter had been sent
+for post-haste to fill the gap created by the absence of the mistress of
+the house. “Good!” I remember crying, “she’ll be put by me;” and my
+apprehension was promptly justified. Mrs. Saltram taken in to dinner,
+and taken in as a consequence of an appeal to her amiability, was Mrs.
+Saltram with a vengeance. I asked myself what Miss Anvoy meant by doing
+such things, but the only answer I arrived at was that Gravener was
+verily fortunate. She hadn’t happened to tell him of her visit to Upper
+Baker Street, but she’d certainly tell him to-morrow; not indeed that
+this would make him like any better her having had the innocence to
+invite such a person as Mrs. Saltram on such an occasion. It could only
+strike me that I had never seen a young woman put such ignorance into her
+cleverness, such freedom into her modesty; this, I think, was when, after
+dinner, she said to me frankly, with almost jubilant mirth: “Oh you don’t
+admire Mrs. Saltram?” Why should I? This was truly a young person
+without guile. I had briefly to consider before I could reply that my
+objection to the lady named was the objection often uttered about people
+met at the social board—I knew all her stories. Then as Miss Anvoy
+remained momentarily vague I added: “Those about her husband.”
+
+“Oh yes, but there are some new ones.”
+
+“None for me. Ah novelty would be pleasant!”
+
+“Doesn’t it appear that of late he has been particularly horrid?”
+
+“His fluctuations don’t matter”, I returned, “for at night all cats are
+grey. You saw the shade of this one the night we waited for him
+together. What will you have? He has no dignity.”
+
+Miss Anvoy, who had been introducing with her American distinctness,
+looked encouragingly round at some of the combinations she had risked.
+“It’s too bad I can’t see him.”
+
+“You mean Gravener won’t let you?”
+
+“I haven’t asked him. He lets me do everything.”
+
+“But you know he knows him and wonders what some of us see in him.”
+
+“We haven’t happened to talk of him,” the girl said.
+
+“Get him to take you some day out to see the Mulvilles.”
+
+“I thought Mr. Saltram had thrown the Mulvilles over.”
+
+“Utterly. But that won’t prevent his being planted there again, to bloom
+like a rose, within a month or two.”
+
+Miss Anvoy thought a moment. Then, “I should like to see them,” she said
+with her fostering smile.
+
+“They’re tremendously worth it. You mustn’t miss them.”
+
+“I’ll make George take me,” she went on as Mrs. Saltram came up to
+interrupt us. She sniffed at this unfortunate as kindly as she had
+smiled at me and, addressing the question to her, continued: “But the
+chance of a lecture—one of the wonderful lectures? Isn’t there another
+course announced?”
+
+“Another? There are about thirty!” I exclaimed, turning away and feeling
+Mrs. Saltram’s little eyes in my back. A few days after this I heard
+that Gravener’s marriage was near at hand—was settled for Whitsuntide;
+but as no invitation had reached me I had my doubts, and there presently
+came to me in fact the report of a postponement. Something was the
+matter; what was the matter was supposed to be that Lady Coxon was now
+critically ill. I had called on her after my dinner in the Regent’s
+Park, but I had neither seen her nor seen Miss Anvoy. I forget to-day
+the exact order in which, at this period, sundry incidents occurred and
+the particular stage at which it suddenly struck me, making me catch my
+breath a little, that the progression, the acceleration, was for all the
+world that of fine drama. This was probably rather late in the day, and
+the exact order doesn’t signify. What had already occurred was some
+accident determining a more patient wait. George Gravener, whom I met
+again, in fact told me as much, but without signs of perturbation. Lady
+Coxon had to be constantly attended to, and there were other good reasons
+as well. Lady Coxon had to be so constantly attended to that on the
+occasion of a second attempt in the Regent’s Park I equally failed to
+obtain a sight of her niece. I judged it discreet in all the conditions
+not to make a third; but this didn’t matter, for it was through Adelaide
+Mulville that the side-wind of the comedy, though I was at first
+unwitting, began to reach me. I went to Wimbledon at times because
+Saltram was there, and I went at others because he wasn’t. The Pudneys,
+who had taken him to Birmingham, had already got rid of him, and we had a
+horrible consciousness of his wandering roofless, in dishonour, about the
+smoky Midlands, almost as the injured Lear wandered on the storm-lashed
+heath. His room, upstairs, had been lately done up (I could hear the
+crackle of the new chintz) and the difference only made his smirches and
+bruises, his splendid tainted genius, the more tragic. If he wasn’t
+barefoot in the mire he was sure to be unconventionally shod. These were
+the things Adelaide and I, who were old enough friends to stare at each
+other in silence, talked about when we didn’t speak. When we spoke it
+was only about the brilliant girl George Gravener was to marry and whom
+he had brought out the other Sunday. I could see that this presentation
+had been happy, for Mrs. Mulville commemorated it after her sole fashion
+of showing confidence in a new relation. “She likes me—she likes me”:
+her native humility exulted in that measure of success. We all knew for
+ourselves how she liked those who liked her, and as regards Ruth Anvoy
+she was more easily won over than Lady Maddock.
+
+
+
+
+VII
+
+
+ONE of the consequences, for the Mulvilles, of the sacrifices they made
+for Frank Saltram was that they had to give up their carriage. Adelaide
+drove gently into London in a one-horse greenish thing, an early
+Victorian landau, hired, near at hand, imaginatively, from a broken-down
+jobmaster whose wife was in consumption—a vehicle that made people turn
+round all the more when her pensioner sat beside her in a soft white hat
+and a shawl, one of the dear woman’s own. This was his position and I
+dare say his costume when on an afternoon in July she went to return Miss
+Anvoy’s visit. The wheel of fate had now revolved, and amid silences
+deep and exhaustive, compunctions and condonations alike unutterable,
+Saltram was reinstated. Was it in pride or in penance that Mrs. Mulville
+had begun immediately to drive him about? If he was ashamed of his
+ingratitude she might have been ashamed of her forgiveness; but she was
+incorrigibly capable of liking him to be conspicuous in the landau while
+she was in shops or with her acquaintance. However, if he was in the
+pillory for twenty minutes in the Regent’s Park—I mean at Lady Coxon’s
+door while his companion paid her call—it wasn’t to the further
+humiliation of any one concerned that she presently came out for him in
+person, not even to show either of them what a fool she was that she drew
+him in to be introduced to the bright young American. Her account of the
+introduction I had in its order, but before that, very late in the
+season, under Gravener’s auspices, I met Miss Anvoy at tea at the House
+of Commons. The member for Clockborough had gathered a group of pretty
+ladies, and the Mulvilles were not of the party. On the great terrace,
+as I strolled off with her a little, the guest of honour immediately
+exclaimed to me: “I’ve seen him, you know—I’ve seen him!” She told me
+about Saltram’s call.
+
+“And how did you find him?”
+
+“Oh so strange!”
+
+“You didn’t like him?”
+
+“I can’t tell till I see him again.”
+
+“You want to do that?”
+
+She had a pause. “Immensely.”
+
+We went no further; I fancied she had become aware Gravener was looking
+at us. She turned back toward the knot of the others, and I said:
+“Dislike him as much as you will—I see you’re bitten.”
+
+“Bitten?” I thought she coloured a little.
+
+“Oh it doesn’t matter!” I laughed; “one doesn’t die of it.”
+
+“I hope I shan’t die of anything before I’ve seen more of Mrs. Mulville.”
+I rejoiced with her over plain Adelaide, whom she pronounced the
+loveliest woman she had met in England; but before we separated I
+remarked to her that it was an act of mere humanity to warn her that if
+she should see more of Frank Saltram—which would be likely to follow on
+any increase of acquaintance with Mrs. Mulville—she might find herself
+flattening her nose against the clear hard pane of an eternal
+question—that of the relative, that of the opposed, importances of virtue
+and brains. She replied that this was surely a subject on which one took
+everything for granted; whereupon I admitted that I had perhaps expressed
+myself ill. What I referred to was what I had referred to the night we
+met in Upper Baker Street—the relative importance (relative to virtue) of
+other gifts. She asked me if I called virtue a gift—a thing handed to us
+in a parcel on our first birthday; and I declared that this very enquiry
+proved to me the problem had already caught her by the skirt. She would
+have help however, the same help I myself had once had, in resisting its
+tendency to make one cross.
+
+“What help do you mean?”
+
+“That of the member for Clockborough.”
+
+She stared, smiled, then returned: “Why my idea has been to help him!”
+
+She had helped him—I had his own word for it that at Clockborough her
+bedevilment of the voters had really put him in. She would do so
+doubtless again and again, though I heard the very next month that this
+fine faculty had undergone a temporary eclipse. News of the catastrophe
+first came to me from Mrs. Saltram, and it was afterwards confirmed at
+Wimbledon: poor Miss Anvoy was in trouble—great disasters in America had
+suddenly summoned her home. Her father, in New York, had suffered
+reverses, lost so much money that it was really vexatious as showing how
+much he had had. It was Adelaide who told me she had gone off alone at
+less than a week’s notice.
+
+“Alone? Gravener has permitted that?”
+
+“What will you have? The House of Commons!”
+
+I’m afraid I cursed the House of Commons: I was so much interested. Of
+course he’d follow her as soon as he was free to make her his wife; only
+she mightn’t now be able to bring him anything like the marriage-portion
+of which he had begun by having the virtual promise. Mrs. Mulville let
+me know what was already said: she was charming, this American girl, but
+really these American fathers—! What was a man to do? Mr. Saltram,
+according to Mrs. Mulville, was of opinion that a man was never to suffer
+his relation to money to become a spiritual relation—he was to keep it
+exclusively material. “Moi pas comprendre!” I commented on this; in
+rejoinder to which Adelaide, with her beautiful sympathy, explained that
+she supposed he simply meant that the thing was to use it, don’t you
+know? but not to think too much about it. “To take it, but not to thank
+you for it?” I still more profanely enquired. For a quarter of an hour
+afterwards she wouldn’t look at me, but this didn’t prevent my asking her
+what had been the result, that afternoon—in the Regent’s Park, of her
+taking our friend to see Miss Anvoy.
+
+“Oh so charming!” she answered, brightening. “He said he recognised in
+her a nature he could absolutely trust.”
+
+“Yes, but I’m speaking of the effect on herself.”
+
+Mrs. Mulville had to remount the stream. “It was everything one could
+wish.”
+
+Something in her tone made me laugh. “Do you mean she gave him—a dole?”
+
+“Well, since you ask me!”
+
+“Right there on the spot?”
+
+Again poor Adelaide faltered. “It was to me of course she gave it.”
+
+I stared; somehow I couldn’t see the scene. “Do you mean a sum of
+money?”
+
+“It was very handsome.” Now at last she met my eyes, though I could see
+it was with an effort. “Thirty pounds.”
+
+“Straight out of her pocket?”
+
+“Out of the drawer of a table at which she had been writing. She just
+slipped the folded notes into my hand. He wasn’t looking; it was while
+he was going back to the carriage.” “Oh,” said Adelaide reassuringly, “I
+take care of it for him!” The dear practical soul thought my agitation,
+for I confess I was agitated, referred to the employment of the money.
+Her disclosure made me for a moment muse violently, and I dare say that
+during that moment I wondered if anything else in the world makes people
+so gross as unselfishness. I uttered, I suppose, some vague synthetic
+cry, for she went on as if she had had a glimpse of my inward amaze at
+such passages. “I assure you, my dear friend, he was in one of his happy
+hours.”
+
+But I wasn’t thinking of that. “Truly indeed these Americans!” I said.
+“With her father in the very act, as it were, of swindling her
+betrothed!”
+
+Mrs. Mulville stared. “Oh I suppose Mr. Anvoy has scarcely gone
+bankrupt—or whatever he has done—on purpose. Very likely they won’t be
+able to keep it up, but there it was, and it was a very beautiful
+impulse.”
+
+“You say Saltram was very fine?”
+
+“Beyond everything. He surprised even me.”
+
+“And I know what you’ve enjoyed.” After a moment I added: “Had he
+peradventure caught a glimpse of the money in the table-drawer?”
+
+At this my companion honestly flushed. “How can you be so cruel when you
+know how little he calculates?”
+
+“Forgive me, I do know it. But you tell me things that act on my nerves.
+I’m sure he hadn’t caught a glimpse of anything but some splendid idea.”
+
+Mrs. Mulville brightly concurred. “And perhaps even of her beautiful
+listening face.”
+
+“Perhaps even! And what was it all about?”
+
+“His talk? It was apropos of her engagement, which I had told him about:
+the idea of marriage, the philosophy, the poetry, the sublimity of it.”
+It was impossible wholly to restrain one’s mirth at this, and some rude
+ripple that I emitted again caused my companion to admonish me. “It
+sounds a little stale, but you know his freshness.”
+
+“Of illustration? Indeed I do!”
+
+“And how he has always been right on that great question.”
+
+“On what great question, dear lady, hasn’t he been right?”
+
+“Of what other great men can you equally say it?—and that he has never,
+but never, had a deflexion?” Mrs. Mulville exultantly demanded.
+
+I tried to think of some other great man, but I had to give it up.
+“Didn’t Miss Anvoy express her satisfaction in any less diffident way
+than by her charming present?” I was reduced to asking instead.
+
+“Oh yes, she overflowed to me on the steps while he was getting into the
+carriage.” These words somehow brushed up a picture of Saltram’s big
+shawled back as he hoisted himself into the green landau. “She said she
+wasn’t disappointed,” Adelaide pursued.
+
+I turned it over. “Did he wear his shawl?”
+
+“His shawl?” She hadn’t even noticed.
+
+“I mean yours.”
+
+“He looked very nice, and you know he’s really clean. Miss Anvoy used
+such a remarkable expression—she said his mind’s like a crystal!”
+
+I pricked up my ears. “A crystal?”
+
+“Suspended in the moral world—swinging and shining and flashing there.
+She’s monstrously clever, you know.”
+
+I thought again. “Monstrously!”
+
+
+
+
+VIII
+
+
+GEORGE GRAVENER didn’t follow her, for late in September, after the House
+had risen, I met him in a railway-carriage. He was coming up from
+Scotland and I had just quitted some relations who lived near Durham.
+The current of travel back to London wasn’t yet strong; at any rate on
+entering the compartment I found he had had it for some time to himself.
+We fared in company, and though he had a blue-book in his lap and the
+open jaws of his bag threatened me with the white teeth of confused
+papers, we inevitably, we even at last sociably conversed. I saw things
+weren’t well with him, but I asked no question till something dropped by
+himself made, as it had made on another occasion, an absence of curiosity
+invidious. He mentioned that he was worried about his good old friend
+Lady Coxon, who, with her niece likely to be detained some time in
+America, lay seriously ill at Clockborough, much on his mind and on his
+hands.
+
+“Ah Miss Anvoy’s in America?”
+
+“Her father has got into horrid straits—has lost no end of money.”
+
+I waited, after expressing due concern, but I eventually said: “I hope
+that raises no objection to your marriage.”
+
+“None whatever; moreover it’s my trade to meet objections. But it may
+create tiresome delays, of which there have been too many, from various
+causes, already. Lady Coxon got very bad, then she got much better.
+Then Mr. Anvoy suddenly began to totter, and now he seems quite on his
+back. I’m afraid he’s really in for some big reverse. Lady Coxon’s
+worse again, awfully upset by the news from America, and she sends me
+word that she _must_ have Ruth. How can I supply her with Ruth? I
+haven’t got Ruth myself!”
+
+“Surely you haven’t lost her?” I returned.
+
+“She’s everything to her wretched father. She writes me every
+post—telling me to smooth her aunt’s pillow. I’ve other things to
+smooth; but the old lady, save for her servants, is really alone. She
+won’t receive her Coxon relations—she’s angry at so much of her money
+going to them. Besides, she’s hopelessly mad,” said Gravener very
+frankly.
+
+I don’t remember whether it was this, or what it was, that made me ask if
+she hadn’t such an appreciation of Mrs. Saltram as might render that
+active person of some use.
+
+He gave me a cold glance, wanting to know what had put Mrs. Saltram into
+my head, and I replied that she was unfortunately never out of it. I
+happened to remember the wonderful accounts she had given me of the
+kindness Lady Coxon had shown her. Gravener declared this to be false;
+Lady Coxon, who didn’t care for her, hadn’t seen her three times. The
+only foundation for it was that Miss Anvoy, who used, poor girl, to chuck
+money about in a manner she must now regret, had for an hour seen in the
+miserable woman—you could never know what she’d see in people—an
+interesting pretext for the liberality with which her nature overflowed.
+But even Miss Anvoy was now quite tired of her. Gravener told me more
+about the crash in New York and the annoyance it had been to him, and we
+also glanced here and there in other directions; but by the time we got
+to Doncaster the principal thing he had let me see was that he was
+keeping something back. We stopped at that station, and, at the
+carriage-door, some one made a movement to get in. Gravener uttered a
+sound of impatience, and I felt sure that but for this I should have had
+the secret. Then the intruder, for some reason, spared us his company;
+we started afresh, and my hope of a disclosure returned. My companion
+held his tongue, however, and I pretended to go to sleep; in fact I
+really dozed for discouragement. When I reopened my eyes he was looking
+at me with an injured air. He tossed away with some vivacity the remnant
+of a cigarette and then said: “If you’re not too sleepy I want to put you
+a case.” I answered that I’d make every effort to attend, and welcomed
+the note of interest when he went on: “As I told you a while ago, Lady
+Coxon, poor dear, is demented.” His tone had much behind it—was full of
+promise. I asked if her ladyship’s misfortune were a trait of her malady
+or only of her character, and he pronounced it a product of both. The
+case he wanted to put to me was a matter on which it concerned him to
+have the impression—the judgement, he might also say—of another person.
+“I mean of the average intelligent man, but you see I take what I can
+get.” There would be the technical, the strictly legal view; then there
+would be the way the question would strike a man of the world. He had
+lighted another cigarette while he talked, and I saw he was glad to have
+it to handle when he brought out at last, with a laugh slightly
+artificial: “In fact it’s a subject on which Miss Anvoy and I are pulling
+different ways.”
+
+“And you want me to decide between you? I decide in advance for Miss
+Anvoy.”
+
+“In advance—that’s quite right. That’s how I decided when I proposed to
+her. But my story will interest you only so far as your mind isn’t made
+up.” Gravener puffed his cigarette a minute and then continued: “Are you
+familiar with the idea of the Endowment of Research?”
+
+“Of Research?” I was at sea a moment.
+
+“I give you Lady Coxon’s phrase. She has it on the brain.”
+
+“She wishes to endow—?”
+
+“Some earnest and ‘loyal’ seeker,” Gravener said. “It was a sketchy
+design of her late husband’s, and he handed it on to her; setting apart
+in his will a sum of money of which she was to enjoy the interest for
+life, but of which, should she eventually see her opportunity—the matter
+was left largely to her discretion—she would best honour his memory by
+determining the exemplary public use. This sum of money, no less than
+thirteen thousand pounds, was to be called The Coxon Fund; and poor Sir
+Gregory evidently proposed to himself that The Coxon Fund should cover
+his name with glory—be universally desired and admired. He left his wife
+a full declaration of his views, so far at least as that term may be
+applied to views vitiated by a vagueness really infantine. A little
+learning’s a dangerous thing, and a good citizen who happens to have been
+an ass is worse for a community than bad sewerage. He’s worst of all
+when he’s dead, because then he can’t be stopped. However, such as they
+were, the poor man’s aspirations are now in his wife’s bosom, or
+fermenting rather in her foolish brain: it lies with her to carry them
+out. But of course she must first catch her hare.”
+
+“Her earnest loyal seeker?”
+
+“The flower that blushes unseen for want of such a pecuniary independence
+as may aid the light that’s in it to shine upon the human race. The
+individual, in a word, who, having the rest of the machinery, the
+spiritual, the intellectual, is most hampered in his search.”
+
+“His search for what?”
+
+“For Moral Truth. That’s what Sir Gregory calls it.”
+
+I burst out laughing. “Delightful munificent Sir Gregory! It’s a
+charming idea.”
+
+“So Miss Anvoy thinks.”
+
+“Has she a candidate for the Fund?”
+
+“Not that I know of—and she’s perfectly reasonable about it. But Lady
+Coxon has put the matter before her, and we’ve naturally had a lot of
+talk.”
+
+“Talk that, as you’ve so interestingly intimated, has landed you in a
+disagreement.”
+
+“She considers there’s something in it,” Gravener said.
+
+“And you consider there’s nothing?”
+
+“It seems to me a piece of solemn twaddle—which can’t fail to be attended
+with consequences certainly grotesque and possibly immoral. To begin
+with, fancy constituting an endowment without establishing a tribunal—a
+bench of competent people, of judges.”
+
+“The sole tribunal is Lady Coxon?”
+
+“And any one she chooses to invite.”
+
+“But she has invited you,” I noted.
+
+“I’m not competent—I hate the thing. Besides, she hasn’t,” my friend
+went on. “The real history of the matter, I take it, is that the
+inspiration was originally Lady Coxon’s own, that she infected him with
+it, and that the flattering option left her is simply his tribute to her
+beautiful, her aboriginal enthusiasm. She came to England forty years
+ago, a thin transcendental Bostonian, and even her odd happy frumpy
+Clockborough marriage never really materialised her. She feels indeed
+that she has become very British—as if that, as a process, as a ‘Werden,’
+as anything but an original sign of grace, were conceivable; but it’s
+precisely what makes her cling to the notion of the ‘Fund’—cling to it as
+to a link with the ideal.”
+
+“How can she cling if she’s dying?”
+
+“Do you mean how can she act in the matter?” Gravener asked. “That’s
+precisely the question. She can’t! As she has never yet caught her
+hare, never spied out her lucky impostor—how should she, with the life
+she has led?—her husband’s intention has come very near lapsing. His
+idea, to do him justice, was that it _should_ lapse if exactly the right
+person, the perfect mixture of genius and chill penury, should fail to
+turn up. Ah the poor dear woman’s very particular—she says there must be
+no mistake.”
+
+I found all this quite thrilling—I took it in with avidity. “And if she
+dies without doing anything, what becomes of the money?” I demanded.
+
+“It goes back to his family, if she hasn’t made some other disposition of
+it.”
+
+“She may do that then—she may divert it?”
+
+“Her hands are not tied. She has a grand discretion. The proof is that
+three months ago she offered to make the proceeds over to her niece.”
+
+“For Miss Anvoy’s own use?”
+
+“For Miss Anvoy’s own use—on the occasion of her prospective marriage.
+She was discouraged—the earnest seeker required so earnest a search. She
+was afraid of making a mistake; every one she could think of seemed
+either not earnest enough or not poor enough. On the receipt of the
+first bad news about Mr. Anvoy’s affairs she proposed to Ruth to make the
+sacrifice for her. As the situation in New York got worse she repeated
+her proposal.”
+
+“Which Miss Anvoy declined?”
+
+“Except as a formal trust.”
+
+“You mean except as committing herself legally to place the money?”
+
+“On the head of the deserving object, the great man frustrated,” said
+Gravener. “She only consents to act in the spirit of Sir Gregory’s
+scheme.”
+
+“And you blame her for that?” I asked with some intensity.
+
+My tone couldn’t have been harsh, but he coloured a little and there was
+a queer light in his eye. “My dear fellow, if I ‘blamed’ the young lady
+I’m engaged to I shouldn’t immediately say it even to so old a friend as
+you.” I saw that some deep discomfort, some restless desire to be sided
+with, reassuringly, approvingly mirrored, had been at the bottom of his
+drifting so far, and I was genuinely touched by his confidence. It was
+inconsistent with his habits; but being troubled about a woman was not,
+for him, a habit: that itself was an inconsistency. George Gravener
+could stand straight enough before any other combination of forces. It
+amused me to think that the combination he had succumbed to had an
+American accent, a transcendental aunt and an insolvent father; but all
+my old loyalty to him mustered to meet this unexpected hint that I could
+help him. I saw that I could from the insincere tone in which he
+pursued: “I’ve criticised her of course, I’ve contended with her, and it
+has been great fun.” Yet it clearly couldn’t have been such great fun as
+to make it improper for me presently to ask if Miss Anvoy had nothing at
+all settled on herself. To this he replied that she had only a trifle
+from her mother—a mere four hundred a year, which was exactly why it
+would be convenient to him that she shouldn’t decline, in the face of
+this total change in her prospects, an accession of income which would
+distinctly help them to marry. When I enquired if there were no other
+way in which so rich and so affectionate an aunt could cause the weight
+of her benevolence to be felt, he answered that Lady Coxon was
+affectionate indeed, but was scarcely to be called rich. She could let
+her project of the Fund lapse for her niece’s benefit, but she couldn’t
+do anything else. She had been accustomed to regard her as tremendously
+provided for, and she was up to her eyes in promises to anxious Coxons.
+She was a woman of an inordinate conscience, and her conscience was now a
+distress to her, hovering round her bed in irreconcilable forms of
+resentful husbands, portionless nieces and undiscoverable philosophers.
+
+We were by this time getting into the whirr of fleeting platforms, the
+multiplication of lights. “I think you’ll find,” I said with a laugh,
+“that your predicament will disappear in the very fact that the
+philosopher _is_ undiscoverable.”
+
+He began to gather up his papers. “Who can set a limit to the ingenuity
+of an extravagant woman?”
+
+“Yes, after all, who indeed?” I echoed as I recalled the extravagance
+commemorated in Adelaide’s anecdote of Miss Anvoy and the thirty pounds.
+
+
+
+
+IX
+
+
+THE thing I had been most sensible of in that talk with George Gravener
+was the way Saltram’s name kept out of it. It seemed to me at the time
+that we were quite pointedly silent about him; but afterwards it appeared
+more probable there had been on my companion’s part no conscious
+avoidance. Later on I was sure of this, and for the best of reasons—the
+simple reason of my perceiving more completely that, for evil as well as
+for good, he said nothing to Gravener’s imagination. That honest man
+didn’t fear him—he was too much disgusted with him. No more did I,
+doubtless, and for very much the same reason. I treated my friend’s
+story as an absolute confidence; but when before Christmas, by Mrs.
+Saltram, I was informed of Lady Coxon’s death without having had news of
+Miss Anvoy’s return, I found myself taking for granted we should hear no
+more of these nuptials, in which, as obscurely unnatural, I now saw I had
+never _too_ disconcertedly believed. I began to ask myself how people
+who suited each other so little could please each other so much. The
+charm was some material charm, some afffinity, exquisite doubtless, yet
+superficial some surrender to youth and beauty and passion, to force and
+grace and fortune, happy accidents and easy contacts. They might dote on
+each other’s persons, but how could they know each other’s souls? How
+could they have the same prejudices, how could they have the same
+horizon? Such questions, I confess, seemed quenched but not answered
+when, one day in February, going out to Wimbledon, I found our young lady
+in the house. A passion that had brought her back across the wintry
+ocean was as much of a passion as was needed. No impulse equally strong
+indeed had drawn George Gravener to America; a circumstance on which,
+however, I reflected only long enough to remind myself that it was none
+of my business. Ruth Anvoy was distinctly different, and I felt that the
+difference was not simply that of her marks of mourning. Mrs. Mulville
+told me soon enough what it was: it was the difference between a handsome
+girl with large expectations and a handsome girl with only four hundred a
+year. This explanation indeed didn’t wholly content me, not even when I
+learned that her mourning had a double cause—learned that poor Mr. Anvoy,
+giving way altogether, buried under the ruins of his fortune and leaving
+next to nothing, had died a few weeks before.
+
+“So she has come out to marry George Gravener?” I commented. “Wouldn’t
+it have been prettier of him to have saved her the trouble?”
+
+“Hasn’t the House just met?” Adelaide replied. “And for Mr. Gravener the
+House—!” Then she added: “I gather that her having come is exactly a
+sign that the marriage is a little shaky. If it were quite all right a
+self-respecting girl like Ruth would have waited for him over there.”
+
+I noted that they were already Ruth and Adelaide, but what I said was:
+“Do you mean she’ll have had to return to _make_ it so?”
+
+“No, I mean that she must have come out for some reason independent of
+it.” Adelaide could only surmise, however, as yet, and there was more,
+as we found, to be revealed. Mrs. Mulville, on hearing of her arrival,
+had brought the young lady out in the green landau for the Sunday. The
+Coxons were in possession of the house in Regent’s Park, and Miss Anvoy
+was in dreary lodgings. George Gravener had been with her when Adelaide
+called, but had assented graciously enough to the little visit at
+Wimbledon. The carriage, with Mr. Saltram in it but not mentioned, had
+been sent off on some errand from which it was to return and pick the
+ladies up. Gravener had left them together, and at the end of an hour,
+on the Saturday afternoon, the party of three had driven out to
+Wimbledon. This was the girl’s second glimpse of our great man, and I
+was interested in asking Mrs. Mulville if the impression made by the
+first appeared to have been confirmed. On her replying after
+consideration, that of course with time and opportunity it couldn’t fail
+to be, but that she was disappointed, I was sufficiently struck with her
+use of this last word to question her further.
+
+“Do you mean you’re disappointed because you judge Miss Anvoy to be?”
+
+“Yes; I hoped for a greater effect last evening. We had two or three
+people, but he scarcely opened his mouth.”
+
+“He’ll be all the better to-night,” I opined after a moment. Then I
+pursued: “What particular importance do you attach to the idea of her
+being impressed?”
+
+Adelaide turned her mild pale eyes on me as for rebuke of my levity.
+“Why the importance of her being as happy as _we_ are!”
+
+I’m afraid that at this my levity grew. “Oh that’s a happiness almost
+too great to wish a person!” I saw she hadn’t yet in her mind what I had
+in mine, and at any rate the visitor’s actual bliss was limited to a walk
+in the garden with Kent Mulville. Later in the afternoon I also took
+one, and I saw nothing of Miss Anvoy till dinner, at which we failed of
+the company of Saltram, who had caused it to be reported that he was
+indisposed and lying down. This made us, most of us—for there were other
+friends present—convey to each other in silence some of the unutterable
+things that in those years our eyes had inevitably acquired the art of
+expressing. If a fine little American enquirer hadn’t been there we
+would have expressed them otherwise, and Adelaide would have pretended
+not to hear. I had seen her, before the very fact, abstract herself
+nobly; and I knew that more than once, to keep it from the servants,
+managing, dissimulating cleverly, she had helped her husband to carry him
+bodily to his room. Just recently he had been so wise and so deep and so
+high that I had begun to get nervous—to wonder if by chance there were
+something behind it, if he were kept straight for instance by the
+knowledge that the hated Pudneys would have more to tell us if they
+chose. He was lying low, but unfortunately it was common wisdom with us
+in this connexion that the biggest splashes took place in the quietest
+pools. We should have had a merry life indeed if all the splashes had
+sprinkled us as refreshingly as the waters we were even then to feel
+about our ears. Kent Mulville had been up to his room, but had come back
+with a face that told as few tales as I had seen it succeed in telling on
+the evening I waited in the lecture-room with Miss Anvoy. I said to
+myself that our friend had gone out, but it was a comfort that the
+presence of a comparative stranger deprived us of the dreary duty of
+suggesting to each other, in respect of his errand, edifying
+possibilities in which we didn’t ourselves believe. At ten o’clock he
+came into the drawing-room with his waistcoat much awry but his eyes
+sending out great signals. It was precisely with his entrance that I
+ceased to be vividly conscious of him. I saw that the crystal, as I had
+called it, had begun to swing, and I had need of my immediate attention
+for Miss Anvoy.
+
+Even when I was told afterwards that he had, as we might have said
+to-day, broken the record, the manner in which that attention had been
+rewarded relieved me of a sense of loss. I had of course a perfect
+general consciousness that something great was going on: it was a little
+like having been etherised to hear Herr Joachim play. The old music was
+in the air; I felt the strong pulse of thought, the sink and swell, the
+flight, the poise, the plunge; but I knew something about one of the
+listeners that nobody else knew, and Saltram’s monologue could reach me
+only through that medium. To this hour I’m of no use when, as a witness,
+I’m appealed to—for they still absurdly contend about it—as to whether or
+no on that historic night he was drunk; and my position is slightly
+ridiculous, for I’ve never cared to tell them what it really was I was
+taken up with. What I got out of it is the only morsel of the total
+experience that is quite my own. The others were shared, but this is
+incommunicable. I feel that now, I’m bound to say, even in thus roughly
+evoking the occasion, and it takes something from my pride of clearness.
+However, I shall perhaps be as clear as is absolutely needful if I remark
+that our young lady was too much given up to her own intensity of
+observation to be sensible of mine. It was plainly not the question of
+her marriage that had brought her back. I greatly enjoyed this discovery
+and was sure that had that question alone been involved she would have
+stirred no step. In this case doubtless Gravener would, in spite of the
+House of Commons, have found means to rejoin her. It afterwards made me
+uncomfortable for her that, alone in the lodging Mrs. Mulville had put
+before me as dreary, she should have in any degree the air of waiting for
+her fate; so that I was presently relieved at hearing of her having gone
+to stay at Coldfield. If she was in England at all while the engagement
+stood the only proper place for her was under Lady Maddock’s wing. Now
+that she was unfortunate and relatively poor, perhaps her prospective
+sister-in-law would be wholly won over.
+
+There would be much to say, if I had space, about the way her behaviour,
+as I caught gleams of it, ministered to the image that had taken birth in
+my mind, to my private amusement, while that other night I listened to
+George Gravener in the railway-carriage. I watched her in the light of
+this queer possibility—a formidable thing certainly to meet—and I was
+aware that it coloured, extravagantly perhaps, my interpretation of her
+very looks and tones. At Wimbledon for instance it had appeared to me
+she was literally afraid of Saltram, in dread of a coercion that she had
+begun already to feel. I had come up to town with her the next day and
+had been convinced that, though deeply interested, she was immensely on
+her guard. She would show as little as possible before she should be
+ready to show everything. What this final exhibition might be on the
+part of a girl perceptibly so able to think things out I found it great
+sport to forecast. It would have been exciting to be approached by her,
+appealed to by her for advice; but I prayed to heaven I mightn’t find
+myself in such a predicament. If there was really a present rigour in
+the situation of which Gravener had sketched for me the elements, she
+would have to get out of her difficulty by herself. It wasn’t I who had
+launched her and it wasn’t I who could help her. I didn’t fail to ask
+myself why, since I couldn’t help her, I should think so much about her.
+It was in part my suspense that was responsible for this; I waited
+impatiently to see whether she wouldn’t have told Mrs. Mulville a portion
+at least of what I had learned from Gravener. But I saw Mrs. Mulville
+was still reduced to wonder what she had come out again for if she hadn’t
+come as a conciliatory bride. That she had come in some other character
+was the only thing that fitted all the appearances. Having for family
+reasons to spend some time that spring in the west of England, I was in a
+manner out of earshot of the great oceanic rumble—I mean of the
+continuous hum of Saltram’s thought—and my uneasiness tended to keep me
+quiet. There was something I wanted so little to have to say that my
+prudence surmounted my curiosity. I only wondered if Ruth Anvoy talked
+over the idea of The Coxon Fund with Lady Maddock, and also somewhat why
+I didn’t hear from Wimbledon. I had a reproachful note about something
+or other from Mrs. Saltram, but it contained no mention of Lady Coxon’s
+niece, on whom her eyes had been much less fixed since the recent
+untoward events.
+
+
+
+
+X
+
+
+POOR Adelaide’s silence was fully explained later—practically explained
+when in June, returning to London, I was honoured by this admirable woman
+with an early visit. As soon as she arrived I guessed everything, and as
+soon as she told me that darling Ruth had been in her house nearly a
+month I had my question ready. “What in the name of maidenly modesty is
+she staying in England for?”
+
+“Because she loves me so!” cried Adelaide gaily. But she hadn’t come to
+see me only to tell me Miss Anvoy loved her: that was quite sufficiently
+established, and what was much more to the point was that Mr. Gravener
+had now raised an objection to it. He had protested at least against her
+being at Wimbledon, where in the innocence of his heart he had originally
+brought her himself; he called on her to put an end to their engagement
+in the only proper, the only happy manner.
+
+“And why in the world doesn’t she do do?” I asked.
+
+Adelaide had a pause. “She says you know.”
+
+Then on my also hesitating she added: “A condition he makes.”
+
+“The Coxon Fund?” I panted.
+
+“He has mentioned to her his having told you about it.”
+
+“Ah but so little! Do you mean she has accepted the trust?”
+
+“In the most splendid spirit—as a duty about which there can be no two
+opinions.” To which my friend added: “Of course she’s thinking of Mr.
+Saltram.”
+
+I gave a quick cry at this, which, in its violence, made my visitor turn
+pale. “How very awful!”
+
+“Awful?”
+
+“Why, to have anything to do with such an idea one’s self.”
+
+“I’m sure _you_ needn’t!” and Mrs. Mulville tossed her head.
+
+“He isn’t good enough!” I went on; to which she opposed a sound almost as
+contentious as my own had been. This made me, with genuine immediate
+horror, exclaim: “You haven’t influenced her, I hope!” and my emphasis
+brought back the blood with a rush to poor Adelaide’s face. She declared
+while she blushed—for I had frightened her again—that she had never
+influenced anybody and that the girl had only seen and heard and judged
+for herself. _He_ had influenced her, if I would, as he did every one
+who had a soul: that word, as we knew, even expressed feebly the power of
+the things he said to haunt the mind. How could she, Adelaide, help it
+if Miss Anvoy’s mind was haunted? I demanded with a groan what right a
+pretty girl engaged to a rising M.P. had to _have_ a mind; but the only
+explanation my bewildered friend could give me was that she was so
+clever. She regarded Mr. Saltram naturally as a tremendous force for
+good. She was intelligent enough to understand him and generous enough
+to admire.
+
+“She’s many things enough, but is she, among them, rich enough?” I
+demanded. “Rich enough, I mean, to sacrifice such a lot of good money?”
+
+“That’s for herself to judge. Besides, it’s not her own money; she
+doesn’t in the least consider it so.”
+
+“And Gravener does, if not _his_ own; and that’s the whole difficulty?”
+
+“The difficulty that brought her back, yes: she had absolutely to see her
+poor aunt’s solicitor. It’s clear that by Lady Coxon’s will she may have
+the money, but it’s still clearer to her conscience that the original
+condition, definite, intensely implied on her uncle’s part, is attached
+to the use of it. She can only take one view of it. It’s for the
+Endowment or it’s for nothing.”
+
+“The Endowment,” I permitted myself to observe, “is a conception
+superficially sublime, but fundamentally ridiculous.”
+
+“Are you repeating Mr. Gravener’s words?” Adelaide asked.
+
+“Possibly, though I’ve not seen him for months. It’s simply the way it
+strikes me too. It’s an old wife’s tale. Gravener made some reference
+to the legal aspect, but such an absurdly loose arrangement has _no_
+legal aspect.”
+
+“Ruth doesn’t insist on that,” said Mrs. Mulville; “and it’s, for her,
+exactly this technical weakness that constitutes the force of the moral
+obligation.”
+
+“Are you repeating _her_ words?” I enquired. I forget what else Adelaide
+said, but she said she was magnificent. I thought of George Gravener
+confronted with such magnificence as that, and I asked what could have
+made two such persons ever suppose they understood each other. Mrs.
+Mulville assured me the girl loved him as such a woman could love and
+that she suffered as such a woman could suffer. Nevertheless she wanted
+to see _me_. At this I sprang up with a groan. “Oh I’m so sorry!—when?”
+Small though her sense of humour, I think Adelaide laughed at my
+sequence. We discussed the day, the nearest it would be convenient I
+should come out; but before she went I asked my visitor how long she had
+been acquainted with these prodigies.
+
+“For several weeks, but I was pledged to secrecy.”
+
+“And that’s why you didn’t write?”
+
+“I couldn’t very well tell you she was with me without telling you that
+no time had even yet been fixed for her marriage. And I couldn’t very
+well tell you as much as that without telling you what I knew of the
+reason of it. It was not till a day or two ago,” Mrs. Mulville went on,
+“that she asked me to ask you if you wouldn’t come and see her. Then at
+last she spoke of your knowing about the idea of the Endowment.”
+
+I turned this over. “Why on earth does she want to see me?”
+
+“To talk with you, naturally, about Mr. Saltram.”
+
+“As a subject for the prize?” This was hugely obvious, and I presently
+returned: “I think I’ll sail to-morrow for Australia.”
+
+“Well then—sail!” said Mrs. Mulville, getting up.
+
+But I frivolously, continued. “On Thursday at five, we said?” The
+appointment was made definite and I enquired how, all this time, the
+unconscious candidate had carried himself.
+
+“In perfection, really, by the happiest of chances: he has positively
+been a dear. And then, as to what we revere him for, in the most
+wonderful form. His very highest—pure celestial light. You _won’t_ do
+him an ill turn?” Adelaide pleaded at the door.
+
+“What danger can equal for him the danger to which he’s exposed from
+himself?” I asked. “Look out sharp, if he has lately been too prim.
+He’ll presently take a day off, treat us to some exhibition that will
+make an Endowment a scandal.”
+
+“A scandal?” Mrs. Mulville dolorously echoed.
+
+“Is Miss Anvoy prepared for that?”
+
+My visitor, for a moment, screwed her parasol into my carpet. “He grows
+bigger every day.”
+
+“So do you!” I laughed as she went off.
+
+That girl at Wimbledon, on the Thursday afternoon, more than justified my
+apprehensions. I recognised fully now the cause of the agitation she had
+produced in me from the first—the faint foreknowledge that there was
+something very stiff I should have to do for her. I felt more than ever
+committed to my fate as, standing before her in the big drawing-room
+where they had tactfully left us to ourselves, I tried with a smile to
+string together the pearls of lucidity which, from her chair, she
+successively tossed me. Pale and bright, in her monotonous mourning, she
+was an image of intelligent purpose, of the passion of duty; but I asked
+myself whether any girl had ever had so charming an instinct as that
+which permitted her to laugh out, as for the joy of her difficulty, into
+the priggish old room. This remarkable young woman could be earnest
+without being solemn, and at moments when I ought doubtless to have
+cursed her obstinacy I found myself watching the unstudied play of her
+eyebrows or the recurrence of a singularly intense whiteness produced by
+the parting of her lips. These aberrations, I hasten to add, didn’t
+prevent my learning soon enough why she had wished to see me. Her reason
+for this was as distinct as her beauty: it was to make me explain what I
+had meant, on the occasion of our first meeting, by Mr. Saltram’s want of
+dignity. It wasn’t that she couldn’t imagine, but she desired it there
+from my lips. What she really desired of course was to know whether
+there was worse about him than what she had found out for herself. She
+hadn’t been a month so much in the house with him without discovering
+that he wasn’t a man of monumental bronze. He was like a jelly minus its
+mould, he had to be embanked; and that was precisely the source of her
+interest in him and the ground of her project. She put her project
+boldly before me: there it stood in its preposterous beauty. She was as
+willing to take the humorous view of it as I could be: the only
+difference was that for her the humorous view of a thing wasn’t
+necessarily prohibitive, wasn’t paralysing.
+
+Moreover she professed that she couldn’t discuss with me the primary
+question—the moral obligation: that was in her own breast. There were
+things she couldn’t go into—injunctions, impressions she had received.
+They were a part of the closest intimacy of her intercourse with her
+aunt, they were absolutely clear to her; and on questions of delicacy,
+the interpretation of a fidelity, of a promise, one had always in the
+last resort to make up one’s mind for one’s self. It was the idea of the
+application to the particular case, such a splendid one at last, that
+troubled her, and she admitted that it stirred very deep things. She
+didn’t pretend that such a responsibility was a simple matter; if it
+_had_ been she wouldn’t have attempted to saddle me with any portion of
+it. The Mulvilles were sympathy itself, but were they absolutely candid?
+Could they indeed be, in their position—would it even have been to be
+desired? Yes, she had sent for me to ask no less than that of me—whether
+there was anything dreadful kept back. She made no allusion whatever to
+George Gravener—I thought her silence the only good taste and her gaiety
+perhaps a part of the very anxiety of that discretion, the effect of a
+determination that people shouldn’t know from herself that her relations
+with the man she was to marry were strained. All the weight, however,
+that she left me to throw was a sufficient implication of the weight _he_
+had thrown in vain. Oh she knew the question of character was immense,
+and that one couldn’t entertain any plan for making merit comfortable
+without running the gauntlet of that terrible procession of
+interrogation-points which, like a young ladies’ school out for a walk,
+hooked their uniform noses at the tail of governess Conduct. But were we
+absolutely to hold that there was never, never, never an exception,
+never, never, never an occasion for liberal acceptance, for clever
+charity, for suspended pedantry—for letting one side, in short,
+outbalance another? When Miss Anvoy threw off this appeal I could have
+embraced her for so delightfully emphasising her unlikeness to Mrs.
+Saltram. “Why not have the courage of one’s forgiveness,” she asked, “as
+well as the enthusiasm of one’s adhesion?”
+
+“Seeing how wonderfully you’ve threshed the whole thing out,” I evasively
+replied, “gives me an extraordinary notion of the point your enthusiasm
+has reached.”
+
+She considered this remark an instant with her eyes on mine, and I
+divined that it struck her I might possibly intend it as a reference to
+some personal subjection to our fat philosopher, to some aberration of
+sensibility, some perversion of taste. At least I couldn’t interpret
+otherwise the sudden flash that came into her face. Such a
+manifestation, as the result of any word of mine, embarrassed me; but
+while I was thinking how to reassure her the flush passed away in a smile
+of exquisite good nature. “Oh you see one forgets so wonderfully how one
+dislikes him!” she said; and if her tone simply extinguished his strange
+figure with the brush of its compassion, it also rings in my ear to-day
+as the purest of all our praises. But with what quick response of fine
+pity such a relegation of the man himself made me privately sigh “Ah poor
+Saltram!” She instantly, with this, took the measure of all I didn’t
+believe, and it enabled her to go on: “What can one do when a person has
+given such a lift to one’s interest in life?”
+
+“Yes, what can one do?” If I struck her as a little vague it was because
+I was thinking of another person. I indulged in another inarticulate
+murmur—“Poor George Gravener!” What had become of the lift _he_ had
+given that interest? Later on I made up my mind that she was sore and
+stricken at the appearance he presented of wanting the miserable money.
+This was the hidden reason of her alienation. The probable sincerity, in
+spite of the illiberality, of his scruples about the particular use of it
+under discussion didn’t efface the ugliness of his demand that they
+should buy a good house with it. Then, as for _his_ alienation, he
+didn’t, pardonably enough, grasp the lift Frank Saltram had given her
+interest in life. If a mere spectator could ask that last question, with
+what rage in his heart the man himself might! He wasn’t, like her, I was
+to see, too proud to show me why he was disappointed.
+
+
+
+
+XI
+
+
+I WAS unable this time to stay to dinner: such at any rate was the plea
+on which I took leave. I desired in truth to get away from my young
+lady, for that obviously helped me not to pretend to satisfy her. How
+_could_ I satisfy her? I asked myself—how could I tell her how much had
+been kept back? I didn’t even know and I certainly didn’t desire to
+know. My own policy had ever been to learn the least about poor
+Saltram’s weaknesses—not to learn the most. A great deal that I had in
+fact learned had been forced upon me by his wife. There was something
+even irritating in Miss Anvoy’s crude conscientiousness, and I wondered
+why, after all, she couldn’t have let him alone and been content to
+entrust George Gravener with the purchase of the good house. I was sure
+he would have driven a bargain, got something excellent and cheap. I
+laughed louder even than she, I temporised, I failed her; I told her I
+must think over her case. I professed a horror of responsibilities and
+twitted her with her own extravagant passion for them. It wasn’t really
+that I was afraid of the scandal, the moral discredit for the Fund; what
+troubled me most was a feeling of a different order. Of course, as the
+beneficiary of the Fund was to enjoy a simple life-interest, as it was
+hoped that new beneficiaries would arise and come up to new standards, it
+wouldn’t be a trifle that the first of these worthies shouldn’t have been
+a striking example of the domestic virtues. The Fund would start badly,
+as it were, and the laurel would, in some respects at least, scarcely be
+greener from the brows of the original wearer. That idea, however, was
+at that hour, as I have hinted, not the source of solicitude it ought
+perhaps to have been, for I felt less the irregularity of Saltram’s
+getting the money than that of this exalted young woman’s giving it up.
+I wanted her to have it for herself, and I told her so before I went
+away. She looked graver at this than she had looked at all, saying she
+hoped such a preference wouldn’t make me dishonest.
+
+It made me, to begin with, very restless—made me, instead of going
+straight to the station, fidget a little about that many-coloured Common
+which gives Wimbledon horizons. There was a worry for me to work off, or
+rather keep at a distance, for I declined even to admit to myself that I
+had, in Miss Anvoy’s phrase, been saddled with it. What could have been
+clearer indeed than the attitude of recognising perfectly what a world of
+trouble The Coxon Fund would in future save us, and of yet liking better
+to face a continuance of that trouble than see, and in fact contribute
+to, a deviation from attainable bliss in the life of two other persons in
+whom I was deeply interested? Suddenly, at the end of twenty minutes,
+there was projected across this clearness the image of a massive
+middle-aged man seated on a bench under a tree, with sad far-wandering
+eyes and plump white hands folded on the head of a stick—a stick I
+recognised, a stout gold-headed staff that I had given him in devoted
+days. I stopped short as he turned his face to me, and it happened that
+for some reason or other I took in as I had perhaps never done before the
+beauty of his rich blank gaze. It was charged with experience as the sky
+is charged with light, and I felt on the instant as if we had been
+overspanned and conjoined by the great arch of a bridge or the great dome
+of a temple. Doubtless I was rendered peculiarly sensitive to it by
+something in the way I had been giving him up and sinking him. While I
+met it I stood there smitten, and I felt myself responding to it with a
+sort of guilty grimace. This brought back his attention in a smile which
+expressed for me a cheerful weary patience, a bruised noble gentleness.
+I had told Miss Anvoy that he had no dignity, but what did he seem to me,
+all unbuttoned and fatigued as he waited for me to come up, if he didn’t
+seem unconcerned with small things, didn’t seem in short majestic? There
+was majesty in his mere unconsciousness of our little conferences and
+puzzlements over his maintenance and his reward.
+
+After I had sat by him a few minutes I passed my arm over his big soft
+shoulder—wherever you touched him you found equally little firmness—and
+said in a tone of which the suppliance fell oddly on my own ear: “Come
+back to town with me, old friend—come back and spend the evening.” I
+wanted to hold him, I wanted to keep him, and at Waterloo, an hour later,
+I telegraphed possessively to the Mulvilles. When he objected, as
+regards staying all night, that he had no things, I asked him if he
+hadn’t everything of mine. I had abstained from ordering dinner, and it
+was too late for preliminaries at a club; so we were reduced to tea and
+fried fish at my rooms—reduced also to the transcendent. Something had
+come up which made me want him to feel at peace with me—and which,
+precisely, was all the dear man himself wanted on any occasion. I had
+too often had to press upon him considerations irrelevant, but it gives
+me pleasure now to think that on that particular evening I didn’t even
+mention Mrs. Saltram and the children. Late into the night we smoked and
+talked; old shames and old rigours fell away from us; I only let him see
+that I was conscious of what I owed him. He was as mild as contrition
+and as copious as faith; he was never so fine as on a shy return, and
+even better at forgiving than at being forgiven. I dare say it was a
+smaller matter than that famous night at Wimbledon, the night of the
+problematical sobriety and of Miss Anvoy’s initiation; but I was as much
+in it on this occasion as I had been out of it then. At about 1.30 he
+was sublime.
+
+He never, in whatever situation, rose till all other risings were over,
+and his breakfasts, at Wimbledon, had always been the principal reason
+mentioned by departing cooks. The coast was therefore clear for me to
+receive her when, early the next morning, to my surprise, it was
+announced to me his wife had called. I hesitated, after she had come up,
+about telling her Saltram was in the house, but she herself settled the
+question, kept me reticent by drawing forth a sealed letter which,
+looking at me very hard in the eyes, she placed, with a pregnant absence
+of comment, in my hand. For a single moment there glimmered before me
+the fond hope that Mrs. Saltram had tendered me, as it were, her
+resignation and desired to embody the act in an unsparing form. To bring
+this about I would have feigned any humiliation; but after my eyes had
+caught the superscription I heard myself say with a flatness that
+betrayed a sense of something very different from relief: “Oh the
+Pudneys!” I knew their envelopes though they didn’t know mine. They
+always used the kind sold at post-offices with the stamp affixed, and as
+this letter hadn’t been posted they had wasted a penny on me. I had seen
+their horrid missives to the Mulvilles, but hadn’t been in direct
+correspondence with them.
+
+“They enclosed it to me, to be delivered. They doubtless explain to you
+that they hadn’t your address.”
+
+I turned the thing over without opening it. “Why in the world should
+they write to me?”
+
+“Because they’ve something to tell you. The worst,” Mrs. Saltram dryly
+added.
+
+It was another chapter, I felt, of the history of their lamentable
+quarrel with her husband, the episode in which, vindictively,
+disingenuously as they themselves had behaved, one had to admit that he
+had put himself more grossly in the wrong than at any moment of his life.
+He had begun by insulting the matchless Mulvilles for these more specious
+protectors, and then, according to his wont at the end of a few months,
+had dug a still deeper ditch for his aberration than the chasm left
+yawning behind. The chasm at Wimbledon was now blessedly closed; but the
+Pudneys, across their persistent gulf, kept up the nastiest fire. I
+never doubted they had a strong case, and I had been from the first for
+not defending him—reasoning that if they weren’t contradicted they’d
+perhaps subside. This was above all what I wanted, and I so far
+prevailed that I did arrest the correspondence in time to save our little
+circle an infliction heavier than it perhaps would have borne. I knew,
+that is I divined, that their allegations had gone as yet only as far as
+their courage, conscious as they were in their own virtue of an exposed
+place in which Saltram could have planted a blow. It was a question with
+them whether a man who had himself so much to cover up would dare his
+blow; so that these vessels of rancour were in a manner afraid of each
+other. I judged that on the day the Pudneys should cease for some reason
+or other to be afraid they would treat us to some revelation more
+disconcerting than any of its predecessors. As I held Mrs. Saltram’s
+letter in my hand it was distinctly communicated to me that the day had
+come—they had ceased to be afraid. “I don’t want to know the worst,” I
+presently declared.
+
+“You’ll have to open the letter. It also contains an enclosure.”
+
+I felt it—it was fat and uncanny. “Wheels within wheels!” I exclaimed.
+“There’s something for me too to deliver.”
+
+“So they tell me—to Miss Anvoy.”
+
+I stared; I felt a certain thrill. “Why don’t they send it to her
+directly?”
+
+Mrs. Saltram hung fire. “Because she’s staying with Mr. and Mrs.
+Mulville.”
+
+“And why should that prevent?”
+
+Again my visitor faltered, and I began to reflect on the grotesque, the
+unconscious perversity of her action. I was the only person save George
+Gravener and the Mulvilles who was aware of Sir Gregory Coxon’s and of
+Miss Anvoy’s strange bounty. Where could there have been a more signal
+illustration of the clumsiness of human affairs than her having
+complacently selected this moment to fly in the face of it? “There’s the
+chance of their seeing her letters. They know Mr. Pudney’s hand.”
+
+Still I didn’t understand; then it flashed upon me. “You mean they might
+intercept it? How can you imply anything so base?” I indignantly
+demanded.
+
+“It’s not I—it’s Mr. Pudney!” cried Mrs. Saltram with a flush. “It’s his
+own idea.”
+
+“Then why couldn’t he send the letter to you to be delivered?”
+
+Mrs. Saltram’s embarrassment increased; she gave me another hard look.
+“You must make that out for yourself.”
+
+I made it out quickly enough. “It’s a denunciation?”
+
+“A real lady doesn’t betray her husband!” this virtuous woman exclaimed.
+
+I burst out laughing, and I fear my laugh may have had an effect of
+impertinence. “Especially to Miss Anvoy, who’s so easily shocked? Why do
+such things concern _her_?” I asked, much at a loss.
+
+“Because she’s there, exposed to all his craft. Mr. and Mrs. Pudney have
+been watching this: they feel she may be taken in.”
+
+“Thank you for all the rest of us! What difference can it make when she
+has lost her power to contribute?”
+
+Again Mrs. Saltram considered; then very nobly: “There are other things
+in the world than money.” This hadn’t occurred to her so long as the
+young lady had any; but she now added, with a glance at my letter, that
+Mr. and Mrs. Pudney doubtless explained their motives. “It’s all in
+kindness,” she continued as she got up.
+
+“Kindness to Miss Anvoy? You took, on the whole, another view of
+kindness before her reverses.”
+
+My companion smiled with some acidity “Perhaps you’re no safer than the
+Mulvilles!”
+
+I didn’t want her to think that, nor that she should report to the
+Pudneys that they had not been happy in their agent; and I well remember
+that this was the moment at which I began, with considerable emotion, to
+promise myself to enjoin upon Miss Anvoy never to open any letter that
+should come to her in one of those penny envelopes. My emotion, and I
+fear I must add my confusion, quickly deepened; I presently should have
+been as glad to frighten Mrs. Saltram as to think I might by some
+diplomacy restore the Pudneys to a quieter vigilance.
+
+“It’s best you should take _my_ view of my safety,” I at any rate soon
+responded. When I saw she didn’t know what I meant by this I added: “You
+may turn out to have done, in bringing me this letter, a thing you’ll
+profoundly regret.” My tone had a significance which, I could see, did
+make her uneasy, and there was a moment, after I had made two or three
+more remarks of studiously bewildering effect, at which her eyes followed
+so hungrily the little flourish of the letter with which I emphasised
+them that I instinctively slipped Mr. Pudney’s communication into my
+pocket. She looked, in her embarrassed annoyance, capable of grabbing it
+to send it back to him. I felt, after she had gone, as if I had almost
+given her my word I wouldn’t deliver the enclosure. The passionate
+movement, at any rate, with which, in solitude, I transferred the whole
+thing, unopened, from my pocket to a drawer which I double-locked would
+have amounted, for an initiated observer, to some such pledge.
+
+
+
+
+XII
+
+
+MRS. SALTRAM left me drawing my breath more quickly and indeed almost in
+pain—as if I had just perilously grazed the loss of something precious.
+I didn’t quite know what it was—it had a shocking resemblance to my
+honour. The emotion was the livelier surely in that my pulses even yet
+vibrated to the pleasure with which, the night before, I had rallied to
+the rare analyst, the great intellectual adventurer and pathfinder. What
+had dropped from me like a cumbersome garment as Saltram appeared before
+me in the afternoon on the heath was the disposition to haggle over his
+value. Hang it, one had to choose, one had to put that value somewhere;
+so I would put it really high and have done with it. Mrs. Mulville drove
+in for him at a discreet hour—the earliest she could suppose him to have
+got up; and I learned that Miss Anvoy would also have come had she not
+been expecting a visit from Mr. Gravener. I was perfectly mindful that I
+was under bonds to see this young lady, and also that I had a letter to
+hand to her; but I took my time, I waited from day to day. I left Mrs.
+Saltram to deal as her apprehensions should prompt with the Pudneys. I
+knew at last what I meant—I had ceased to wince at my responsibility. I
+gave this supreme impression of Saltram time to fade if it would; but it
+didn’t fade, and, individually, it hasn’t faded even now. During the
+month that I thus invited myself to stiffen again, Adelaide Mulville,
+perplexed by my absence, wrote to me to ask why I _was_ so stiff. At
+that season of the year I was usually oftener “with” them. She also
+wrote that she feared a real estrangement had set in between Mr. Gravener
+and her sweet young friend—a state of things but half satisfactory to her
+so long as the advantage resulting to Mr. Saltram failed to disengage
+itself from the merely nebulous state. She intimated that her sweet
+young friend was, if anything, a trifle too reserved; she also intimated
+that there might now be an opening for another clever young man. There
+never was the slightest opening, I may here parenthesise, and of course
+the question can’t come up to-day. These are old frustrations now. Ruth
+Anvoy hasn’t married, I hear, and neither have I. During the month,
+toward the end, I wrote to George Gravener to ask if, on a special
+errand, I might come to see him, and his answer was to knock the very
+next day at my door. I saw he had immediately connected my enquiry with
+the talk we had had in the railway-carriage, and his promptitude showed
+that the ashes of his eagerness weren’t yet cold. I told him there was
+something I felt I ought in candour to let him know—I recognised the
+obligation his friendly confidence had laid on me.
+
+“You mean Miss Anvoy has talked to you? She has told me so herself,” he
+said.
+
+“It wasn’t to tell you so that I wanted to see you,” I replied; “for it
+seemed to me that such a communication would rest wholly with herself.
+If however she did speak to you of our conversation she probably told you
+I was discouraging.”
+
+“Discouraging?”
+
+“On the subject of a present application of The Coxon Fund.”
+
+“To the case of Mr. Saltram? My dear fellow, I don’t know what you call
+discouraging!” Gravener cried.
+
+“Well I thought I was, and I thought she thought I was.”
+
+“I believe she did, but such a thing’s measured by the effect. She’s not
+‘discouraged,’” he said.
+
+“That’s her own affair. The reason I asked you to see me was that it
+appeared to me I ought to tell you frankly that—decidedly!—I can’t
+undertake to produce that effect. In fact I don’t want to!”
+
+“It’s very good of you, damn you!” my visitor laughed, red and really
+grave. Then he said: “You’d like to see that scoundrel publicly
+glorified—perched on the pedestal of a great complimentary pension?”
+
+I braced myself. “Taking one form of public recognition with another it
+seems to me on the whole I should be able to bear it. When I see the
+compliments that _are_ paid right and left I ask myself why this one
+shouldn’t take its course. This therefore is what you’re entitled to
+have looked to me to mention to you. I’ve some evidence that perhaps
+would be really dissuasive, but I propose to invite Mss Anvoy to remain
+in ignorance of it.”
+
+“And to invite me to do the same?”
+
+“Oh you don’t require it—you’ve evidence enough. I speak of a sealed
+letter that I’ve been requested to deliver to her.”
+
+“And you don’t mean to?”
+
+“There’s only one consideration that would make me,” I said.
+
+Gravener’s clear handsome eyes plunged into mine a minute, but evidently
+without fishing up a clue to this motive—a failure by which I was almost
+wounded. “What does the letter contain?”
+
+“It’s sealed, as I tell you, and I don’t know what it contains.”
+
+“Why is it sent through you?”
+
+“Rather than you?” I wondered how to put the thing. “The only
+explanation I can think of is that the person sending it may have
+imagined your relations with Miss Anvoy to be at an end—may have been
+told this is the case by Mrs. Saltram.”
+
+“My relations with Miss Anvoy are not at an end,” poor Gravener
+stammered.
+
+Again for an instant I thought. “The offer I propose to make you gives
+me the right to address you a question remarkably direct. Are you still
+engaged to Miss Anvoy?”
+
+“No, I’m not,” he slowly brought out. “But we’re perfectly good
+friends.”
+
+“Such good friends that you’ll again become prospective husband and wife
+if the obstacle in your path be removed?”
+
+“Removed?” he anxiously repeated.
+
+“If I send Miss Anvoy the letter I speak of she may give up her idea.”
+
+“Then for God’s sake send it!”
+
+“I’ll do so if you’re ready to assure me that her sacrifice would now
+presumably bring about your marriage.”
+
+“I’d marry her the next day!” my visitor cried.
+
+“Yes, but would she marry _you_? What I ask of you of course is nothing
+less than your word of honour as to your conviction of this. If you give
+it me,” I said, “I’ll engage to hand her the letter before night.”
+
+Gravener took up his hat; turning it mechanically round he stood looking
+a moment hard at its unruffled perfection. Then very angrily honestly
+and gallantly, “Hand it to the devil!” he broke out; with which he
+clapped the hat on his head and left me.
+
+“Will you read it or not?” I said to Ruth Anvoy, at Wimbledon, when I had
+told her the story of Mrs. Saltram’s visit.
+
+She debated for a time probably of the briefest, but long enough to make
+me nervous. “Have you brought it with you?”
+
+“No indeed. It’s at home, locked up.”
+
+There was another great silence, and then she said “Go back and destroy
+it.”
+
+I went back, but I didn’t destroy it till after Saltram’s death, when I
+burnt it unread. The Pudneys approached her again pressingly, but,
+prompt as they were, The Coxon Fund had already become an operative
+benefit and a general amaze: Mr. Saltram, while we gathered about, as it
+were, to watch the manna descend, had begun to draw the magnificent
+income. He drew it as he had always drawn everything, with a grand
+abstracted gesture. Its magnificence, alas, as all the world now knows,
+quite quenched him; it was the beginning of his decline. It was also
+naturally a new grievance for his wife, who began to believe in him as
+soon as he was blighted, and who at this hour accuses us of having bribed
+him, on the whim of a meddlesome American, to renounce his glorious
+office, to become, as she says, like everybody else. The very day he
+found himself able to publish he wholly ceased to produce. This deprived
+us, as may easily be imagined, of much of our occupation, and especially
+deprived the Mulvilles, whose want of self-support I never measured till
+they lost their great inmate. They’ve no one to live on now. Adelaide’s
+most frequent reference to their destitution is embodied in the remark
+that dear far-away Ruth’s intentions were doubtless good. She and Kent
+are even yet looking for another prop, but no one presents a true sphere
+of usefulness. They complain that people are self-sufficing. With
+Saltram the fine type of the child of adoption was scattered, the
+grander, the elder style. They’ve got their carriage back, but what’s an
+empty carriage? In short I think we were all happier as well as poorer
+before; even including George Gravener, who by the deaths of his brother
+and his nephew has lately become Lord Maddock. His wife, whose fortune
+clears the property, is criminally dull; he hates being in the Upper
+House, and hasn’t yet had high office. But what are these accidents,
+which I should perhaps apologise for mentioning, in the light of the
+great eventual boon promised the patient by the rate at which The Coxon
+Fund must be rolling up?
+
+Now, answer the question based on the story asconcisely as you can, using a single phrase if possible. Do not provide any explanation.
+
+Question: What is Saltram's living situation?
+
+Answer:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
diff --git a/LeanK/run_example.py b/LeanK/run_example.py
new file mode 100644
index 00000000..93c8495e
--- /dev/null
+++ b/LeanK/run_example.py
@@ -0,0 +1,30 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from minference import MInference
+
+prompt = open("narrativeqa_example.txt").read()
+
+# model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+model_name = "Qwen/Qwen2.5-7B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map="cuda",
+ _attn_implementation="flash_attention_2",
+)
+
+# Patch model with dense prefill and leank decoding
+minference_patch = MInference(attn_type="dense", model_name=model_name, kv_type="leank")
+model = minference_patch(model)
+
+batch_inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+print("Input sequence length", batch_inputs.input_ids.shape[1])
+outputs = model.generate(**batch_inputs, max_new_tokens=512)
+generated_text = tokenizer.decode(
+ outputs[0][batch_inputs["input_ids"].shape[1] :], skip_special_tokens=True
+)
+print(f"Generated text: {generated_text!r}")
diff --git a/LeanK/scripts/llama3.1-8b.yaml b/LeanK/scripts/llama3.1-8b.yaml
new file mode 100644
index 00000000..761a91f7
--- /dev/null
+++ b/LeanK/scripts/llama3.1-8b.yaml
@@ -0,0 +1,28 @@
+training:
+ stage1:
+ model_name: meta-llama/Meta-Llama-3.1-8B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 96000
+ reg_weight: 0.06
+ lr: 0.02
+ dataset_format: passkey_retrival_mixed_tasks
+ steps: 2000
+ context_lengths_num_intervals: 100
+ output_dir: patterns/llama/stage1-lr=0.02-reg=0.06-ctx=4000_96000-data=passkey_retrival_mixed_tasks
+ exp_name: llama-stage1-lr=0.02-reg=0.06-ctx=4000_96000-data=passkey_retrival_mixed_tasks
+ stage2:
+ model_name: meta-llama/Meta-Llama-3.1-8B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 96000
+ reg_weight: 0.
+ lr: 0.01
+ dataset_format: passkey_retrival_mixed_tasks
+ steps: 200
+ context_lengths_num_intervals: 100
+ align: 32
+ ratio: 0.7
+ stage1_rst_path: patterns/llama/lr=0.02-reg=0.06-ctx=4000_96000-data=passkey_retrival_mixed_tasks
+ output_dir: patterns/llama/stage2-lr=0.01-reg=0-ctx=4000_96000-data=passkey_retrival_mixed_tasks
+ exp_name: llama-stage2-lr=0.01-reg=0-ctx=4000_96000-data=passkey_retrival_mixed_tasks-align=32-ratio=0.7
diff --git a/LeanK/scripts/qwen-duo-attn-task.yaml b/LeanK/scripts/qwen-duo-attn-task.yaml
new file mode 100644
index 00000000..2ea1ec9d
--- /dev/null
+++ b/LeanK/scripts/qwen-duo-attn-task.yaml
@@ -0,0 +1,28 @@
+training:
+ stage1:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.2
+ lr: 0.08
+ dataset_format: duo_data
+ steps: 2000
+ context_lengths_num_intervals: 100
+ output_dir: patterns/qwen/stage1-lr=0.08-reg=0.2-ctx=4000_32000-data=duo_data
+ exp_name: qwen-stage1-lr=0.08-reg=0.2-ctx=4000_32000-data=duo_data
+ stage2:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.
+ lr: 0.02
+ dataset_format: duo_data
+ steps: 200
+ context_lengths_num_intervals: 100
+ align: 32
+ ratio: 0.6
+ stage1_rst_path: patterns/qwen/stage1-lr=0.08-reg=0.2-ctx=4000_32000-data=duo_data
+ output_dir: patterns/qwen/stage2-lr=0.02-reg=0-ctx=4000_32000-data=duo_data
+ exp_name: qwen-stage2-lr=0.02-reg=0-ctx=4000_32000-data=duo_data-align=32-ratio=0.6
diff --git a/LeanK/scripts/qwen-task1-only.yaml b/LeanK/scripts/qwen-task1-only.yaml
new file mode 100644
index 00000000..274693dc
--- /dev/null
+++ b/LeanK/scripts/qwen-task1-only.yaml
@@ -0,0 +1,28 @@
+training:
+ stage1:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.06
+ lr: 0.04
+ dataset_format: multi_key_retrival
+ steps: 2000
+ context_lengths_num_intervals: 100
+ output_dir: patterns/qwen/stage1-lr=0.04-reg=0.06-ctx=4000_32000-data=multi_key_retrival
+ exp_name: qwen-stage1-lr=0.04-reg=0.06-ctx=4000_32000-data=multi_key_retrival
+ stage2:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.
+ lr: 0.02
+ dataset_format: multi_key_retrival
+ steps: 200
+ context_lengths_num_intervals: 100
+ align: 32
+ ratio: 0.6
+ stage1_rst_path: patterns/qwen/stage1-lr=0.04-reg=0.06-ctx=4000_32000-data=multi_key_retrival
+ output_dir: patterns/qwen/stage2-lr=0.02-reg=0-ctx=4000_32000-data=multi_key_retrival
+ exp_name: qwen-stage2-lr=0.02-reg=0-ctx=4000_32000-data=multi_key_retrival-align=32-ratio=0.6
diff --git a/LeanK/scripts/qwen-task2-only.yaml b/LeanK/scripts/qwen-task2-only.yaml
new file mode 100644
index 00000000..2573234e
--- /dev/null
+++ b/LeanK/scripts/qwen-task2-only.yaml
@@ -0,0 +1,28 @@
+training:
+ stage1:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.08
+ lr: 0.04
+ dataset_format: multi_val_retrival
+ steps: 2000
+ context_lengths_num_intervals: 100
+ output_dir: patterns/qwen/stage1-lr=0.04-reg=0.08-ctx=4000_32000-data=multi_val_retrival
+ exp_name: qwen-stage1-lr=0.04-reg=0.08-ctx=4000_32000-data=multi_val_retrival
+ stage2:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.
+ lr: 0.02
+ dataset_format: multi_val_retrival
+ steps: 200
+ context_lengths_num_intervals: 100
+ align: 32
+ ratio: 0.6
+ stage1_rst_path: patterns/qwen/stage1-lr=0.04-reg=0.08-ctx=4000_32000-data=multi_val_retrival
+ output_dir: patterns/qwen/stage2-lr=0.02-reg=0-ctx=4000_32000-data=multi_val_retrival
+ exp_name: qwen-stage2-lr=0.02-reg=0-ctx=4000_32000-data=multi_val_retrival-align=32-ratio=0.6
diff --git a/LeanK/scripts/qwen2.5-7b-yarn.yaml b/LeanK/scripts/qwen2.5-7b-yarn.yaml
new file mode 100644
index 00000000..be436c1c
--- /dev/null
+++ b/LeanK/scripts/qwen2.5-7b-yarn.yaml
@@ -0,0 +1,28 @@
+training:
+ stage1:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 64000
+ context_length_max: 96000
+ reg_weight: 0.06
+ lr: 0.04
+ dataset_format: passkey_retrival_mixed_tasks
+ steps: 2000
+ context_lengths_num_intervals: 100
+ output_dir: patterns/qwen_yarn/stage1-lr=0.04-reg=0.06-ctx=64000_96000-data=passkey_retrival_mixed_tasks
+ exp_name: qwen_yarn-stage1-lr=0.02-reg=0.06-ctx=64000_96000-data=passkey_retrival_mixed_tasks
+ stage2:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.
+ lr: 0.02
+ dataset_format: passkey_retrival_mixed_tasks
+ steps: 200
+ context_lengths_num_intervals: 100
+ align: 32
+ ratio: 0.6
+ stage1_rst_path: patterns/qwen_yarn/stage1-lr=0.04-reg=0.06-ctx=64000_96000-data=passkey_retrival_mixed_tasks
+ output_dir: patterns/qwen_yarn/stage2-lr=0.02-reg=0-ctx=64000_96000-data=passkey_retrival_mixed_tasks
+ exp_name: qwen_yarn-stage2-lr=0.02-reg=0-ctx=64000_96000-data=passkey_retrival_mixed_tasks-align=32-ratio=0.7
diff --git a/LeanK/scripts/qwen2.5-7b.yaml b/LeanK/scripts/qwen2.5-7b.yaml
new file mode 100644
index 00000000..ba95a6b2
--- /dev/null
+++ b/LeanK/scripts/qwen2.5-7b.yaml
@@ -0,0 +1,28 @@
+training:
+ stage1:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.06
+ lr: 0.04
+ dataset_format: passkey_retrival_mixed_tasks
+ steps: 2000
+ context_lengths_num_intervals: 100
+ output_dir: patterns/qwen/stage1-lr=0.04-reg=0.06-ctx=4000_32000-data=passkey_retrival_mixed_tasks
+ exp_name: qwen-stage1-lr=0.04-reg=0.06-ctx=4000_32000-data=passkey_retrival_mixed_tasks
+ stage2:
+ model_name: Qwen/Qwen2.5-7B-Instruct
+ batch_size: 1
+ context_length_min: 4000
+ context_length_max: 32000
+ reg_weight: 0.
+ lr: 0.02
+ dataset_format: passkey_retrival_mixed_tasks
+ steps: 200
+ context_lengths_num_intervals: 100
+ align: 32
+ ratio: 0.6
+ stage1_rst_path: patterns/qwen/stage1-lr=0.04-reg=0.06-ctx=4000_32000-data=passkey_retrival_mixed_tasks
+ output_dir: patterns/qwen/stage2-lr=0.02-reg=0-ctx=4000_32000-data=passkey_retrival_mixed_tasks
+ exp_name: qwen-stage2-lr=0.02-reg=0-ctx=4000_32000-data=passkey_retrival_mixed_tasks-align=32-ratio=0.6
diff --git a/LeanK/train.py b/LeanK/train.py
new file mode 100644
index 00000000..c1433eb6
--- /dev/null
+++ b/LeanK/train.py
@@ -0,0 +1,516 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import json
+import os
+import types
+
+import matplotlib.pyplot as plt
+import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+import wandb
+import yaml
+from leank.data import (
+ MultiplePasskeyRetrievalDataset,
+ PasskeyRetrievalDataset,
+ get_dataset,
+ get_supervised_dataloader,
+)
+from leank.loss import l1_loss
+from leank.patch import (
+ enable_training,
+ full_attn_forward_llama,
+ full_attn_forward_qwen,
+ get_scaling_factors,
+ load_scaling_factors,
+ map_scaling_factors,
+ scaled_attn_forward_llama,
+ scaled_attn_forward_qwen,
+)
+from leank.utils import (
+ convert_to_list,
+ get_tokenizer,
+ parse_args,
+ save_scaling_factors,
+ seed_everything,
+ sparsify_scaling_factors,
+ visualize_patterns,
+)
+from torch.distributed._composable.fsdp import MixedPrecisionPolicy, fully_shard
+from torch.distributed._tensor import DeviceMesh
+from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import (
+ apply_activation_checkpointing,
+)
+from tqdm import tqdm
+from transformers import AutoConfig, AutoModelForCausalLM
+from transformers.models.llama.modeling_llama import LlamaAttention, LlamaDecoderLayer
+from transformers.models.qwen2.modeling_qwen2 import Qwen2Attention, Qwen2DecoderLayer
+
+
+def setup():
+ # initialize the process group
+ dist.init_process_group("nccl")
+
+
+def cleanup():
+ dist.destroy_process_group()
+
+
+def apply_fsdp(model: torch.nn.Module, mesh, mp_policy, modules_to_shard):
+ """
+ Apply data parallelism to the model. FSDP2 is used here.
+ """
+ fsdp_config = {"mp_policy": mp_policy, "mesh": mesh, "reshard_after_forward": True}
+
+ for module in model.modules():
+ if any([isinstance(module, m) for m in modules_to_shard]):
+ fully_shard(module, **fsdp_config)
+ fully_shard(model, **fsdp_config)
+
+
+def train(
+ args, model, rank, world_size, train_dataloader, optimizer, scheduler, resume_step
+):
+ model.train()
+
+ if rank == 0:
+ pbar = tqdm(range(args.num_steps))
+
+ local_rank = int(os.environ["LOCAL_RANK"])
+
+ global_step = 0
+ local_step = 0
+
+ while True:
+ if global_step >= args.num_steps:
+ break
+ for step, batch in enumerate(train_dataloader):
+ if global_step <= resume_step:
+ global_step += 1
+ if rank == 0:
+ pbar.update(1)
+ pbar.set_description(
+ f"Skipping step {global_step} to resume to {resume_step}"
+ )
+ continue
+
+ @torch.no_grad()
+ def clamp_(x, min_val, max_val):
+ x.clamp_(min_val, max_val)
+
+ map_scaling_factors(model, func=lambda x: clamp_(x, 0, 1))
+
+ length_context = batch["length_context"]
+ batch.pop("length_context")
+
+ batch = {k: v.to(f"cuda:{local_rank}") for k, v in batch.items()}
+
+ input_ids = batch["input_ids"]
+
+ n_seq = (input_ids.shape[-1] + world_size - 1) // world_size
+ pad_len = n_seq * world_size - input_ids.shape[-1]
+ input_ids = torch.cat(
+ (
+ torch.Tensor([0] * pad_len)
+ .unsqueeze(0)
+ .to(input_ids.device)
+ .to(input_ids.dtype),
+ input_ids,
+ ),
+ dim=-1,
+ )
+ sample_len = input_ids.shape[-1]
+
+ seq_parallel_chunk_start = n_seq * rank
+ seq_parallel_chunk_end = seq_parallel_chunk_start + n_seq
+ position_ids = torch.arange(
+ seq_parallel_chunk_start,
+ seq_parallel_chunk_end,
+ device=input_ids.device,
+ ).unsqueeze(0)
+
+ scaling_factors_list = []
+
+ for layer in model.layers:
+ module = layer.self_attn
+ if isinstance(module._checkpoint_wrapped_module, LlamaAttention):
+ module.forward = types.MethodType(full_attn_forward_llama, module)
+ elif isinstance(module._checkpoint_wrapped_module, Qwen2Attention):
+ module.forward = types.MethodType(full_attn_forward_qwen, module)
+ else:
+ assert False
+ scaling_factors_list.append(
+ module.scaling_factors.full_tensor().to(model.device)
+ )
+
+ with torch.no_grad():
+ outputs = model(
+ input_ids=input_ids[
+ :, seq_parallel_chunk_start:seq_parallel_chunk_end
+ ],
+ position_ids=position_ids,
+ length_context=length_context[0] + pad_len,
+ )
+ if args.stage2:
+ mask_round = sparsify_scaling_factors(
+ scaling_factors_list, args.ratio, args.align
+ )
+
+ original_hidden_states = outputs[0]
+
+ for i, layer in enumerate(model.layers):
+ module = layer.self_attn
+ if isinstance(module._checkpoint_wrapped_module, LlamaAttention):
+ module.forward = types.MethodType(scaled_attn_forward_llama, module)
+ elif isinstance(module._checkpoint_wrapped_module, Qwen2Attention):
+ module.forward = types.MethodType(scaled_attn_forward_qwen, module)
+ else:
+ assert False
+ module.mask_round = mask_round[i] if args.stage2 else None
+
+ outputs = model(
+ input_ids=input_ids[:, seq_parallel_chunk_start:seq_parallel_chunk_end],
+ position_ids=position_ids,
+ length_context=length_context[0] + pad_len,
+ )
+ pruned_hidden_states = outputs[0]
+
+ labels = batch["labels"]
+ labels = torch.cat(
+ (
+ torch.Tensor([-100] * pad_len)
+ .unsqueeze(0)
+ .to(labels.device)
+ .to(labels.dtype),
+ labels,
+ ),
+ dim=-1,
+ )
+ labels = labels[:, seq_parallel_chunk_start:seq_parallel_chunk_end]
+ label_mask = labels != -100
+ num_labels = label_mask.sum()
+ global_num_labels = num_labels.clone().detach()
+ dist.all_reduce(global_num_labels)
+
+ # filter out label == IGNORE_INDEX (-100)
+ original_hidden_states = original_hidden_states[label_mask].float()
+ pruned_hidden_states = pruned_hidden_states[label_mask].float()
+
+ distill_loss = (
+ (original_hidden_states - pruned_hidden_states)
+ .pow(2)
+ .mean(dim=-1)
+ .sum()
+ * world_size
+ / global_num_labels
+ )
+
+ scaling_factors = get_scaling_factors(model)
+ scaling_factors = [
+ h.full_tensor().to(model.device) for h in scaling_factors
+ ]
+
+ reg_loss = l1_loss(torch.cat(scaling_factors).float())
+
+ loss = distill_loss + args.reg_weight * reg_loss
+
+ dist.all_reduce(loss, op=dist.ReduceOp.AVG)
+ dist.all_reduce(distill_loss, op=dist.ReduceOp.AVG)
+ dist.all_reduce(reg_loss, op=dist.ReduceOp.AVG)
+
+ loss.backward()
+
+ local_step = (local_step + 1) % args.gradient_accumulation_steps
+
+ if local_step != 0:
+ continue
+
+ optimizer.step()
+ scheduler.step()
+ optimizer.zero_grad()
+
+ global_step += 1
+ if rank == 0:
+ scaling_factors_list = convert_to_list(scaling_factors)
+
+ if not args.disable_wandb:
+ fig = visualize_patterns(scaling_factors_list)
+
+ wandb.log(
+ {
+ "distill_loss": distill_loss.item(),
+ "reg_loss": reg_loss.item(),
+ "loss": loss.item(),
+ "attn_heads": fig,
+ "step": global_step,
+ "sample_len": sample_len,
+ "lr": optimizer.param_groups[0]["lr"],
+ },
+ step=global_step,
+ )
+
+ plt.close(fig)
+ del fig
+
+ pbar.set_description(
+ f"Len={sample_len}/{global_num_labels}|Dloss={distill_loss.item():.3f}|Rloss={reg_loss.item():.3f}|Loss={loss.item(): 3f}|LR={optimizer.param_groups[0]['lr']:.2e}"
+ )
+ pbar.update(1)
+
+ if args.output_dir is not None and global_step % args.save_steps == 0:
+ if rank == 0:
+ save_scaling_factors(
+ scaling_factors_list,
+ os.path.join(
+ args.output_dir,
+ f"scaling_factors_step={global_step}.tsv",
+ ),
+ )
+ os.system(f"rm {args.output_dir}/scaling_factors_latest.tsv")
+ os.system(
+ f"cp {args.output_dir}/scaling_factors_step={global_step}.tsv {args.output_dir}/scaling_factors_latest.tsv"
+ )
+
+ # save scheduler and optimizer state
+ torch.save(
+ {
+ "optimizer": optimizer.state_dict(),
+ "scheduler": scheduler.state_dict(),
+ "global_step": global_step,
+ },
+ os.path.join(
+ args.output_dir,
+ f"optimizer_scheduler_state-step={global_step}-rank={rank}.pt",
+ ),
+ )
+
+ # copy the scaling_factors and optimizer_scheduler_state to the latest state, replacing the old one
+ # remove the previous latest state
+ os.system(
+ f"rm {args.output_dir}/optimizer_scheduler_state_latest-rank={rank}.pt"
+ )
+ os.system(
+ f"cp {args.output_dir}/optimizer_scheduler_state-step={global_step}-rank={rank}.pt {args.output_dir}/optimizer_scheduler_state_latest-rank={rank}.pt"
+ )
+
+ if global_step >= args.num_steps:
+ break
+
+ torch.cuda.empty_cache()
+
+ if rank == 0:
+ pbar.close()
+
+
+def main(args):
+ local_rank = int(os.environ["LOCAL_RANK"])
+ rank = int(os.environ["RANK"])
+ world_size = int(os.environ["WORLD_SIZE"])
+
+ if rank == 0:
+ if args.output_dir is not None:
+ os.makedirs(args.output_dir, exist_ok=True)
+
+ tokenizer = get_tokenizer(args.model_name)
+
+ if args.config_name is not None:
+ config = AutoConfig.from_pretrained(args.config_name)
+ else:
+ config = AutoConfig.from_pretrained(args.model_name)
+
+ if args.rope_theta is not None:
+ print(f"Setting rope_theta from {config.rope_theta} to {args.rope_theta}")
+ config.rope_theta = args.rope_theta
+
+ model = AutoModelForCausalLM.from_pretrained(
+ args.model_name,
+ config=config,
+ torch_dtype="auto",
+ low_cpu_mem_usage=False,
+ attn_implementation="flash_attention_2",
+ )
+
+ if args.resume and os.path.exists(
+ os.path.join(
+ args.output_dir, f"optimizer_scheduler_state_latest-rank={rank}.pt"
+ )
+ ):
+ # load the latest state in the output_dir
+ state = torch.load(
+ os.path.join(
+ args.output_dir, f"optimizer_scheduler_state_latest-rank={rank}.pt"
+ )
+ )
+ resume_step = state["global_step"]
+ scaling_factors = load_scaling_factors(
+ args.output_dir, filename=f"scaling_factors_step={resume_step}.tsv"
+ )
+ else:
+ resume_step = -1
+ scaling_factors = None
+
+ if args.stage2:
+ scaling_factors = load_scaling_factors(
+ args.stage1_rst_path, filename=f"scaling_factors.tsv"
+ )
+ else:
+ scaling_factors = None
+
+ enable_training(
+ model,
+ args.sink_size,
+ args.recent_size,
+ initial_value=args.initial_value,
+ enable_ulysses_attention=True,
+ scaling_factors=scaling_factors,
+ )
+
+ model = model.model
+
+ for param in model.parameters():
+ param.requires_grad = False
+
+ num_attn_heads = 0
+ for name, param in model.named_parameters():
+ if "scaling_factors" in name:
+ param.requires_grad = True
+ num_attn_heads += param.numel()
+
+ setup()
+
+ torch.cuda.set_device(local_rank)
+ mp_policy = MixedPrecisionPolicy(
+ param_dtype=torch.bfloat16,
+ reduce_dtype=torch.bfloat16,
+ )
+
+ apply_activation_checkpointing(model)
+
+ mesh = DeviceMesh(device_type="cuda", mesh=[i for i in range(world_size)])
+
+ apply_fsdp(
+ model,
+ mesh,
+ mp_policy,
+ modules_to_shard={LlamaDecoderLayer, Qwen2DecoderLayer},
+ )
+
+ if rank == 0:
+ print(model)
+ for name, param in model.named_parameters():
+ if param.requires_grad:
+ print(
+ f"Trainable parameter: {name} with shape {param.shape}, dtype {param.dtype}, device {param.device}"
+ )
+
+ if args.dataset_format == "passkey_retrival_mixed_tasks":
+ train_dataset = PasskeyRetrievalDataset(
+ tokenizer,
+ context_length_min=args.context_length_min,
+ context_length_max=args.context_length_max,
+ )
+ elif args.dataset_format == "multi_key_retrival":
+ train_dataset = PasskeyRetrievalDataset(
+ tokenizer,
+ context_length_min=args.context_length_min,
+ context_length_max=args.context_length_max,
+ type_haystack=["needle"],
+ )
+ elif args.dataset_format == "multi_val_retrival":
+ train_dataset = PasskeyRetrievalDataset(
+ tokenizer,
+ context_length_min=args.context_length_min,
+ context_length_max=args.context_length_max,
+ type_haystack=["essay"],
+ )
+ elif args.dataset_format == "duo_data":
+ haystack_dataset = get_dataset("leank/data/booksum.jsonl.zst", split="train")
+ train_dataset = MultiplePasskeyRetrievalDataset(
+ haystack_dataset,
+ tokenizer,
+ context_length_min=args.context_length_min,
+ context_length_max=args.context_length_max,
+ )
+ else:
+ raise ValueError(f"Invalid dataset format: {args.dataset_format}")
+
+ train_dataloader = get_supervised_dataloader(
+ train_dataset, tokenizer, args.batch_size, shuffle=True
+ )
+
+ optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0)
+
+ scheduler = torch.optim.lr_scheduler.LambdaLR(
+ optimizer,
+ lr_lambda=lambda step: min(
+ 1,
+ max((step + 1) / (args.num_steps // 5), 0.1),
+ max((args.num_steps - step) / (args.num_steps // 5), 0.1),
+ ),
+ )
+
+ if rank == 0:
+ experiment_config = vars(args)
+ if not args.disable_wandb:
+ wandb.init(project="LeanK", config=experiment_config)
+ if args.exp_name is not None:
+ wandb.run.name = args.exp_name
+
+ if args.output_dir is not None:
+ with open(os.path.join(args.output_dir, "config.json"), "w") as f:
+ json.dump(experiment_config, f)
+
+ train(
+ args,
+ model,
+ rank,
+ world_size,
+ train_dataloader,
+ optimizer,
+ scheduler,
+ resume_step,
+ )
+
+ scaling_factors = get_scaling_factors(model)
+ scaling_factors = [h.full_tensor() for h in scaling_factors]
+
+ if rank == 0:
+ print("Training finished")
+ if args.output_dir is not None:
+ if args.stage2:
+ with torch.no_grad():
+ mask_final = sparsify_scaling_factors(
+ scaling_factors, args.ratio, args.align
+ )
+ torch.save(
+ mask_final,
+ os.path.join(
+ args.output_dir,
+ f"mask_ratio{args.ratio}_align{args.align}.pth",
+ ),
+ )
+
+ scaling_factors_list = convert_to_list(scaling_factors)
+ save_scaling_factors(
+ scaling_factors_list,
+ os.path.join(args.output_dir, "scaling_factors.tsv"),
+ )
+
+ dist.barrier()
+ cleanup()
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ with open(args.config, "r") as f:
+ config = yaml.safe_load(f)
+ if args.stage2:
+ config = config["training"]["stage2"]
+ else:
+ config = config["training"]["stage1"]
+ for key, value in config.items():
+ if hasattr(args, key):
+ setattr(args, key, value)
+ seed_everything(args.seed)
+ main(args)
diff --git a/README.md b/README.md
index 0e144638..701feb2e 100644
--- a/README.md
+++ b/README.md
@@ -10,19 +10,32 @@
| Project Page |
Paper |
HF Demo |
+ SCBench |
+ MMInference |
https://github.com/microsoft/MInference/assets/30883354/52613efc-738f-4081-8367-7123c81d6b19
_Now, you can process **1M context 10x faster in a single A100** using Long-context LLMs like LLaMA-3-8B-1M, GLM-4-1M, with even **better accuracy**, try **MInference 1.0** right now!_
-## News
-- 🧤 [24/09/26] MInference has been accepted as **spotlight** at **NeurIPS'24**. See you in Vancouver!
-- 👘 [24/09/16] We are pleased to announce the release of our KV cache offloading work, [RetrievalAttention](https://aka.ms/RetrievalAttention), which accelerates long-context LLM inference via vector retrieval.
-- 🥤 [24/07/24] MInference support [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) now.
-- 🪗 [24/07/07] Thanks @AK for sponsoring. You can now use MInference online in the [HF Demo](https://huggingface.co/spaces/microsoft/MInference) with ZeroGPU.
-- 📃 [24/07/03] Due to an issue with arXiv, the PDF is currently unavailable there. You can find the paper at this [link](https://export.arxiv.org/pdf/2407.02490).
-- 🧩 [24/07/03] We will present **MInference 1.0** at the _**Microsoft Booth**_ and _**ES-FoMo**_ at ICML'24. See you in Vienna!
+## 📰 News
+- 🐝 [25/05/02] MMInference has been accepted at **ICML'25**.
+- 👨💻 [25/04/14] [SGLang](https://github.com/sgl-project/sglang/pull/5327) and [vLLM](https://github.com/vllm-project/flash-attention/pull/33) have merged the MInference sparse attention kernel. _MInference already supports the optimized kernels._ Just try `pip install sglang`. You can achieve up to **1.64× (64K), 2.4× (96K), 2.9× (128K), 5.2× (256K), 8× (512K), and 15× (1M)** speedup. Notably, SGLang also adapted it for FlashAttention-3. Special thanks to @zhyncs and @yinfan98 for their contributions!
+- 👾 [25/04/23] We are excited to announce the release of our multi-modality work, [MMInference](https://aka.ms/2504.16083), which use **modality-aware permutation sparse attention** to accelerate long-context VLMs. We'll present MMInference at **Microsoft Booth** and **FW-Wild at ICLR'25**. See you in Singapore!
+- 🤗 [25/01/27] MInference has been integrated into [Qwen2.5-1M](https://qwenlm.github.io/blog/qwen2.5-1m/) and online services. For details, refer to the [paper](https://arxiv.org/abs/2501.15383) and the [vLLM implementation](https://github.com/vllm-project/vllm/pull/11844).
+- 🪸 [25/01/23] SCBench has been accepted at **ICLR'25**.
+
+More News
+
+ - 🍩 [24/12/13] We are excited to announce the release of our KV cache-centric analysis work, SCBench, which evaluates long-context methods from a KV cache perspective.
+ - 🧤 [24/09/26] MInference has been accepted as spotlight at NeurIPS'24. See you in Vancouver!
+ - 👘 [24/09/16] We are pleased to announce the release of our KV cache offloading work, RetrievalAttention, which accelerates long-context LLM inference via vector retrieval.
+ - 🥤 [24/07/24] MInference supports meta-llama/Meta-Llama-3.1-8B-Instruct now.
+ - 🪗 [24/07/07] Thanks @AK for sponsoring. You can now use MInference online in the HF Demo with ZeroGPU.
+ - 📃 [24/07/03] Due to an issue with arXiv, the PDF is currently unavailable there. You can find the paper at this link.
+ - 🧩 [24/07/03] We will present MInference 1.0 at the Microsoft Booth and ES-FoMo at ICML'24. See you in Vienna!
+
+
## TL;DR
@@ -31,10 +44,22 @@ _Now, you can process **1M context 10x faster in a single A100** using Long-cont
- [MInference 1.0: Accelerating Pre-filling for Long-Context LLMs via Dynamic Sparse Attention](https://arxiv.org/abs/2407.02490) (NeurIPS'24 **spotlight**, ES-FoMo @ ICML'24)
_Huiqiang Jiang†, Yucheng Li†, Chengruidong Zhang†, Qianhui Wu, Xufang Luo, Surin Ahn, Zhenhua Han, Amir H. Abdi, Dongsheng Li, Chin-Yew Lin, Yuqing Yang and Lili Qiu_
+**SCBench** analyzes long-context methods from a **KV cache-centric perspective** across the full KV cache lifecycle (e.g., KV cache generation, compression, retrieval, and loading). It evaluates 12 tasks under two shared context modes, covering four categories of long-context capabilities: string retrieval, semantic retrieval, global information, and multi-task scenarios.
+
+- [SCBench: A KV Cache-Centric Analysis of Long-Context Methods](https://arxiv.org/abs/2412.10319) (ICLR'25, ENLSP @ NeurIPS'24)
+ _Yucheng Li, Huiqiang Jiang, Qianhui Wu, Xufang Luo, Surin Ahn, Chengruidong Zhang, Amir H. Abdi, Dongsheng Li, Jianfeng Gao, Yuqing Yang and Lili Qiu_
+
+**MMInference** use **modality-aware permutation sparse attention** to accelerate long-context VLMs inference in prefilling-stage. Specifically, we implement three distinct permutation-based sparse attention mechanisms, with FlashAttention, FlashDecoding and PIT, to address the grid patterns in vision inputs and the modality boundary issues in mixed-modality scenarios.
+
+- [MMInference: Accelerating Pre-filling for Long-Context VLMs via Modality-Aware Permutation Sparse Attention](https://arxiv.org/abs/2504.16083) (ICML'25, FM-Wild @ ICLR'25)
+ _Yucheng Li, Huiqiang Jiang, Chengruidong Zhang, Qianhui Wu, Xufang Luo, Surin Ahn, Amir H. Abdi, Dongsheng Li, Jianfeng Gao, Yuqing Yang and Lili Qiu_
+
## 🎥 Overview
+
+
## 🎯 Quick Start
@@ -42,7 +67,8 @@ _Now, you can process **1M context 10x faster in a single A100** using Long-cont
- Torch
- FlashAttention-2 (Optional)
-- Triton == 2.1.0
+- Triton
+- **Transformers >= 4.46.0**
To get started with MInference, simply install it using pip:
@@ -50,6 +76,24 @@ To get started with MInference, simply install it using pip:
pip install minference
```
+### Supported Efficient Methods
+
+You can get the complete list of supported efficient methods by running the following code:
+```python
+from minference import MInferenceConfig
+supported_attn_types = MInferenceConfig.get_available_attn_types()
+supported_kv_types = MInferenceConfig.get_available_kv_types()
+```
+
+Currently, we support the following long-context methods:
+
+- **[① KV Cache Generation]:** [MInference](https://arxiv.org/abs/2407.02490), [xAttention](https://arxiv.org/abs/2503.16428), [FlexPrefill](https://arxiv.org/abs/2502.20766), [A-shape](https://arxiv.org/abs/2309.17453), [Tri-shape](https://arxiv.org/abs/2412.10319), [MInference w/ static](https://arxiv.org/abs/2407.02490), [Dilated](https://arxiv.org/abs/2004.05150), [Strided](https://arxiv.org/abs/1904.10509)
+- **[② KV Cache Compression]:** [StreamingLLM](https://arxiv.org/abs/2309.17453), [SnapKV](https://arxiv.org/abs/2404.14469), [PyramidKV](https://arxiv.org/abs/2406.02069), [KIVI](https://arxiv.org/abs/2402.02750)
+- **[③ KV Cache Retrieval]:** [CacheBlend](https://arxiv.org/abs/2405.16444)
+- **[④ KV Cache Loading]:** [Quest](https://arxiv.org/abs/2406.10774), [RetrievalAttention](https://arxiv.org/abs/2409.10516)
+
+For more details about the KV cache lifecycle, please refer to [**SCBench**](https://arxiv.org/abs/2412.10319). Note that some modes are supported by vLLM, while all modes are supported by HF.
+
### Supported Models
General *MInference* **supports any decoding LLMs**, including LLaMA-style models, and Phi models.
@@ -63,7 +107,8 @@ get_support_models()
```
Currently, we support the following LLMs:
-- LLaMA-3.1: [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct)
+- Qwen2.5: [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct), [Qwen/Qwen2.5-32B-Instruct](https://huggingface.co/Qwen/Qwen2.5-32B-Instruct), [Qwen/Qwen2.5-72B-Instruct](https://huggingface.co/Qwen/Qwen2.5-72B-Instruct), [Qwen/Qwen2.5-7B-Instruct-1M](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-1M), [Qwen/Qwen2.5-14B-Instruct-1M](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct-1M)
+- LLaMA-3.1: [meta-llama/Meta-Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct), [meta-llama/Meta-Llama-3.1-70B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct)
- LLaMA-3: [gradientai/Llama-3-8B-Instruct-262k](https://huggingface.co/gradientai/Llama-3-8B-Instruct-262k), [gradientai/Llama-3-8B-Instruct-Gradient-1048k](https://huggingface.co/gradientai/Llama-3-8B-Instruct-Gradient-1048k), [gradientai/Llama-3-8B-Instruct-Gradient-4194k](https://huggingface.co/gradientai/Llama-3-8B-Instruct-Gradient-4194k), [gradientai/Llama-3-70B-Instruct-Gradient-262k](https://huggingface.co/gradientai/Llama-3-70B-Instruct-Gradient-262k), [gradientai/Llama-3-70B-Instruct-Gradient-1048k](https://huggingface.co/gradientai/Llama-3-70B-Instruct-Gradient-1048k)
- GLM-4: [THUDM/glm-4-9b-chat-1m](https://huggingface.co/THUDM/glm-4-9b-chat-1m)
- Yi: [01-ai/Yi-9B-200K](https://huggingface.co/01-ai/Yi-9B-200K)
@@ -72,6 +117,19 @@ Currently, we support the following LLMs:
### How to use MInference
+> [!TIP]
+> To benefit from fast kernel implementations, we recommend installing **SGLang** or **vLLM**.
+> for sglang
+> ```bash
+> uv pip install "sglang[all]>=0.4.6.post4"
+> ```
+>
+> for vllm
+> ```bash
+> uv pip install "vllm>=0.9.0"
+> uv pip install git+https://github.com/vllm-project/flash-attention
+> ```
+
for HF,
```diff
from transformers import pipeline
@@ -84,6 +142,12 @@ pipe = pipeline("text-generation", model=model_name, torch_dtype="auto", device_
+minference_patch = MInference("minference", model_name)
+pipe.model = minference_patch(pipe.model)
+pipe(prompt, max_length=10)
+
+# Using sparse kv methods, e.g. snapkv, quest, retr_attn, kivi
++minference_patch = MInference(attn_type="minference", model_name=model_name, kv_type="quest")
++pipe.model = minference_patch(pipe.model)
+
pipe(prompt, max_length=10)
```
@@ -94,7 +158,27 @@ for vLLM,
from vllm import LLM, SamplingParams
+ from minference import MInference
-llm = LLM(model_name, max_num_seqs=1, enforce_eager=True, max_model_len=128000)
+llm = LLM(model_name, enforce_eager=True, max_model_len=128_000, enable_chunked_prefill=False)
+
+# Patch MInference Module,
+# If you use the local path, please use the model_name from HF when initializing MInference.
++minference_patch = MInference("vllm", model_name)
++llm = minference_patch(llm)
+
+outputs = llm.generate(prompts, sampling_params)
+```
+
+for vLLM w/ TP,
+
+1. Copy `minference_patch_vllm_tp` and `minference_patch_vllm_executor` from `minference/patch.py` to the end of the `Worker` class in `vllm/worker/worker.py`. Make sure to indent `minference_patch_vllm_tp`.
+2. When calling VLLM, ensure `enable_chunked_prefill=False` is set.
+3. Refer to the script in https://github.com/microsoft/MInference/blob/main/experiments/benchmarks/run_e2e_vllm_tp.sh
+
+```diff
+from vllm import LLM, SamplingParams
++ from minference import MInference
+
+llm = LLM(model_name, enforce_eager=True, max_model_len=128_000, enable_chunked_prefill=False, tensor_parallel_size=2)
# Patch MInference Module,
# If you use the local path, please use the model_name from HF when initializing MInference.
@@ -125,6 +209,62 @@ python app.py
For more details, please refer to our [Examples](https://github.com/microsoft/MInference/tree/main/examples) and [Experiments](https://github.com/microsoft/MInference/tree/main/experiments). You can find more information about the dynamic compiler PIT in this [paper](https://dl.acm.org/doi/10.1145/3600006.3613139) and on [GitHub](https://github.com/microsoft/SparTA/tree/pit_artifact).
+## SCBench
+
+> [!Note]
+> - **datasets >= 2.15.0**
+
+### Load Data
+You can download and load the **SCBench** data through the Hugging Face datasets ([🤗 HF Repo](https://huggingface.co/datasets/microsoft/SCBench)):
+```python
+from datasets import load_dataset
+
+datasets = ["scbench_kv", "scbench_prefix_suffix", "scbench_vt", "scbench_repoqa", "scbench_qa_eng", "scbench_qa_chn", "scbench_choice_eng", "scbench_many_shot", "scbench_summary", "scbench_mf", "scbench_summary_with_needles", "scbench_repoqa_and_kv"]
+
+for dataset in datasets:
+ data = load_dataset("microsoft/SCBench", dataset, split="test")
+```
+
+### Data Format
+
+All data in **SCBench** are standardized to the following format:
+
+```json
+{
+ "id": "Random id for each piece of data.",
+ "context": "The long context required for the task, such as repo-code, long-document, and many-shot.",
+ "multi_turns": [{"input": "multi-turn question.", "answer": "multi-turn reference answer."}],
+}
+```
+
+### Experiments
+
+We implement **Multi-Turn** and **Multi-Request** modes with HF and vLLM in [`GreedySearch`](https://github.com/microsoft/MInference/blob/yucheng/kvcompression/scbench/eval_utils.py#L1160) and [`GreedySearch_vllm`](https://github.com/microsoft/MInference/blob/yucheng/kvcompression/scbench/eval_utils.py#L1070) two class. Please refer the follow scripts to run the experiments.
+
+for all methods,
+```bash
+cd scbench
+# Single-GPU, in Multi-Turn Mode
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 1 multi-turn
+# Multi-GPU, in Multi-Turn Mode
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 2 multi-turn
+# Multi-GPU, in Multi-Request Mode
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 2 scdq
+```
+
+for single methods,
+```bash
+cd scbench
+# Single-GPU, in Multi-Turn Mode, using attn_type: vllm, kv_type: dense
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 1 multi-turn vllm dense
+# Multi-GPU, in Multi-Turn Mode, using attn_type: vllm, kv_type: dense
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 2 multi-turn vllm dense
+# Multi-GPU, in Multi-Request Mode, using attn_type: vllm, kv_type: dense
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 2 scdq vllm dense
+```
+
+More details about **attn_type** and **kv_type**, please refer to this section: [Supported Efficient Methods](https://github.com/microsoft/MInference/tree/main?tab=readme-ov-file#supported-efficient-methods).
+
## FAQ
For more insights and answers, visit our [FAQ section](https://github.com/microsoft/MInference/blob/main/Transparency_FAQ.md).
@@ -153,16 +293,41 @@ Similar vertical and slash line sparse patterns have been discovered in BERT[1]
All four approaches (MInference, SSM, Linear Attention, and Sparse Attention) efficiently optimize attention complexity in Transformers, each introducing inductive bias differently. The latter three require training from scratch. Recent works like Mamba-2 and Unified Implicit Attention Representation unify SSM and Linear Attention as static sparse attention, with Mamba-2 itself being a block-wise sparse method. While these approaches show potential due to sparse redundancy in attention, static sparse attention may struggle with dynamic semantic associations in complex tasks. In contrast, dynamic sparse attention is better suited for managing these relationships.
+**Q5**: CUDA Out of Memory in in `_prepare_4d_causal_attention_mask_with_cache_position`
+
+_Solution_: Set the Hugging Face model's attention backend to FlashAttention-2 by adding the following argument during model initialization: `_attn_implementation="flash_attention_2",`.
+
+**Q6**: CUDA Out of Memory in in `logits = self.lm_head(hidden_states[:, -logits_to_keep:, :])`
+
+_Solution_: Set the `logits_to_keep=1` in model forward.
+
## Citation
If you find MInference useful or relevant to your project and research, please kindly cite our paper:
```bibtex
-@article{jiang2024minference,
- title={MInference 1.0: Accelerating Pre-filling for Long-Context LLMs via Dynamic Sparse Attention},
- author={Jiang, Huiqiang and Li, Yucheng and Zhang, Chengruidong and Wu, Qianhui and Luo, Xufang and Ahn, Surin and Han, Zhenhua and Abdi, Amir H and Li, Dongsheng and Lin, Chin-Yew and Yang, Yuqing and Qiu, Lili},
- journal={arXiv preprint arXiv:2407.02490},
- year={2024}
+@inproceedings{jiang2024minference,
+ author = {Huiqiang Jiang and Yucheng Li and Chengruidong Zhang and Qianhui Wu and Xufang Luo and Surin Ahn and Zhenhua Han and Amir H. Abdi and Dongsheng Li and Chin-Yew Lin and Yuqing Yang and Lili Qiu},
+ booktitle = {The Thirty-eighth Annual Conference on Neural Information Processing Systems},
+ title = {{MI}nference 1.0: Accelerating Pre-filling for Long-Context {LLM}s via Dynamic Sparse Attention},
+ url = {https://openreview.net/forum?id=fPBACAbqSN},
+ year = {2024}
+}
+
+@inproceedings{li2025scbench,
+ title={{SCB}ench: A {KV} Cache-Centric Analysis of Long-Context Methods},
+ author={Yucheng Li and Huiqiang Jiang and Qianhui Wu and Xufang Luo and Surin Ahn and Chengruidong Zhang and Amir H. Abdi and Dongsheng Li and Jianfeng Gao and Yuqing Yang and Lili Qiu},
+ booktitle={The Thirteenth International Conference on Learning Representations},
+ year={2025},
+ url={https://openreview.net/forum?id=gkUyYcY1W9}
+}
+
+@inproceedings{li2025mminference,
+ title={{MMI}ference: Accelerating Pre-filling for Long-Context VLMs via Modality-Aware Permutation Sparse Attention},
+ author={Li, Yucheng and Jiang, Huiqiang and Zhang, Chengruidong and Wu, Qianhui and Luo, Xufang and Ahn, Surin and Abdi, Amir H and Li, Dongsheng and Gao, Jianfeng and Yang, Yuqing and Qiu, Lili},
+ booktitle={Forty-second International Conference on Machine Learning},
+ year={2025},
+ url={https://openreview.net/forum?id=me6PfbATWM}
}
```
diff --git a/TriangleMix/.gitignore b/TriangleMix/.gitignore
new file mode 100644
index 00000000..8cdad5e8
--- /dev/null
+++ b/TriangleMix/.gitignore
@@ -0,0 +1,2 @@
+!*.json
+!ruler/data
diff --git a/TriangleMix/README.md b/TriangleMix/README.md
new file mode 100644
index 00000000..220ccc7c
--- /dev/null
+++ b/TriangleMix/README.md
@@ -0,0 +1,117 @@
+# TriangleMix: A Lossless and Efficient Attention Pattern for Long Context Prefilling
+
+We propose `TriangleMix`, a training-free static attention pattern for efficient long context prefilling.
+
+
+ 
+
+
+1. `TriangleMix` applies dense attention in the shallow layers and transitions to triangle attention in the deeper layers. Notably, triangle attention **reduces the $\mathcal{O}(N^2)$ attention complexity to $\mathcal{O}(N)$**, which is a significant complexity decrease, especially for long input sequences.
+2. `TriangleMix` can be integrated with dynamic attention methods (e.g., `MInference` or `FlexPrefill`) by utilizing dynamic attention in the shallow layers and switching to triangle sparse attention in the deeper layers.
+
+Extensive experiments demonstrate that `TriangleMix` reduces attention overhead by **3.7× to 15.3×** in deep layers, and **decreases overall Time-to-First-Token (TTFT) by 12% to 32%** for sequence lengths ranging from 32K to 128K, without sacrificing model accuracy. Moreover, the integration with dynamic sparsity methods to achieve further speedup, e.g. **accelerating MInference by 19% at 128K**, for example, highlighting its potential to enhance LLM inference efficiency.
+
+
+
+ 
+
+
+
+We discover this pattern using a novel **gradient-based method**. The causal attention is divided into three sections. The gradient measures the importance of each section relative to the outputs. We observe that the importance of the middle Q-K section drops significantly in deeper layers.
+
+Our hypothesis is that this arises from a **Train–Test Misalignment**:
+
+- **Training**: The loss is applied uniformly to all positions in the input. For example, if the input has 4000 tokens, the model is trained to predict token 2001 given tokens 1–2000.
+- **Inference**: We only care about predicting tokens after the prompt, so predicting tokens inside the prompt (like token 2001) is irrelevant.
+
+We find the Middle Q-K region is important primarily for predicting tokens within the prompt, while for generation tasks focused on tokens after the prompt, the Middle Q-K in deeper layers is largely redundant. Detailed analysis on this topic is provided in the paper.
+
+## Quick Start
+
+Make sure you have installed the latest `minference`:
+
+```bash
+conda create -n minference python=3.11
+conda activate minference
+pip install "transformers[torch]"
+pip install flash-attn --no-build-isolation
+git clone https://github.com/microsoft/MInference.git ~/MInference
+cd ~/MInference
+pip install -e .
+```
+
+Then you can use the `tri_mix` methods:
+
+```diff
+from transformers import AutoModelForCausalLM, AutoTokenizer
++ from minference import MInference
+
+model_name = "meta-llama/Llama-3.1-8B-Instruct"
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
+model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ trust_remote_code=True,
+ attn_implementation="flash_attention_2",
+)
+
++minference_patch = MInference(
++ attn_type="tri_mix",
++ model_name=model_name,
++ attn_kwargs={"last_n": 128, "starting_layer": 16, "n_local": 512, "n_init": 8},
++)
++model = minference_patch(model)
+
+prompt = "your prompt here"
+inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(model.device)
+output = model.generate(**inputs, do_sample=False, max_new_tokens=50)
+```
+
+## Reproduce Ruler Performance
+
+First, setup ruler environments. See [setup_ruler.sh](./setup_ruler.sh) for details.
+
+Then, change directory to `/TriangleMix/ruler/`.
+
+Run dense attention:
+
+```bash
+mkdir -p ./results/dense
+bash run_dense.sh meta-llama/Llama-3.1-8B-Instruct minference ./results/dense
+```
+
+Run `TriangleMix`:
+
+```bash
+mkdir -p ./results/tri_mix
+bash run_tri_mix.sh meta-llama/Llama-3.1-8B-Instruct minference ./results/tri_mix 16
+```
+
+Results on 128K context length (Minimal drop from 77.6 to 77.3):
+
+| Method | AVG | NI.SG1 | NI.SG2 | NI.SG3 | NI.MK1 | NI.MK2 | NI.MK3 | NI.MV | NI.MQ | VT | CWE | FWE | QA1 | QA2 |
+|-------------|------|--------|--------|--------|--------|--------|--------|--------|--------|------|------|-------|------|------|
+| Dense | 77.6 | 100.0 | 96.0 | 100.0 | 95.0 | 91.0 | 62.0 | 97.25 | 98.5 | 90.0 | 2.2 | 57.33 | 77.0 | 43.0 |
+| TriangleMix | 77.3 | 100.0 | 96.0 | 100.0 | 95.0 | 91.0 | 63.0 | 97.75 | 98.0 | 93.2 | 0.0 | 50.67 | 77.0 | 43.0 |
+
+
+## Reproduce Efficiency Metrics
+
+We provide a speed test script `speed_test.py`. This script measures the TTFT (time-to-first-token).
+
+```bash
+python speed_test.py --method dense --model_name meta-llama/Llama-3.1-8B-Instruct
+python speed_test.py --method tri_mix --model_name meta-llama/Llama-3.1-8B-Instruct
+python speed_test.py --method tri_mix_minfernece --model_name meta-llama/Llama-3.1-8B-Instruct
+```
+
+TTFT in seconds on A100 80GB:
+
+| Method | 32K | 48K | 64K | 80K | 96K | 112K | 128K |
+|--------------------|-------------|-------------|-------------|-------------|-------------|-------------|-------------|
+| Dense | 4.1 | 7.3 | 11.2 | 15.9 | 21.3 | 27.5 | 34.5 |
+| MInference | 5.5 (+34%) | 7.8 (+7%) | 10.1 (-10%) | 12.3 (-23%) | 13.4 (-37%) | 15.9 (-42%) | 18.0 (-48%) |
+| TriangleMix | **3.6 (-12%)** | **5.9 (-19%)** | 8.6 (-23%) | 11.7 (-26%) | 15.2 (-29%) | 19.1 (-31%) | 23.4 (-32%) |
+| Ours + MInference | 4.2 (+2%) | 6.0 (-18%) | **7.7 (-31%)** | **9.5 (-40%)** | **10.9 (-49%)** | **12.7 (-54%)** | **14.5 (-58%)** |
diff --git a/TriangleMix/images/drop.jpg b/TriangleMix/images/drop.jpg
new file mode 100644
index 00000000..944021f5
Binary files /dev/null and b/TriangleMix/images/drop.jpg differ
diff --git a/TriangleMix/images/overview.jpg b/TriangleMix/images/overview.jpg
new file mode 100644
index 00000000..32ede051
Binary files /dev/null and b/TriangleMix/images/overview.jpg differ
diff --git a/TriangleMix/ruler/config_models.sh b/TriangleMix/ruler/config_models.sh
new file mode 100644
index 00000000..13f4954e
--- /dev/null
+++ b/TriangleMix/ruler/config_models.sh
@@ -0,0 +1,160 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+TEMPERATURE="0.0" # greedy
+TOP_P="1.0"
+TOP_K="32"
+SEQ_LENGTHS=(
+ 4096
+ 8192
+ 16384
+ 32768
+ 65536
+ 131072
+)
+
+MODEL_SELECT() {
+ MODEL_NAME=$1
+ MODEL_DIR=$2
+ ENGINE_DIR=$3
+
+ case $MODEL_NAME in
+ llama2-7b-chat)
+ MODEL_PATH="${MODEL_DIR}/llama2-7b-chat-hf"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="vllm"
+ ;;
+ llama-3-262k)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="vllm"
+ ;;
+ llama-3-262k-minference)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="minference"
+ ;;
+ llama-inf-llm)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="InfLLM"
+ ;;
+ yi-inf-llm)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="InfLLM"
+ ;;
+ llama-3-262k-dilated1)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="dilated1"
+ ;;
+ MInference6KoP)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="MInference6KoP"
+ ;;
+ MInference6KOPYi)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="MInference6KOPYi"
+ ;;
+ llama-3-262k-streaming)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="dilated1"
+ ;;
+ Yi-dilated1)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="dilated1"
+ ;;
+ Yi-dilated2)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="dilated2"
+ ;;
+ Yi-static)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="YiStatic"
+ ;;
+ MInference6KOPYi)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="MInference6KOPYi"
+ ;;
+ llama-static)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="LlamaStatic"
+ ;;
+ Yi-streaming)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="a_shape"
+ ;;
+ OPYiHalfV2)
+ MODEL_PATH="01-ai/Yi-9B-200K"
+ MODEL_TEMPLATE_TYPE="meta-chat"
+ MODEL_FRAMEWORK="OPYiHalfV2"
+ ;;
+ llama-3-262k-dilated2)
+ MODEL_PATH="gradientai/Llama-3-8B-Instruct-262k"
+ MODEL_TEMPLATE_TYPE="llama-3"
+ MODEL_FRAMEWORK="dilated2"
+ ;;
+ gpt-3.5-turbo)
+ MODEL_PATH="gpt-3.5-turbo-0125"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="openai"
+ TOKENIZER_PATH="cl100k_base"
+ TOKENIZER_TYPE="openai"
+ OPENAI_API_KEY=""
+ AZURE_ID=""
+ AZURE_SECRET=""
+ AZURE_ENDPOINT=""
+ ;;
+ gpt-4-turbo)
+ MODEL_PATH="gpt-4"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="openai"
+ TOKENIZER_PATH="cl100k_base"
+ TOKENIZER_TYPE="openai"
+ OPENAI_API_KEY=""
+ AZURE_ID=""
+ AZURE_SECRET=""
+ AZURE_ENDPOINT=""
+ ;;
+ gemini_1.0_pro)
+ MODEL_PATH="gemini-1.0-pro-latest"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="gemini"
+ TOKENIZER_PATH=$MODEL_PATH
+ TOKENIZER_TYPE="gemini"
+ GEMINI_API_KEY=""
+ ;;
+ gemini_1.5_pro)
+ MODEL_PATH="gemini-1.5-pro-latest"
+ MODEL_TEMPLATE_TYPE="base"
+ MODEL_FRAMEWORK="gemini"
+ TOKENIZER_PATH=$MODEL_PATH
+ TOKENIZER_TYPE="gemini"
+ GEMINI_API_KEY=""
+ ;;
+ esac
+
+
+ if [ -z "${TOKENIZER_PATH}" ]; then
+ if [ -f ${MODEL_PATH}/tokenizer.model ]; then
+ TOKENIZER_PATH=${MODEL_PATH}/tokenizer.model
+ TOKENIZER_TYPE="nemo"
+ else
+ TOKENIZER_PATH=${MODEL_PATH}
+ TOKENIZER_TYPE="hf"
+ fi
+ fi
+
+
+ echo "$MODEL_PATH:$MODEL_TEMPLATE_TYPE:$MODEL_FRAMEWORK:$TOKENIZER_PATH:$TOKENIZER_TYPE:$OPENAI_API_KEY:$GEMINI_API_KEY:$AZURE_ID:$AZURE_SECRET:$AZURE_ENDPOINT"
+}
diff --git a/TriangleMix/ruler/config_tasks.sh b/TriangleMix/ruler/config_tasks.sh
new file mode 100644
index 00000000..0199ff02
--- /dev/null
+++ b/TriangleMix/ruler/config_tasks.sh
@@ -0,0 +1,35 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+NUM_SAMPLES=80
+REMOVE_NEWLINE_TAB=false
+STOP_WORDS=""
+
+if [ -z "${STOP_WORDS}" ]; then
+ STOP_WORDS=""
+else
+ STOP_WORDS="--stop_words \"${STOP_WORDS}\""
+fi
+
+if [ "${REMOVE_NEWLINE_TAB}" = false ]; then
+ REMOVE_NEWLINE_TAB=""
+else
+ REMOVE_NEWLINE_TAB="--remove_newline_tab"
+fi
+
+# task name in `synthetic.yaml`
+synthetic=(
+ "niah_single_1"
+ "niah_single_2"
+ "niah_single_3"
+ "niah_multikey_1"
+ "niah_multikey_2"
+ "niah_multikey_3"
+ "niah_multivalue"
+ "niah_multiquery"
+ "vt"
+ "cwe"
+ "fwe"
+ "qa_1"
+ "qa_2"
+)
diff --git a/TriangleMix/ruler/data/prepare.py b/TriangleMix/ruler/data/prepare.py
new file mode 100644
index 00000000..467606be
--- /dev/null
+++ b/TriangleMix/ruler/data/prepare.py
@@ -0,0 +1,167 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Prepare jsonl with field `input` and `outputs`.
+{
+ "index" int,
+ "input": str,
+ "outputs": [str],
+}
+
+python prepare.py \
+ --save_dir ./ \
+ --benchmark synthetic \
+ --task niah_single_1 \
+ --tokenizer_path tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --model_template_type base \
+ --num_samples 10 \
+"""
+import argparse
+import importlib
+import json
+import math
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+
+import yaml
+from template import Templates
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--benchmark", type=str, default="synthetic", help="Options: [synthetic]"
+)
+parser.add_argument("--task", type=str, required=True, help="tasks in benchmark")
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--num_samples",
+ type=int,
+ default=500,
+ help="maximum number of samples we want to test",
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument(
+ "--model_template_type", type=str, default="base", help="Options in `template.py`"
+)
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+parser.add_argument(
+ "--chunk_idx", type=int, default=0, help="index of current split chunk"
+)
+parser.add_argument("--chunk_amount", type=int, default=1, help="size of split chunk")
+
+args = parser.parse_args()
+
+
+def main():
+ start_time = time.time()
+ curr_folder = os.path.dirname(os.path.abspath(__file__))
+
+ try:
+ module = importlib.import_module(f"{args.benchmark}.constants")
+ except ImportError:
+ print(f"Module data.{args.benchmark}.constants not found.")
+
+ tasks_base = module.TASKS
+ with open(os.path.join(curr_folder, f"../{args.benchmark}.yaml"), "r") as f:
+ tasks_customized = yaml.safe_load(f)
+
+ if args.task not in tasks_customized:
+ raise ValueError(f"{args.task} is not found in config_tasks.yaml")
+
+ config = tasks_customized.get(args.task)
+ config.update(tasks_base[config["task"]])
+
+ # Add templates
+ assert args.model_template_type in Templates, print(
+ f"{args.model_template_type} is not found in {Templates.keys()}"
+ )
+ model_template = Templates[args.model_template_type]
+ task_template = config["template"]
+
+ # Add answer prefix for all models
+ answer_prefix = config["answer_prefix"] if "answer_prefix" in config else ""
+ config["template"] = (
+ model_template.format(task_template=task_template) + answer_prefix
+ )
+
+ # Split task into multiple chunks
+ chunks = [
+ (args.num_samples // args.chunk_amount)
+ + (1 if i < args.num_samples % args.chunk_amount else 0)
+ for i in range(args.chunk_amount)
+ ]
+ num_samples = chunks[args.chunk_idx]
+ pre_samples = sum(chunks[: args.chunk_idx])
+
+ random_seed = 42 + args.chunk_idx
+
+ try:
+ script = os.path.join(curr_folder, args.benchmark, f"{config['task']}.py")
+ additional_args = " ".join([f"--{k} {v}" for k, v in config["args"].items()])
+ command = f"""python {script} \
+ --save_dir {args.save_dir} \
+ --save_name {args.task} \
+ --subset {args.subset} \
+ --tokenizer_path {args.tokenizer_path} \
+ --tokenizer_type {args.tokenizer_type} \
+ --max_seq_length {args.max_seq_length} \
+ --tokens_to_generate {config['tokens_to_generate']} \
+ --num_samples {num_samples} \
+ --random_seed {random_seed} \
+ {additional_args} \
+ {f"--remove_newline_tab" if args.remove_newline_tab else ""} \
+ {f"--pre_samples {pre_samples}" if config['task'] == 'qa' else ""} \
+ --template "{config['template']}"
+ """
+ print(command)
+ result = subprocess.run(
+ command,
+ shell=True,
+ check=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ )
+
+ if result.returncode == 0:
+ print("Output:")
+ print(result.stdout)
+ else:
+ print("Error:")
+ print(result.stderr)
+ except subprocess.CalledProcessError as e:
+ print("Error output:", e.stderr)
+
+ save_file = args.save_dir / args.task / f"{args.subset}.jsonl"
+ print(f"Prepare {args.task} with lines: {args.num_samples} to {save_file}")
+ print(f"Used time: {round((time.time() - start_time) / 60, 1)} minutes")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/data/synthetic/common_words_extraction.py b/TriangleMix/ruler/data/synthetic/common_words_extraction.py
new file mode 100644
index 00000000..c33d28ff
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/common_words_extraction.py
@@ -0,0 +1,207 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Create a dataset jsonl file for common words extraction.
+
+python common_words_extraction.py \
+ --save_dir=./ \
+ --save_name=vt \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --tokens_to_generate 30 \
+ --num_samples 10 \
+ --random_seed 42 \
+ -freq_cw 30 --freq_ucw 3 --num_cw 10 \
+ --template "[INST] Below is a numbered list of words. In these words, some appear more often than others. Memorize the ones that appear most often.\n{context}\nQuestion: What are the 10 most common words in the above list? [/INST] Answer: The top 10 words that appear most often in the list are:"
+"""
+
+import argparse
+import os
+import random
+import sys
+from pathlib import Path
+
+import wonderwords
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate",
+ type=int,
+ required=True,
+ help="expected generated token amount.",
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+parser.add_argument("--freq_cw", type=int, default=30)
+parser.add_argument("--freq_ucw", type=int, default=3)
+parser.add_argument("--num_cw", type=int, default=10)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+nouns = wonderwords.random_word._get_words_from_text_file("nounlist.txt")
+adjs = wonderwords.random_word._get_words_from_text_file("adjectivelist.txt")
+verbs = wonderwords.random_word._get_words_from_text_file("verblist.txt")
+words = nouns + adjs + verbs
+words = sorted(list(set(words)))
+random.Random(args.random_seed).shuffle(words)
+
+
+def get_example(num_words, common_repeats=30, uncommon_repeats=3, common_nums=10):
+ word_list_full = random.sample(words, num_words)
+ common, uncommon = word_list_full[:common_nums], word_list_full[common_nums:]
+ word_list = common * int(common_repeats) + uncommon * int(uncommon_repeats)
+ random.Random(args.random_seed).shuffle(word_list)
+
+ # Formatting the word list as "1. word1 2. word2 3. word3 ..."
+ context = " ".join([f"{i + 1}. {word}" for i, word in enumerate(word_list)])
+
+ return context, common
+
+
+def generate_input_output(num_words):
+ if args.max_seq_length < 4096:
+ context_example, answer_example = get_example(20, 3, 1, args.num_cw)
+ context, answer = get_example(num_words, 6, 1, args.num_cw)
+ else:
+ context_example, answer_example = get_example(40, 10, 3, args.num_cw)
+ context, answer = get_example(
+ num_words, args.freq_cw, args.freq_ucw, args.num_cw
+ )
+
+ template = args.template
+
+ input_example = template.format(
+ context=context_example,
+ query="",
+ ) + " ".join([f"{i + 1}. {word}" for i, word in enumerate(answer_example)])
+
+ input_text = template.format(
+ context=context,
+ query="",
+ )
+
+ return input_example + "\n" + input_text, answer
+
+
+def sys_word_pair_random(
+ num_samples: int, max_seq_length: int, save_dir: str, incremental: int = 10
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ # Find the perfect num_words
+ num_words = incremental
+
+ total_tokens = 0
+ while total_tokens + tokens_to_generate < max_seq_length:
+ input_text, answer = generate_input_output(num_words)
+ # Calculate the number of tokens in the example
+ total_tokens = len(
+ TOKENIZER.text_to_tokens(
+ input_text
+ + " "
+ + " ".join([f"{i + 1}. {word}" for i, word in enumerate(answer)])
+ )
+ )
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Words: {num_words}"
+ )
+ if total_tokens + tokens_to_generate > max_seq_length:
+ num_words -= incremental
+ break
+
+ num_words += incremental
+ if num_words > len(words):
+ num_words = len(words)
+ break
+
+ print("num_words:", num_words)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_words = num_words
+ while True:
+ try:
+ input_text, answer = generate_input_output(used_words)
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_words > incremental:
+ used_words -= incremental
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ write_jsons = sys_word_pair_random(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ save_dir=args.save_dir,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/data/synthetic/constants.py b/TriangleMix/ruler/data/synthetic/constants.py
new file mode 100644
index 00000000..eb8ffd2b
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/constants.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Add a new task (required arguments):
+
+TASK_NAME: {
+ 'tokens_to_generate': how many tokens we want to generate.
+ 'template': the template with at least {context} and {query}.
+}
+"""
+
+TASKS = {
+ "niah": {
+ "tokens_to_generate": 128,
+ "template": """Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned in the provided text?""",
+ "answer_prefix": """ The special magic {type_needle_v} for {query} mentioned in the provided text are""",
+ },
+ "variable_tracking": {
+ "tokens_to_generate": 30,
+ "template": """Memorize and track the chain(s) of variable assignment hidden in the following text.\n\n{context}\nQuestion: Find all variables that are assigned the value {query} in the text above.""",
+ "answer_prefix": """ Answer: According to the chain(s) of variable assignment in the text above, {num_v} variables are assgined the value {query}, they are: """,
+ },
+ "common_words_extraction": {
+ "tokens_to_generate": 120,
+ "template": """Below is a numbered list of words. In these words, some appear more often than others. Memorize the ones that appear most often.\n{context}\nQuestion: What are the 10 most common words in the above list?""",
+ "answer_prefix": """ Answer: The top 10 words that appear most often in the list are:""",
+ },
+ "freq_words_extraction": {
+ "tokens_to_generate": 50,
+ "template": """Read the following coded text and track the frequency of each coded word. Find the three most frequently appeared coded words. {context}\nQuestion: Do not provide any explanation. Please ignore the dots '....'. What are the three most frequently appeared words in the above coded text?""",
+ "answer_prefix": """ Answer: According to the coded text above, the three most frequently appeared words are:""",
+ },
+ "qa": {
+ "tokens_to_generate": 32,
+ "template": """Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query}""",
+ "answer_prefix": """ Answer:""",
+ },
+}
diff --git a/TriangleMix/ruler/data/synthetic/freq_words_extraction.py b/TriangleMix/ruler/data/synthetic/freq_words_extraction.py
new file mode 100644
index 00000000..1b3625fb
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/freq_words_extraction.py
@@ -0,0 +1,190 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Create a dataset jsonl file for frequent words extraction.
+
+python freq_words_extraction.py \
+ --save_dir=./ \
+ --save_name=vt \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --tokens_to_generate 30 \
+ --num_samples 10 \
+ --random_seed 42 \
+ --alpha 2.0 \
+ --template "[INST] Read the following coded text and track the frequency of each coded word. Find the three most frequently appeared coded words. {context}\nQuestion: Do not provide any explanation. Please ignore the dots '....'. What are the three most frequently appeared words in the above coded text? [/INST] Answer: According to the coded text above, the three most frequently appeared words are:"
+"""
+
+import argparse
+import os
+import random
+import string
+import sys
+from pathlib import Path
+
+import numpy as np
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from scipy.special import zeta
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate", type=int, default=50, help="number of tokens to generate"
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+parser.add_argument(
+ "--coded_wordlen", type=int, default=6, help="length of synthetic word"
+)
+parser.add_argument(
+ "--vocab_size", type=int, default=-1, help="synthetic vocab size to sample from"
+)
+parser.add_argument("--alpha", type=float, default=2.0, help="zeta distribution alpha")
+parser.add_argument("--add_fewshot", action="store_true", default=False)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+
+def generate_input_output(
+ max_len, num_words=-1, coded_wordlen=6, vocab_size=2000, incremental=10, alpha=2.0
+):
+ # generate vocab
+ vocab = [
+ "".join(random.choices(string.ascii_lowercase, k=coded_wordlen))
+ for _ in range(vocab_size)
+ ]
+ while len(set(vocab)) < vocab_size:
+ vocab.append("".join(random.choices(string.ascii_lowercase, k=coded_wordlen)))
+ vocab = sorted(list(set(vocab)))
+ random.Random(args.random_seed).shuffle(vocab)
+ vocab[0] = "..." # treat the top ranked as noise
+
+ # sample words
+ template = args.template
+
+ def gen_text(num_words):
+ k = np.arange(1, len(vocab) + 1)
+ sampled_cnt = num_words * (k**-alpha) / zeta(alpha)
+ sampled_words = [[w] * zi for w, zi in zip(vocab, sampled_cnt.astype(int))]
+ sampled_words = [x for wlst in sampled_words for x in wlst]
+ random.Random(args.random_seed).shuffle(sampled_words)
+ return template.format(context=" ".join(sampled_words), query=""), vocab[1:4]
+
+ if num_words > 0:
+ num_words = num_words
+ text, answer = gen_text(num_words)
+ while len(TOKENIZER.text_to_tokens(text)) > max_len:
+ num_words -= incremental
+ text, answer = gen_text(num_words)
+ else:
+ num_words = max_len // coded_wordlen # init
+ text, answer = gen_text(num_words)
+ while len(TOKENIZER.text_to_tokens(text)) < max_len:
+ num_words += incremental
+ text, answer = gen_text(num_words)
+ num_words -= incremental
+ text, answer = gen_text(num_words)
+ return text, answer, num_words
+
+
+def sys_kwext(num_samples: int, max_seq_length: int, incremental: int = 10):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ vocab_size = max_seq_length // 50 if args.vocab_size == -1 else args.vocab_size
+
+ # get number of words
+ input_max_len = max_seq_length
+ _, _, num_example_words = generate_input_output(
+ input_max_len,
+ coded_wordlen=args.coded_wordlen,
+ vocab_size=vocab_size,
+ incremental=input_max_len // 32,
+ alpha=args.alpha,
+ )
+ print("num_example_words:", num_example_words)
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ # construct input
+ input_max_len = max_seq_length
+ input_text, answer, _ = generate_input_output(
+ input_max_len,
+ num_words=num_example_words,
+ coded_wordlen=args.coded_wordlen,
+ vocab_size=vocab_size,
+ incremental=input_max_len // 32,
+ alpha=args.alpha,
+ )
+
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+ write_jsons = sys_kwext(
+ num_samples=args.num_samples, max_seq_length=args.max_seq_length, incremental=10
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/data/synthetic/json/PaulGrahamEssays_URLs.txt b/TriangleMix/ruler/data/synthetic/json/PaulGrahamEssays_URLs.txt
new file mode 100644
index 00000000..b9a38e44
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/json/PaulGrahamEssays_URLs.txt
@@ -0,0 +1,218 @@
+http://www.paulgraham.com/13sentences.html
+http://www.paulgraham.com/5founders.html
+http://www.paulgraham.com/6631327.html
+http://www.paulgraham.com/95.html
+http://www.paulgraham.com/ace.html
+http://www.paulgraham.com/airbnb.html
+http://www.paulgraham.com/airbnbs.html
+http://www.paulgraham.com/alien.html
+http://www.paulgraham.com/altair.html
+http://www.paulgraham.com/ambitious.html
+http://www.paulgraham.com/america.html
+http://www.paulgraham.com/angelinvesting.html
+http://www.paulgraham.com/artistsship.html
+http://www.paulgraham.com/badeconomy.html
+http://www.paulgraham.com/better.html
+http://www.paulgraham.com/bronze.html
+http://www.paulgraham.com/bubble.html
+http://www.paulgraham.com/charisma.html
+http://www.paulgraham.com/cities.html
+http://www.paulgraham.com/college.html
+http://www.paulgraham.com/colleges.html
+http://www.paulgraham.com/conformism.html
+http://www.paulgraham.com/control.html
+http://www.paulgraham.com/convergence.html
+http://www.paulgraham.com/convince.html
+http://www.paulgraham.com/cred.html
+http://www.paulgraham.com/credentials.html
+http://www.paulgraham.com/determination.html
+http://www.paulgraham.com/die.html
+http://www.paulgraham.com/disagree.html
+http://www.paulgraham.com/disc.html
+http://www.paulgraham.com/discover.html
+http://www.paulgraham.com/distraction.html
+http://www.paulgraham.com/divergence.html
+http://www.paulgraham.com/donate.html
+http://www.paulgraham.com/ds.html
+http://www.paulgraham.com/early.html
+http://www.paulgraham.com/earnest.html
+http://www.paulgraham.com/equity.html
+http://www.paulgraham.com/essay.html
+http://www.paulgraham.com/ffb.html
+http://www.paulgraham.com/fh.html
+http://www.paulgraham.com/fix.html
+http://www.paulgraham.com/fn.html
+http://www.paulgraham.com/foundersatwork.html
+http://www.paulgraham.com/fp.html
+http://www.paulgraham.com/fr.html
+http://www.paulgraham.com/fundraising.html
+http://www.paulgraham.com/future.html
+http://www.paulgraham.com/genius.html
+http://www.paulgraham.com/getideas.html
+http://www.paulgraham.com/good.html
+http://www.paulgraham.com/goodart.html
+http://www.paulgraham.com/googles.html
+http://www.paulgraham.com/greatwork.html
+http://www.paulgraham.com/growth.html
+http://www.paulgraham.com/guidetoinvestors.html
+http://www.paulgraham.com/hackernews.html
+http://www.paulgraham.com/head.html
+http://www.paulgraham.com/herd.html
+http://www.paulgraham.com/heresy.html
+http://www.paulgraham.com/heroes.html
+http://www.paulgraham.com/highres.html
+http://www.paulgraham.com/hiresfund.html
+http://www.paulgraham.com/hiring.html
+http://www.paulgraham.com/hp.html
+http://www.paulgraham.com/hs.html
+http://www.paulgraham.com/hundred.html
+http://www.paulgraham.com/hw.html
+http://www.paulgraham.com/hwh.html
+http://www.paulgraham.com/icad.html
+http://www.paulgraham.com/ideas.html
+http://www.paulgraham.com/identity.html
+http://www.paulgraham.com/ineq.html
+http://www.paulgraham.com/inequality.html
+http://www.paulgraham.com/investors.html
+http://www.paulgraham.com/invtrend.html
+http://www.paulgraham.com/javacover.html
+http://www.paulgraham.com/jessica.html
+http://www.paulgraham.com/judgement.html
+http://www.paulgraham.com/kate.html
+http://www.paulgraham.com/kids.html
+http://www.paulgraham.com/ladder.html
+http://www.paulgraham.com/lesson.html
+http://www.paulgraham.com/lies.html
+http://www.paulgraham.com/lwba.html
+http://www.paulgraham.com/mac.html
+http://www.paulgraham.com/makersschedule.html
+http://www.paulgraham.com/marginal.html
+http://www.paulgraham.com/maybe.html
+http://www.paulgraham.com/mean.html
+http://www.paulgraham.com/microsoft.html
+http://www.paulgraham.com/mit.html
+http://www.paulgraham.com/name.html
+http://www.paulgraham.com/nerds.html
+http://www.paulgraham.com/newthings.html
+http://www.paulgraham.com/noob.html
+http://www.paulgraham.com/noop.html
+http://www.paulgraham.com/notnot.html
+http://www.paulgraham.com/nov.html
+http://www.paulgraham.com/nthings.html
+http://www.paulgraham.com/opensource.html
+http://www.paulgraham.com/organic.html
+http://www.paulgraham.com/orth.html
+http://www.paulgraham.com/own.html
+http://www.paulgraham.com/patentpledge.html
+http://www.paulgraham.com/pgh.html
+http://www.paulgraham.com/pinch.html
+http://www.paulgraham.com/polls.html
+http://www.paulgraham.com/power.html
+http://www.paulgraham.com/prcmc.html
+http://www.paulgraham.com/procrastination.html
+http://www.paulgraham.com/progbot.html
+http://www.paulgraham.com/prop62.html
+http://www.paulgraham.com/property.html
+http://www.paulgraham.com/publishing.html
+http://www.paulgraham.com/pypar.html
+http://www.paulgraham.com/ramenprofitable.html
+http://www.paulgraham.com/randomness.html
+http://www.paulgraham.com/re.html
+http://www.paulgraham.com/read.html
+http://www.paulgraham.com/real.html
+http://www.paulgraham.com/really.html
+http://www.paulgraham.com/relres.html
+http://www.paulgraham.com/revolution.html
+http://www.paulgraham.com/richnow.html
+http://www.paulgraham.com/road.html
+http://www.paulgraham.com/ronco.html
+http://www.paulgraham.com/safe.html
+http://www.paulgraham.com/say.html
+http://www.paulgraham.com/schlep.html
+http://www.paulgraham.com/seesv.html
+http://www.paulgraham.com/segway.html
+http://www.paulgraham.com/selfindulgence.html
+http://www.paulgraham.com/sfp.html
+http://www.paulgraham.com/simply.html
+http://www.paulgraham.com/smart.html
+http://www.paulgraham.com/softwarepatents.html
+http://www.paulgraham.com/spam.html
+http://www.paulgraham.com/speak.html
+http://www.paulgraham.com/start.html
+http://www.paulgraham.com/startupfunding.html
+http://www.paulgraham.com/startuphubs.html
+http://www.paulgraham.com/startupideas.html
+http://www.paulgraham.com/startupmistakes.html
+http://www.paulgraham.com/stuff.html
+http://www.paulgraham.com/superlinear.html
+http://www.paulgraham.com/swan.html
+http://www.paulgraham.com/tablets.html
+http://www.paulgraham.com/talk.html
+http://www.paulgraham.com/taste.html
+http://www.paulgraham.com/think.html
+http://www.paulgraham.com/top.html
+http://www.paulgraham.com/trolls.html
+http://www.paulgraham.com/twitter.html
+http://www.paulgraham.com/usa.html
+http://www.paulgraham.com/users.html
+http://www.paulgraham.com/venturecapital.html
+http://www.paulgraham.com/wealth.html
+http://www.paulgraham.com/webstartups.html
+http://www.paulgraham.com/whyyc.html
+http://www.paulgraham.com/word.html
+http://www.paulgraham.com/words.html
+http://www.paulgraham.com/work.html
+http://www.paulgraham.com/writing44.html
+http://www.paulgraham.com/wtax.html
+http://www.paulgraham.com/yahoo.html
+http://www.paulgraham.com/ycombinator.html
+http://www.paulgraham.com/ycstart.html
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/addiction.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/aord.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/apple.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/avg.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/before.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/bias.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/boss.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/copy.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/corpdev.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/desres.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/diff.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/ecw.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/founders.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/foundervisa.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gap.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gba.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/gh.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/goodtaste.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/hubs.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/iflisp.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/island.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/know.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/langdes.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/laundry.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/love.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/mod.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/newideas.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/nft.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/philosophy.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/popular.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/pow.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/rootsoflisp.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/rss.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/siliconvalley.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/startuplessons.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/submarine.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/sun.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/superangels.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/todo.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/unions.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/useful.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vb.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vcsqueeze.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/vw.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/want.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/web20.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/weird.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/wisdom.txt
+https://github.com/gkamradt/LLMTest_NeedleInAHaystack/raw/main/needlehaystack/PaulGrahamEssays/worked.txt
diff --git a/TriangleMix/ruler/data/synthetic/json/download_paulgraham_essay.py b/TriangleMix/ruler/data/synthetic/json/download_paulgraham_essay.py
new file mode 100644
index 00000000..52bb6078
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/json/download_paulgraham_essay.py
@@ -0,0 +1,74 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import glob
+import json
+import os
+import shutil
+import urllib.request
+
+import html2text
+from bs4 import BeautifulSoup
+from tqdm import tqdm
+
+temp_folder_repo = "essay_repo"
+temp_folder_html = "essay_html"
+os.makedirs(temp_folder_repo, exist_ok=True)
+os.makedirs(temp_folder_html, exist_ok=True)
+
+h = html2text.HTML2Text()
+h.ignore_images = True
+h.ignore_tables = True
+h.escape_all = True
+h.reference_links = False
+h.mark_code = False
+
+with open("PaulGrahamEssays_URLs.txt") as f:
+ urls = [line.strip() for line in f]
+
+for url in tqdm(urls):
+ if ".html" in url:
+ filename = url.split("/")[-1].replace(".html", ".txt")
+ try:
+ with urllib.request.urlopen(url) as website:
+ content = website.read().decode("unicode_escape", "utf-8")
+ soup = BeautifulSoup(content, "html.parser")
+ specific_tag = soup.find("font")
+ parsed = h.handle(str(specific_tag))
+
+ with open(os.path.join(temp_folder_html, filename), "w") as file:
+ file.write(parsed)
+
+ except Exception as e:
+ print(f"Fail download {filename}, ({e})")
+
+ else:
+ filename = url.split("/")[-1]
+ try:
+ with urllib.request.urlopen(url) as website:
+ content = website.read().decode("utf-8")
+
+ with open(os.path.join(temp_folder_repo, filename), "w") as file:
+ file.write(content)
+
+ except Exception as e:
+ print(f"Fail download {filename}, ({e})")
+
+files_repo = glob.glob(os.path.join(temp_folder_repo, "*.txt"))
+files_html = glob.glob(os.path.join(temp_folder_html, "*.txt"))
+print(
+ f"Download {len(files_repo)} essays from `https://github.com/gkamradt/LLMTest_NeedleInAHaystack/`"
+)
+print(f"Download {len(files_html)} essays from `http://www.paulgraham.com/`")
+
+text = ""
+for file in files_repo + files_html:
+ with open(file, "r") as f:
+ text += f.read()
+
+with open("PaulGrahamEssays.json", "w") as f:
+ json.dump({"text": text}, f)
+
+
+shutil.rmtree(temp_folder_repo)
+shutil.rmtree(temp_folder_html)
diff --git a/TriangleMix/ruler/data/synthetic/json/download_qa_dataset.sh b/TriangleMix/ruler/data/synthetic/json/download_qa_dataset.sh
new file mode 100644
index 00000000..21838b64
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/json/download_qa_dataset.sh
@@ -0,0 +1,5 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json -O squad.json
+wget http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_distractor_v1.json -O hotpotqa.json
diff --git a/TriangleMix/ruler/data/synthetic/niah.py b/TriangleMix/ruler/data/synthetic/niah.py
new file mode 100644
index 00000000..5d80e965
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/niah.py
@@ -0,0 +1,332 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Create a dataset jsonl file for needle in a haystack.
+
+python niah.py \
+ --save_dir=./ \
+ --save_name=niah_single \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type=nemo \
+ --max_seq_length=4096 \
+ --tokens_to_generate=128 \
+ --num_samples=10 \
+ --template="Some special magic {type_needle_v} are hidden within the following text. Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned in the provided text? The special magic {type_needle_v} for {query} mentioned in the provided text are"
+"""
+import argparse
+import importlib
+import json
+import os
+import random
+import re
+import sys
+import uuid
+from pathlib import Path
+
+import numpy as np
+import wonderwords
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from nltk.tokenize import sent_tokenize
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+# Basic Configurations
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate",
+ type=int,
+ required=True,
+ help="expected generated token amount.",
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+# Complexity Configurations
+parser.add_argument("--num_needle_k", type=int, default=1)
+parser.add_argument("--num_needle_v", type=int, default=1)
+parser.add_argument("--num_needle_q", type=int, default=1)
+parser.add_argument(
+ "--type_haystack",
+ type=str,
+ default="essay",
+ help="[Options] repeat, essay, needle.",
+)
+parser.add_argument(
+ "--type_needle_k",
+ type=str,
+ default="words",
+ help="[Options] numbers, words, uuids.",
+)
+parser.add_argument(
+ "--type_needle_v",
+ type=str,
+ default="numbers",
+ help="[Options] numbers, words, uuids.",
+)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+args.num_needle_k = max(args.num_needle_k, args.num_needle_q)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+# Define Needle/Haystack Format
+needle = "One of the special magic {type_needle_v} for {key} is: {value}."
+if args.type_haystack == "essay":
+ essay = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), "json/PaulGrahamEssays.json"
+ )
+ essay = json.load(open(essay))["text"]
+ haystack = re.sub(r"\s+", " ", essay).split(" ")
+elif args.type_haystack == "repeat":
+ haystack = "The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again."
+elif args.type_haystack == "needle":
+ haystack = needle
+else:
+ raise NotImplementedError(f"{args.type_haystack} is not implemented.")
+
+
+# Words
+nouns = wonderwords.random_word._get_words_from_text_file("nounlist.txt")
+adjs = wonderwords.random_word._get_words_from_text_file("adjectivelist.txt")
+# verbs = wonderwords.random_word._get_words_from_text_file("verblist.txt")
+words = [f"{adj}-{noun}" for adj in adjs for noun in nouns]
+words = sorted(list(set(words)))
+
+
+# Positions
+DEPTHS = list(np.round(np.linspace(0, 100, num=40, endpoint=True)).astype(int))
+
+
+def generate_random_number(num_digits=7):
+ lower_bound = 10 ** (num_digits - 1)
+ upper_bound = 10**num_digits - 1
+ return str(random.randint(lower_bound, upper_bound))
+
+
+def generate_random_word():
+ word = random.choice(words)
+ return word
+
+
+def generate_random_uuid():
+ return str(uuid.UUID(int=random.getrandbits(128), version=4))
+
+
+def generate_random(type_needle: str):
+ if type_needle == "numbers":
+ return generate_random_number()
+ elif type_needle == "words":
+ return generate_random_word()
+ elif type_needle == "uuids":
+ return generate_random_uuid()
+ else:
+ raise NotImplementedError(f"{args.type_needle} is not implemented.")
+
+
+def generate_input_output(num_haystack):
+ keys, values, needles = [], [], []
+ for _ in range(args.num_needle_k):
+ keys.append(generate_random(args.type_needle_k))
+ value = []
+ for _ in range(args.num_needle_v):
+ value.append(generate_random(args.type_needle_v))
+ needles.append(
+ needle.format(
+ type_needle_v=args.type_needle_v,
+ key=keys[-1],
+ value=value[-1],
+ )
+ )
+ values.append(value)
+
+ random.Random(args.random_seed).shuffle(needles)
+
+ # Context
+ if args.type_haystack == "essay":
+ text = " ".join(haystack[:num_haystack])
+ document_sents = sent_tokenize(text.strip())
+ insertion_positions = (
+ [0]
+ + sorted(
+ [
+ int(len(document_sents) * (depth / 100))
+ for depth in random.sample(DEPTHS, len(needles))
+ ]
+ )
+ + [len(document_sents)]
+ )
+ document_sents_list = []
+ for i in range(1, len(insertion_positions)):
+ last_pos = insertion_positions[i - 1]
+ next_pos = insertion_positions[i]
+ document_sents_list.append(" ".join(document_sents[last_pos:next_pos]))
+ if i - 1 < len(needles):
+ document_sents_list.append(needles[i - 1])
+ context = " ".join(document_sents_list)
+
+ else:
+ if args.type_haystack == "repeat":
+ sentences = [haystack] * num_haystack
+ elif args.type_haystack == "needle":
+ sentences = [
+ haystack.format(
+ type_needle_v=args.type_needle_v,
+ key=generate_random(args.type_needle_k),
+ value=generate_random(args.type_needle_v),
+ )
+ for _ in range(num_haystack)
+ ]
+
+ indexes = sorted(random.sample(range(num_haystack), len(needles)), reverse=True)
+ for index, element in zip(indexes, needles):
+ sentences.insert(index, element)
+ context = "\n".join(sentences)
+
+ ## Query and Answer
+ indices = random.sample(range(args.num_needle_k), args.num_needle_q)
+ queries = [keys[i] for i in indices]
+ answers = [a for i in indices for a in values[i]]
+ query = (
+ ", ".join(queries[:-1]) + ", and " + queries[-1]
+ if len(queries) > 1
+ else queries[0]
+ )
+
+ template = args.template
+ type_needle_v = args.type_needle_v
+ if args.num_needle_q * args.num_needle_v == 1:
+ template = template.replace("Some", "A")
+ template = template.replace("are all", "is")
+ template = template.replace("are", "is")
+ template = template.replace("answers", "answer")
+ type_needle_v = type_needle_v[:-1] # remove "s"
+
+ input_text = template.format(
+ type_needle_v=type_needle_v,
+ context=context,
+ query=query,
+ )
+
+ return input_text, answers
+
+
+def generate_samples(
+ num_samples: int, max_seq_length: int, save_dir: str, incremental: int = 500
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ if args.type_haystack == "essay":
+ incremental = 500
+ elif args.type_haystack == "repeat":
+ incremental = 25
+ elif args.type_haystack == "needle":
+ incremental = 25
+
+ if args.type_haystack != "essay" and args.max_seq_length < 4096:
+ incremental = 5
+
+ num_haystack = incremental
+
+ total_tokens = 0 # Track the total tokens generated for the first example
+ while total_tokens + tokens_to_generate < max_seq_length:
+ input_text, answer = generate_input_output(num_haystack)
+ # Calculate the number of tokens in the example
+ total_tokens = len(TOKENIZER.text_to_tokens(input_text + " ".join(answer)))
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Haystack: {num_haystack}"
+ )
+ if total_tokens + tokens_to_generate > max_seq_length:
+ num_haystack -= incremental
+ break
+
+ if args.type_haystack == "essay" and num_haystack > len(haystack):
+ num_haystack = len(haystack)
+ break
+
+ num_haystack += incremental
+
+ print("Num haystack:", num_haystack)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_haystack = num_haystack
+ while True:
+ try:
+ input_text, answer = generate_input_output(used_haystack)
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_haystack > incremental:
+ used_haystack -= incremental
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ write_jsons = generate_samples(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ save_dir=args.save_dir,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/data/synthetic/qa.py b/TriangleMix/ruler/data/synthetic/qa.py
new file mode 100644
index 00000000..7aa79739
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/qa.py
@@ -0,0 +1,262 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Create a dataset jsonl file for QA task.
+
+python qa.py \
+ --save_dir=./ \
+ --save_name=niah_single \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type=nemo \
+ --max_seq_length=4096 \
+ --tokens_to_generate=128 \
+ --num_samples=10 \
+ --template="Answer the question based on the given documents. Only give me the answer and do not output any other words.\n\nThe following are given documents.\n\n{context}\n\nAnswer the question based on the given documents. Only give me the answer and do not output any other words.\n\nQuestion: {query} Answer:"
+"""
+import argparse
+import json
+import os
+import random
+import re
+import sys
+from pathlib import Path
+
+import numpy as np
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+# Basic Configurations
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate",
+ type=int,
+ required=True,
+ help="expected generated token amount.",
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument(
+ "--pre_samples", type=int, default=0, help="number of samples are already generated"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, required=True, help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+# Complexity Configurations
+parser.add_argument("--dataset", type=str, required=True, help="dataset file")
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+
+# Read SQuAD QA dataset
+def read_squad(file):
+ with open(file) as f:
+ data = json.load(f)
+
+ total_docs = [p["context"] for d in data["data"] for p in d["paragraphs"]]
+ total_docs = sorted(list(set(total_docs)))
+ total_docs_dict = {c: idx for idx, c in enumerate(total_docs)}
+
+ total_qas = []
+ for d in data["data"]:
+ more_docs = [total_docs_dict[p["context"]] for p in d["paragraphs"]]
+ for p in d["paragraphs"]:
+ for qas in p["qas"]:
+ if not qas["is_impossible"]:
+ total_qas.append(
+ {
+ "query": qas["question"],
+ "outputs": [a["text"] for a in qas["answers"]],
+ "context": [total_docs_dict[p["context"]]],
+ "more_context": [
+ idx
+ for idx in more_docs
+ if idx != total_docs_dict[p["context"]]
+ ],
+ }
+ )
+
+ return total_qas, total_docs
+
+
+# Read Hotpot QA dataset
+def read_hotpotqa(file):
+ with open(file) as f:
+ data = json.load(f)
+
+ total_docs = [f"{t}\n{''.join(p)}" for d in data for t, p in d["context"]]
+ total_docs = sorted(list(set(total_docs)))
+ total_docs_dict = {c: idx for idx, c in enumerate(total_docs)}
+
+ total_qas = []
+ for d in data:
+ total_qas.append(
+ {
+ "query": d["question"],
+ "outputs": [d["answer"]],
+ "context": [
+ total_docs_dict[f"{t}\n{''.join(p)}"] for t, p in d["context"]
+ ],
+ }
+ )
+
+ return total_qas, total_docs
+
+
+DOCUMENT_PROMPT = "Document {i}:\n{document}"
+if args.dataset == "squad":
+ QAS, DOCS = read_squad(
+ os.path.join(os.path.dirname(os.path.abspath(__file__)), "json/squad.json")
+ )
+elif args.dataset == "hotpotqa":
+ QAS, DOCS = read_hotpotqa(
+ os.path.join(os.path.dirname(os.path.abspath(__file__)), "json/hotpotqa.json")
+ )
+else:
+ raise NotImplementedError(f"{args.dataset} is not implemented.")
+
+
+def generate_input_output(index, num_docs):
+ curr_q = QAS[index]["query"]
+ curr_a = QAS[index]["outputs"]
+ curr_docs = QAS[index]["context"]
+ curr_more = QAS[index].get("more_context", [])
+ if num_docs < len(DOCS):
+ if (num_docs - len(curr_docs)) > len(curr_more):
+ addition_docs = [
+ i for i, d in enumerate(DOCS) if i not in curr_docs + curr_more
+ ]
+ all_docs = (
+ curr_docs
+ + curr_more
+ + random.sample(
+ addition_docs, max(0, num_docs - len(curr_docs) - len(curr_more))
+ )
+ )
+ else:
+ all_docs = curr_docs + random.sample(curr_more, num_docs - len(curr_docs))
+
+ all_docs = [DOCS[idx] for idx in all_docs]
+ else:
+ all_docs = DOCS
+
+ random.Random(args.random_seed).shuffle(all_docs)
+
+ context = "\n\n".join(
+ [DOCUMENT_PROMPT.format(i=i + 1, document=d) for i, d in enumerate(all_docs)]
+ )
+ input_text = args.template.format(context=context, query=curr_q)
+ return input_text, curr_a
+
+
+def generate_samples(
+ num_samples: int, max_seq_length: int, save_dir: str, incremental: int = 10
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ # Find the perfect num_docs
+ num_docs = incremental
+
+ total_tokens = 0 # Track the total tokens generated for this example
+ while total_tokens + tokens_to_generate < max_seq_length:
+ input_text, answer = generate_input_output(0, num_docs)
+ # Calculate the number of tokens in the example
+ total_tokens = len(TOKENIZER.text_to_tokens(input_text + f" {answer}"))
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Docs: {num_docs}"
+ )
+ if total_tokens + tokens_to_generate > max_seq_length:
+ num_docs -= incremental
+ break
+
+ num_docs += incremental
+ if num_docs > len(DOCS):
+ num_docs = len(DOCS)
+ break
+ print("Number of documents:", num_docs)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_docs = num_docs
+ while True:
+ try:
+ input_text, answer = generate_input_output(
+ index + args.pre_samples, used_docs
+ )
+ length = len(TOKENIZER.text_to_tokens(input_text)) + tokens_to_generate
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_docs > incremental:
+ used_docs -= incremental
+
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ write_jsons = generate_samples(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ save_dir=args.save_dir,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/data/synthetic/variable_tracking.py b/TriangleMix/ruler/data/synthetic/variable_tracking.py
new file mode 100644
index 00000000..7e0778cc
--- /dev/null
+++ b/TriangleMix/ruler/data/synthetic/variable_tracking.py
@@ -0,0 +1,261 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Create a dataset jsonl file for variable tracking.
+
+python variable_tracking.py \
+ --save_dir=./ \
+ --save_name=vt \
+ --tokenizer_path=tokenizer.model \
+ --tokenizer_type nemo \
+ --max_seq_length 4096 \
+ --tokens_to_generate 30 \
+ --num_samples 10 \
+ --random_seed 42 \
+ --num_chains 1 --num_hops 4 \
+ --template "[INST] Memorize and track the chain(s) of variable assignment hidden in the following text.\n\n{context}\nQuestion: Find all variables that are assigned the value {query} in the text above. [/INST] Answer: According to the chain(s) of variable assignment in the text above, {num_v} variables are assgined the value {query}, they are: "
+"""
+import argparse
+import os
+import random
+import string
+import sys
+from pathlib import Path
+
+from constants import TASKS
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
+import numpy as np
+from tokenizer import select_tokenizer
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--save_dir", type=Path, required=True, help="dataset folder to save dataset"
+)
+parser.add_argument(
+ "--save_name", type=str, required=True, help="name of the save dataset jsonl file"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--tokenizer_path", type=str, required=True, help="path to the tokenizer model"
+)
+parser.add_argument(
+ "--tokenizer_type", type=str, default="nemo", help="[Options] nemo, hf, openai."
+)
+parser.add_argument(
+ "--max_seq_length",
+ type=int,
+ required=True,
+ help="max sequence length including all input tokens and generated tokens.",
+)
+parser.add_argument(
+ "--tokens_to_generate", type=int, default=120, help="number of tokens to generate"
+)
+parser.add_argument(
+ "--num_samples", type=int, required=True, help="number of samples to generate"
+)
+parser.add_argument("--random_seed", type=int, default=42)
+parser.add_argument("--template", type=str, default="", help="prompt template")
+parser.add_argument(
+ "--remove_newline_tab",
+ action="store_true",
+ help="remove `\n` and `\t` in all strings.",
+)
+
+parser.add_argument(
+ "--num_chains", type=int, default=1, help="number of inserted variable chains"
+)
+parser.add_argument(
+ "--num_hops", type=int, default=4, help="number of hops in each chain"
+)
+parser.add_argument("--add_fewshot", action="store_true", default=False)
+
+args = parser.parse_args()
+random.seed(args.random_seed)
+np.random.seed(args.random_seed)
+
+# Load Tokenizer
+TOKENIZER = select_tokenizer(args.tokenizer_type, args.tokenizer_path)
+
+
+def generate_chains(num_chains, num_hops, is_icl=False):
+ vars_all = []
+ k = 5 if not is_icl else 3
+ num_hops = num_hops if not is_icl else min(10, num_hops)
+ vars_all = [
+ "".join(random.choices(string.ascii_uppercase, k=k)).upper()
+ for _ in range((num_hops + 1) * num_chains)
+ ]
+ while len(set(vars_all)) < num_chains * (num_hops + 1):
+ vars_all.append("".join(random.choices(string.ascii_uppercase, k=k)).upper())
+
+ vars_ret = []
+ chains_ret = []
+ for i in range(0, len(vars_all), num_hops + 1):
+ this_vars = vars_all[i : i + num_hops + 1]
+ vars_ret.append(this_vars)
+ this_chain = [f"VAR {this_vars[0]} = {np.random.randint(10000, 99999)}"]
+ for j in range(num_hops):
+ this_chain.append(f"VAR {this_vars[j+1]} = VAR {this_vars[j]} ")
+ chains_ret.append(this_chain)
+ return vars_ret, chains_ret
+
+
+def generate_input_output(num_noises, num_chains, num_hops, is_icl=False):
+ vars, chains = generate_chains(num_chains, num_hops, is_icl=is_icl)
+
+ noise = "The grass is green. The sky is blue. The sun is yellow. Here we go. There and back again.\n"
+
+ # Create a list of the repeated noise
+ sentences = [noise] * num_noises
+ if len(sentences) <= len(chains[0]):
+ sentences = [
+ n + "." if len(n.strip()) > 0 else n
+ for n in [x for noise in sentences for x in noise.split(".")]
+ ]
+ try:
+ assert len(sentences) > len(
+ chains[0]
+ ), "Noises too short, unable to generate data"
+ except:
+ print("reduces chain length for not enough noises")
+ chains = [chain[: len(sentences) - 1] for chain in chains]
+ # sample random positions to insert variable assignment
+ for chain_i in chains:
+ # sample random positions (sorted) to insert variable assignment
+ positions = list(sorted(random.sample(range(len(sentences)), len(chain_i))))
+ for insert_pi, j in zip(positions, range(len(chain_i))):
+ sentences.insert(insert_pi + j, chain_i[j])
+
+ # Insert the passkey sentence at the random position
+ context = " ".join(sentences)
+ context = context.replace(". \n", ".\n")
+
+ template = args.template
+ if is_icl:
+ # remove model template
+ cutoff = template.index(TASKS["variable_tracking"]["template"][:20])
+ cutoff_ans = template.index(TASKS["variable_tracking"]["answer_prefix"][:10])
+ template = (
+ " ".join(template[cutoff:cutoff_ans].split()[:-1]) + template[cutoff_ans:]
+ )
+
+ value = chains[0][0].split("=")[-1].strip()
+ input_text = template.format(context=context, query=value, num_v=num_hops + 1)
+
+ return input_text, vars[0]
+
+
+def sys_vartrack_w_noise_random(
+ num_samples: int,
+ max_seq_length: int,
+ incremental: int = 10,
+ num_chains: int = 1,
+ num_hops: int = 4,
+ add_fewshot: bool = True,
+ icl_example: str = None,
+):
+ write_jsons = []
+ tokens_to_generate = args.tokens_to_generate
+
+ # Find the perfect num_noises
+ num_noises = incremental
+
+ total_tokens = 0 # Track the total tokens generated for this example
+ example_tokens = 0
+ if add_fewshot and (icl_example is not None):
+ icl_example_out = " ".join(icl_example["outputs"])
+ icl_example = icl_example["input"] + " " + icl_example_out + "\n\n"
+ example_tokens = len(TOKENIZER.text_to_tokens(icl_example))
+
+ while total_tokens + tokens_to_generate + example_tokens < max_seq_length:
+ input_text, answer = generate_input_output(
+ num_noises, num_chains, num_hops, is_icl=add_fewshot & (icl_example is None)
+ )
+ # Calculate the number of tokens in the example
+ total_tokens = len(TOKENIZER.text_to_tokens(input_text + f" {answer}"))
+ print(
+ f"Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate + example_tokens} | Noises: {num_noises}"
+ )
+ if total_tokens + tokens_to_generate + example_tokens > max_seq_length:
+ num_noises -= incremental
+ break
+ num_noises += incremental
+ print("Num noises:", num_noises)
+
+ # Generate samples
+ for index in tqdm(range(num_samples)):
+ used_noises = num_noises
+ while True:
+ try:
+ input_text, answer = generate_input_output(
+ num_noises,
+ num_chains,
+ num_hops,
+ is_icl=add_fewshot & (icl_example is None),
+ )
+ length = (
+ len(TOKENIZER.text_to_tokens(input_text))
+ + tokens_to_generate
+ + example_tokens
+ )
+ assert length <= max_seq_length, f"{length} exceeds max_seq_length."
+ break
+ except:
+ if used_noises > incremental:
+ used_noises -= incremental
+
+ if add_fewshot and (icl_example is not None):
+ # insert icl_example between model template and input
+ cutoff = input_text.index(TASKS["variable_tracking"]["template"][:20])
+ input_text = (
+ input_text[:cutoff] + " " + icl_example + "\n\n" + input_text[cutoff:]
+ )
+ if args.remove_newline_tab:
+ input_text = " ".join(
+ input_text.replace("\n", " ").replace("\t", " ").strip().split()
+ )
+
+ formatted_output = {
+ "index": index,
+ "input": input_text,
+ "outputs": answer,
+ "length": length,
+ }
+ write_jsons.append(formatted_output)
+
+ return write_jsons
+
+
+def main():
+ save_file = args.save_dir / f"{args.save_name}" / f"{args.subset}.jsonl"
+ save_file.parent.mkdir(parents=True, exist_ok=True)
+
+ icl_example = sys_vartrack_w_noise_random(
+ num_samples=1,
+ max_seq_length=500,
+ incremental=5,
+ num_chains=args.num_chains,
+ num_hops=args.num_hops,
+ )[0]
+ write_jsons = sys_vartrack_w_noise_random(
+ num_samples=args.num_samples,
+ max_seq_length=args.max_seq_length,
+ num_chains=args.num_chains,
+ num_hops=args.num_hops,
+ icl_example=icl_example,
+ )
+
+ write_manifest(save_file, write_jsons)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/data/template.py b/TriangleMix/ruler/data/template.py
new file mode 100644
index 00000000..d059f983
--- /dev/null
+++ b/TriangleMix/ruler/data/template.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+Templates = {
+ "base": "{task_template}",
+ "meta-chat": "[INST] {task_template} [/INST]",
+ "vicuna-chat": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {task_template} ASSISTANT:",
+ "lwm-chat": "You are a helpful assistant. USER: {task_template} ASSISTANT: ",
+ "command-r-chat": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{task_template}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",
+ "chatglm-chat": "[gMASK]sop<|user|> \n {task_template}<|assistant|> \n ",
+ "RWKV": "User: hi\n\nAssistant: Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it\n\nUser: {task_template}\n\nAssistant:",
+ "llama-3": "<|start_header_id|>user<|end_header_id|> {task_template}<|eot_id|><|start_header_id|>assistant<|end_header_id|>",
+ "glm-4": "[gMASK]<|user|>\n {task_template}<|assistant|>\n",
+}
diff --git a/TriangleMix/ruler/data/tokenizer.py b/TriangleMix/ruler/data/tokenizer.py
new file mode 100644
index 00000000..3612769c
--- /dev/null
+++ b/TriangleMix/ruler/data/tokenizer.py
@@ -0,0 +1,102 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+
+import os
+from typing import List
+
+from tenacity import retry, stop_after_attempt, wait_fixed, wait_random
+
+
+def select_tokenizer(tokenizer_type, tokenizer_path):
+ if tokenizer_type == "nemo":
+ return NeMoSentencePieceTokenizer(model_path=tokenizer_path)
+ elif tokenizer_type == "hf":
+ return HFTokenizer(model_path=tokenizer_path)
+ elif tokenizer_type == "openai":
+ return OpenAITokenizer(model_path=tokenizer_path)
+ elif tokenizer_type == "gemini":
+ return GeminiTokenizer(model_path=tokenizer_path)
+ else:
+ raise ValueError(f"Unknown tokenizer_type {tokenizer_type}")
+
+
+class NeMoSentencePieceTokenizer:
+ """
+ Tokenizer from NeMo SentencePieceTokenizer
+ """
+
+ def __init__(self, model_path) -> None:
+ from nemo.collections.common.tokenizers.sentencepiece_tokenizer import (
+ SentencePieceTokenizer,
+ )
+
+ self.tokenizer = SentencePieceTokenizer(model_path=model_path)
+
+ def text_to_tokens(self, text: str) -> List[str]:
+ tokens = self.tokenizer.text_to_tokens(text)
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ text = self.tokenizer.tokens_to_text(tokens)
+ return text
+
+
+class HFTokenizer:
+ """
+ Tokenizer from HF models
+ """
+
+ def __init__(self, model_path) -> None:
+ from transformers import AutoTokenizer
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ model_path, trust_remote_code=True
+ )
+
+ def text_to_tokens(self, text: str) -> List[str]:
+ tokens = self.tokenizer.tokenize(text)
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ text = self.tokenizer.convert_tokens_to_string(tokens)
+ return text
+
+
+class OpenAITokenizer:
+ """
+ Tokenizer from tiktoken
+ """
+
+ def __init__(self, model_path="cl100k_base") -> None:
+ import tiktoken
+
+ self.tokenizer = tiktoken.get_encoding(model_path)
+
+ def text_to_tokens(self, text: str) -> List[int]:
+ tokens = self.tokenizer.encode(text)
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ text = self.tokenizer.decode(tokens)
+ return text
+
+
+class GeminiTokenizer:
+ """
+ Tokenizer from gemini
+ """
+
+ def __init__(self, model_path="gemini-1.5-pro-latest") -> None:
+ import google.generativeai as genai
+
+ genai.configure(api_key=os.environ["GEMINI_API_KEY"])
+ self.model = genai.GenerativeModel(model_path)
+
+ @retry(wait=wait_fixed(60) + wait_random(0, 10), stop=stop_after_attempt(3))
+ def text_to_tokens(self, text: str) -> List[int]:
+ tokens = list(range(self.model.count_tokens(text).total_tokens))
+ return tokens
+
+ def tokens_to_text(self, tokens: List[int]) -> str:
+ pass
diff --git a/TriangleMix/ruler/eval/evaluate.py b/TriangleMix/ruler/eval/evaluate.py
new file mode 100644
index 00000000..ddae8a9d
--- /dev/null
+++ b/TriangleMix/ruler/eval/evaluate.py
@@ -0,0 +1,227 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Get summary.csv with score and null predictions amount.
+
+Running
+```
+python evaluate.py \
+ --data_dir /path/to/your/prediction_jsonl_folder \
+ --benchmark synthetic
+```
+"""
+
+import argparse
+import os
+import re
+
+import nltk
+
+try:
+ nltk.data.find("tokenizers/punkt")
+except LookupError:
+ nltk.download("punkt")
+
+import importlib
+from collections import defaultdict
+from pathlib import Path
+
+import pandas as pd
+import yaml
+from nemo.collections.asr.parts.utils.manifest_utils import (
+ read_manifest,
+ write_manifest,
+)
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--data_dir", type=str, required=True, help="path to the prediction jsonl files"
+)
+parser.add_argument(
+ "--benchmark", type=str, default="synthetic", help="Options: [synthetic]"
+)
+parser.add_argument(
+ "--verbose", type=int, default=0, help="how many lines you want to display."
+)
+args = parser.parse_args()
+
+
+def postprocess_pred(predict_str: str, task_config: dict):
+ predict_str = predict_str.strip()
+
+ # Remove all non-printable characters
+ np_pattern = re.compile(r"[\x00-\x1f]")
+ predict_str = np_pattern.sub("\n", predict_str).strip()
+
+ return predict_str
+
+
+def get_pred_and_ref(
+ predictions_file: str,
+ task_config: dict,
+ input_field: str = "input",
+ references_field: str = "outputs",
+ prediction_field: str = "pred",
+ metadata_field: str = "others",
+):
+ lines = read_manifest(predictions_file)
+
+ inputs = []
+ predicts = []
+ references = []
+ indices = []
+
+ for line in tqdm(lines):
+ input = line[input_field]
+ predict = line[prediction_field]
+ predict = postprocess_pred(predict, task_config)
+ reference = line.get(references_field, [line.get("output", "")])
+ index = line[metadata_field].get("id", line["index"])
+
+ inputs.append(input)
+ predicts.append(predict)
+ references.append(reference)
+ indices.append(index)
+
+ return inputs, predicts, references, indices
+
+
+def run_evaluation_per_task(task_config: dict, predictions_file: str, verbose: int = 0):
+ inputs, predicts, references, indices = get_pred_and_ref(
+ predictions_file=predictions_file,
+ task_config=task_config,
+ )
+
+ task_nulls = f"{sum([len(x)==0 for x in predicts])}/{len(predicts)}"
+
+ if len(references) > 0 and references[0][0] is not None:
+ task_score = task_config["metric_fn"](predicts, references)
+ else:
+ task_score = 0.0
+
+ if verbose != 0:
+ print("=" * 40)
+ for i, (input, reference, predict) in enumerate(
+ zip(inputs, references, predicts)
+ ):
+ print(f"Input : {input}")
+ print(f"Reference : {reference}")
+ print(f"Prediction: {predict}")
+ print("=" * 40)
+ if i > verbose:
+ break
+
+ return task_score, task_nulls, predicts, indices
+
+
+def write_evaluation(results: dict):
+ tasks = list(results.keys())
+ score = [results[task]["score"] for task in tasks]
+ nulls = [results[task]["nulls"] for task in tasks]
+ dfs = [
+ ["Tasks"] + tasks,
+ ["Score"] + score,
+ ["Nulls"] + nulls,
+ ]
+
+ output_file = os.path.join(
+ args.data_dir, "summary.csv" if len(tasks) > 1 else f"summary-{tasks[0]}.csv"
+ )
+ df = pd.DataFrame(dfs)
+ df.to_csv(output_file, index=False)
+ print("\n=============================================\n")
+ print(df)
+ print(f"\nSaved eval results to {output_file}")
+
+
+def write_submission(results: dict):
+ COLUMNS = ["Task", "ID", "Prediction"]
+ dfs = pd.DataFrame(columns=COLUMNS, data=[])
+
+ for task, result in results.items():
+ df = pd.DataFrame(
+ {"Task": task, "ID": result["indices"], "Prediction": result["predicts"]}
+ )
+ dfs = pd.concat((dfs, df[COLUMNS]))
+
+ output_file = os.path.join(args.data_dir, "submission.csv")
+ dfs = dfs.reset_index(drop=True)
+ dfs.to_csv(output_file, index=False)
+ print(f"\nSaved submission results to {output_file}")
+
+
+def aggregate_chunk(folder):
+ jsonl_files = [file for file in os.listdir(folder) if Path(file).suffix == ".jsonl"]
+ chunk_files = sorted(
+ [file for file in jsonl_files if re.match(r".*[^_]+-\d+\.jsonl", file)]
+ )
+ chunk_files_dict = defaultdict(list)
+ for file in chunk_files:
+ task = "-".join(file.split("-")[:-1])
+ chunk_files_dict[task].append(file)
+
+ for task, files in chunk_files_dict.items():
+ lines = []
+ for file in sorted(files):
+ file = os.path.join(folder, file)
+ lines += read_manifest(file)
+ os.remove(file) # Remove chunk files
+ write_manifest(os.path.join(folder, f"{task}.jsonl"), lines)
+
+
+def main():
+ curr_folder = os.path.dirname(os.path.abspath(__file__))
+
+ try:
+ module = importlib.import_module(f"{args.benchmark}.constants")
+ except ImportError:
+ print(f"Module eval.{args.benchmark}.constants not found.")
+
+ tasks_base = module.TASKS
+ with open(os.path.join(curr_folder, f"../{args.benchmark}.yaml"), "r") as f:
+ tasks_customized = yaml.safe_load(f)
+
+ TASKS = tasks_customized
+ for _, config in TASKS.items():
+ config.update(tasks_base[config["task"]])
+
+ print(f"Total tasks: {list(TASKS.keys())}")
+
+ # Aggregate all prediction files
+ aggregate_chunk(args.data_dir)
+
+ # Get scores and nulls
+ jsonl_files = [
+ file for file in os.listdir(args.data_dir) if Path(file).suffix == ".jsonl"
+ ]
+ eval_results = {}
+ subm_results = {}
+
+ for task, config in TASKS.items():
+ if f"{task}.jsonl" not in jsonl_files:
+ print(f"Prediction file {task}.jsonl is not found.")
+ continue
+
+ print(f"Evaluate task {task}...")
+ task_score, task_nulls, predicts, indices = run_evaluation_per_task(
+ predictions_file=os.path.join(args.data_dir, f"{task}.jsonl"),
+ task_config=config,
+ )
+ eval_results[task] = {
+ "score": task_score,
+ "nulls": task_nulls,
+ }
+ subm_results[task] = {
+ "predicts": predicts,
+ "indices": indices,
+ }
+
+ # Write to csv
+ write_evaluation(eval_results)
+ write_submission(subm_results)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/eval/synthetic/constants.py b/TriangleMix/ruler/eval/synthetic/constants.py
new file mode 100644
index 00000000..fb3bbd7c
--- /dev/null
+++ b/TriangleMix/ruler/eval/synthetic/constants.py
@@ -0,0 +1,55 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Add a new task:
+
+TASK_NAME: {
+ 'metric_fn': the metric function with input (predictions: [str], references: [[str]]) to compute score.
+}
+"""
+
+
+def string_match_part(preds, refs):
+ score = (
+ sum(
+ [
+ max([1.0 if r.lower() in pred.lower() else 0.0 for r in ref])
+ for pred, ref in zip(preds, refs)
+ ]
+ )
+ / len(preds)
+ * 100
+ )
+ return round(score, 2)
+
+
+def string_match_all(preds, refs):
+ score = (
+ sum(
+ [
+ sum([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) / len(ref)
+ for pred, ref in zip(preds, refs)
+ ]
+ )
+ / len(preds)
+ * 100
+ )
+ return round(score, 2)
+
+
+TASKS = {
+ "niah": {
+ "metric_fn": string_match_all,
+ },
+ "variable_tracking": {
+ "metric_fn": string_match_all,
+ },
+ "common_words_extraction": {
+ "metric_fn": string_match_all,
+ },
+ "freq_words_extraction": {"metric_fn": string_match_all},
+ "qa": {
+ "metric_fn": string_match_part,
+ },
+}
diff --git a/TriangleMix/ruler/pred/call_api.py b/TriangleMix/ruler/pred/call_api.py
new file mode 100644
index 00000000..a37a10e1
--- /dev/null
+++ b/TriangleMix/ruler/pred/call_api.py
@@ -0,0 +1,349 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""
+Prepare prediction jsonl with field `pred` .
+dataset jsonl:
+{
+ "index" int,
+ "input": str,
+ "outputs": [str],
+}
+
+prediction jsonl:
+{
+ "index" int,
+ "input": str,
+ "outputs": [str],
+ "pred": str,
+}
+"""
+
+import argparse
+import importlib
+import json
+import math
+import os
+import sys
+import threading
+import time
+import traceback
+from pathlib import Path
+
+import yaml
+from nemo.collections.asr.parts.utils.manifest_utils import read_manifest
+from tqdm import tqdm
+
+SERVER_TYPES = (
+ "trtllm",
+ "vllm",
+ "openai",
+ "gemini",
+ "hf",
+ "minference",
+ "a_shape",
+ "InfLLM",
+ "minference_with_dense",
+)
+
+
+class ServerAction(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ namespace.server_type = values
+
+
+parser = argparse.ArgumentParser()
+# Data
+parser.add_argument(
+ "--data_dir", type=Path, required=True, help="path to load the dataset jsonl files"
+)
+parser.add_argument(
+ "--save_dir",
+ type=Path,
+ required=True,
+ help="path to save the prediction jsonl files",
+)
+parser.add_argument(
+ "--benchmark", type=str, default="synthetic", help="Options: [synthetic]"
+)
+parser.add_argument(
+ "--task", type=str, required=True, help="Options: tasks in benchmark"
+)
+parser.add_argument(
+ "--subset", type=str, default="validation", help="Options: validation or test"
+)
+parser.add_argument(
+ "--chunk_idx", type=int, default=0, help="index of current split chunk"
+)
+parser.add_argument("--chunk_amount", type=int, default=1, help="size of split chunk")
+
+# Server
+parser.add_argument(
+ "--server_type", default="nemo", action=ServerAction, choices=SERVER_TYPES
+)
+parser.add_argument("--server_host", type=str, default="127.0.0.1")
+parser.add_argument("--server_port", type=str, default="5000")
+parser.add_argument("--ssh_server", type=str)
+parser.add_argument("--ssh_key_path", type=str)
+parser.add_argument(
+ "--model_name_or_path",
+ type=str,
+ default="gpt-3.5-turbo",
+ help="supported models from OpenAI or HF (provide a key or a local path to the checkpoint)",
+)
+
+# Inference
+parser.add_argument("--temperature", type=float, default=1.0)
+parser.add_argument("--top_k", type=int, default=32)
+parser.add_argument("--top_p", type=float, default=1.0)
+parser.add_argument("--random_seed", type=int, default=0)
+parser.add_argument("--stop_words", type=str, default="")
+parser.add_argument("--sliding_window_size", type=int)
+parser.add_argument("--threads", type=int, default=4)
+
+# MInference
+parser.add_argument("--config_path", type=str)
+parser.add_argument("--starting_layer", type=int, default=-1)
+parser.add_argument("--kv_cache_cpu", action="store_true")
+parser.add_argument("--kv_cache_cpu_device", type=str, default="cpu")
+parser.add_argument("--kv_type", type=str, default="dense")
+parser.add_argument("--trust_remote_code", action="store_true")
+parser.add_argument("--attn_type", type=str, default="minference")
+parser.add_argument("--attn_kwargs", type=json.loads, default={})
+
+args = parser.parse_args()
+args.stop_words = list(filter(None, args.stop_words.split(",")))
+# if args.server_type == 'hf' or args.server_type == 'gemini' or args.server_type == 'minference':
+if args.server_type in [
+ "hf",
+ "gemini",
+ "minference",
+ "a_shape",
+ "InfLLM",
+ "minference_with_dense",
+]:
+ args.threads = 1
+
+
+def get_llm(tokens_to_generate):
+ if args.server_type == "trtllm":
+ from client_wrappers import TRTLLMClient
+
+ llm = TRTLLMClient(
+ server_host=args.server_host,
+ server_port=args.server_port,
+ ssh_server=args.ssh_server,
+ ssh_key_path=args.ssh_key_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ max_attention_window_size=args.sliding_window_size,
+ )
+
+ elif args.server_type == "vllm":
+ from client_wrappers import VLLMClient
+
+ llm = VLLMClient(
+ server_host=args.server_host,
+ server_port=args.server_port,
+ ssh_server=args.ssh_server,
+ ssh_key_path=args.ssh_key_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ )
+
+ elif args.server_type == "openai":
+ from client_wrappers import OpenAIClient
+
+ llm = OpenAIClient(
+ model_name=args.model_name_or_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ )
+
+ elif args.server_type == "gemini":
+ from client_wrappers import GeminiClient
+
+ llm = GeminiClient(
+ model_name=args.model_name_or_path,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ random_seed=args.random_seed,
+ stop=args.stop_words,
+ tokens_to_generate=tokens_to_generate,
+ )
+
+ elif args.server_type == "hf":
+ from model_wrappers import HuggingFaceModel
+
+ llm = HuggingFaceModel(
+ name_or_path=args.model_name_or_path,
+ do_sample=args.temperature > 0,
+ repetition_penalty=1,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ stop=args.stop_words,
+ max_new_tokens=tokens_to_generate,
+ )
+
+ elif args.server_type == "minference":
+ from model_wrappers import MInferenceModel
+
+ llm = MInferenceModel(
+ name_or_path=args.model_name_or_path,
+ do_sample=args.temperature > 0,
+ repetition_penalty=1,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ stop=args.stop_words,
+ max_new_tokens=tokens_to_generate,
+ config_path=args.config_path,
+ kv_cache_cpu=args.kv_cache_cpu,
+ kv_cache_cpu_device=args.kv_cache_cpu_device,
+ kv_type=args.kv_type,
+ trust_remote_code=args.trust_remote_code,
+ starting_layer=args.starting_layer,
+ attn_type=args.attn_type,
+ attn_kwargs=args.attn_kwargs,
+ )
+
+ elif args.server_type == "InfLLM":
+ from model_wrappers import InfLLM
+
+ llm = InfLLM(
+ name_or_path=args.model_name_or_path,
+ do_sample=args.temperature > 0,
+ repetition_penalty=1,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ stop=args.stop_words,
+ max_new_tokens=tokens_to_generate,
+ )
+
+ elif args.server_type == "a_shape":
+ from model_wrappers import Streaming
+
+ llm = Streaming(
+ name_or_path=args.model_name_or_path,
+ do_sample=args.temperature > 0,
+ repetition_penalty=1,
+ temperature=args.temperature,
+ top_k=args.top_k,
+ top_p=args.top_p,
+ stop=args.stop_words,
+ max_new_tokens=tokens_to_generate,
+ config_path=args.config_path,
+ kv_cache_cpu=args.kv_cache_cpu,
+ kv_cache_cpu_device=args.kv_cache_cpu_device,
+ kv_type=args.kv_type,
+ trust_remote_code=args.trust_remote_code,
+ starting_layer=args.starting_layer,
+ )
+
+ else:
+ raise RuntimeError(f"Unsupported server type {args.server_type}")
+
+ return llm
+
+
+def main():
+ start_time = time.time()
+
+ curr_folder = os.path.dirname(os.path.abspath(__file__))
+
+ try:
+ sys.path.append(os.path.dirname(curr_folder))
+ module = importlib.import_module(f"data.{args.benchmark}.constants")
+ except ImportError:
+ print(f"Module data.{args.benchmark}.constants not found.")
+
+ tasks_base = module.TASKS
+ with open(os.path.join(curr_folder, f"../{args.benchmark}.yaml"), "r") as f:
+ tasks_customized = yaml.safe_load(f)
+
+ if args.task not in tasks_customized:
+ raise ValueError(f"{args.task} is not found in config_tasks.yaml")
+
+ config = tasks_customized.get(args.task)
+ config.update(tasks_base[config["task"]])
+
+ task_file = args.data_dir / args.task / f"{args.subset}.jsonl"
+
+ if args.chunk_amount > 1:
+ pred_file = args.save_dir / f"{args.task}-{args.chunk_idx}.jsonl"
+ else:
+ pred_file = args.save_dir / f"{args.task}.jsonl"
+
+ print(f"Predict {args.task} \nfrom {task_file}\nto {pred_file}")
+ pred_file.parent.mkdir(parents=True, exist_ok=True)
+
+ # Load data
+ if os.path.exists(pred_file):
+ pred_index = [sample["index"] for sample in read_manifest(pred_file)]
+ data = [
+ sample
+ for sample in read_manifest(task_file)
+ if sample["index"] not in pred_index
+ ]
+ else:
+ data = read_manifest(task_file)
+
+ # Load api
+ llm = get_llm(config["tokens_to_generate"])
+
+ def get_output(idx, index, input, outputs, others, truncation, length):
+ while True:
+ try:
+ pred = llm(prompt=input)
+ break
+ except Exception as e:
+ traceback.print_exc()
+
+ if len(pred["text"]) > 0:
+ outputs_parallel[idx] = {
+ "index": index,
+ "pred": pred["text"][0],
+ "input": input,
+ "outputs": outputs,
+ "others": others,
+ "truncation": truncation,
+ "length": length,
+ }
+
+ threads = []
+ outputs_parallel = [{} for _ in range(len(data))]
+ # setting buffering=1 to force to dump the output after every line, so that we can see intermediate generations
+ with open(pred_file, "at", encoding="utf-8", buffering=1) as fout:
+ for idx, data_point in tqdm(enumerate(data), total=len(data)):
+ get_output(
+ idx,
+ data_point["index"],
+ data_point["input"],
+ data_point["outputs"],
+ data_point.get("others", {}),
+ data_point.get("truncation", -1),
+ data_point.get("length", -1),
+ )
+
+ fout.write(json.dumps(outputs_parallel[idx]) + "\n")
+
+ print(f"Used time: {round((time.time() - start_time) / 60, 1)} minutes")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/TriangleMix/ruler/pred/client_wrappers.py b/TriangleMix/ruler/pred/client_wrappers.py
new file mode 100644
index 00000000..c5c882ad
--- /dev/null
+++ b/TriangleMix/ruler/pred/client_wrappers.py
@@ -0,0 +1,361 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+
+import abc
+import json
+import os
+import re
+import sys
+import time
+import traceback
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import List, Tuple, Union
+
+import requests
+from tenacity import retry, stop_after_attempt, wait_random_exponential
+
+
+class Client(abc.ABC):
+ def __init__(
+ self,
+ server_host,
+ server_port="5000",
+ ssh_server=None,
+ ssh_key_path=None,
+ **generation_kwargs,
+ ):
+ self.server_host = server_host
+ self.server_port = server_port
+ self.ssh_server = os.getenv("SSH_SERVER", ssh_server)
+ self.ssh_key_path = os.getenv("SSH_KEY_PATH", ssh_key_path)
+ self.generation_kwargs = generation_kwargs
+
+ @abc.abstractmethod
+ def _single_call(
+ self,
+ prompts,
+ ):
+ pass
+
+ def __call__(self, prompt: str, **kwargs):
+ request = self.generation_kwargs
+ # prompts are added later
+ request["prompts"] = [f"{prompt}"]
+ if "others" in kwargs:
+ requeset["others"] = kwargs["others"]
+
+ outputs = self._single_call(**request)
+ response = {"text": outputs}
+ return response
+
+ @retry(wait=wait_random_exponential(min=15, max=60), stop=stop_after_attempt(3))
+ def _send_request(self, request, route="generate"):
+ if self.ssh_server and self.ssh_key_path:
+ import sshtunnel_requests
+
+ sshtunnel_request = sshtunnel_requests.from_url(
+ f"ssh://{self.ssh_server}:22", self.ssh_key_path
+ )
+ outputs = sshtunnel_request.put(
+ url="http://{}:{}/{}".format(self.server_host, self.server_port, route),
+ data=json.dumps(request),
+ headers={"Content-Type": "application/json"},
+ ).json()
+ else:
+ outputs = requests.put(
+ url="http://{}:{}/{}".format(self.server_host, self.server_port, route),
+ data=json.dumps(request),
+ headers={"Content-Type": "application/json"},
+ ).json()
+ return outputs
+
+
+class TRTLLMClient(Client):
+ def _single_call(
+ self,
+ prompts,
+ tokens_to_generate,
+ temperature,
+ top_p,
+ top_k,
+ random_seed,
+ stop: List[str],
+ max_attention_window_size=None,
+ ):
+ request = {
+ "prompts": prompts,
+ "tokens_to_generate": tokens_to_generate,
+ "temperature": temperature,
+ "top_k": top_k,
+ "top_p": top_p,
+ "random_seed": random_seed,
+ "stop_words_list": ",".join(stop),
+ }
+ if max_attention_window_size:
+ request["max_attention_window_size"] = max_attention_window_size
+
+ outputs = self._send_request(request)
+ return outputs
+
+
+class VLLMClient(Client):
+ def _single_call(
+ self,
+ prompts,
+ tokens_to_generate,
+ temperature,
+ top_p,
+ top_k,
+ random_seed,
+ stop: List[str],
+ ):
+ request = {
+ "prompt": prompts[0],
+ "max_tokens": tokens_to_generate,
+ "temperature": temperature,
+ "top_k": top_k,
+ "top_p": top_p,
+ "stop": stop,
+ }
+ # TODO: random seed is not supported?
+ outputs = self._send_request(request)
+ outputs = outputs["text"]
+ return outputs
+
+
+class OpenAIClient:
+ def __init__(self, model_name, **generation_kwargs):
+ model2length = {
+ # OpenAI
+ "gpt-4": 8192,
+ "gpt-4-0613": 8192,
+ "gpt-4-1106-preview": 128000,
+ "gpt-4-0125-preview": 128000,
+ "gpt-4-turbo-preview": 128000,
+ "gpt-3.5-turbo-0125": 16385,
+ "gpt-3.5-turbo-1106": 16385,
+ "gpt-3.5-turbo-0613": 4096,
+ "gpt-3.5-turbo": 16385,
+ "gpt-3.5-turbo-16k": 16385,
+ "gpt-3.5-turbo-16k-0613": 16385,
+ # Azure
+ "gpt-4-32k": 32768,
+ "gpt-4": 128000,
+ "gpt-35-turbo-16k": 16384,
+ }
+ self.openai_api_key = os.environ["OPENAI_API_KEY"]
+ self.azure_api_id = os.environ["AZURE_API_ID"]
+ self.azure_api_secret = os.environ["AZURE_API_SECRET"]
+ self.azure_api_endpoint = os.environ["AZURE_API_ENDPOINT"]
+ self.model_name = model_name
+
+ # Azure
+ if self.azure_api_id and self.azure_api_secret:
+ if "gpt-3.5" in model_name:
+ self.model_name = "gpt-35-turbo-16k"
+ if "gpt-4" in model_name:
+ self.model_name = "gpt-4"
+
+ import tiktoken
+
+ self.encoding = tiktoken.get_encoding("cl100k_base")
+ self.max_length = model2length[self.model_name]
+ self.generation_kwargs = generation_kwargs
+ self._create_client()
+
+ def _create_client(
+ self,
+ ):
+ from openai import AzureOpenAI, OpenAI
+
+ # OpenAI
+ if self.openai_api_key:
+ self.client = OpenAI(api_key=self.openai_api_key)
+
+ # Azure
+ elif self.azure_api_id and self.azure_api_secret:
+ self.client = AzureOpenAI(
+ api_key=self.get_azure_api_key(
+ self.azure_api_id,
+ self.azure_api_secret,
+ self.azure_api_endpoint,
+ ),
+ api_version="2024-02-15-preview",
+ azure_endpoint=os.path.join(self.azure_api_endpoint, "llm/v1/azure"),
+ )
+
+ def _count_tokens(self, messages):
+ tokens_per_message = 3
+ tokens_per_name = 1
+ num_tokens = 0
+ for message in messages:
+ num_tokens += tokens_per_message
+ for key, value in message.items():
+ num_tokens += len(self.encoding.encode(value))
+ if key == "name":
+ num_tokens += tokens_per_name
+ num_tokens += 3 # every reply is primed with <|start|>assistant<|message|>
+ return num_tokens
+
+ @retry(wait=wait_random_exponential(min=15, max=60), stop=stop_after_attempt(3))
+ def _send_request(self, request):
+ try:
+ response = self.client.chat.completions.create(
+ model=self.model_name,
+ messages=request["msgs"],
+ max_tokens=request["tokens_to_generate"],
+ temperature=request["temperature"],
+ seed=request["random_seed"],
+ top_p=request["top_p"],
+ stop=request["stop"],
+ )
+ except Exception as e:
+ print(f"Error occurred while calling OpenAI: {e}")
+ if self.azure_api_id and self.azure_api_secret and e.status_code == 401:
+ # token expired
+ self._create_client()
+
+ return response
+
+ def __call__(
+ self,
+ prompt: str,
+ ):
+ # system_msg = [{"role": "system", "content": ""}]
+ system_msg = []
+ user_assistant_msgs = [{"role": "user", "content": prompt}]
+ msgs = system_msg + user_assistant_msgs
+ openai_length = self._count_tokens(msgs)
+ request = self.generation_kwargs
+
+ tokens_to_generate_new = self.max_length - openai_length
+ if tokens_to_generate_new < request["tokens_to_generate"]:
+ print(
+ f"Reduce generate tokens from {request['tokens_to_generate']} to {tokens_to_generate_new}"
+ )
+ request["tokens_to_generate"] = tokens_to_generate_new
+
+ request["msgs"] = msgs
+ outputs = self._send_request(request)
+ response = {"text": [outputs.choices[0].message.content]}
+ return response
+
+ def get_azure_api_key(
+ self,
+ p_client_id,
+ p_client_secret,
+ p_token_url,
+ p_scope="azureopenai-readwrite",
+ cache_file="azure_openai_key.json",
+ ):
+ base_path = Path(__file__).parent
+ file_path = Path.joinpath(base_path, cache_file)
+
+ # Check if the token is cached
+ renew = True
+ if os.path.exists(file_path):
+ with open(file_path, "r") as f:
+ token = json.load(f)
+ renew = True if time.time() > token["expires_in"] else False
+
+ if renew:
+ # Get a new token from the OAuth server
+ response = requests.post(
+ os.path.join(p_token_url, "oauth/api/v1/ssa/default/token"),
+ data={
+ "grant_type": "client_credentials",
+ "client_id": p_client_id,
+ "client_secret": p_client_secret,
+ "scope": p_scope,
+ },
+ )
+ response.raise_for_status()
+ token = response.json()
+ token["expires_in"] += time.time()
+ with open(file_path, "w") as f:
+ json.dump(token, f)
+
+ authToken = token["access_token"]
+ return authToken
+
+
+class GeminiClient:
+ def __init__(self, model_name, **generation_kwargs):
+ model2length = {
+ "gemini-1.0-pro-latest": (30720, 2048),
+ "gemini-1.5-pro-latest": (1048576, 8192),
+ }
+
+ self.model_name = model_name
+ self.model = self._initialize_model()
+ self.max_input_length = model2length[model_name][0]
+ self.max_output_length = model2length[model_name][1]
+ assert generation_kwargs["tokens_to_generate"] < self.max_output_length, print(
+ f"tokens_to_generate exceeds {self.max_output_length}"
+ )
+
+ import google.generativeai as genai
+
+ self.config = genai.GenerationConfig(
+ candidate_count=1,
+ stop_sequences=generation_kwargs["stop"],
+ max_output_tokens=generation_kwargs["tokens_to_generate"],
+ temperature=generation_kwargs["temperature"],
+ top_p=generation_kwargs["top_p"],
+ top_k=generation_kwargs["top_k"],
+ )
+
+ from google.generativeai.types import HarmBlockThreshold, HarmCategory
+
+ self.safety_settings = {
+ HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
+ HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
+ HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
+ HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
+ }
+
+ @retry(wait=wait_random_exponential(min=60, max=60), stop=stop_after_attempt(3))
+ def _send_request(self, request):
+ try:
+ response = self.model.generate_content(
+ request["prompt"],
+ generation_config=request["config"],
+ safety_settings=self.safety_settings,
+ )
+ except Exception as e:
+ traceback.print_exc()
+ return None
+ return response
+
+ def __call__(
+ self,
+ prompt: str,
+ ):
+ assert (
+ self.model.count_tokens(prompt).total_tokens < self.max_input_length
+ ), print(f"input length exceeds {self.max_input_length}")
+
+ request = {
+ "prompt": prompt,
+ "config": self.config,
+ }
+
+ outputs = self._send_request(request)
+
+ try:
+ response = {"text": [outputs.candidates[0].content.parts[0].text]}
+ except Exception as e:
+ response = {"text": []}
+ print(outputs)
+ traceback.print_exc()
+
+ return response
+
+ def _initialize_model(self):
+ import google.generativeai as genai
+
+ genai.configure(api_key=os.environ["GEMINI_API_KEY"])
+ return genai.GenerativeModel(self.model_name)
diff --git a/TriangleMix/ruler/pred/model_wrappers.py b/TriangleMix/ruler/pred/model_wrappers.py
new file mode 100644
index 00000000..77705491
--- /dev/null
+++ b/TriangleMix/ruler/pred/model_wrappers.py
@@ -0,0 +1,308 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import json
+import logging
+from typing import Dict, List, Optional
+
+import requests
+import torch
+
+
+class HuggingFaceModel:
+ def __init__(self, name_or_path: str, **generation_kwargs) -> None:
+ from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ name_or_path, trust_remote_code=True
+ )
+
+ if "Yarn-Llama" in name_or_path:
+ model_kwargs = None
+ else:
+ model_kwargs = {"attn_implementation": "flash_attention_2"}
+
+ try:
+ self.pipeline = pipeline(
+ "text-generation",
+ model=name_or_path,
+ tokenizer=self.tokenizer,
+ trust_remote_code=True,
+ device_map="auto",
+ torch_dtype=torch.bfloat16,
+ model_kwargs=model_kwargs,
+ )
+ except:
+ self.pipeline = None
+ self.model = AutoModelForCausalLM.from_pretrained(
+ name_or_path,
+ trust_remote_code=True,
+ device_map="auto",
+ torch_dtype=torch.bfloat16,
+ _attn_implementation="flash_attention_2",
+ )
+
+ self.generation_kwargs = generation_kwargs
+ self.stop = self.generation_kwargs.pop("stop")
+
+ def __call__(self, prompt: str, **kwargs) -> Dict[str, List[str]]:
+ if self.pipeline is None:
+ inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+ output = self.model.generate(**inputs, **self.generation_kwargs)
+ generated_text = self.tokenizer.decode(
+ output[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
+ )
+ else:
+ output = self.pipeline(
+ text_inputs=prompt,
+ **self.generation_kwargs,
+ )
+ assert len(output) == 1
+ generated_text = output[0]["generated_text"]
+
+ # remove the input form the generated text
+ if generated_text.startswith(prompt):
+ generated_text = generated_text[len(prompt) :]
+
+ if self.stop is not None:
+ for s in self.stop:
+ generated_text = generated_text.split(s)[0]
+ return {"text": [generated_text]}
+
+
+class MInferenceModel:
+ def __init__(
+ self,
+ name_or_path: str,
+ config_path: str,
+ do_sample: bool = False,
+ repetition_penalty: float = 1.0,
+ temperature: float = 0.0,
+ top_k: int = 32,
+ top_p: float = 0.9,
+ stop: Optional[List[str]] = None,
+ max_new_tokens: int = 100,
+ starting_layer: int = -1,
+ kv_cache_cpu: bool = False,
+ kv_cache_cpu_device: str = None,
+ kv_type: str = "",
+ trust_remote_code: bool = False,
+ attn_type: str = "minference",
+ attn_kwargs: dict = {},
+ ) -> None:
+ from transformers import (
+ AutoConfig,
+ AutoModelForCausalLM,
+ AutoTokenizer,
+ GenerationConfig,
+ )
+
+ from minference import MInference
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ name_or_path,
+ trust_remote_code=trust_remote_code,
+ resume_download=None,
+ )
+ model = AutoModelForCausalLM.from_pretrained(
+ name_or_path,
+ torch_dtype="auto",
+ device_map="cuda",
+ resume_download=None,
+ trust_remote_code=trust_remote_code,
+ _attn_implementation="flash_attention_2",
+ )
+ minference_patch = MInference(
+ attn_type,
+ name_or_path,
+ config_path=config_path,
+ starting_layer=starting_layer,
+ kv_type=kv_type,
+ kv_cache_cpu=kv_cache_cpu,
+ kv_cache_cpu_device=kv_cache_cpu_device,
+ is_search=False,
+ attn_kwargs=attn_kwargs,
+ )
+ self.model = minference_patch(model)
+
+ self.pipeline = None
+ generation_config = GenerationConfig(
+ do_sample=do_sample,
+ repetition_penalty=repetition_penalty,
+ max_new_tokens=max_new_tokens,
+ )
+ if do_sample:
+ generation_config.top_k = top_k
+ generation_config.top_p = top_p
+ generation_config.temperature = temperature
+
+ self.generation_config = generation_config
+
+ self.stop = stop
+
+ def __call__(self, prompt: str, **kwargs) -> Dict[str, List[str]]:
+ torch.cuda.empty_cache()
+ inputs = self.tokenizer(
+ prompt, return_tensors="pt", return_attention_mask=False
+ ).to(self.model.device)
+ output = self.model.generate(**inputs, generation_config=self.generation_config)
+ generated_text = self.tokenizer.decode(
+ output[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
+ )
+
+ # remove the input form the generated text
+ if generated_text.startswith(prompt):
+ generated_text = generated_text[len(prompt) :]
+
+ if self.stop is not None:
+ for s in self.stop:
+ generated_text = generated_text.split(s)[0]
+ return {"text": [generated_text]}
+
+
+class InfLLM(MInferenceModel):
+ def __init__(self, name_or_path: str, **generation_kwargs) -> None:
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+ from minference import MInference
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ name_or_path, trust_remote_code=True
+ )
+ self.model = AutoModelForCausalLM.from_pretrained(
+ name_or_path,
+ torch_dtype="auto",
+ device_map="auto",
+ resume_download=None,
+ trust_remote_code=True,
+ )
+ minference_patch = MInference("inf_llm", name_or_path, None, starting_layer=0)
+ self.model = minference_patch.patch_model(self.model)
+ self.pipeline = None
+ self.generation_kwargs = generation_kwargs
+ self.stop = self.generation_kwargs.pop("stop")
+
+ def __call__(self, prompt: str, **kwargs) -> Dict[str, List[str]]:
+ inputs = self.tokenizer(
+ prompt, return_tensors="pt", return_attention_mask=False
+ ).to(self.model.device)
+ output = self.model.generate(
+ **inputs,
+ max_new_tokens=self.generation_kwargs["max_new_tokens"],
+ )
+ generated_text = self.tokenizer.decode(
+ output[0][inputs.input_ids.shape[1] :], skip_special_tokens=True
+ )
+
+ # remove the input form the generated text
+ if generated_text.startswith(prompt):
+ generated_text = generated_text[len(prompt) :]
+
+ if self.stop is not None:
+ for s in self.stop:
+ generated_text = generated_text.split(s)[0]
+ return {"text": [generated_text]}
+
+
+class Streaming(MInferenceModel):
+ def __init__(
+ self,
+ name_or_path: str,
+ config_path: str,
+ do_sample: bool = False,
+ repetition_penalty: float = 1.0,
+ temperature: float = 0.0,
+ top_k: int = 32,
+ top_p: float = 0.9,
+ stop: Optional[List[str]] = None,
+ max_new_tokens: int = 100,
+ starting_layer: int = -1,
+ kv_cache_cpu: bool = False,
+ kv_cache_cpu_device: str = None,
+ kv_type: str = "",
+ trust_remote_code: bool = False,
+ ) -> None:
+ from transformers import (
+ AutoConfig,
+ AutoModelForCausalLM,
+ AutoTokenizer,
+ GenerationConfig,
+ )
+
+ from minference import MInference
+
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ name_or_path,
+ trust_remote_code=trust_remote_code,
+ resume_download=None,
+ )
+ model = AutoModelForCausalLM.from_pretrained(
+ name_or_path,
+ torch_dtype="auto",
+ device_map="cuda",
+ resume_download=None,
+ trust_remote_code=trust_remote_code,
+ _attn_implementation="flash_attention_2",
+ )
+ minference_patch = MInference(
+ "a_shape",
+ name_or_path,
+ config_path=config_path,
+ starting_layer=starting_layer,
+ kv_type=kv_type,
+ kv_cache_cpu=kv_cache_cpu,
+ kv_cache_cpu_device=kv_cache_cpu_device,
+ is_search=False,
+ )
+ self.model = minference_patch(model)
+
+ self.pipeline = None
+ generation_config = GenerationConfig(
+ do_sample=do_sample,
+ repetition_penalty=repetition_penalty,
+ max_new_tokens=max_new_tokens,
+ )
+ if do_sample:
+ generation_config.top_k = top_k
+ generation_config.top_p = top_p
+ generation_config.temperature = temperature
+
+ self.generation_config = generation_config
+
+ self.stop = stop
+
+
+class MambaModel:
+ def __init__(self, name_or_path: str, **generation_kwargs) -> None:
+ from mamba_ssm.models.mixer_seq_simple import MambaLMHeadModel
+ from transformers import AutoTokenizer
+
+ self.tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+ self.device = "cuda"
+ self.model = MambaLMHeadModel.from_pretrained(
+ name_or_path, device=self.device, dtype=torch.bfloat16
+ )
+ self.generation_kwargs = generation_kwargs
+ self.stop = self.generation_kwargs.pop("stop")
+ self.max_genlen = self.generation_kwargs.pop("max_new_tokens")
+ self.minp = 0.0
+
+ def __call__(self, prompt: str, **kwargs) -> Dict[str, List[str]]:
+ # tokenize
+ tokens = self.tokenizer(prompt, return_tensors="pt")
+ input_ids = tokens.input_ids.to(self.device)
+ max_length = input_ids.shape[1] + self.max_genlen
+
+ # generate
+ out = self.model.generate(
+ input_ids=input_ids,
+ max_length=max_length,
+ cg=True,
+ return_dict_in_generate=True,
+ output_scores=True,
+ enable_timing=False,
+ **self.generation_kwargs,
+ )
+ assert len(out.sequences) == 1
+ # detok
+ return {"text": [self.tokenizer.decode(out.sequences[0][input_ids.shape[1] :])]}
diff --git a/TriangleMix/ruler/pred/serve_trt.py b/TriangleMix/ruler/pred/serve_trt.py
new file mode 100644
index 00000000..d9360ee0
--- /dev/null
+++ b/TriangleMix/ruler/pred/serve_trt.py
@@ -0,0 +1,265 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# adapted from https://github.com/Kipok/NeMo-Skills/blob/v0.1/nemo_skills/inference/server/serve_trt.py
+
+import json
+import logging
+import sys
+from argparse import ArgumentParser
+from pathlib import Path
+
+import numpy as np
+import tensorrt_llm
+import torch
+from flask import Flask, jsonify, request
+from flask_restful import Api, Resource
+from mpi4py import MPI
+from tensorrt_llm.runtime import ModelRunnerCpp
+from transformers import AutoTokenizer
+
+
+class TritonServerGenerate(Resource):
+ def __init__(self, model):
+ self.model = model
+ self.comm = MPI.COMM_WORLD
+
+ def generate(
+ self,
+ prompts,
+ max_new_tokens,
+ temperature,
+ top_k,
+ top_p,
+ repetition_penalty,
+ random_seed,
+ stop_words_list,
+ max_attention_window_size=None,
+ ):
+ output = self.model.forward(
+ prompts,
+ max_output_token=max_new_tokens,
+ top_k=top_k,
+ top_p=top_p,
+ temperature=temperature,
+ repetition_penalty=repetition_penalty,
+ random_seed=random_seed,
+ stop_words_list=stop_words_list,
+ max_attention_window_size=max_attention_window_size,
+ )
+ return output
+
+ def put(self):
+ logging.info("request IP: " + str(request.remote_addr))
+ logging.info(json.dumps(request.get_json()))
+
+ input_request = request.get_json()
+
+ tokens_to_generate = input_request.get("tokens_to_generate", 64)
+ temperature = input_request.get("temperature", 1.0)
+ top_k = input_request.get("top_k", 0)
+ top_p = input_request.get("top_p", 1.0)
+ repetition_penalty = input_request.get("repetition_penalty", 1.2)
+ stop_words_list = input_request.get("stop_words_list")
+ max_attention_window_size = input_request.get("max_attention_window_size")
+ random_seed = input_request.get("random_seed", 0)
+ prompts = input_request["prompts"]
+
+ data = dict(
+ prompts=prompts,
+ max_new_tokens=tokens_to_generate,
+ temperature=temperature,
+ top_k=top_k,
+ top_p=top_p,
+ repetition_penalty=repetition_penalty,
+ random_seed=random_seed,
+ stop_words_list=stop_words_list,
+ max_attention_window_size=max_attention_window_size,
+ )
+ self.comm.Barrier()
+ data = self.comm.bcast(data, root=0)
+
+ out = self.generate(**data)
+ return jsonify(out)
+
+
+def parse_input(input_texts: str, tokenizer):
+ batch_input_ids = [
+ tokenizer.encode(
+ input_text,
+ add_special_tokens=False, # TODO: does this need to be true?
+ )
+ for input_text in input_texts
+ ]
+ batch_input_ids = [
+ torch.tensor(x, dtype=torch.int32, device="cuda") for x in batch_input_ids
+ ]
+ input_lengths = [x.size(0) for x in batch_input_ids]
+
+ return batch_input_ids, input_lengths
+
+
+def get_output(output_ids, input_lengths, max_output_len, tokenizer, eos_token):
+ num_beams = output_ids.size(1)
+ assert num_beams == 1
+ output_texts = []
+ for idx, input_len in enumerate(input_lengths):
+ output_begin = input_len
+ output_end = input_len + max_output_len
+ outputs = output_ids[idx][0][output_begin:output_end]
+ eos_ids = (outputs == eos_token).nonzero(as_tuple=True)[-1]
+ if len(eos_ids) > 0:
+ outputs = outputs[: eos_ids[0]]
+ outputs = outputs.tolist()
+ output_texts.append(tokenizer.decode(outputs))
+ return output_texts
+
+
+def prepare_stop_words(stop_words_list, tokenizer):
+ # adapted from https://github.com/NVIDIA/TensorRT-LLM/blob/b310ec675145c9ee7668592549f733df4abf1e94/tensorrt_llm/runtime/generation.py#L46
+ flat_ids = []
+ offsets = []
+ for batch_stop_words in stop_words_list:
+ item_flat_ids = []
+ item_offsets = []
+
+ for word in batch_stop_words:
+ # there is a known issue in TensorRT-LLM that word ids are not unique and might change depending on
+ # where in the text it appears. In our case we mainly need to stop on ids as they appear in the middle
+ # of the text. The following is a workaround to get such ids that works for both kind of stop
+ # words as well as newlines that we commonly use. But note that it's not a universal fix, so this might
+ # require refactoring if different stop words are used in the future.
+ # Eventually, this needs to be fixed inside TensorRT-LLM itself.
+ ids = tokenizer.encode("magic" + word)
+ ids = ids[2:] # skipping "magic"
+
+ if len(ids) == 0:
+ continue
+
+ item_flat_ids += ids
+ item_offsets.append(len(ids))
+
+ flat_ids.append(np.array(item_flat_ids))
+ offsets.append(np.cumsum(np.array(item_offsets)))
+
+ pad_to = max(1, max(len(ids) for ids in flat_ids))
+
+ for i, (ids, offs) in enumerate(zip(flat_ids, offsets)):
+ flat_ids[i] = np.pad(ids, (0, pad_to - len(ids)), constant_values=0)
+ offsets[i] = np.pad(offs, (0, pad_to - len(offs)), constant_values=-1)
+
+ stop_words = np.array([flat_ids, offsets], dtype="int32").transpose((1, 0, 2))
+ return torch.Tensor(stop_words).to(torch.int32).to("cuda").contiguous()
+
+
+def load_tokenizer(tokenizer_dir: str):
+ tokenizer = AutoTokenizer.from_pretrained(
+ tokenizer_dir,
+ legacy=False,
+ trust_remote_code=True,
+ )
+
+ if tokenizer.pad_token_id is None:
+ tokenizer.pad_token_id = tokenizer.eos_token_id
+ pad_id = tokenizer.pad_token_id
+ end_id = tokenizer.eos_token_id
+
+ return tokenizer, pad_id, end_id
+
+
+class TensorRTLLM:
+ def __init__(self, model_path: str):
+ self.tokenizer, self.pad_id, self.end_id = load_tokenizer(
+ tokenizer_dir=model_path
+ )
+ self.runner = ModelRunnerCpp.from_dir(
+ engine_dir=model_path, rank=tensorrt_llm.mpi_rank()
+ )
+
+ @torch.no_grad()
+ def forward(
+ self,
+ input_texts,
+ max_output_token,
+ top_k,
+ top_p,
+ temperature,
+ repetition_penalty,
+ random_seed,
+ stop_words_list,
+ max_attention_window_size,
+ ):
+ batch_input_ids, input_lengths = parse_input(input_texts, self.tokenizer)
+
+ stop_words_list = [stop_words_list for _ in range(len(input_texts))]
+ stop_words_list = prepare_stop_words(stop_words_list, self.tokenizer)
+
+ # TODO: return dictionary with a proper error reporting
+ try:
+ output_ids = self.runner.generate(
+ batch_input_ids,
+ max_new_tokens=max_output_token,
+ end_id=self.end_id,
+ pad_id=self.pad_id,
+ temperature=temperature,
+ top_k=top_k,
+ top_p=top_p,
+ repetition_penalty=repetition_penalty,
+ random_seed=random_seed,
+ stop_words_list=stop_words_list,
+ max_attention_window_size=max_attention_window_size,
+ return_dict=False,
+ )
+ torch.cuda.synchronize()
+
+ output = get_output(
+ output_ids, input_lengths, max_output_token, self.tokenizer, self.end_id
+ )
+ except RuntimeError as e:
+ logging.error("RuntimeError: %s", e)
+ output = [f"RuntimeError: {e}"] * len(input_texts)
+
+ return output
+
+
+class WrapperServer:
+ def __init__(self, model_path: str):
+ self.comm = MPI.COMM_WORLD
+ self.rank = self.comm.Get_rank()
+
+ self.model = TensorRTLLM(model_path=model_path)
+
+ if self.rank == 0:
+ self.app = Flask(__file__, static_url_path="")
+ api = Api(self.app)
+ api.add_resource(
+ TritonServerGenerate, "/generate", resource_class_args=[self.model]
+ )
+
+ def run(self, url, port=5000):
+ if self.rank == 0:
+ self.app.run(url, threaded=True, port=port, debug=False)
+ else:
+ self.worker_loop()
+
+ def worker_loop(self):
+ triton = TritonServerGenerate(self.model)
+ while True:
+ self.comm.Barrier()
+ data = None
+ data = self.comm.bcast(data, root=0)
+ triton.generate(**data)
+
+
+if __name__ == "__main__":
+ # TODO: can we reuse normal logger here?
+ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
+
+ parser = ArgumentParser()
+ parser.add_argument("--model_path", required=True)
+ parser.add_argument("--host", type=str, default="0.0.0.0")
+ parser.add_argument("--port", type=int, default=5000)
+ args = parser.parse_args()
+
+ server = WrapperServer(model_path=args.model_path)
+ server.run(args.host, args.port)
diff --git a/TriangleMix/ruler/pred/serve_vllm.py b/TriangleMix/ruler/pred/serve_vllm.py
new file mode 100644
index 00000000..9464bca7
--- /dev/null
+++ b/TriangleMix/ruler/pred/serve_vllm.py
@@ -0,0 +1,100 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# adapted from https://github.com/vllm-project/vllm/blob/v0.4.0/vllm/entrypoints/api_server.py
+
+import argparse
+import json
+from typing import AsyncGenerator
+
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse, Response, StreamingResponse
+from vllm.engine.arg_utils import AsyncEngineArgs
+from vllm.engine.async_llm_engine import AsyncLLMEngine
+from vllm.sampling_params import SamplingParams
+from vllm.utils import random_uuid
+
+TIMEOUT_KEEP_ALIVE = 5 # seconds.
+app = FastAPI()
+engine = None
+
+
+@app.get("/health")
+async def health() -> Response:
+ """Health check."""
+ return Response(status_code=200)
+
+
+@app.put("/generate")
+async def generate(request: Request) -> Response:
+ """Generate completion for the request.
+
+ The request should be a JSON object with the following fields:
+ - prompt: the prompt to use for the generation.
+ - stream: whether to stream the results or not.
+ - other fields: the sampling parameters (See `SamplingParams` for details).
+ """
+ request_dict = await request.json()
+ prompt = request_dict.pop("prompt")
+ stream = request_dict.pop("stream", False)
+ sampling_params = SamplingParams(**request_dict)
+ request_id = random_uuid()
+
+ results_generator = engine.generate(prompt, sampling_params, request_id)
+
+ # Streaming case
+ async def stream_results() -> AsyncGenerator[bytes, None]:
+ async for request_output in results_generator:
+ prompt = request_output.prompt
+ text_outputs = [prompt + output.text for output in request_output.outputs]
+ ret = {"text": text_outputs}
+ yield (json.dumps(ret) + "\0").encode("utf-8")
+
+ if stream:
+ return StreamingResponse(stream_results())
+
+ # Non-streaming case
+ final_output = None
+ async for request_output in results_generator:
+ if await request.is_disconnected():
+ # Abort the request if the client disconnects.
+ await engine.abort(request_id)
+ return Response(status_code=499)
+ final_output = request_output
+ assert final_output is not None
+ text_outputs = [output.text for output in final_output.outputs]
+ ret = {"text": text_outputs}
+ return JSONResponse(ret)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--host", type=str, default="0.0.0.0")
+ parser.add_argument("--port", type=int, default=5000)
+ parser.add_argument("--ssl-keyfile", type=str, default=None)
+ parser.add_argument("--ssl-certfile", type=str, default=None)
+ parser.add_argument(
+ "--root-path",
+ type=str,
+ default=None,
+ help="FastAPI root_path when app is behind a path based routing proxy",
+ )
+ parser = AsyncEngineArgs.add_cli_args(parser)
+ args = parser.parse_args()
+
+ engine_args = AsyncEngineArgs.from_cli_args(args)
+ engine_args.max_model_len = 140000
+ engine_args.disable_log_requests = True
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
+
+ app.root_path = args.root_path
+ uvicorn.run(
+ app,
+ host=args.host,
+ port=args.port,
+ log_level="warning",
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
+ ssl_keyfile=args.ssl_keyfile,
+ ssl_certfile=args.ssl_certfile,
+ )
diff --git a/TriangleMix/ruler/requirements.txt b/TriangleMix/ruler/requirements.txt
new file mode 100644
index 00000000..889ed5f4
--- /dev/null
+++ b/TriangleMix/ruler/requirements.txt
@@ -0,0 +1,11 @@
+flask
+flask_restful
+sshtunnel_requests
+tritonclient[all]
+wonderwords
+tenacity
+huggingface_hub
+causal-conv1d>=1.2.0
+mamba-ssm
+html2text
+google-generativeai
diff --git a/TriangleMix/ruler/run_dense.sh b/TriangleMix/ruler/run_dense.sh
new file mode 100644
index 00000000..bf66039a
--- /dev/null
+++ b/TriangleMix/ruler/run_dense.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+export TOKENIZERS_PARALLELISM=false
+RULER_PATH=$(dirname $0)
+python -c "import nltk; nltk.download('punkt')"
+
+SEQ_LENGTHS=(
+ 4096
+ 8192
+ 16384
+ 32768
+ 65536
+ 131072
+)
+
+TASKS=(
+ "niah_single_1"
+ "niah_single_2"
+ "niah_single_3"
+ "niah_multikey_1"
+ "niah_multikey_2"
+ "niah_multikey_3"
+ "niah_multivalue"
+ "niah_multiquery"
+ "vt"
+ "cwe"
+ "fwe"
+ "qa_1"
+ "qa_2"
+)
+
+# Experiment Setup
+NUM_SAMPLES=100
+TEMPERATURE="0.0"
+TOP_P="1.0"
+TOP_K="32"
+
+# The model
+MODEL_NAME=$1
+BENCHMARK="synthetic"
+MODEL_TEMPLATE_TYPE="base"
+MODEL_FRAMEWORK=$2
+
+# MInference
+STARTING_LAYER=-1
+KV_CACHE_CPU="false"
+USE_SNAPKV="false"
+TRUST_REMOTE_CODE="true"
+
+if [ "${MODEL_FRAMEWORK}" == "minference" ]; then
+ MINFERENCE_PARAMS="--starting_layer ${STARTING_LAYER}"
+
+ if [ -n "${CONFIG_PATH}" ]; then
+ MINFERENCE_PARAMS="${MINFERENCE_PARAMS} --config_path ${CONFIG_PATH}"
+ fi
+
+ if [ "${USE_SNAPKV}" == "true" ]; then
+ MINFERENCE_PARAMS="${MINFERENCE_PARAMS} --use_snapkv"
+ fi
+
+ echo "MInference enabled with params: ${MINFERENCE_PARAMS}"
+fi
+
+if [ "${TRUST_REMOTE_CODE}" == "true" ]; then
+ EXTRA_PARAMS="${EXTRA_PARAMS} --trust_remote_code"
+fi
+
+if [ "${KV_CACHE_CPU}" == "true" ]; then
+ EXTRA_PARAMS="${EXTRA_PARAMS} --kv_cache_cpu --kv_cache_cpu_device cpu"
+fi
+
+# Gpu and output path
+GPUS="1" # GPU size for tensor_parallel.
+ROOT_DIR=$3 # the path that stores generated task samples and model predictions.
+
+for MAX_SEQ_LENGTH in "${SEQ_LENGTHS[@]}"; do
+
+ RESULTS_DIR="${ROOT_DIR}/${MODEL_NAME}_${MODEL_FRAMEWORK}/${BENCHMARK}/${MAX_SEQ_LENGTH}"
+ DATA_DIR="${RESULTS_DIR}/data"
+ PRED_DIR="${RESULTS_DIR}/pred"
+ mkdir -p ${DATA_DIR}
+ mkdir -p ${PRED_DIR}
+
+ for TASK in "${TASKS[@]}"; do
+ python ${RULER_PATH}/data/prepare.py \
+ --save_dir ${DATA_DIR} \
+ --benchmark ${BENCHMARK} \
+ --task ${TASK} \
+ --tokenizer_path ${MODEL_NAME} \
+ --tokenizer_type "hf" \
+ --max_seq_length ${MAX_SEQ_LENGTH} \
+ --model_template_type ${MODEL_TEMPLATE_TYPE} \
+ --num_samples ${NUM_SAMPLES} \
+ ${REMOVE_NEWLINE_TAB}
+
+ python ${RULER_PATH}/pred/call_api.py \
+ --data_dir ${DATA_DIR} \
+ --save_dir ${PRED_DIR} \
+ --benchmark ${BENCHMARK} \
+ --task ${TASK} \
+ --server_type ${MODEL_FRAMEWORK} \
+ --attn_type dense \
+ --model_name_or_path ${MODEL_NAME} \
+ --temperature ${TEMPERATURE} \
+ --top_k ${TOP_K} \
+ --top_p ${TOP_P} \
+ ${MINFERENCE_PARAMS} \
+ ${EXTRA_PARAMS} \
+ ${STOP_WORDS}
+ done
+
+ python ${RULER_PATH}/eval/evaluate.py \
+ --data_dir ${PRED_DIR} \
+ --benchmark ${BENCHMARK}
+done
diff --git a/TriangleMix/ruler/run_tri_mix.sh b/TriangleMix/ruler/run_tri_mix.sh
new file mode 100644
index 00000000..a541666f
--- /dev/null
+++ b/TriangleMix/ruler/run_tri_mix.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+export TOKENIZERS_PARALLELISM=false
+RULER_PATH=$(dirname $0)
+python -c "import nltk; nltk.download('punkt')"
+
+SEQ_LENGTHS=(
+ 4096
+ 8192
+ 16384
+ 32768
+ 65536
+ 131072
+)
+
+TASKS=(
+ "niah_single_1"
+ "niah_single_2"
+ "niah_single_3"
+ "niah_multikey_1"
+ "niah_multikey_2"
+ "niah_multikey_3"
+ "niah_multivalue"
+ "niah_multiquery"
+ "vt"
+ "cwe"
+ "fwe"
+ "qa_1"
+ "qa_2"
+)
+
+# Experiment Setup
+NUM_SAMPLES=100
+TEMPERATURE="0.0"
+TOP_P="1.0"
+TOP_K="32"
+
+# The model
+MODEL_NAME=$1
+BENCHMARK="synthetic"
+MODEL_TEMPLATE_TYPE="base"
+MODEL_FRAMEWORK=$2
+
+# MInference
+STARTING_LAYER=-1
+KV_CACHE_CPU="false"
+USE_SNAPKV="false"
+TRUST_REMOTE_CODE="true"
+
+if [ "${MODEL_FRAMEWORK}" == "minference" ]; then
+ MINFERENCE_PARAMS="--starting_layer ${STARTING_LAYER}"
+
+ if [ -n "${CONFIG_PATH}" ]; then
+ MINFERENCE_PARAMS="${MINFERENCE_PARAMS} --config_path ${CONFIG_PATH}"
+ fi
+
+ if [ "${USE_SNAPKV}" == "true" ]; then
+ MINFERENCE_PARAMS="${MINFERENCE_PARAMS} --use_snapkv"
+ fi
+
+ echo "MInference enabled with params: ${MINFERENCE_PARAMS}"
+fi
+
+if [ "${TRUST_REMOTE_CODE}" == "true" ]; then
+ EXTRA_PARAMS="${EXTRA_PARAMS} --trust_remote_code"
+fi
+
+if [ "${KV_CACHE_CPU}" == "true" ]; then
+ EXTRA_PARAMS="${EXTRA_PARAMS} --kv_cache_cpu --kv_cache_cpu_device cpu"
+fi
+
+# Gpu and output path
+GPUS="1" # GPU size for tensor_parallel.
+ROOT_DIR=$3 # the path that stores generated task samples and model predictions.
+STARTING_LAYER_TRI_MIX=$4
+
+for MAX_SEQ_LENGTH in "${SEQ_LENGTHS[@]}"; do
+
+ RESULTS_DIR="${ROOT_DIR}/${MODEL_NAME}_${MODEL_FRAMEWORK}/${BENCHMARK}/${MAX_SEQ_LENGTH}"
+ DATA_DIR="${RESULTS_DIR}/data"
+ PRED_DIR="${RESULTS_DIR}/pred"
+ mkdir -p ${DATA_DIR}
+ mkdir -p ${PRED_DIR}
+
+ for TASK in "${TASKS[@]}"; do
+ python ${RULER_PATH}/data/prepare.py \
+ --save_dir ${DATA_DIR} \
+ --benchmark ${BENCHMARK} \
+ --task ${TASK} \
+ --tokenizer_path ${MODEL_NAME} \
+ --tokenizer_type "hf" \
+ --max_seq_length ${MAX_SEQ_LENGTH} \
+ --model_template_type ${MODEL_TEMPLATE_TYPE} \
+ --num_samples ${NUM_SAMPLES} \
+ ${REMOVE_NEWLINE_TAB}
+
+ python ${RULER_PATH}/pred/call_api.py \
+ --data_dir ${DATA_DIR} \
+ --save_dir ${PRED_DIR} \
+ --benchmark ${BENCHMARK} \
+ --task ${TASK} \
+ --server_type ${MODEL_FRAMEWORK} \
+ --attn_type tri_mix \
+ --attn_kwargs "{\"starting_layer\": ${STARTING_LAYER_TRI_MIX}, \"n_local\": 512, \"n_init\": 8, \"last_n\": 128}" \
+ --model_name_or_path ${MODEL_NAME} \
+ --temperature ${TEMPERATURE} \
+ --top_k ${TOP_K} \
+ --top_p ${TOP_P} \
+ ${MINFERENCE_PARAMS} \
+ ${EXTRA_PARAMS} \
+ ${STOP_WORDS}
+ done
+
+ python ${RULER_PATH}/eval/evaluate.py \
+ --data_dir ${PRED_DIR} \
+ --benchmark ${BENCHMARK}
+done
diff --git a/TriangleMix/ruler/synthetic.yaml b/TriangleMix/ruler/synthetic.yaml
new file mode 100644
index 00000000..6aa05622
--- /dev/null
+++ b/TriangleMix/ruler/synthetic.yaml
@@ -0,0 +1,110 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+niah_single_1:
+ task: niah
+ args:
+ type_haystack: repeat
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_single_2:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_single_3:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: uuids
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multikey_1:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 4
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multikey_2:
+ task: niah
+ args:
+ type_haystack: needle
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multikey_3:
+ task: niah
+ args:
+ type_haystack: needle
+ type_needle_k: uuids
+ type_needle_v: uuids
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 1
+
+niah_multivalue:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 4
+ num_needle_q: 1
+
+niah_multiquery:
+ task: niah
+ args:
+ type_haystack: essay
+ type_needle_k: words
+ type_needle_v: numbers
+ num_needle_k: 1
+ num_needle_v: 1
+ num_needle_q: 4
+
+vt:
+ task: variable_tracking
+ args:
+ num_chains: 1
+ num_hops: 4
+
+cwe:
+ task: common_words_extraction
+ args:
+ freq_cw: 30
+ freq_ucw: 3
+ num_cw: 10
+
+fwe:
+ task: freq_words_extraction
+ args:
+ alpha: 2.0
+
+qa_1:
+ task: qa
+ args:
+ dataset: squad
+
+qa_2:
+ task: qa
+ args:
+ dataset: hotpotqa
diff --git a/TriangleMix/setup_ruler.sh b/TriangleMix/setup_ruler.sh
new file mode 100644
index 00000000..d0351c24
--- /dev/null
+++ b/TriangleMix/setup_ruler.sh
@@ -0,0 +1,23 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+pushd $(dirname "$0") > /dev/null
+
+ pip install transformers
+
+ pip install jieba rouge
+ pip install tree-sitter==0.21.3
+
+ pip install datasets jsonlines fire matplotlib pandas seaborn tqdm
+ pip install accelerate
+ pip install jieba mysql-connector-python fuzzywuzzy rouge jsonlines SentencePiece
+ pip install git+https://github.com/NVIDIA/NeMo.git
+ pip install nltk hydra-core wonderwords lightning lhotse jiwer librosa pyannote-core webdataset editdistance pyannote.metrics tenacity xopen
+ pip install html2text bs4
+ python -c 'import nltk; nltk.download("punkt_tab")'
+
+ cd ruler/data/synthetic/json
+ python download_paulgraham_essay.py
+ bash download_qa_dataset.sh
+
+popd > /dev/null
diff --git a/TriangleMix/speed_test.py b/TriangleMix/speed_test.py
new file mode 100644
index 00000000..1a4c043f
--- /dev/null
+++ b/TriangleMix/speed_test.py
@@ -0,0 +1,292 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import copy
+import gc
+import random
+import time
+import uuid
+
+import fire
+import numpy as np
+import pandas as pd
+import torch
+from tqdm import tqdm
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from minference import MInference
+
+kv_retrieval_prompt_template = (
+ """
+ Extract the value corresponding to the specified key in the data below.
+
+Data:
+{formatted_kv_records}
+
+Extract the value corresponding to this key:
+key: {key}
+corresponding value:
+""".strip()
+ + " "
+)
+
+kv_retrieval_prompt_template_llama2_chat = (
+ """
+ [INST] Extract the value corresponding to the specified key in the data below.
+
+Data:
+{formatted_kv_records}
+
+Extract the value corresponding to this key:
+key: {key}
+
+Please directly output the corresponding value without outputing anything else. [/INST] Sure! The value corresponding to the key "{key}" is:
+""".strip()
+ + "\n\nvalue: "
+)
+
+
+kv_retrieval_prompt_template_llama3_instruct = (
+ """
+<|begin_of_text|><|start_header_id|>user<|end_header_id|>
+
+Extract the value corresponding to the specified key in the data below.
+
+Data:
+{formatted_kv_records}
+
+Extract the value corresponding to this key:
+key: {key}
+
+Please directly output the corresponding value without outputing anything else.
+value:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+""".strip()
+ + "\n\n"
+)
+
+kv_retrieval_prompt_template_qwen_instruct = (
+ """
+<|im_start|>system
+You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>
+<|im_start|>user
+Extract the value corresponding to the specified key in the data below.
+
+Data:
+{formatted_kv_records}
+
+Extract the value corresponding to this key:
+key: {key}
+
+Please directly output the corresponding value without outputing anything else.<|im_end|>
+<|im_start|>nuser
+""".strip()
+ + "\n"
+)
+
+
+def get_kv_retrieval_prompt(
+ data,
+ key,
+ model_name,
+):
+ # Format the KV data into a string
+ formatted_kv_records = ""
+ for index, record in enumerate(data):
+ data_string = f"key: {record[0]} value: {record[1]}\n"
+ formatted_kv_records += data_string
+
+ if model_name == "meta-llama/Llama-2-7b-chat-hf":
+ prompt_template = kv_retrieval_prompt_template_llama2_chat
+ elif (
+ model_name == "meta-llama/Meta-Llama-3-8B-Instruct"
+ or model_name == "gradientai/Llama-3-8B-Instruct-262k"
+ or model_name == "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
+ or model_name == "meta-llama/Llama-3.1-8B-Instruct"
+ ):
+ prompt_template = kv_retrieval_prompt_template_llama3_instruct
+ elif (
+ model_name == "meta-llama/Llama-2-7b-hf"
+ or model_name == "meta-llama/Meta-Llama-3-8B"
+ ):
+ prompt_template = kv_retrieval_prompt_template
+ elif model_name == "Qwen/Qwen2.5-7B-Instruct":
+ prompt_template = kv_retrieval_prompt_template_qwen_instruct
+
+ return prompt_template.format(formatted_kv_records=formatted_kv_records, key=key)
+
+
+def quick_get_random_kv_samples(
+ model_name, tokenizer, gold_index, n_kv_num=10, n_sample=100
+):
+ samples = []
+ sample_counter = 0
+ # create n_kv_num key-value pairs
+ for idx in range(n_sample):
+ ordered_kv_records = [
+ [str(uuid.uuid4()), str(uuid.uuid4())] for _ in range(n_kv_num)
+ ]
+ key = str(uuid.uuid4())
+ value = str(uuid.uuid4())
+ ordered_kv_records.insert(gold_index, [key, value])
+ kv_prompt = get_kv_retrieval_prompt(
+ data=ordered_kv_records,
+ key=key,
+ model_name=model_name,
+ )
+ input_ids = tokenizer(kv_prompt, add_special_tokens=False)["input_ids"]
+ samples.append(
+ {
+ "input_ids": input_ids,
+ "key": key,
+ "value": value,
+ }
+ )
+ return samples
+
+
+def main(
+ model_name="meta-llama/Llama-3.1-8B-Instruct",
+ method="dense",
+ starting_layer=None,
+ gamma=None,
+):
+ seq_len_list = [
+ 32000,
+ 48000,
+ 64000,
+ 80000,
+ 96000,
+ 112000,
+ 128000,
+ ]
+ n_times = 3
+ if starting_layer is None:
+ if model_name == "meta-llama/Llama-3.1-8B-Instruct":
+ starting_layer = 16
+ elif model_name == "gradientai/Llama-3-8B-Instruct-262k":
+ starting_layer = 16
+ elif model_name == "Qwen/Qwen2.5-7B-Instruct":
+ starting_layer = 20
+ else:
+ raise NotImplementedError
+ if gamma is None:
+ gamma = 0.95
+ if method == "dense":
+ kwargs = dict(attn_type="dense")
+ elif method == "tri_mix":
+ kwargs = dict(
+ attn_type="tri_mix",
+ attn_kwargs={
+ "last_n": 128,
+ "starting_layer": starting_layer,
+ "n_local": 512,
+ "n_init": 8,
+ },
+ )
+ elif method == "flexprefill":
+ kwargs = dict(
+ attn_type="flexprefill",
+ attn_kwargs={"gamma": gamma},
+ )
+ elif method == "minference":
+ kwargs = dict(
+ attn_type="minference",
+ )
+ elif method == "tri_mix_minference":
+ kwargs = dict(
+ attn_type="tri_mix_minference",
+ attn_kwargs={
+ "last_n": 128,
+ "starting_layer": starting_layer,
+ "n_local": 512,
+ "n_init": 8,
+ },
+ )
+ else:
+ raise NotImplementedError
+
+ model_name_to_saving_name = {
+ "meta-llama/Llama-3.1-8B-Instruct": "Llama-3.1-8B-Instruct",
+ "gradientai/Llama-3-8B-Instruct-262k": "Llama-3-8B-Instruct-262k",
+ "Qwen/Qwen2.5-7B-Instruct": "Qwen2.5-7B-Instruct",
+ }
+ tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
+
+ minference_patch = MInference(
+ model_name=model_name,
+ config_path=None,
+ starting_layer=-1,
+ kv_type="dense",
+ is_search=False,
+ kv_cache_cpu=False,
+ kv_cache_cpu_device="cpu",
+ **kwargs,
+ )
+
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ trust_remote_code=True,
+ attn_implementation="flash_attention_2",
+ )
+
+ model = minference_patch(model)
+ samples = quick_get_random_kv_samples(
+ model_name, tokenizer, 3000, n_kv_num=6000, n_sample=n_times
+ )
+
+ # warmup
+ for seq_len in seq_len_list:
+ input_ids = samples[0]["input_ids"][:seq_len]
+ input_ids = torch.tensor([input_ids], device=model.device)
+ with torch.no_grad():
+ model(input_ids, use_cache=False)
+ torch.cuda.empty_cache()
+
+ # start test
+ ret_list = []
+ print("---------------------------")
+ for seq_len in seq_len_list:
+ dur_list = []
+ for i in range(n_times):
+ sample = samples[i]
+ input_ids = sample["input_ids"]
+ # print(len(input_ids))
+ assert len(input_ids) >= seq_len
+ input_ids = input_ids[:seq_len]
+ input_ids = torch.tensor([input_ids], device=model.device)
+
+ with torch.no_grad():
+ torch.cuda.synchronize(device=model.device)
+ start_event = torch.cuda.Event(enable_timing=True)
+ end_event = torch.cuda.Event(enable_timing=True)
+ start_event.record()
+ model(input_ids, use_cache=False)
+ torch.cuda.synchronize(device=model.device)
+ end_event.record()
+ torch.cuda.synchronize(device=model.device)
+ elapsed_time_ms = start_event.elapsed_time(end_event)
+ torch.cuda.empty_cache()
+ dur_list.append((elapsed_time_ms) / 1000.0)
+ print("seq_len: {:<20} time: {:.2f}s".format(seq_len, np.mean(dur_list)))
+ print("---------------------------")
+ ret_list.append(
+ {
+ "model": model_name_to_saving_name[model_name],
+ "method": method,
+ "seq_len": seq_len,
+ "time": np.mean(dur_list),
+ }
+ )
+
+ if gamma != 0.95:
+ method = "{}_{:.2f}".format(method, gamma)
+ pd.DataFrame(ret_list).to_csv(
+ f"speed_test_{model_name_to_saving_name[model_name]}_result_{method}.csv",
+ index=False,
+ )
+
+
+if __name__ == "__main__":
+ fire.Fire(main)
diff --git a/assets/license_header.txt b/assets/license_header.txt
new file mode 100644
index 00000000..5efc35fc
--- /dev/null
+++ b/assets/license_header.txt
@@ -0,0 +1,2 @@
+Copyright (c) 2025 Microsoft
+Licensed under The MIT License [see LICENSE for details]
diff --git a/examples/run_hf.py b/examples/run_hf.py
index 19bf71fe..b770e2de 100644
--- a/examples/run_hf.py
+++ b/examples/run_hf.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -13,10 +13,13 @@
model_name,
torch_dtype="auto",
device_map="cuda",
+ _attn_implementation="flash_attention_2",
)
# Patch MInference Module
-minference_patch = MInference("minference", model_name)
+minference_patch = MInference(
+ attn_type="minference", model_name=model_name, kv_type="dense"
+)
model = minference_patch(model)
batch_inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
diff --git a/examples/run_hf_streaming.py b/examples/run_hf_streaming.py
index 18d91e86..5488c3d0 100644
--- a/examples/run_hf_streaming.py
+++ b/examples/run_hf_streaming.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import warnings
@@ -6,14 +6,9 @@
warnings.filterwarnings("ignore")
import argparse
-import json
-import os
-import re
-import sys
import time
import torch
-from tqdm import tqdm
from transformers import AutoModelForCausalLM, AutoTokenizer
from minference import MInference
@@ -107,11 +102,12 @@ def main(args):
model_name_or_path,
torch_dtype="auto",
device_map="cuda",
+ _attn_implementation="flash_attention_2",
)
# Patch MInference Module
minference_patch = MInference(
- args.attn_type if args.attn_type != "hf" else "minference_with_dense",
+ args.attn_type,
model_name_or_path,
kv_cache_cpu=True,
)
@@ -148,12 +144,13 @@ def main(args):
type=str,
required=True,
choices=[
- "vllm",
"hf",
- "streaming",
+ "a_shape",
+ "tri_shape",
"minference",
"inf_llm",
- "minference_with_dense",
+ "dense",
+ "flexprefill",
],
)
args = parser.parse_args()
diff --git a/experiments/README.md b/experiments/README.md
index 72e152a5..7a1c4b5e 100644
--- a/experiments/README.md
+++ b/experiments/README.md
@@ -42,6 +42,19 @@ Environment parameters:
To demonstrate the efficiency of our method, we conducted end-to-end latency tests using the [LLaMA-3-8B-Instruct-1M](https://huggingface.co/gradientai/Llama-3-8B-Instruct-Gradient-1048k) model. The prompts were trimmed to different target token numbers, and we measured the pre-filling stage latency without using KV cache.
+> [!TIP]
+> To benefit from fast kernel implementations, we recommend installing **SGLang** or **vLLM**.
+> for sglang
+> ```bash
+> uv pip install "sglang[all]>=0.4.6.post4"
+> ```
+>
+> for vllm
+> ```bash
+> uv pip install "vllm>=0.9.0"
+> uv pip install git+https://github.com/vllm-project/flash-attention
+> ```
+
1. Download the prompt:
```bash
@@ -67,15 +80,15 @@ python experiments/benchmarks/benchmark_e2e.py --run_benchmark
4. After that, you should get the end-to-end latency results like this:
```json
- FlashAttention-2 StreamingLLM InfLLM MInference
-1K 0.54565 1.07110 2.94495 2.96450
-10K 0.97590 1.18339 2.21052 2.77618
-50K 8.52933 5.47972 14.63624 7.54537
-100K 24.88319 10.86379 27.67215 13.98508
-200K 79.39184 21.61490 55.64703 26.81303
-300K 169.62441 32.44844 80.74326 41.09374
-500K 456.78353 54.15910 167.91472 66.27691
-1000K 1765.56387 107.85639 328.58551 179.12031
+ FlashAttention-2 A-Shape InfLLM MInference MInference w/ SGLang
+1K 0.54565 1.07110 2.94495 2.96450 1.25478
+10K 0.97590 1.18339 2.21052 2.77618 2.43798
+50K 8.52933 5.47972 14.63624 7.54537 6.19598
+100K 24.88319 10.86379 27.67215 13.98508 10.81580
+200K 79.39184 21.61490 55.64703 26.81303 20.62303
+300K 169.62441 32.44844 80.74326 41.09374 31.01629
+500K 456.78353 54.15910 167.91472 66.27691 51.96293
+1000K 1765.56387 107.85639 328.58551 179.12031 112.37610
```
> [!TIP]
diff --git a/experiments/benchmarks/benchmark_e2e.py b/experiments/benchmarks/benchmark_e2e.py
index b67ed5cd..565e803f 100644
--- a/experiments/benchmarks/benchmark_e2e.py
+++ b/experiments/benchmarks/benchmark_e2e.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import argparse
@@ -30,7 +30,12 @@ def run_target_length(m: int, model, attn_type: str):
start = time.time()
with torch.no_grad():
if attn_type != "inf_llm":
- model(input_ids, attention_mask, use_cache=False)
+ model(
+ input_ids,
+ attention_mask,
+ use_cache=False,
+ logits_to_keep=1,
+ )
else:
model.generate(
input_ids, generation_config=GenerationConfig(max_new_tokens=1)
@@ -43,10 +48,10 @@ def run_target_length(m: int, model, attn_type: str):
def run_benchmark(model_name: str):
TARGET_LENS = [l * 1000 for l in [10, 50, 100, 200, 300, 500, 1000]]
- ATTN_TYPES = ["minference_with_dense", "streaming", "minference"]
+ ATTN_TYPES = ["dense", "a_shape", "minference"]
ATTN_TYPES2NAME = {
- "minference_with_dense": "FlashAttention-2",
- "streaming": "StreamingLLM",
+ "dense": "FlashAttention-2",
+ "a_shape": "A-Shape",
"inf_llm": "InfLLM",
"minference": "MInference",
}
@@ -57,6 +62,7 @@ def run_benchmark(model_name: str):
model_name,
torch_dtype="auto",
device_map="auto",
+ _attn_implementation="flash_attention_2",
)
attn_kwargs = {} if args.attn_type != "inf_llm" else {"dense_decoding": False}
if attn_type != "hf":
@@ -93,19 +99,20 @@ def run_benchmark(model_name: str):
"--model_name",
type=str,
default="gradientai/Llama-3-8B-Instruct-Gradient-1048k",
+ # default="Qwen/Qwen2.5-7B-Instruct",
)
args.add_argument(
"--attn_type",
type=str,
choices=[
"hf",
- "streaming",
+ "a_shape",
"minference",
- "minference_with_dense",
+ "dense",
"inf_llm",
],
)
- args.add_argument("--context_window", type=int, default=100000)
+ args.add_argument("--context_window", type=int, default=100_000)
args.add_argument("--run_benchmark", action="store_true")
args.add_argument("--kv_cache_cpu", action="store_true")
args.add_argument("--trust_remote_code", action="store_true")
@@ -123,6 +130,7 @@ def run_benchmark(model_name: str):
torch_dtype="auto",
device_map="auto",
trust_remote_code=args.trust_remote_code,
+ _attn_implementation="flash_attention_2",
)
attn_kwargs = {} if args.attn_type != "inf_llm" else {"dense_decoding": False}
if args.attn_type != "hf":
diff --git a/experiments/benchmarks/benchmark_e2e_vllm.py b/experiments/benchmarks/benchmark_e2e_vllm.py
index d0c62b50..054f8874 100644
--- a/experiments/benchmarks/benchmark_e2e_vllm.py
+++ b/experiments/benchmarks/benchmark_e2e_vllm.py
@@ -23,13 +23,14 @@ def run_target_length(m: int, model, sampling_params, attn_type: str):
s = 0
T = 10
- for _ in range(T):
+ for _ in range(T + 1):
torch.cuda.synchronize()
start = time.time()
with torch.no_grad():
outputs = llm.generate([prompt], sampling_params)
torch.cuda.synchronize()
- s += time.time() - start
+ if _:
+ s += time.time() - start
print(attn_type, m, s / T)
return s / T
@@ -60,14 +61,14 @@ def run_target_length(m: int, model, sampling_params, attn_type: str):
llm = LLM(
model_name,
- max_num_seqs=1,
enforce_eager=True,
- max_model_len=129000,
+ max_model_len=args.context_window + 10_000,
+ enable_chunked_prefill=False,
)
# Patch MInference Module
if args.attn_type == "minference":
- minference_patch = MInference("vllm", model_name)
+ minference_patch = MInference("vllm_minference", model_name)
llm = minference_patch(llm)
run_target_length(args.context_window, llm, sampling_params, args.attn_type)
diff --git a/experiments/benchmarks/benchmark_e2e_vllm_tp.py b/experiments/benchmarks/benchmark_e2e_vllm_tp.py
new file mode 100644
index 00000000..267162fb
--- /dev/null
+++ b/experiments/benchmarks/benchmark_e2e_vllm_tp.py
@@ -0,0 +1,76 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+import time
+
+import torch
+from transformers import AutoTokenizer
+from vllm import LLM, SamplingParams
+
+from minference import MInference
+
+
+def run_target_length(m: int, model, sampling_params, attn_type: str):
+ # wget https://raw.githubusercontent.com/FranxYao/chain-of-thought-hub/main/gsm8k/lib_prompt/prompt_hardest.txt
+ prompt_complex = open("./prompt_hardest.txt").read()
+ input_ids = tokenizer(prompt_complex)["input_ids"]
+ n = len(input_ids)
+ b = m // n + 1
+
+ new_input_ids = (input_ids * b)[:m]
+ prompt = tokenizer.decode(new_input_ids)
+
+ s = 0
+ T = 10
+ for _ in range(T + 1):
+ torch.cuda.synchronize()
+ start = time.time()
+ with torch.no_grad():
+ outputs = llm.generate([prompt], sampling_params)
+ torch.cuda.synchronize()
+ if _:
+ s += time.time() - start
+ print(attn_type, m, s / T)
+ return s / T
+
+
+if __name__ == "__main__":
+ args = argparse.ArgumentParser()
+ args.add_argument(
+ "--model_name",
+ type=str,
+ default="gradientai/Llama-3-8B-Instruct-Gradient-1048k",
+ )
+ args.add_argument(
+ "--attn_type",
+ type=str,
+ choices=["flash_attn", "minference"],
+ )
+ args.add_argument("--context_window", type=int, default=100_000)
+ args.add_argument("--tensor_parallel_size", type=int, default=2)
+ args = args.parse_args()
+
+ model_name = args.model_name
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+ sampling_params = SamplingParams(
+ temperature=0.8,
+ top_p=0.95,
+ max_tokens=1,
+ )
+
+ llm = LLM(
+ model_name,
+ enforce_eager=True,
+ max_model_len=args.context_window + 10_000,
+ enable_chunked_prefill=False,
+ tensor_parallel_size=args.tensor_parallel_size,
+ )
+
+ # Patch MInference Module
+ if args.attn_type == "minference":
+ minference_patch = MInference("vllm_minference", model_name)
+ llm = minference_patch(llm)
+
+ run_target_length(args.context_window, llm, sampling_params, args.attn_type)
diff --git a/experiments/benchmarks/run_e2e_vllm_tp.sh b/experiments/benchmarks/run_e2e_vllm_tp.sh
new file mode 100644
index 00000000..ceb0d4ba
--- /dev/null
+++ b/experiments/benchmarks/run_e2e_vllm_tp.sh
@@ -0,0 +1,10 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Load data
+wget https://raw.githubusercontent.com/FranxYao/chain-of-thought-hub/main/gsm8k/lib_prompt/prompt_hardest.txt
+
+VLLM_WORKER_MULTIPROC_METHOD=spawn python experiments/benchmarks/benchmark_e2e_vllm_tp.py \
+ --attn_type minference \
+ --context_window 100_000 \
+ --tensor_parallel_size 4
diff --git a/experiments/infinite_bench/args.py b/experiments/infinite_bench/args.py
index 9ef2e99e..3f0d1aef 100644
--- a/experiments/infinite_bench/args.py
+++ b/experiments/infinite_bench/args.py
@@ -5,6 +5,8 @@
from eval_utils import DATA_NAME_TO_MAX_NEW_TOKENS
+from minference import MInferenceConfig
+
def parse_args() -> Namespace:
p = ArgumentParser()
@@ -58,23 +60,19 @@ def parse_args() -> Namespace:
p.add_argument("--topk_dims_file_path", type=str, default=None)
p.add_argument("--kv_cache_cpu", action="store_true")
p.add_argument("--kv_cache_cpu_device", type=str, default="cpu")
- p.add_argument("--use_snapkv", action="store_true")
p.add_argument("--trust_remote_code", action="store_true")
+ p.add_argument("--tensor_parallel_size", type=int, default=1)
p.add_argument(
"--attn_type",
type=str,
- choices=[
- "vllm",
- "hf",
- "streaming",
- "inf_llm",
- "flash_attn",
- "minference",
- "minference_with_dense",
- "dilated1",
- "dilated2",
- ],
+ choices=MInferenceConfig.get_available_attn_types(),
default="hf",
)
+ p.add_argument(
+ "--kv_type",
+ type=str,
+ default="dense",
+ choices=MInferenceConfig.get_available_kv_types(),
+ )
p.add_argument("--is_search", action="store_true")
return p.parse_args()
diff --git a/experiments/infinite_bench/compute_scores.py b/experiments/infinite_bench/compute_scores.py
index a0a2941a..32b7a8ef 100644
--- a/experiments/infinite_bench/compute_scores.py
+++ b/experiments/infinite_bench/compute_scores.py
@@ -144,6 +144,8 @@ def get_score_one_kv_retrieval(pred, label, model_name: str) -> bool:
for c in ["\n", ":", '"', "'", ".", ",", "?", "!", "{", "}", "", "The", "To"]:
pred = pred.replace(c, " ")
words = pred.split()
+ if isinstance(label, list):
+ label = label[0]
return label in words
diff --git a/experiments/infinite_bench/eval_utils.py b/experiments/infinite_bench/eval_utils.py
index 00bc7dc3..6a956365 100644
--- a/experiments/infinite_bench/eval_utils.py
+++ b/experiments/infinite_bench/eval_utils.py
@@ -34,9 +34,9 @@
}
DATA_NAME_TO_MAX_NEW_TOKENS = {
- "passkey": 6,
- "number_string": 12,
- "kv_retrieval": 50,
+ "passkey": 15,
+ "number_string": 20,
+ "kv_retrieval": 80,
"longbook_sum_eng": 1200,
"longbook_choice_eng": 40,
"longbook_qa_eng": 40,
diff --git a/experiments/infinite_bench/requirements.txt b/experiments/infinite_bench/requirements.txt
new file mode 100644
index 00000000..b2bc53ed
--- /dev/null
+++ b/experiments/infinite_bench/requirements.txt
@@ -0,0 +1,5 @@
+evaluate
+jieba
+nltk
+rouge
+rouge_score
diff --git a/experiments/infinite_bench/run_infinitebench.py b/experiments/infinite_bench/run_infinitebench.py
index fd695517..113c80c1 100644
--- a/experiments/infinite_bench/run_infinitebench.py
+++ b/experiments/infinite_bench/run_infinitebench.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
from __future__ import annotations
@@ -81,7 +81,7 @@ def get_pred(
print("...")
print(tok.decode(input_tokens[-200:]))
print("=====================================")
- if attn_type == "vllm":
+ if "vllm" in attn_type:
if len(input_tokens) != 1:
input_tokens = [input_tokens]
outputs = model.generate(
@@ -117,37 +117,43 @@ def load_model(
attn_type: str = "vllm",
max_seq_length: int = None,
is_search: bool = False,
- use_snapkv: bool = False,
+ kv_type: str = "",
trust_remote_code: bool = False,
kv_cache_cpu: bool = False,
kv_cache_cpu_device: str = "cpu",
+ tensor_parallel_size: int = 1,
):
tok = AutoTokenizer.from_pretrained(
model_name, resume_download=None, trust_remote_code=trust_remote_code
)
tok.pad_token = tok.eos_token
+
minference_patch = MInference(
attn_type,
model_name,
config_path=topk_dims_file_path,
starting_layer=starting_layer,
- use_snapkv=use_snapkv,
+ kv_type=kv_type,
is_search=is_search,
kv_cache_cpu=kv_cache_cpu,
kv_cache_cpu_device=kv_cache_cpu_device,
)
- if attn_type == "vllm":
+ if "vllm" in attn_type:
llm = LLM(
- model_name,
- max_num_seqs=1,
- swap_space=64,
- gpu_memory_utilization=0.98,
+ model=model_name,
max_model_len=max_seq_length,
+ enable_chunked_prefill=False,
+ tensor_parallel_size=tensor_parallel_size,
+ trust_remote_code=trust_remote_code,
+ swap_space=64,
)
else:
config = AutoConfig.from_pretrained(
- model_name, resume_download=None, trust_remote_code=trust_remote_code
+ model_name,
+ resume_download=None,
+ trust_remote_code=trust_remote_code,
+ _attn_implementation="flash_attention_2",
)
if "LWM" in model_name:
c = {
@@ -170,7 +176,9 @@ def load_model(
resume_download=None,
trust_remote_code=trust_remote_code,
)
- llm = minference_patch(llm)
+
+ if attn_type not in ["vllm", "hf"]:
+ llm = minference_patch(llm)
print("Model and tokenizer loaded.")
return llm, tok
@@ -200,10 +208,11 @@ def load_model(
attn_type=args.attn_type,
max_seq_length=max_seq_length,
is_search=args.is_search,
- use_snapkv=args.use_snapkv,
+ kv_type=args.kv_type,
trust_remote_code=args.trust_remote_code,
kv_cache_cpu=args.kv_cache_cpu,
kv_cache_cpu_device=args.kv_cache_cpu_device,
+ tensor_parallel_size=args.tensor_parallel_size,
)
results = {}
@@ -212,7 +221,7 @@ def load_model(
if max_new_tokens >= max_seq_length:
max_new_tokens = 500
- if args.attn_type == "vllm":
+ if "vllm" in args.attn_type:
generation_config = SamplingParams(
temperature=0,
max_tokens=max_new_tokens,
@@ -257,6 +266,11 @@ def load_model(
ground_truth = get_answer(eg, data_name)
# print(input_text.index(ground_truth), len(input_text), input_text.index(ground_truth) / len(input_text))
# print(f"====== Example {i} ======")
+
+ msgs = [dict(role="system", content=input_text)]
+ input_text = tok.apply_chat_template(
+ msgs, add_generation_prompt=True, tokenize=False
+ )
pred = get_pred(
model,
tok,
diff --git a/experiments/needle_in_a_haystack/needle_summary.py b/experiments/needle_in_a_haystack/needle_summary.py
index deed952a..56622fc6 100644
--- a/experiments/needle_in_a_haystack/needle_summary.py
+++ b/experiments/needle_in_a_haystack/needle_summary.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import argparse
@@ -13,7 +13,7 @@ def summary(run_name: str, output_path: str):
datas, cs = [], set()
for path in pathlist:
if run_name in path:
- data = json.load(open(dirs + path))
+ data = json.load(open(os.path.join(output_path, path)))
if data[0]["context_length"] in cs:
continue
datas.extend(data)
@@ -25,7 +25,12 @@ def summary(run_name: str, output_path: str):
if ii["correct"] is False:
print(ii["response"])
sorted(res.items())
- with open("{output_path}/{run_name}.json", "w") as json_file:
+ length_res = Counter()
+ for (le, dep), cor in sorted(res.items()):
+ length_res[le] += cor
+ for le, cor in length_res.items():
+ print(le / 1024, cor / 50)
+ with open(f"{output_path}/{run_name}.json", "w") as json_file:
json.dump(datas, json_file)
diff --git a/experiments/needle_in_a_haystack/needle_test.py b/experiments/needle_in_a_haystack/needle_test.py
index 61390827..e2c2dc80 100644
--- a/experiments/needle_in_a_haystack/needle_test.py
+++ b/experiments/needle_in_a_haystack/needle_test.py
@@ -9,6 +9,8 @@
from needle_tools import LLMNeedleHaystackTester
from needle_viz import plot_needle_viz
+from minference import MInferenceConfig
+
@dataclass
class Config:
@@ -29,6 +31,7 @@ class Config:
kv_cache_cpu: bool = False
trust_remote_code: bool = False
kv_cache_cpu_device: str = "cpu"
+ kv_type: str = "dense"
def __post_init__(self):
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -51,6 +54,7 @@ def main(
kv_cache_cpu: bool = False,
trust_remote_code: bool = False,
kv_cache_cpu_device: str = "cpu",
+ kv_type: str = "dense",
):
config = Config(
model_name=model_name,
@@ -65,6 +69,7 @@ def main(
kv_cache_cpu=kv_cache_cpu,
trust_remote_code=trust_remote_code,
kv_cache_cpu_device=kv_cache_cpu_device,
+ kv_type=kv_type,
)
kwargs = {
"swap_space": 64,
@@ -95,15 +100,14 @@ def main(
args.add_argument(
"--attn_type",
type=str,
- required=True,
- choices=[
- "vllm",
- "hf",
- "streaming",
- "minference",
- "inf_llm",
- "minference_with_dense",
- ],
+ choices=MInferenceConfig.get_available_attn_types(),
+ default="hf",
+ )
+ args.add_argument(
+ "--kv_type",
+ type=str,
+ default="dense",
+ choices=MInferenceConfig.get_available_kv_types(),
)
args.add_argument("--output_path", type=str, default="results/needle/")
args.add_argument("--pattern_path", type=str, default=None)
@@ -120,6 +124,7 @@ def main(
model_name=args.model_name,
run_name=args.run_name,
attn_type=args.attn_type,
+ kv_type=args.kv_type,
output_path=args.output_path,
pattern_path=args.pattern_path,
rounds=args.rounds,
diff --git a/experiments/needle_in_a_haystack/needle_tools.py b/experiments/needle_in_a_haystack/needle_tools.py
index 2e1c38b4..8f2f7893 100644
--- a/experiments/needle_in_a_haystack/needle_tools.py
+++ b/experiments/needle_in_a_haystack/needle_tools.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import json
@@ -212,8 +212,9 @@ def __init__(
attn_kwargs=(
{} if self.config.attn_type != "inf_llm" else {"dense_decoding": False}
),
+ kv_type=self.config.kv_type,
)
- if self.config.attn_type == "vllm":
+ if "vllm" in self.config.attn_type:
#### use vllm implementation
self.model = LLM(
model=self.config.model_name,
@@ -228,6 +229,7 @@ def __init__(
torch_dtype="auto",
device_map="cuda",
trust_remote_code=config.trust_remote_code,
+ _attn_implementation="flash_attention_2",
**kwargs,
)
self.model = minference_patch(self.model)
diff --git a/experiments/ppl/run_ppl.py b/experiments/ppl/run_ppl.py
index 3115d2b6..528fe225 100644
--- a/experiments/ppl/run_ppl.py
+++ b/experiments/ppl/run_ppl.py
@@ -18,7 +18,7 @@
LlamaForCausalLM,
)
-from minference import MInference
+from minference import MInference, MInferenceConfig
class LongPPL:
@@ -33,6 +33,7 @@ def __init__(
output_path: str = "results/long-ppl/",
data_path: str = "liyucheng/pg19-4k",
num_eval_examples: int = 100,
+ kv_type: str = "dense",
**kwargs,
) -> None:
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -43,7 +44,7 @@ def __init__(
intervals,
num_eval_examples,
)
- self.load_model(model_name, attn_type, **kwargs)
+ self.load_model(model_name, attn_type, kv_type, **kwargs)
if not os.path.exists(output_path):
os.makedirs(output_path)
self.output_path = os.path.join(
@@ -58,9 +59,10 @@ def load_model(
self,
model_name: str,
attn_type: str = "vllm",
+ kv_type: str = "dense",
**kwargs,
):
- if attn_type == "vllm":
+ if "vllm" in attn_type:
pass
else:
topk_dims_file_path = kwargs.get("topk_dims_file_path", None)
@@ -70,6 +72,7 @@ def load_model(
model_name,
topk_dims_file_path,
starting_layer=topk_from_layer,
+ kv_type=kv_type,
)
self.model = LlamaForCausalLM.from_pretrained(
model_name,
@@ -229,15 +232,15 @@ def viz_results(
args.add_argument(
"--attn_type",
type=str,
- choices=[
- "hf",
- "streaming",
- "minference",
- "dilated1",
- "dilated2",
- ],
+ choices=MInferenceConfig.get_available_attn_types(),
default="hf",
)
+ args.add_argument(
+ "--kv_type",
+ type=str,
+ default="dense",
+ choices=MInferenceConfig.get_available_kv_types(),
+ )
args.add_argument("--do_plot", action="store_true")
args.add_argument("--min_seq_length", type=int, default=1_000)
args.add_argument("--max_seq_length", type=int, default=100_000)
@@ -264,6 +267,7 @@ def viz_results(
topk=args.topk,
topk_from_layer=args.topk_from_layer,
topk_dims_file_path=args.topk_dims_file_path,
+ kv_type=args.kv_type,
)
test.start_test()
diff --git a/experiments/ruler/config_models.sh b/experiments/ruler/config_models.sh
index bc1e44be..eb6c6927 100644
--- a/experiments/ruler/config_models.sh
+++ b/experiments/ruler/config_models.sh
@@ -92,7 +92,7 @@ MODEL_SELECT() {
Yi-streaming)
MODEL_PATH="01-ai/Yi-9B-200K"
MODEL_TEMPLATE_TYPE="meta-chat"
- MODEL_FRAMEWORK="streaming"
+ MODEL_FRAMEWORK="a_shape"
;;
OPYiHalfV2)
MODEL_PATH="01-ai/Yi-9B-200K"
diff --git a/experiments/ruler/pred/call_api.py b/experiments/ruler/pred/call_api.py
index 997ac177..bead9c3f 100644
--- a/experiments/ruler/pred/call_api.py
+++ b/experiments/ruler/pred/call_api.py
@@ -41,8 +41,9 @@
"gemini",
"hf",
"minference",
- "streaming",
+ "a_shape",
"InfLLM",
+ "minference_with_dense",
)
@@ -105,7 +106,7 @@ def __call__(self, parser, namespace, values, option_string=None):
parser.add_argument("--starting_layer", type=int, default=-1)
parser.add_argument("--kv_cache_cpu", action="store_true")
parser.add_argument("--kv_cache_cpu_device", type=str, default="cpu")
-parser.add_argument("--use_snapkv", action="store_true")
+parser.add_argument("--kv_type", type=str, default="dense")
parser.add_argument("--trust_remote_code", action="store_true")
args = parser.parse_args()
@@ -115,8 +116,9 @@ def __call__(self, parser, namespace, values, option_string=None):
"hf",
"gemini",
"minference",
- "streaming",
+ "a_shape",
"InfLLM",
+ "minference_with_dense",
]:
args.threads = 1
@@ -210,7 +212,7 @@ def get_llm(tokens_to_generate):
config_path=args.config_path,
kv_cache_cpu=args.kv_cache_cpu,
kv_cache_cpu_device=args.kv_cache_cpu_device,
- use_snapkv=args.use_snapkv,
+ kv_type=args.kv_type,
trust_remote_code=args.trust_remote_code,
starting_layer=args.starting_layer,
)
@@ -229,7 +231,7 @@ def get_llm(tokens_to_generate):
max_new_tokens=tokens_to_generate,
)
- elif args.server_type == "streaming":
+ elif args.server_type == "a_shape":
from model_wrappers import Streaming
llm = Streaming(
@@ -244,7 +246,7 @@ def get_llm(tokens_to_generate):
config_path=args.config_path,
kv_cache_cpu=args.kv_cache_cpu,
kv_cache_cpu_device=args.kv_cache_cpu_device,
- use_snapkv=args.use_snapkv,
+ kv_type=args.kv_type,
trust_remote_code=args.trust_remote_code,
starting_layer=args.starting_layer,
)
diff --git a/experiments/ruler/pred/model_wrappers.py b/experiments/ruler/pred/model_wrappers.py
index b4b975f4..841e4c7e 100644
--- a/experiments/ruler/pred/model_wrappers.py
+++ b/experiments/ruler/pred/model_wrappers.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import json
@@ -39,6 +39,7 @@ def __init__(self, name_or_path: str, **generation_kwargs) -> None:
trust_remote_code=True,
device_map="auto",
torch_dtype=torch.bfloat16,
+ _attn_implementation="flash_attention_2",
)
self.generation_kwargs = generation_kwargs
@@ -84,8 +85,9 @@ def __init__(
starting_layer: int = -1,
kv_cache_cpu: bool = False,
kv_cache_cpu_device: str = None,
- use_snapkv: bool = False,
+ kv_type: str = "",
trust_remote_code: bool = False,
+ attn_type: str = "minference",
) -> None:
from transformers import (
AutoConfig,
@@ -107,13 +109,14 @@ def __init__(
device_map="cuda",
resume_download=None,
trust_remote_code=trust_remote_code,
+ _attn_implementation="flash_attention_2",
)
minference_patch = MInference(
- "minference",
+ attn_type,
name_or_path,
config_path=config_path,
starting_layer=starting_layer,
- use_snapkv=use_snapkv,
+ kv_type=kv_type,
kv_cache_cpu=kv_cache_cpu,
kv_cache_cpu_device=kv_cache_cpu_device,
is_search=False,
@@ -214,7 +217,7 @@ def __init__(
starting_layer: int = -1,
kv_cache_cpu: bool = False,
kv_cache_cpu_device: str = None,
- use_snapkv: bool = False,
+ kv_type: str = "",
trust_remote_code: bool = False,
) -> None:
from transformers import (
@@ -237,13 +240,14 @@ def __init__(
device_map="cuda",
resume_download=None,
trust_remote_code=trust_remote_code,
+ _attn_implementation="flash_attention_2",
)
minference_patch = MInference(
- "streaming",
+ "a_shape",
name_or_path,
config_path=config_path,
starting_layer=starting_layer,
- use_snapkv=use_snapkv,
+ kv_type=kv_type,
kv_cache_cpu=kv_cache_cpu,
kv_cache_cpu_device=kv_cache_cpu_device,
is_search=False,
diff --git a/images/MMInference_onepage.png b/images/MMInference_onepage.png
new file mode 100644
index 00000000..f2869159
Binary files /dev/null and b/images/MMInference_onepage.png differ
diff --git a/images/SCBench_onepage.png b/images/SCBench_onepage.png
new file mode 100644
index 00000000..825c1e77
Binary files /dev/null and b/images/SCBench_onepage.png differ
diff --git a/images/mtraining/Framework.png b/images/mtraining/Framework.png
new file mode 100644
index 00000000..45ed0454
Binary files /dev/null and b/images/mtraining/Framework.png differ
diff --git a/images/scbench/comparison.png b/images/scbench/comparison.png
new file mode 100644
index 00000000..cec82684
Binary files /dev/null and b/images/scbench/comparison.png differ
diff --git a/images/scbench/overview.png b/images/scbench/overview.png
new file mode 100644
index 00000000..2396580b
Binary files /dev/null and b/images/scbench/overview.png differ
diff --git a/minference/__init__.py b/minference/__init__.py
index 54089cdc..76c8f08b 100644
--- a/minference/__init__.py
+++ b/minference/__init__.py
@@ -12,7 +12,7 @@
from .patch import (
minference_patch,
minference_patch_kv_cache_cpu,
- minference_patch_with_snapkv,
+ minference_patch_with_kvcompress,
patch_hf,
)
from .version import VERSION as __version__
@@ -22,7 +22,7 @@
"MInferenceConfig",
"minference_patch",
"minference_patch_kv_cache_cpu",
- "minference_patch_with_snapkv",
+ "minference_patch_with_kvcompress",
"patch_hf",
"vertical_slash_sparse_attention",
"block_sparse_attention",
diff --git a/minference/configs/Llama_3.1_70B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json b/minference/configs/Llama_3.1_70B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json
new file mode 100644
index 00000000..5212471a
--- /dev/null
+++ b/minference/configs/Llama_3.1_70B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json
@@ -0,0 +1 @@
+[{"0": ["vertical_and_slash", 1000, 6096, 0.38275444507598877], "1": ["vertical_and_slash", 1000, 6096, 0.3826100528240204], "2": ["vertical_and_slash", 1000, 6096, 0.37926939129829407], "3": ["vertical_and_slash", 1000, 6096, 0.3949490487575531], "4": ["vertical_and_slash", 1000, 6096, 0.38691017031669617], "5": ["vertical_and_slash", 1000, 6096, 0.3833230137825012], "6": ["vertical_and_slash", 1000, 6096, 0.3769170939922333], "7": ["vertical_and_slash", 1000, 6096, 0.3871508240699768], "8": ["vertical_and_slash", 1000, 6096, 0.3663298785686493], "9": ["vertical_and_slash", 1000, 6096, 0.3623872995376587], "10": ["vertical_and_slash", 1000, 6096, 0.35321760177612305], "11": ["vertical_and_slash", 1000, 6096, 0.36464789509773254], "12": ["vertical_and_slash", 1000, 6096, 0.35689157247543335], "13": ["vertical_and_slash", 1000, 6096, 0.3610193431377411], "14": ["vertical_and_slash", 1000, 6096, 0.3654029667377472], "15": ["vertical_and_slash", 1000, 6096, 0.36772286891937256], "16": ["vertical_and_slash", 1000, 6096, 0.35992830991744995], "17": ["vertical_and_slash", 1000, 6096, 0.3629811406135559], "18": ["vertical_and_slash", 1000, 6096, 0.37045222520828247], "19": ["vertical_and_slash", 1000, 6096, 0.36531585454940796], "20": ["vertical_and_slash", 1000, 6096, 0.34927937388420105], "21": ["vertical_and_slash", 1000, 6096, 0.37366679310798645], "22": ["vertical_and_slash", 1000, 6096, 0.36565107107162476], "23": ["vertical_and_slash", 1000, 6096, 0.3697221875190735], "24": ["vertical_and_slash", 1000, 6096, 0.4699176549911499], "25": ["vertical_and_slash", 1000, 6096, 0.40619152784347534], "26": ["vertical_and_slash", 1000, 6096, 0.5786199569702148], "27": ["vertical_and_slash", 1000, 6096, 0.4302184581756592], "28": ["vertical_and_slash", 1000, 6096, 0.6581957340240479], "29": ["vertical_and_slash", 1000, 6096, 0.593205451965332], "30": ["vertical_and_slash", 1000, 6096, 0.5827254056930542], "31": ["vertical_and_slash", 1000, 6096, 0.7082818746566772], "32": ["vertical_and_slash", 1000, 6096, 0.4427865445613861], "33": ["vertical_and_slash", 1000, 6096, 0.38276875019073486], "34": ["vertical_and_slash", 1000, 6096, 0.369324654340744], "35": ["vertical_and_slash", 1000, 6096, 0.3842734098434448], "36": ["vertical_and_slash", 1000, 6096, 0.43138888478279114], "37": ["vertical_and_slash", 100, 800, 1.0], "38": ["vertical_and_slash", 1000, 6096, 0.39381271600723267], "39": ["vertical_and_slash", 1000, 6096, 0.36797329783439636], "40": ["vertical_and_slash", 1000, 6096, 0.3926314115524292], "41": ["vertical_and_slash", 1000, 6096, 0.3932630121707916], "42": ["vertical_and_slash", 1000, 6096, 0.40250080823898315], "43": ["vertical_and_slash", 1000, 6096, 0.4032178521156311], "44": ["vertical_and_slash", 1000, 6096, 0.3852488696575165], "45": ["vertical_and_slash", 1000, 6096, 0.3947626054286957], "46": ["vertical_and_slash", 1000, 6096, 0.396650493144989], "47": ["vertical_and_slash", 1000, 6096, 0.3970978260040283], "48": ["vertical_and_slash", 1000, 6096, 0.40175575017929077], "49": ["vertical_and_slash", 1000, 6096, 0.3886752128601074], "50": ["vertical_and_slash", 1000, 6096, 0.3942696452140808], "51": ["vertical_and_slash", 1000, 6096, 0.3881273865699768], "52": ["vertical_and_slash", 1000, 6096, 0.39494937658309937], "53": ["vertical_and_slash", 1000, 6096, 0.39482244849205017], "54": ["vertical_and_slash", 1000, 6096, 0.3959391713142395], "55": ["vertical_and_slash", 1000, 6096, 0.40950101613998413], "56": ["vertical_and_slash", 1000, 6096, 0.41832900047302246], "57": ["vertical_and_slash", 1000, 6096, 0.5111724734306335], "58": ["vertical_and_slash", 1000, 6096, 0.4218967854976654], "59": ["vertical_and_slash", 3500, 100, 0.4713592827320099], "60": ["vertical_and_slash", 1000, 6096, 0.4746357202529907], "61": ["vertical_and_slash", 1000, 6096, 0.42022737860679626], "62": ["vertical_and_slash", 1000, 6096, 0.41058939695358276], "63": ["vertical_and_slash", 1000, 6096, 0.40124738216400146]}, {"0": ["vertical_and_slash", 30, 800, 0.996486485004425], "1": ["vertical_and_slash", 30, 800, 0.9925103783607483], "2": ["vertical_and_slash", 30, 800, 0.9929607510566711], "3": ["vertical_and_slash", 30, 800, 0.9575023055076599], "4": ["vertical_and_slash", 30, 800, 0.9914794564247131], "5": ["vertical_and_slash", 30, 800, 0.9968298077583313], "6": ["vertical_and_slash", 1000, 6096, 0.9882379174232483], "7": ["vertical_and_slash", 30, 800, 0.9786902070045471], "8": ["vertical_and_slash", 1000, 6096, 0.997696578502655], "9": ["vertical_and_slash", 1000, 6096, 0.9729989767074585], "10": ["vertical_and_slash", 3500, 100, 0.9680017828941345], "11": ["vertical_and_slash", 1000, 6096, 0.9922733902931213], "12": ["vertical_and_slash", 30, 800, 0.9776073694229126], "13": ["vertical_and_slash", 1000, 6096, 0.9969093203544617], "14": ["vertical_and_slash", 30, 800, 0.9880983233451843], "15": ["vertical_and_slash", 3500, 100, 0.9585568904876709], "16": ["vertical_and_slash", 30, 800, 0.8946104049682617], "17": ["vertical_and_slash", 30, 800, 0.9261633157730103], "18": ["vertical_and_slash", 30, 800, 0.9954677820205688], "19": ["vertical_and_slash", 30, 800, 0.9914458394050598], "20": ["vertical_and_slash", 1000, 6096, 0.9742741584777832], "21": ["vertical_and_slash", 30, 800, 0.9837812781333923], "22": ["vertical_and_slash", 30, 800, 0.9730805158615112], "23": ["vertical_and_slash", 500, 700, 0.965228796005249], "24": ["vertical_and_slash", 30, 800, 0.9641847014427185], "25": ["vertical_and_slash", 30, 800, 0.9744320511817932], "26": ["vertical_and_slash", 30, 800, 0.9736859202384949], "27": ["vertical_and_slash", 30, 800, 0.98580002784729], "28": ["vertical_and_slash", 30, 800, 0.9825156331062317], "29": ["vertical_and_slash", 30, 800, 0.9944815039634705], "30": ["vertical_and_slash", 30, 800, 0.9822251796722412], "31": ["vertical_and_slash", 30, 800, 0.99454265832901], "32": ["vertical_and_slash", 30, 800, 0.98089599609375], "33": ["vertical_and_slash", 30, 800, 0.9526920318603516], "34": ["vertical_and_slash", 30, 800, 0.9748521447181702], "35": ["vertical_and_slash", 30, 800, 0.9733909368515015], "36": ["vertical_and_slash", 30, 800, 0.8622740507125854], "37": ["vertical_and_slash", 30, 800, 0.9803394079208374], "38": ["vertical_and_slash", 30, 800, 0.9642454981803894], "39": ["vertical_and_slash", 30, 800, 0.9734092950820923], "40": ["vertical_and_slash", 30, 800, 0.9721104502677917], "41": ["vertical_and_slash", 30, 800, 0.9240614771842957], "42": ["vertical_and_slash", 30, 800, 0.9703289270401001], "43": ["vertical_and_slash", 30, 800, 0.965050458908081], "44": ["vertical_and_slash", 30, 800, 0.9598613381385803], "45": ["vertical_and_slash", 30, 800, 0.9792070388793945], "46": ["vertical_and_slash", 30, 800, 0.963590145111084], "47": ["vertical_and_slash", 30, 800, 0.961765468120575], "48": ["vertical_and_slash", 1000, 6096, 0.975767970085144], "49": ["vertical_and_slash", 30, 800, 0.995331883430481], "50": ["vertical_and_slash", 1000, 6096, 0.9783875346183777], "51": ["vertical_and_slash", 3500, 100, 0.977223813533783], "52": ["vertical_and_slash", 30, 800, 0.9763361811637878], "53": ["vertical_and_slash", 30, 800, 0.9443849325180054], "54": ["vertical_and_slash", 30, 800, 0.994658887386322], "55": ["vertical_and_slash", 30, 800, 0.9769454002380371], "56": ["vertical_and_slash", 30, 800, 0.968227744102478], "57": ["vertical_and_slash", 30, 800, 0.965366542339325], "58": ["vertical_and_slash", 30, 800, 0.970639705657959], "59": ["vertical_and_slash", 30, 800, 0.9717962145805359], "60": ["vertical_and_slash", 30, 800, 0.9713809490203857], "61": ["vertical_and_slash", 30, 800, 0.9464559555053711], "62": ["vertical_and_slash", 30, 800, 0.889991044998169], "63": ["vertical_and_slash", 30, 800, 0.8131658434867859]}, {"0": ["vertical_and_slash", 30, 800, 0.9830862879753113], "1": ["vertical_and_slash", 30, 800, 0.9744600653648376], "2": ["vertical_and_slash", 30, 800, 0.9770146012306213], "3": ["vertical_and_slash", 30, 800, 0.9875395894050598], "4": ["vertical_and_slash", 30, 800, 0.9877269268035889], "5": ["vertical_and_slash", 30, 800, 0.9922516942024231], "6": ["vertical_and_slash", 30, 800, 0.9952882528305054], "7": ["vertical_and_slash", 30, 800, 0.9926594495773315], "8": ["vertical_and_slash", 30, 800, 0.9893272519111633], "9": ["vertical_and_slash", 30, 800, 0.9018037915229797], "10": ["vertical_and_slash", 30, 800, 0.9784188866615295], "11": ["vertical_and_slash", 30, 800, 0.8872243165969849], "12": ["vertical_and_slash", 30, 800, 0.984529972076416], "13": ["vertical_and_slash", 30, 800, 0.9815819263458252], "14": ["vertical_and_slash", 30, 800, 0.988295316696167], "15": ["vertical_and_slash", 30, 800, 0.9647165536880493], "16": ["vertical_and_slash", 30, 800, 0.9709293246269226], "17": ["vertical_and_slash", 30, 800, 0.979886531829834], "18": ["vertical_and_slash", 30, 800, 0.9860535264015198], "19": ["vertical_and_slash", 30, 800, 0.9128928780555725], "20": ["vertical_and_slash", 30, 800, 0.9872728586196899], "21": ["vertical_and_slash", 30, 800, 0.9809684157371521], "22": ["vertical_and_slash", 30, 800, 0.8684135675430298], "23": ["vertical_and_slash", 30, 800, 0.9797967672348022], "24": ["vertical_and_slash", 30, 800, 0.9910656213760376], "25": ["vertical_and_slash", 30, 800, 0.9788667559623718], "26": ["vertical_and_slash", 30, 800, 0.9927509427070618], "27": ["vertical_and_slash", 30, 800, 0.9858380556106567], "28": ["vertical_and_slash", 30, 800, 0.9887804985046387], "29": ["vertical_and_slash", 30, 800, 0.9673082828521729], "30": ["vertical_and_slash", 30, 800, 0.9925527572631836], "31": ["vertical_and_slash", 30, 800, 0.990679144859314], "32": ["vertical_and_slash", 30, 800, 0.9958600997924805], "33": ["vertical_and_slash", 30, 800, 0.9839220643043518], "34": ["vertical_and_slash", 30, 800, 0.9868613481521606], "35": ["vertical_and_slash", 30, 800, 0.9915750622749329], "36": ["vertical_and_slash", 30, 800, 0.9960166811943054], "37": ["vertical_and_slash", 30, 800, 0.9867536425590515], "38": ["vertical_and_slash", 30, 800, 0.9954875707626343], "39": ["vertical_and_slash", 30, 800, 0.9831879734992981], "40": ["vertical_and_slash", 500, 700, 0.9818185567855835], "41": ["vertical_and_slash", 30, 800, 0.9936351776123047], "42": ["vertical_and_slash", 30, 800, 0.9951633214950562], "43": ["vertical_and_slash", 30, 800, 0.993500828742981], "44": ["vertical_and_slash", 30, 800, 0.9607060551643372], "45": ["vertical_and_slash", 30, 800, 0.9960837960243225], "46": ["vertical_and_slash", 30, 800, 0.9967908263206482], "47": ["vertical_and_slash", 500, 700, 0.9786515831947327], "48": ["vertical_and_slash", 30, 800, 0.9884766340255737], "49": ["vertical_and_slash", 30, 800, 0.9804601073265076], "50": ["vertical_and_slash", 30, 800, 0.9770196676254272], "51": ["vertical_and_slash", 30, 800, 0.9934117197990417], "52": ["vertical_and_slash", 30, 800, 0.9913334250450134], "53": ["vertical_and_slash", 30, 800, 0.9895665645599365], "54": ["vertical_and_slash", 30, 800, 0.9798051118850708], "55": ["vertical_and_slash", 30, 800, 0.9271437525749207], "56": ["vertical_and_slash", 1000, 6096, 0.9507524371147156], "57": ["vertical_and_slash", 1000, 6096, 0.9587556719779968], "58": ["vertical_and_slash", 1000, 6096, 0.9850970506668091], "59": ["vertical_and_slash", 30, 800, 0.9723434448242188], "60": ["vertical_and_slash", 1000, 6096, 0.9936138987541199], "61": ["vertical_and_slash", 30, 800, 0.9045288562774658], "62": ["vertical_and_slash", 1000, 6096, 0.9903751611709595], "63": ["vertical_and_slash", 30, 800, 0.9232630729675293]}, {"0": ["vertical_and_slash", 30, 800, 0.9831088185310364], "1": ["vertical_and_slash", 30, 800, 0.9527093172073364], "2": ["vertical_and_slash", 30, 800, 0.9745521545410156], "3": ["vertical_and_slash", 30, 800, 0.9660399556159973], "4": ["vertical_and_slash", 30, 800, 0.8777262568473816], "5": ["vertical_and_slash", 30, 800, 0.9534850120544434], "6": ["vertical_and_slash", 30, 800, 0.9777054190635681], "7": ["vertical_and_slash", 30, 800, 0.8617303371429443], "8": ["vertical_and_slash", 30, 800, 0.9471170902252197], "9": ["vertical_and_slash", 30, 800, 0.9809898734092712], "10": ["vertical_and_slash", 30, 800, 0.9348939657211304], "11": ["vertical_and_slash", 1000, 6096, 0.9420511722564697], "12": ["vertical_and_slash", 1000, 6096, 0.9728347659111023], "13": ["vertical_and_slash", 500, 700, 0.9674691557884216], "14": ["vertical_and_slash", 1000, 6096, 0.9638375639915466], "15": ["vertical_and_slash", 1000, 6096, 0.9691937565803528], "16": ["vertical_and_slash", 30, 800, 0.9912046194076538], "17": ["vertical_and_slash", 30, 800, 0.9780941009521484], "18": ["vertical_and_slash", 30, 800, 0.9949376583099365], "19": ["vertical_and_slash", 30, 800, 0.9854227900505066], "20": ["vertical_and_slash", 30, 800, 0.9778551459312439], "21": ["vertical_and_slash", 100, 800, 0.95703125], "22": ["vertical_and_slash", 30, 800, 0.9908679723739624], "23": ["vertical_and_slash", 30, 800, 0.9846581816673279], "24": ["vertical_and_slash", 30, 800, 0.9938567876815796], "25": ["vertical_and_slash", 30, 800, 0.982925295829773], "26": ["vertical_and_slash", 30, 800, 0.9619585275650024], "27": ["vertical_and_slash", 30, 800, 0.9869666695594788], "28": ["vertical_and_slash", 30, 800, 0.9915645718574524], "29": ["vertical_and_slash", 30, 800, 0.9726572632789612], "30": ["vertical_and_slash", 30, 800, 0.9877031445503235], "31": ["vertical_and_slash", 30, 800, 0.9805050492286682], "32": ["vertical_and_slash", 30, 800, 0.9137957096099854], "33": ["vertical_and_slash", 30, 800, 0.9689615368843079], "34": ["vertical_and_slash", 30, 800, 0.9676137566566467], "35": ["vertical_and_slash", 30, 800, 0.9801207184791565], "36": ["vertical_and_slash", 30, 800, 0.9873160719871521], "37": ["vertical_and_slash", 30, 800, 0.9808938503265381], "38": ["vertical_and_slash", 30, 800, 0.9844655394554138], "39": ["vertical_and_slash", 30, 800, 0.9728024005889893], "40": ["vertical_and_slash", 30, 800, 0.9646276235580444], "41": ["vertical_and_slash", 30, 800, 0.9900849461555481], "42": ["vertical_and_slash", 30, 800, 0.9906730055809021], "43": ["vertical_and_slash", 30, 800, 0.9890775680541992], "44": ["vertical_and_slash", 30, 800, 0.9883682131767273], "45": ["vertical_and_slash", 30, 800, 0.9917970299720764], "46": ["vertical_and_slash", 30, 800, 0.9511682987213135], "47": ["vertical_and_slash", 30, 800, 0.9875025749206543], "48": ["vertical_and_slash", 30, 800, 0.9769311547279358], "49": ["vertical_and_slash", 30, 800, 0.9852514266967773], "50": ["vertical_and_slash", 3500, 100, 0.9479455351829529], "51": ["vertical_and_slash", 30, 800, 0.9617856740951538], "52": ["vertical_and_slash", 30, 800, 0.9782654643058777], "53": ["vertical_and_slash", 30, 800, 0.8911669850349426], "54": ["vertical_and_slash", 30, 800, 0.9047389030456543], "55": ["vertical_and_slash", 30, 800, 0.9901536107063293], "56": ["vertical_and_slash", 30, 800, 0.9582361578941345], "57": ["vertical_and_slash", 30, 800, 0.9573557376861572], "58": ["vertical_and_slash", 30, 800, 0.9311973452568054], "59": ["vertical_and_slash", 30, 800, 0.986301839351654], "60": ["vertical_and_slash", 30, 800, 0.9024320244789124], "61": ["vertical_and_slash", 30, 800, 0.9546028971672058], "62": ["vertical_and_slash", 30, 800, 0.8656956553459167], "63": ["vertical_and_slash", 30, 800, 0.9548195004463196]}, {"0": ["vertical_and_slash", 30, 800, 0.9929259419441223], "1": ["vertical_and_slash", 30, 800, 0.9840747714042664], "2": ["vertical_and_slash", 30, 800, 0.9480395913124084], "3": ["vertical_and_slash", 30, 800, 0.9878013134002686], "4": ["vertical_and_slash", 30, 800, 0.9942758083343506], "5": ["vertical_and_slash", 30, 800, 0.9859737753868103], "6": ["vertical_and_slash", 30, 800, 0.985692024230957], "7": ["vertical_and_slash", 30, 800, 0.9844392538070679], "8": ["vertical_and_slash", 30, 800, 0.988520622253418], "9": ["vertical_and_slash", 30, 800, 0.9679064154624939], "10": ["vertical_and_slash", 500, 700, 0.8913217782974243], "11": ["vertical_and_slash", 30, 800, 0.9927842020988464], "12": ["vertical_and_slash", 30, 800, 0.9765784740447998], "13": ["vertical_and_slash", 30, 800, 0.992040753364563], "14": ["vertical_and_slash", 30, 800, 0.9932926893234253], "15": ["vertical_and_slash", 30, 800, 0.9865526556968689], "16": ["vertical_and_slash", 30, 800, 0.9967150092124939], "17": ["vertical_and_slash", 30, 800, 0.995030403137207], "18": ["vertical_and_slash", 30, 800, 0.9936914443969727], "19": ["vertical_and_slash", 1000, 6096, 0.9546133279800415], "20": ["vertical_and_slash", 30, 800, 0.9964591264724731], "21": ["vertical_and_slash", 30, 800, 0.9958589673042297], "22": ["vertical_and_slash", 30, 800, 0.9602380394935608], "23": ["vertical_and_slash", 30, 800, 0.9963759779930115], "24": ["vertical_and_slash", 30, 800, 0.9729142189025879], "25": ["vertical_and_slash", 30, 800, 0.9686161279678345], "26": ["vertical_and_slash", 30, 800, 0.9870795011520386], "27": ["vertical_and_slash", 1000, 6096, 0.9934161305427551], "28": ["vertical_and_slash", 30, 800, 0.9851441383361816], "29": ["vertical_and_slash", 30, 800, 0.9969308376312256], "30": ["vertical_and_slash", 30, 800, 0.9943761229515076], "31": ["vertical_and_slash", 30, 800, 0.996328592300415], "32": ["vertical_and_slash", 30, 800, 0.9359889030456543], "33": ["vertical_and_slash", 30, 800, 0.8803390860557556], "34": ["vertical_and_slash", 30, 800, 0.9919400215148926], "35": ["vertical_and_slash", 30, 800, 0.9875897169113159], "36": ["vertical_and_slash", 1000, 6096, 0.9825850129127502], "37": ["vertical_and_slash", 30, 800, 0.9928356409072876], "38": ["vertical_and_slash", 30, 800, 0.9928852319717407], "39": ["vertical_and_slash", 30, 800, 0.971994936466217], "40": ["vertical_and_slash", 30, 800, 0.983616054058075], "41": ["vertical_and_slash", 30, 800, 0.9917797446250916], "42": ["vertical_and_slash", 30, 800, 0.9746459126472473], "43": ["vertical_and_slash", 30, 800, 0.9773445725440979], "44": ["vertical_and_slash", 30, 800, 0.9699975848197937], "45": ["vertical_and_slash", 30, 800, 0.9938631057739258], "46": ["vertical_and_slash", 30, 800, 0.992953360080719], "47": ["vertical_and_slash", 30, 800, 0.9934280514717102], "48": ["vertical_and_slash", 30, 800, 0.9812237024307251], "49": ["vertical_and_slash", 100, 800, 0.95703125], "50": ["vertical_and_slash", 100, 800, 0.93359375], "51": ["vertical_and_slash", 30, 800, 0.9914011359214783], "52": ["vertical_and_slash", 30, 800, 0.9936418533325195], "53": ["vertical_and_slash", 30, 800, 0.9895716905593872], "54": ["vertical_and_slash", 30, 800, 0.9903197288513184], "55": ["vertical_and_slash", 30, 800, 0.9873484373092651], "56": ["vertical_and_slash", 1000, 6096, 0.9261571168899536], "57": ["vertical_and_slash", 500, 700, 0.9682961106300354], "58": ["vertical_and_slash", 30, 800, 0.9859751462936401], "59": ["vertical_and_slash", 30, 800, 0.9697057008743286], "60": ["vertical_and_slash", 1000, 6096, 0.9922568202018738], "61": ["vertical_and_slash", 30, 800, 0.9945876002311707], "62": ["vertical_and_slash", 30, 800, 0.9981052875518799], "63": ["vertical_and_slash", 30, 800, 0.9959244132041931]}, {"0": ["vertical_and_slash", 30, 800, 0.9773167967796326], "1": ["vertical_and_slash", 1000, 6096, 0.963140606880188], "2": ["vertical_and_slash", 500, 700, 0.9685677886009216], "3": ["vertical_and_slash", 30, 800, 0.9823920726776123], "4": ["vertical_and_slash", 1000, 6096, 0.9752131104469299], "5": ["vertical_and_slash", 1000, 6096, 0.9783498644828796], "6": ["vertical_and_slash", 1000, 6096, 0.9076468348503113], "7": ["vertical_and_slash", 1000, 6096, 0.9876634478569031], "8": ["vertical_and_slash", 30, 800, 0.9007910490036011], "9": ["vertical_and_slash", 30, 800, 0.9722722172737122], "10": ["vertical_and_slash", 1000, 6096, 0.9815333485603333], "11": ["vertical_and_slash", 30, 800, 0.9691078662872314], "12": ["vertical_and_slash", 1000, 6096, 0.9807459115982056], "13": ["vertical_and_slash", 1000, 6096, 0.9261451363563538], "14": ["vertical_and_slash", 30, 800, 0.9052866101264954], "15": ["vertical_and_slash", 30, 800, 0.9554086327552795], "16": ["vertical_and_slash", 30, 800, 0.9866732954978943], "17": ["vertical_and_slash", 30, 800, 0.989112377166748], "18": ["vertical_and_slash", 30, 800, 0.9851261377334595], "19": ["vertical_and_slash", 30, 800, 0.9896757006645203], "20": ["vertical_and_slash", 30, 800, 0.9861950874328613], "21": ["vertical_and_slash", 30, 800, 0.9819672703742981], "22": ["vertical_and_slash", 30, 800, 0.9928784370422363], "23": ["vertical_and_slash", 30, 800, 0.9861727356910706], "24": ["vertical_and_slash", 100, 800, 0.9609375], "25": ["vertical_and_slash", 30, 800, 0.9807386994361877], "26": ["vertical_and_slash", 1000, 6096, 0.9790823459625244], "27": ["vertical_and_slash", 30, 800, 0.9774340391159058], "28": ["vertical_and_slash", 100, 800, 0.890625], "29": ["vertical_and_slash", 1000, 6096, 0.9598889946937561], "30": ["vertical_and_slash", 30, 800, 0.971238911151886], "31": ["vertical_and_slash", 30, 800, 0.9892426133155823], "32": ["vertical_and_slash", 30, 800, 0.9813048243522644], "33": ["vertical_and_slash", 30, 800, 0.9886040687561035], "34": ["vertical_and_slash", 30, 800, 0.9810928702354431], "35": ["vertical_and_slash", 30, 800, 0.9680299758911133], "36": ["vertical_and_slash", 30, 800, 0.9805577993392944], "37": ["vertical_and_slash", 30, 800, 0.9802305698394775], "38": ["vertical_and_slash", 30, 800, 0.9690309166908264], "39": ["vertical_and_slash", 30, 800, 0.917763888835907], "40": ["vertical_and_slash", 1000, 6096, 0.9946138858795166], "41": ["vertical_and_slash", 1000, 6096, 0.9664149284362793], "42": ["vertical_and_slash", 30, 800, 0.9904451370239258], "43": ["vertical_and_slash", 1000, 6096, 0.9941055774688721], "44": ["vertical_and_slash", 1000, 6096, 0.992802083492279], "45": ["vertical_and_slash", 1000, 6096, 0.9935398697853088], "46": ["vertical_and_slash", 1000, 6096, 0.9536246061325073], "47": ["vertical_and_slash", 1000, 6096, 0.9662259817123413], "48": ["vertical_and_slash", 30, 800, 0.9715701341629028], "49": ["vertical_and_slash", 30, 800, 0.988308310508728], "50": ["vertical_and_slash", 100, 750, 0.9207593202590942], "51": ["vertical_and_slash", 30, 800, 0.9625194668769836], "52": ["vertical_and_slash", 30, 800, 0.9922022819519043], "53": ["vertical_and_slash", 30, 800, 0.9742878079414368], "54": ["vertical_and_slash", 30, 800, 0.9892542958259583], "55": ["vertical_and_slash", 30, 800, 0.9813191294670105], "56": ["vertical_and_slash", 30, 800, 0.9002098441123962], "57": ["vertical_and_slash", 30, 800, 0.9852099418640137], "58": ["vertical_and_slash", 30, 800, 0.9928624629974365], "59": ["vertical_and_slash", 30, 800, 0.9649196863174438], "60": ["vertical_and_slash", 30, 800, 0.989941418170929], "61": ["vertical_and_slash", 30, 800, 0.9355747103691101], "62": ["vertical_and_slash", 30, 800, 0.9788942933082581], "63": ["vertical_and_slash", 30, 800, 0.9838983416557312]}, {"0": ["vertical_and_slash", 30, 800, 0.9438474178314209], "1": ["vertical_and_slash", 30, 800, 0.9880681037902832], "2": ["vertical_and_slash", 30, 800, 0.9504310488700867], "3": ["vertical_and_slash", 100, 800, 0.8671875], "4": ["vertical_and_slash", 3500, 100, 0.9712250828742981], "5": ["vertical_and_slash", 100, 800, 0.9609375], "6": ["vertical_and_slash", 100, 800, 0.9765625], "7": ["vertical_and_slash", 100, 800, 0.9609375], "8": ["vertical_and_slash", 30, 800, 0.9296700954437256], "9": ["vertical_and_slash", 30, 800, 0.9648227095603943], "10": ["vertical_and_slash", 30, 800, 0.9840734004974365], "11": ["vertical_and_slash", 30, 800, 0.9900840520858765], "12": ["vertical_and_slash", 30, 800, 0.9844371676445007], "13": ["vertical_and_slash", 30, 800, 0.9876707792282104], "14": ["vertical_and_slash", 30, 800, 0.9890128970146179], "15": ["vertical_and_slash", 30, 800, 0.9050971865653992], "16": ["vertical_and_slash", 1000, 6096, 0.9785110354423523], "17": ["vertical_and_slash", 30, 800, 0.9922237992286682], "18": ["vertical_and_slash", 1000, 6096, 0.9729659557342529], "19": ["vertical_and_slash", 30, 800, 0.9900347590446472], "20": ["vertical_and_slash", 1000, 6096, 0.987310528755188], "21": ["vertical_and_slash", 1000, 6096, 0.9649333953857422], "22": ["vertical_and_slash", 1000, 6096, 0.992845892906189], "23": ["vertical_and_slash", 30, 800, 0.9884690046310425], "24": ["vertical_and_slash", 30, 800, 0.9893113970756531], "25": ["vertical_and_slash", 30, 800, 0.9722930192947388], "26": ["vertical_and_slash", 30, 800, 0.9777242541313171], "27": ["vertical_and_slash", 30, 800, 0.9582722187042236], "28": ["vertical_and_slash", 30, 800, 0.9880891442298889], "29": ["vertical_and_slash", 30, 800, 0.9862256646156311], "30": ["vertical_and_slash", 30, 800, 0.9498446583747864], "31": ["vertical_and_slash", 30, 800, 0.9576575756072998], "32": ["vertical_and_slash", 1000, 6096, 0.9827015399932861], "33": ["vertical_and_slash", 1000, 6096, 0.9837527275085449], "34": ["vertical_and_slash", 1000, 6096, 0.9899967312812805], "35": ["vertical_and_slash", 1000, 6096, 0.9868255853652954], "36": ["vertical_and_slash", 1000, 6096, 0.9866345524787903], "37": ["vertical_and_slash", 1000, 6096, 0.9922086000442505], "38": ["vertical_and_slash", 1000, 6096, 0.9896307587623596], "39": ["vertical_and_slash", 1000, 6096, 0.9835390448570251], "40": ["vertical_and_slash", 30, 800, 0.9618409276008606], "41": ["vertical_and_slash", 30, 800, 0.9836397767066956], "42": ["vertical_and_slash", 30, 800, 0.977034330368042], "43": ["vertical_and_slash", 30, 800, 0.9774137139320374], "44": ["vertical_and_slash", 30, 800, 0.9796658158302307], "45": ["vertical_and_slash", 30, 800, 0.9565669894218445], "46": ["vertical_and_slash", 30, 800, 0.9769356846809387], "47": ["vertical_and_slash", 30, 800, 0.9794620275497437], "48": ["vertical_and_slash", 1000, 6096, 0.9831351041793823], "49": ["vertical_and_slash", 100, 800, 0.984375], "50": ["vertical_and_slash", 100, 800, 0.9921875], "51": ["vertical_and_slash", 100, 800, 0.89453125], "52": ["vertical_and_slash", 30, 800, 0.9934444427490234], "53": ["vertical_and_slash", 30, 800, 0.9929190278053284], "54": ["vertical_and_slash", 30, 800, 0.9540435671806335], "55": ["vertical_and_slash", 1000, 6096, 0.9778568744659424], "56": ["vertical_and_slash", 30, 800, 0.9876856207847595], "57": ["vertical_and_slash", 30, 800, 0.9862412214279175], "58": ["vertical_and_slash", 30, 800, 0.9658806324005127], "59": ["vertical_and_slash", 30, 800, 0.9913110136985779], "60": ["vertical_and_slash", 30, 800, 0.9893407225608826], "61": ["vertical_and_slash", 30, 800, 0.9894734025001526], "62": ["vertical_and_slash", 30, 800, 0.9903742671012878], "63": ["vertical_and_slash", 30, 800, 0.9760178923606873]}, {"0": ["vertical_and_slash", 100, 800, 0.9609375], "1": ["vertical_and_slash", 30, 800, 0.9571090340614319], "2": ["vertical_and_slash", 30, 800, 0.9890798926353455], "3": ["vertical_and_slash", 30, 800, 0.9866421222686768], "4": ["vertical_and_slash", 30, 800, 0.9923579692840576], "5": ["vertical_and_slash", 1000, 6096, 0.98585045337677], "6": ["vertical_and_slash", 30, 800, 0.9880631566047668], "7": ["vertical_and_slash", 100, 800, 0.98828125], "8": ["vertical_and_slash", 1000, 6096, 0.9905042052268982], "9": ["vertical_and_slash", 30, 800, 0.9942885041236877], "10": ["vertical_and_slash", 1000, 6096, 0.9950703382492065], "11": ["vertical_and_slash", 30, 800, 0.9683127403259277], "12": ["vertical_and_slash", 1000, 6096, 0.9945932030677795], "13": ["vertical_and_slash", 30, 800, 0.9383946657180786], "14": ["vertical_and_slash", 30, 800, 0.993502140045166], "15": ["vertical_and_slash", 30, 800, 0.9898476004600525], "16": ["vertical_and_slash", 100, 800, 0.98046875], "17": ["vertical_and_slash", 30, 800, 0.9534019231796265], "18": ["vertical_and_slash", 30, 800, 0.9676714539527893], "19": ["vertical_and_slash", 100, 800, 0.9609375], "20": ["vertical_and_slash", 30, 800, 0.9509228467941284], "21": ["vertical_and_slash", 30, 800, 0.9895873069763184], "22": ["vertical_and_slash", 100, 800, 0.8828125], "23": ["vertical_and_slash", 100, 800, 0.95703125], "24": ["vertical_and_slash", 1000, 6096, 0.9768097996711731], "25": ["vertical_and_slash", 1000, 6096, 0.9811352491378784], "26": ["vertical_and_slash", 1000, 6096, 0.974529504776001], "27": ["vertical_and_slash", 1000, 6096, 0.9700140357017517], "28": ["vertical_and_slash", 1000, 6096, 0.9627239108085632], "29": ["vertical_and_slash", 1000, 6096, 0.9724246263504028], "30": ["vertical_and_slash", 1000, 6096, 0.9805324077606201], "31": ["vertical_and_slash", 1000, 6096, 0.9949752688407898], "32": ["vertical_and_slash", 1000, 6096, 0.9538902044296265], "33": ["vertical_and_slash", 1000, 6096, 0.993955135345459], "34": ["vertical_and_slash", 1000, 6096, 0.9828385710716248], "35": ["vertical_and_slash", 1000, 6096, 0.9825907349586487], "36": ["vertical_and_slash", 1000, 6096, 0.9962171912193298], "37": ["vertical_and_slash", 1000, 6096, 0.975330650806427], "38": ["vertical_and_slash", 1000, 6096, 0.9895368814468384], "39": ["vertical_and_slash", 1000, 6096, 0.9914214611053467], "40": ["vertical_and_slash", 3500, 100, 0.9675924777984619], "41": ["vertical_and_slash", 3500, 100, 0.9751370549201965], "42": ["vertical_and_slash", 3500, 100, 0.962358295917511], "43": ["vertical_and_slash", 3500, 100, 0.9621944427490234], "44": ["vertical_and_slash", 500, 700, 0.9730700850486755], "45": ["vertical_and_slash", 3500, 100, 0.9558079242706299], "46": ["vertical_and_slash", 1000, 6096, 0.9799144268035889], "47": ["vertical_and_slash", 3500, 100, 0.9627045392990112], "48": ["vertical_and_slash", 30, 800, 0.9853342175483704], "49": ["vertical_and_slash", 1000, 6096, 0.9924542903900146], "50": ["vertical_and_slash", 500, 700, 0.9768703579902649], "51": ["vertical_and_slash", 30, 800, 0.9897663593292236], "52": ["vertical_and_slash", 30, 800, 0.9974449872970581], "53": ["vertical_and_slash", 30, 800, 0.9899056553840637], "54": ["vertical_and_slash", 100, 800, 0.99609375], "55": ["vertical_and_slash", 30, 800, 0.9649139046669006], "56": ["vertical_and_slash", 30, 800, 0.9885178804397583], "57": ["vertical_and_slash", 30, 800, 0.9876024127006531], "58": ["vertical_and_slash", 30, 800, 0.9805638194084167], "59": ["vertical_and_slash", 30, 800, 0.9865089654922485], "60": ["vertical_and_slash", 30, 800, 0.9729229211807251], "61": ["vertical_and_slash", 30, 800, 0.9878937005996704], "62": ["vertical_and_slash", 30, 800, 0.9818024635314941], "63": ["vertical_and_slash", 30, 800, 0.9579209089279175]}, {"0": ["vertical_and_slash", 30, 800, 0.9606621265411377], "1": ["vertical_and_slash", 30, 800, 0.9883118867874146], "2": ["vertical_and_slash", 30, 800, 0.9834955334663391], "3": ["vertical_and_slash", 30, 800, 0.9642312526702881], "4": ["vertical_and_slash", 30, 800, 0.9728757739067078], "5": ["vertical_and_slash", 30, 800, 0.9865761995315552], "6": ["vertical_and_slash", 30, 800, 0.9865660667419434], "7": ["vertical_and_slash", 30, 800, 0.9870844483375549], "8": ["vertical_and_slash", 1000, 6096, 0.987014651298523], "9": ["vertical_and_slash", 30, 800, 0.9893589615821838], "10": ["vertical_and_slash", 1000, 6096, 0.9651404619216919], "11": ["vertical_and_slash", 30, 800, 0.9807274341583252], "12": ["vertical_and_slash", 30, 800, 0.9782439470291138], "13": ["vertical_and_slash", 100, 800, 0.98046875], "14": ["vertical_and_slash", 1000, 6096, 0.9626099467277527], "15": ["vertical_and_slash", 30, 800, 0.8739796876907349], "16": ["vertical_and_slash", 30, 800, 0.9672869443893433], "17": ["vertical_and_slash", 30, 800, 0.9530259966850281], "18": ["vertical_and_slash", 30, 800, 0.9728261828422546], "19": ["vertical_and_slash", 100, 800, 0.89453125], "20": ["vertical_and_slash", 100, 800, 0.91015625], "21": ["vertical_and_slash", 30, 800, 0.9608057141304016], "22": ["vertical_and_slash", 30, 800, 0.9573055505752563], "23": ["vertical_and_slash", 30, 800, 0.9583159685134888], "24": ["vertical_and_slash", 30, 800, 0.954291820526123], "25": ["vertical_and_slash", 30, 800, 0.924720048904419], "26": ["vertical_and_slash", 30, 800, 0.9889179468154907], "27": ["vertical_and_slash", 30, 800, 0.9555203914642334], "28": ["vertical_and_slash", 30, 800, 0.9820623397827148], "29": ["vertical_and_slash", 30, 800, 0.965025007724762], "30": ["vertical_and_slash", 30, 800, 0.9843047857284546], "31": ["vertical_and_slash", 30, 800, 0.9847993850708008], "32": ["vertical_and_slash", 30, 800, 0.993728756904602], "33": ["vertical_and_slash", 30, 800, 0.9885037541389465], "34": ["vertical_and_slash", 30, 800, 0.9865205883979797], "35": ["vertical_and_slash", 30, 800, 0.9802103638648987], "36": ["vertical_and_slash", 30, 800, 0.9761874079704285], "37": ["vertical_and_slash", 30, 800, 0.9894469976425171], "38": ["vertical_and_slash", 30, 800, 0.9819567203521729], "39": ["vertical_and_slash", 30, 800, 0.9747263193130493], "40": ["vertical_and_slash", 30, 800, 0.9020816087722778], "41": ["vertical_and_slash", 30, 800, 0.9882520437240601], "42": ["vertical_and_slash", 30, 800, 0.9924037456512451], "43": ["vertical_and_slash", 30, 800, 0.9362927675247192], "44": ["vertical_and_slash", 30, 800, 0.9709300398826599], "45": ["vertical_and_slash", 3500, 100, 0.9593322277069092], "46": ["vertical_and_slash", 30, 800, 0.9866483211517334], "47": ["vertical_and_slash", 1000, 6096, 0.9898219704627991], "48": ["vertical_and_slash", 30, 800, 0.9906712770462036], "49": ["vertical_and_slash", 100, 800, 0.99609375], "50": ["vertical_and_slash", 30, 800, 0.9905202388763428], "51": ["vertical_and_slash", 30, 800, 0.9897274374961853], "52": ["vertical_and_slash", 500, 700, 0.7587087750434875], "53": ["vertical_and_slash", 100, 800, 0.97265625], "54": ["vertical_and_slash", 100, 800, 0.9921875], "55": ["vertical_and_slash", 30, 800, 0.9691149592399597], "56": ["vertical_and_slash", 100, 800, 0.97265625], "57": ["vertical_and_slash", 30, 800, 0.9922186732292175], "58": ["vertical_and_slash", 30, 800, 0.9941394925117493], "59": ["vertical_and_slash", 30, 800, 0.9971425533294678], "60": ["vertical_and_slash", 30, 800, 0.9923318028450012], "61": ["vertical_and_slash", 100, 800, 0.97265625], "62": ["vertical_and_slash", 30, 800, 0.9932958483695984], "63": ["vertical_and_slash", 30, 800, 0.9924513101577759]}, {"0": ["vertical_and_slash", 30, 800, 0.9804450869560242], "1": ["vertical_and_slash", 30, 800, 0.9681053757667542], "2": ["vertical_and_slash", 30, 800, 0.9022992253303528], "3": ["vertical_and_slash", 500, 700, 0.7609636783599854], "4": ["vertical_and_slash", 1000, 6096, 0.9800698757171631], "5": ["vertical_and_slash", 30, 800, 0.9699357748031616], "6": ["vertical_and_slash", 1000, 6096, 0.9864827990531921], "7": ["vertical_and_slash", 1000, 6096, 0.9697545766830444], "8": ["vertical_and_slash", 30, 800, 0.9630061388015747], "9": ["vertical_and_slash", 30, 800, 0.9822589755058289], "10": ["vertical_and_slash", 1000, 6096, 0.9955841898918152], "11": ["vertical_and_slash", 30, 800, 0.9806007146835327], "12": ["vertical_and_slash", 1000, 6096, 0.9855539798736572], "13": ["vertical_and_slash", 30, 800, 0.9855115413665771], "14": ["vertical_and_slash", 30, 800, 0.9841164946556091], "15": ["vertical_and_slash", 30, 800, 0.960256040096283], "16": ["vertical_and_slash", 30, 800, 0.9784650206565857], "17": ["vertical_and_slash", 30, 800, 0.9307375550270081], "18": ["vertical_and_slash", 30, 800, 0.9725090265274048], "19": ["vertical_and_slash", 30, 800, 0.9620038270950317], "20": ["vertical_and_slash", 30, 800, 0.9600361585617065], "21": ["vertical_and_slash", 30, 800, 0.9751067757606506], "22": ["vertical_and_slash", 30, 800, 0.8909373879432678], "23": ["vertical_and_slash", 30, 800, 0.9723846316337585], "24": ["vertical_and_slash", 30, 800, 0.9733383655548096], "25": ["vertical_and_slash", 30, 800, 0.9448040723800659], "26": ["vertical_and_slash", 30, 800, 0.978152334690094], "27": ["vertical_and_slash", 30, 800, 0.9761038422584534], "28": ["vertical_and_slash", 30, 800, 0.8595834374427795], "29": ["vertical_and_slash", 30, 800, 0.9753406047821045], "30": ["vertical_and_slash", 30, 800, 0.9619556069374084], "31": ["vertical_and_slash", 30, 800, 0.9267991781234741], "32": ["vertical_and_slash", 30, 800, 0.9982537031173706], "33": ["vertical_and_slash", 30, 800, 0.9250535368919373], "34": ["vertical_and_slash", 30, 800, 0.9055324196815491], "35": ["vertical_and_slash", 30, 800, 0.995537519454956], "36": ["vertical_and_slash", 30, 800, 0.9844455718994141], "37": ["vertical_and_slash", 100, 800, 0.99609375], "38": ["vertical_and_slash", 3500, 100, 0.9578905701637268], "39": ["vertical_and_slash", 100, 800, 0.921875], "40": ["vertical_and_slash", 30, 800, 0.9783494472503662], "41": ["vertical_and_slash", 30, 800, 0.933086633682251], "42": ["vertical_and_slash", 30, 800, 0.9793987274169922], "43": ["vertical_and_slash", 30, 800, 0.9774795174598694], "44": ["vertical_and_slash", 30, 800, 0.979150652885437], "45": ["vertical_and_slash", 30, 800, 0.8678485751152039], "46": ["vertical_and_slash", 30, 800, 0.9680995941162109], "47": ["vertical_and_slash", 30, 800, 0.9773225784301758], "48": ["vertical_and_slash", 30, 800, 0.9787015318870544], "49": ["vertical_and_slash", 30, 800, 0.9778434634208679], "50": ["vertical_and_slash", 30, 800, 0.9270550012588501], "51": ["vertical_and_slash", 30, 800, 0.9787206053733826], "52": ["vertical_and_slash", 30, 800, 0.9367327690124512], "53": ["vertical_and_slash", 30, 800, 0.9791626930236816], "54": ["vertical_and_slash", 30, 800, 0.9696778059005737], "55": ["vertical_and_slash", 30, 800, 0.9777553081512451], "56": ["vertical_and_slash", 30, 800, 0.9886125922203064], "57": ["vertical_and_slash", 30, 800, 0.993108332157135], "58": ["vertical_and_slash", 30, 800, 0.9644357562065125], "59": ["vertical_and_slash", 30, 800, 0.9815438985824585], "60": ["vertical_and_slash", 30, 800, 0.9933310151100159], "61": ["vertical_and_slash", 30, 800, 0.9636465311050415], "62": ["vertical_and_slash", 30, 800, 0.9642088413238525], "63": ["vertical_and_slash", 30, 800, 0.9895989298820496]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9891161918640137], "1": ["vertical_and_slash", 1000, 6096, 0.977868914604187], "2": ["vertical_and_slash", 1000, 6096, 0.9851346611976624], "3": ["vertical_and_slash", 1000, 6096, 0.9981952905654907], "4": ["vertical_and_slash", 1000, 6096, 0.9920579195022583], "5": ["vertical_and_slash", 1000, 6096, 0.9846153259277344], "6": ["vertical_and_slash", 1000, 6096, 0.9927820563316345], "7": ["vertical_and_slash", 100, 800, 0.91015625], "8": ["vertical_and_slash", 1000, 6096, 0.9943665862083435], "9": ["vertical_and_slash", 1000, 6096, 0.9880841970443726], "10": ["vertical_and_slash", 1000, 6096, 0.963089644908905], "11": ["vertical_and_slash", 30, 800, 0.9896222949028015], "12": ["vertical_and_slash", 1000, 6096, 0.9605267643928528], "13": ["vertical_and_slash", 1000, 6096, 0.9863606095314026], "14": ["vertical_and_slash", 30, 800, 0.9835635423660278], "15": ["vertical_and_slash", 1000, 6096, 0.987554669380188], "16": ["vertical_and_slash", 1000, 6096, 0.9972808361053467], "17": ["vertical_and_slash", 1000, 6096, 0.994634211063385], "18": ["vertical_and_slash", 1000, 6096, 0.9900151491165161], "19": ["vertical_and_slash", 1000, 6096, 0.9925185441970825], "20": ["vertical_and_slash", 3500, 100, 0.9969769716262817], "21": ["vertical_and_slash", 1000, 6096, 0.9891420006752014], "22": ["vertical_and_slash", 1000, 6096, 0.9954760074615479], "23": ["vertical_and_slash", 1000, 6096, 0.9936257600784302], "24": ["vertical_and_slash", 1000, 6096, 0.8665743470191956], "25": ["vertical_and_slash", 1000, 6096, 0.9844601154327393], "26": ["vertical_and_slash", 1000, 6096, 0.9694246649742126], "27": ["vertical_and_slash", 1000, 6096, 0.994500458240509], "28": ["vertical_and_slash", 1000, 6096, 0.9928778409957886], "29": ["vertical_and_slash", 1000, 6096, 0.9679787158966064], "30": ["vertical_and_slash", 1000, 6096, 0.9927939176559448], "31": ["vertical_and_slash", 1000, 6096, 0.9249139428138733], "32": ["vertical_and_slash", 1000, 6096, 0.9731390476226807], "33": ["vertical_and_slash", 1000, 6096, 0.9752944707870483], "34": ["vertical_and_slash", 1000, 6096, 0.9423426389694214], "35": ["vertical_and_slash", 1000, 6096, 0.9969841241836548], "36": ["vertical_and_slash", 1000, 6096, 0.9947851300239563], "37": ["vertical_and_slash", 1000, 6096, 0.9970207810401917], "38": ["vertical_and_slash", 1000, 6096, 0.9899964332580566], "39": ["vertical_and_slash", 1000, 6096, 0.9846908450126648], "40": ["vertical_and_slash", 1000, 6096, 0.9684431552886963], "41": ["vertical_and_slash", 1000, 6096, 0.99799644947052], "42": ["vertical_and_slash", 1000, 6096, 0.9785661697387695], "43": ["vertical_and_slash", 1000, 6096, 0.9890553951263428], "44": ["vertical_and_slash", 1000, 6096, 0.9770569801330566], "45": ["vertical_and_slash", 1000, 6096, 0.9899786710739136], "46": ["vertical_and_slash", 1000, 6096, 0.9817518591880798], "47": ["vertical_and_slash", 1000, 6096, 0.9987812638282776], "48": ["vertical_and_slash", 1000, 6096, 0.9520062804222107], "49": ["vertical_and_slash", 1000, 6096, 0.9851368069648743], "50": ["vertical_and_slash", 1000, 6096, 0.9431411027908325], "51": ["vertical_and_slash", 1000, 6096, 0.9962673783302307], "52": ["vertical_and_slash", 1000, 6096, 0.9855237007141113], "53": ["vertical_and_slash", 1000, 6096, 0.9969472289085388], "54": ["vertical_and_slash", 1000, 6096, 0.9974427819252014], "55": ["vertical_and_slash", 1000, 6096, 0.9249348044395447], "56": ["vertical_and_slash", 30, 800, 0.9653343558311462], "57": ["vertical_and_slash", 30, 800, 0.9269059896469116], "58": ["vertical_and_slash", 30, 800, 0.9682387709617615], "59": ["vertical_and_slash", 100, 800, 0.85546875], "60": ["vertical_and_slash", 30, 800, 0.9738735556602478], "61": ["vertical_and_slash", 30, 800, 0.8736201524734497], "62": ["vertical_and_slash", 30, 800, 0.975856602191925], "63": ["vertical_and_slash", 30, 800, 0.9663101434707642]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9473738670349121], "1": ["vertical_and_slash", 1000, 6096, 0.9921725988388062], "2": ["vertical_and_slash", 3500, 100, 0.9432547092437744], "3": ["vertical_and_slash", 100, 800, 0.98828125], "4": ["vertical_and_slash", 30, 800, 0.9618416428565979], "5": ["vertical_and_slash", 30, 800, 0.9888799786567688], "6": ["vertical_and_slash", 30, 800, 0.8952481150627136], "7": ["vertical_and_slash", 30, 800, 0.983036994934082], "8": ["vertical_and_slash", 1000, 6096, 0.9856565594673157], "9": ["vertical_and_slash", 30, 800, 0.9888338446617126], "10": ["vertical_and_slash", 500, 700, 0.9514689445495605], "11": ["vertical_and_slash", 30, 800, 0.9891687631607056], "12": ["vertical_and_slash", 1000, 6096, 0.9931038618087769], "13": ["vertical_and_slash", 500, 700, 0.9745760560035706], "14": ["vertical_and_slash", 1000, 6096, 0.9410001039505005], "15": ["vertical_and_slash", 30, 800, 0.9334917664527893], "16": ["vertical_and_slash", 30, 800, 0.9823850393295288], "17": ["vertical_and_slash", 30, 800, 0.956362247467041], "18": ["vertical_and_slash", 30, 800, 0.9820957183837891], "19": ["vertical_and_slash", 30, 800, 0.9728254675865173], "20": ["vertical_and_slash", 30, 800, 0.9420588612556458], "21": ["vertical_and_slash", 500, 700, 0.9830220341682434], "22": ["vertical_and_slash", 30, 800, 0.9808295965194702], "23": ["vertical_and_slash", 30, 800, 0.9138931035995483], "24": ["vertical_and_slash", 1000, 6096, 0.9875288605690002], "25": ["vertical_and_slash", 1000, 6096, 0.964775800704956], "26": ["vertical_and_slash", 1000, 6096, 0.9726804494857788], "27": ["vertical_and_slash", 1000, 6096, 0.9929394125938416], "28": ["vertical_and_slash", 1000, 6096, 0.9702687859535217], "29": ["vertical_and_slash", 1000, 6096, 0.9722868204116821], "30": ["vertical_and_slash", 1000, 6096, 0.9877567291259766], "31": ["vertical_and_slash", 1000, 6096, 0.9766455292701721], "32": ["vertical_and_slash", 1000, 6096, 0.9983102083206177], "33": ["vertical_and_slash", 1000, 6096, 0.991988480091095], "34": ["vertical_and_slash", 1000, 6096, 0.9952487945556641], "35": ["vertical_and_slash", 1000, 6096, 0.9968292713165283], "36": ["vertical_and_slash", 1000, 6096, 0.9972949624061584], "37": ["vertical_and_slash", 1000, 6096, 0.9927100539207458], "38": ["vertical_and_slash", 1000, 6096, 0.9958603382110596], "39": ["vertical_and_slash", 1000, 6096, 0.992963969707489], "40": ["vertical_and_slash", 1000, 6096, 0.9907368421554565], "41": ["vertical_and_slash", 1000, 6096, 0.9516956806182861], "42": ["vertical_and_slash", 1000, 6096, 0.9962841868400574], "43": ["vertical_and_slash", 1000, 6096, 0.9968675971031189], "44": ["vertical_and_slash", 30, 800, 0.8786101937294006], "45": ["vertical_and_slash", 30, 800, 0.9942981600761414], "46": ["vertical_and_slash", 500, 700, 0.9860331416130066], "47": ["vertical_and_slash", 30, 800, 0.990585446357727], "48": ["vertical_and_slash", 1000, 6096, 0.8935190439224243], "49": ["vertical_and_slash", 30, 800, 0.9900436401367188], "50": ["vertical_and_slash", 1000, 6096, 0.9856786131858826], "51": ["vertical_and_slash", 1000, 6096, 0.9884719848632812], "52": ["vertical_and_slash", 1000, 6096, 0.950843095779419], "53": ["vertical_and_slash", 1000, 6096, 0.8995730876922607], "54": ["vertical_and_slash", 30, 800, 0.990696907043457], "55": ["vertical_and_slash", 1000, 6096, 0.9863711595535278], "56": ["vertical_and_slash", 30, 800, 0.9874786138534546], "57": ["vertical_and_slash", 30, 800, 0.9936750531196594], "58": ["vertical_and_slash", 30, 800, 0.9851971864700317], "59": ["vertical_and_slash", 30, 800, 0.9900017976760864], "60": ["vertical_and_slash", 30, 800, 0.9622085690498352], "61": ["vertical_and_slash", 30, 800, 0.9890133142471313], "62": ["vertical_and_slash", 30, 800, 0.9744436740875244], "63": ["vertical_and_slash", 30, 800, 0.964031994342804]}, {"0": ["vertical_and_slash", 100, 800, 0.94140625], "1": ["vertical_and_slash", 30, 800, 0.9874139428138733], "2": ["vertical_and_slash", 30, 800, 0.9310460090637207], "3": ["vertical_and_slash", 30, 800, 0.939209520816803], "4": ["vertical_and_slash", 1000, 6096, 0.9422671794891357], "5": ["vertical_and_slash", 30, 800, 0.9779161214828491], "6": ["vertical_and_slash", 30, 800, 0.9670752286911011], "7": ["vertical_and_slash", 30, 800, 0.9483434557914734], "8": ["vertical_and_slash", 1000, 6096, 0.9910116195678711], "9": ["vertical_and_slash", 1000, 6096, 0.8873341083526611], "10": ["vertical_and_slash", 30, 800, 0.9793561100959778], "11": ["vertical_and_slash", 30, 800, 0.9334904551506042], "12": ["vertical_and_slash", 30, 800, 0.9918155074119568], "13": ["vertical_and_slash", 30, 800, 0.9655446410179138], "14": ["vertical_and_slash", 30, 800, 0.9626305103302002], "15": ["vertical_and_slash", 30, 800, 0.9554268717765808], "16": ["vertical_and_slash", 30, 800, 0.9737954139709473], "17": ["vertical_and_slash", 30, 800, 0.967167854309082], "18": ["vertical_and_slash", 30, 800, 0.9683127999305725], "19": ["vertical_and_slash", 30, 800, 0.963815450668335], "20": ["vertical_and_slash", 30, 800, 0.9537728428840637], "21": ["vertical_and_slash", 30, 800, 0.9703646302223206], "22": ["vertical_and_slash", 30, 800, 0.959972083568573], "23": ["vertical_and_slash", 30, 800, 0.9592294096946716], "24": ["vertical_and_slash", 1000, 6096, 0.9449480175971985], "25": ["vertical_and_slash", 1000, 6096, 0.9859976172447205], "26": ["vertical_and_slash", 1000, 6096, 0.9314061403274536], "27": ["vertical_and_slash", 1000, 6096, 0.9673770070075989], "28": ["vertical_and_slash", 1000, 6096, 0.9781957268714905], "29": ["vertical_and_slash", 1000, 6096, 0.9713826775550842], "30": ["vertical_and_slash", 1000, 6096, 0.9927789568901062], "31": ["vertical_and_slash", 1000, 6096, 0.9633471369743347], "32": ["vertical_and_slash", 30, 800, 0.986078143119812], "33": ["vertical_and_slash", 30, 800, 0.9896390438079834], "34": ["vertical_and_slash", 30, 800, 0.9891392588615417], "35": ["vertical_and_slash", 30, 800, 0.9381373524665833], "36": ["vertical_and_slash", 100, 800, 0.9609375], "37": ["vertical_and_slash", 30, 800, 0.9910147786140442], "38": ["vertical_and_slash", 30, 800, 0.9860256314277649], "39": ["vertical_and_slash", 30, 800, 0.9968755841255188], "40": ["vertical_and_slash", 30, 800, 0.9818190932273865], "41": ["vertical_and_slash", 1000, 6096, 0.9675905108451843], "42": ["vertical_and_slash", 30, 800, 0.9685260653495789], "43": ["vertical_and_slash", 30, 800, 0.9840868711471558], "44": ["vertical_and_slash", 30, 800, 0.9657062888145447], "45": ["vertical_and_slash", 30, 800, 0.9616870880126953], "46": ["vertical_and_slash", 30, 800, 0.9230520725250244], "47": ["vertical_and_slash", 30, 800, 0.9755227565765381], "48": ["vertical_and_slash", 1000, 6096, 0.967684805393219], "49": ["vertical_and_slash", 100, 800, 0.92578125], "50": ["vertical_and_slash", 1000, 6096, 0.9991117715835571], "51": ["vertical_and_slash", 100, 800, 1.0], "52": ["vertical_and_slash", 30, 800, 0.9921984672546387], "53": ["vertical_and_slash", 3500, 100, 0.9990168809890747], "54": ["vertical_and_slash", 3500, 100, 0.9951124787330627], "55": ["vertical_and_slash", 100, 800, 0.99609375], "56": ["vertical_and_slash", 1000, 6096, 0.8777572512626648], "57": ["vertical_and_slash", 30, 800, 0.9743943810462952], "58": ["vertical_and_slash", 30, 800, 0.9624928832054138], "59": ["vertical_and_slash", 30, 800, 0.9549711346626282], "60": ["vertical_and_slash", 30, 800, 0.9888890981674194], "61": ["vertical_and_slash", 30, 800, 0.982546865940094], "62": ["vertical_and_slash", 30, 800, 0.8805662989616394], "63": ["vertical_and_slash", 100, 800, 0.8203125]}, {"0": ["vertical_and_slash", 30, 800, 0.9789323210716248], "1": ["vertical_and_slash", 1000, 6096, 0.581502377986908], "2": ["vertical_and_slash", 30, 800, 0.9263228178024292], "3": ["vertical_and_slash", 30, 800, 0.9822036623954773], "4": ["vertical_and_slash", 30, 800, 0.9730559587478638], "5": ["vertical_and_slash", 30, 800, 0.9449484348297119], "6": ["vertical_and_slash", 30, 800, 0.9713903665542603], "7": ["vertical_and_slash", 30, 800, 0.9878142476081848], "8": ["vertical_and_slash", 30, 800, 0.9493753910064697], "9": ["vertical_and_slash", 30, 800, 0.9632876515388489], "10": ["vertical_and_slash", 30, 800, 0.977073609828949], "11": ["vertical_and_slash", 30, 800, 0.97398442029953], "12": ["vertical_and_slash", 30, 800, 0.9877112507820129], "13": ["vertical_and_slash", 1000, 6096, 0.9693617224693298], "14": ["vertical_and_slash", 30, 800, 0.9793475866317749], "15": ["vertical_and_slash", 30, 800, 0.9599741697311401], "16": ["vertical_and_slash", 30, 800, 0.8823153972625732], "17": ["vertical_and_slash", 30, 800, 0.878853976726532], "18": ["vertical_and_slash", 30, 800, 0.8993266820907593], "19": ["vertical_and_slash", 1000, 6096, 0.867080807685852], "20": ["vertical_and_slash", 30, 800, 0.892155110836029], "21": ["vertical_and_slash", 30, 800, 0.8958237767219543], "22": ["vertical_and_slash", 30, 800, 0.9502267241477966], "23": ["vertical_and_slash", 30, 800, 0.9562439918518066], "24": ["vertical_and_slash", 1000, 6096, 0.869123101234436], "25": ["vertical_and_slash", 1000, 6096, 0.9018592238426208], "26": ["vertical_and_slash", 30, 800, 0.9347516298294067], "27": ["vertical_and_slash", 30, 800, 0.8818024396896362], "28": ["vertical_and_slash", 30, 800, 0.902928352355957], "29": ["vertical_and_slash", 30, 800, 0.9519749283790588], "30": ["vertical_and_slash", 30, 800, 0.9627676010131836], "31": ["vertical_and_slash", 30, 800, 0.9452598094940186], "32": ["vertical_and_slash", 30, 800, 0.9377075433731079], "33": ["vertical_and_slash", 30, 800, 0.9745929837226868], "34": ["vertical_and_slash", 30, 800, 0.9563027620315552], "35": ["vertical_and_slash", 30, 800, 0.9676364660263062], "36": ["vertical_and_slash", 30, 800, 0.9637657403945923], "37": ["vertical_and_slash", 30, 800, 0.9617603421211243], "38": ["vertical_and_slash", 30, 800, 0.9629600048065186], "39": ["vertical_and_slash", 30, 800, 0.9549747109413147], "40": ["vertical_and_slash", 30, 800, 0.9904516339302063], "41": ["vertical_and_slash", 30, 800, 0.9937340617179871], "42": ["vertical_and_slash", 30, 800, 0.8953508138656616], "43": ["vertical_and_slash", 30, 800, 0.9900045990943909], "44": ["vertical_and_slash", 30, 800, 0.9851028919219971], "45": ["vertical_and_slash", 30, 800, 0.9703664779663086], "46": ["vertical_and_slash", 1000, 6096, 0.9873459935188293], "47": ["vertical_and_slash", 30, 800, 0.9905288219451904], "48": ["vertical_and_slash", 30, 800, 0.9511098861694336], "49": ["vertical_and_slash", 30, 800, 0.9803453683853149], "50": ["vertical_and_slash", 30, 800, 0.9871240258216858], "51": ["vertical_and_slash", 30, 800, 0.9755739569664001], "52": ["vertical_and_slash", 30, 800, 0.9894148707389832], "53": ["vertical_and_slash", 30, 800, 0.9745233058929443], "54": ["vertical_and_slash", 30, 800, 0.9816275238990784], "55": ["vertical_and_slash", 30, 800, 0.9888976216316223], "56": ["vertical_and_slash", 30, 800, 0.9816218018531799], "57": ["vertical_and_slash", 30, 800, 0.9889175891876221], "58": ["vertical_and_slash", 30, 800, 0.9881533980369568], "59": ["vertical_and_slash", 30, 800, 0.9856588244438171], "60": ["vertical_and_slash", 500, 700, 0.9076676368713379], "61": ["vertical_and_slash", 100, 800, 0.8203125], "62": ["vertical_and_slash", 30, 800, 0.9655718803405762], "63": ["vertical_and_slash", 1000, 6096, 0.9720288515090942]}, {"0": ["vertical_and_slash", 30, 800, 0.9753791689872742], "1": ["vertical_and_slash", 1000, 6096, 0.904731810092926], "2": ["vertical_and_slash", 30, 800, 0.9768797755241394], "3": ["vertical_and_slash", 30, 800, 0.9897578954696655], "4": ["vertical_and_slash", 1000, 6096, 0.8303517699241638], "5": ["vertical_and_slash", 1000, 6096, 0.996037483215332], "6": ["vertical_and_slash", 1000, 6096, 0.8902897238731384], "7": ["vertical_and_slash", 1000, 6096, 0.9908125996589661], "8": ["vertical_and_slash", 30, 800, 0.9854862689971924], "9": ["vertical_and_slash", 30, 800, 0.975682258605957], "10": ["vertical_and_slash", 500, 700, 0.9548078179359436], "11": ["vertical_and_slash", 30, 800, 0.9861394166946411], "12": ["vertical_and_slash", 30, 800, 0.9750257730484009], "13": ["vertical_and_slash", 30, 800, 0.946538507938385], "14": ["vertical_and_slash", 30, 800, 0.9819313883781433], "15": ["vertical_and_slash", 1000, 6096, 0.9859066009521484], "16": ["vertical_and_slash", 3500, 100, 0.988437831401825], "17": ["vertical_and_slash", 1000, 6096, 0.9825325608253479], "18": ["vertical_and_slash", 1000, 6096, 0.9963878989219666], "19": ["vertical_and_slash", 1000, 6096, 0.9642111659049988], "20": ["vertical_and_slash", 1000, 6096, 0.9528767466545105], "21": ["vertical_and_slash", 1000, 6096, 0.7593525052070618], "22": ["vertical_and_slash", 1000, 6096, 0.983763575553894], "23": ["vertical_and_slash", 1000, 6096, 0.7601028084754944], "24": ["vertical_and_slash", 30, 800, 0.9254814386367798], "25": ["vertical_and_slash", 30, 800, 0.9688318371772766], "26": ["vertical_and_slash", 30, 800, 0.979478120803833], "27": ["vertical_and_slash", 30, 800, 0.9839501976966858], "28": ["vertical_and_slash", 30, 800, 0.9754621982574463], "29": ["vertical_and_slash", 30, 800, 0.9357173442840576], "30": ["vertical_and_slash", 30, 800, 0.9750608205795288], "31": ["vertical_and_slash", 30, 800, 0.9821027517318726], "32": ["vertical_and_slash", 30, 800, 0.962790310382843], "33": ["vertical_and_slash", 1000, 6096, 0.9204011559486389], "34": ["vertical_and_slash", 100, 800, 0.984375], "35": ["vertical_and_slash", 3500, 100, 0.928776204586029], "36": ["vertical_and_slash", 1000, 6096, 0.9757254123687744], "37": ["vertical_and_slash", 1000, 6096, 0.933245837688446], "38": ["vertical_and_slash", 30, 800, 0.8713231086730957], "39": ["vertical_and_slash", 30, 800, 0.9476797580718994], "40": ["vertical_and_slash", 1000, 6096, 0.9899322986602783], "41": ["vertical_and_slash", 1000, 6096, 0.9956942200660706], "42": ["vertical_and_slash", 1000, 6096, 0.9652335047721863], "43": ["vertical_and_slash", 1000, 6096, 0.9858012795448303], "44": ["vertical_and_slash", 1000, 6096, 0.9860623478889465], "45": ["vertical_and_slash", 1000, 6096, 0.9152382016181946], "46": ["vertical_and_slash", 1000, 6096, 0.9756700992584229], "47": ["vertical_and_slash", 1000, 6096, 0.9512685537338257], "48": ["vertical_and_slash", 500, 700, 0.9084185361862183], "49": ["vertical_and_slash", 1000, 6096, 0.9692651033401489], "50": ["vertical_and_slash", 1000, 6096, 0.97395920753479], "51": ["vertical_and_slash", 30, 800, 0.9904205203056335], "52": ["vertical_and_slash", 30, 800, 0.9665771126747131], "53": ["vertical_and_slash", 1000, 6096, 0.9527360796928406], "54": ["vertical_and_slash", 1000, 6096, 0.9617610573768616], "55": ["vertical_and_slash", 30, 800, 0.9501744508743286], "56": ["vertical_and_slash", 30, 800, 0.9788270592689514], "57": ["vertical_and_slash", 30, 800, 0.967989981174469], "58": ["vertical_and_slash", 30, 800, 0.9662458300590515], "59": ["vertical_and_slash", 30, 800, 0.9804056286811829], "60": ["vertical_and_slash", 30, 800, 0.9653540849685669], "61": ["vertical_and_slash", 100, 800, 0.859375], "62": ["vertical_and_slash", 30, 800, 0.9533804059028625], "63": ["vertical_and_slash", 30, 800, 0.980417013168335]}, {"0": ["vertical_and_slash", 500, 700, 0.9670920968055725], "1": ["vertical_and_slash", 1000, 6096, 0.9964378476142883], "2": ["vertical_and_slash", 1000, 6096, 0.9885406494140625], "3": ["vertical_and_slash", 1000, 6096, 1.0000731945037842], "4": ["vertical_and_slash", 1000, 6096, 0.9935339689254761], "5": ["vertical_and_slash", 1000, 6096, 0.9632836580276489], "6": ["vertical_and_slash", 1000, 6096, 0.9641180038452148], "7": ["vertical_and_slash", 1000, 6096, 0.9972721934318542], "8": ["vertical_and_slash", 1000, 6096, 0.989022970199585], "9": ["vertical_and_slash", 1000, 6096, 0.9927436709403992], "10": ["vertical_and_slash", 1000, 6096, 0.9898725152015686], "11": ["vertical_and_slash", 30, 800, 0.9846010208129883], "12": ["vertical_and_slash", 1000, 6096, 0.9608356952667236], "13": ["vertical_and_slash", 1000, 6096, 0.9842557311058044], "14": ["vertical_and_slash", 30, 800, 0.9821281433105469], "15": ["vertical_and_slash", 1000, 6096, 0.9349039196968079], "16": ["vertical_and_slash", 1000, 6096, 0.9510119557380676], "17": ["vertical_and_slash", 1000, 6096, 0.9734442830085754], "18": ["vertical_and_slash", 1000, 6096, 0.8859578371047974], "19": ["vertical_and_slash", 1000, 6096, 0.9378039836883545], "20": ["vertical_and_slash", 1000, 6096, 0.9047842025756836], "21": ["vertical_and_slash", 1000, 6096, 0.849723756313324], "22": ["vertical_and_slash", 1000, 6096, 0.8725143671035767], "23": ["vertical_and_slash", 1000, 6096, 0.8746799826622009], "24": ["vertical_and_slash", 1000, 6096, 0.9604934453964233], "25": ["vertical_and_slash", 1000, 6096, 0.994159996509552], "26": ["vertical_and_slash", 1000, 6096, 0.9344531893730164], "27": ["vertical_and_slash", 1000, 6096, 0.9727720618247986], "28": ["vertical_and_slash", 1000, 6096, 0.9539724588394165], "29": ["vertical_and_slash", 1000, 6096, 1.0003288984298706], "30": ["vertical_and_slash", 1000, 6096, 0.9921741485595703], "31": ["vertical_and_slash", 1000, 6096, 0.9989508390426636], "32": ["vertical_and_slash", 1000, 6096, 0.9893904328346252], "33": ["vertical_and_slash", 100, 800, 0.97265625], "34": ["vertical_and_slash", 1000, 6096, 0.967260479927063], "35": ["vertical_and_slash", 30, 800, 0.9586178660392761], "36": ["vertical_and_slash", 500, 700, 0.9884262681007385], "37": ["vertical_and_slash", 1000, 6096, 0.9599385857582092], "38": ["vertical_and_slash", 30, 800, 0.8346987962722778], "39": ["vertical_and_slash", 30, 800, 0.9851138591766357], "40": ["vertical_and_slash", 1000, 6096, 0.9012252688407898], "41": ["vertical_and_slash", 1000, 6096, 0.9944440126419067], "42": ["vertical_and_slash", 30, 800, 0.9408630132675171], "43": ["vertical_and_slash", 1000, 6096, 0.9721562266349792], "44": ["vertical_and_slash", 30, 800, 0.980770468711853], "45": ["vertical_and_slash", 30, 800, 0.9704340696334839], "46": ["vertical_and_slash", 30, 800, 0.9724265336990356], "47": ["vertical_and_slash", 1000, 6096, 0.9579482674598694], "48": ["vertical_and_slash", 30, 800, 0.9366120100021362], "49": ["vertical_and_slash", 30, 800, 0.9867264628410339], "50": ["vertical_and_slash", 100, 800, 0.96484375], "51": ["vertical_and_slash", 30, 800, 0.9583899974822998], "52": ["vertical_and_slash", 30, 800, 0.9424249529838562], "53": ["vertical_and_slash", 1000, 6096, 0.9481703042984009], "54": ["vertical_and_slash", 1000, 6096, 0.9842755794525146], "55": ["vertical_and_slash", 30, 800, 0.9735766649246216], "56": ["vertical_and_slash", 30, 800, 0.9594770073890686], "57": ["vertical_and_slash", 30, 800, 0.9624528288841248], "58": ["vertical_and_slash", 30, 800, 0.9334046244621277], "59": ["vertical_and_slash", 30, 800, 0.9416961669921875], "60": ["vertical_and_slash", 30, 800, 0.9485501646995544], "61": ["vertical_and_slash", 30, 800, 0.9158278107643127], "62": ["vertical_and_slash", 30, 800, 0.9705098271369934], "63": ["vertical_and_slash", 30, 800, 0.962493360042572]}, {"0": ["vertical_and_slash", 30, 800, 0.934903085231781], "1": ["vertical_and_slash", 30, 800, 0.9757312536239624], "2": ["vertical_and_slash", 1000, 6096, 0.9776284694671631], "3": ["vertical_and_slash", 30, 800, 0.8883693814277649], "4": ["vertical_and_slash", 30, 800, 0.9510934948921204], "5": ["vertical_and_slash", 30, 800, 0.9093663692474365], "6": ["vertical_and_slash", 30, 800, 0.9023526310920715], "7": ["vertical_and_slash", 30, 800, 0.9631701111793518], "8": ["vertical_and_slash", 1000, 6096, 0.966623067855835], "9": ["vertical_and_slash", 1000, 6096, 0.9557743072509766], "10": ["vertical_and_slash", 1000, 6096, 0.9910546541213989], "11": ["vertical_and_slash", 1000, 6096, 0.9526346921920776], "12": ["vertical_and_slash", 1000, 6096, 0.8689665198326111], "13": ["vertical_and_slash", 1000, 6096, 0.956572949886322], "14": ["vertical_and_slash", 1000, 6096, 0.9921777844429016], "15": ["vertical_and_slash", 1000, 6096, 0.9692468643188477], "16": ["vertical_and_slash", 1000, 6096, 0.9611530900001526], "17": ["vertical_and_slash", 1000, 6096, 0.9604033827781677], "18": ["vertical_and_slash", 1000, 6096, 0.961546778678894], "19": ["vertical_and_slash", 1000, 6096, 0.9675098657608032], "20": ["vertical_and_slash", 3500, 100, 0.9936402440071106], "21": ["vertical_and_slash", 1000, 6096, 0.9862780570983887], "22": ["vertical_and_slash", 1000, 6096, 0.9912402629852295], "23": ["vertical_and_slash", 1000, 6096, 0.9825507998466492], "24": ["vertical_and_slash", 30, 800, 0.9614688158035278], "25": ["vertical_and_slash", 30, 800, 0.8531063795089722], "26": ["vertical_and_slash", 30, 800, 0.9192854762077332], "27": ["vertical_and_slash", 30, 800, 0.8997114300727844], "28": ["vertical_and_slash", 30, 800, 0.9635742902755737], "29": ["vertical_and_slash", 30, 800, 0.8879558444023132], "30": ["vertical_and_slash", 100, 800, 0.92578125], "31": ["vertical_and_slash", 100, 800, 0.8984375], "32": ["vertical_and_slash", 30, 800, 0.743541955947876], "33": ["vertical_and_slash", 30, 800, 0.9351573586463928], "34": ["vertical_and_slash", 30, 800, 0.9739220142364502], "35": ["vertical_and_slash", 1000, 6096, 0.8084629774093628], "36": ["vertical_and_slash", 1000, 6096, 0.9444695711135864], "37": ["vertical_and_slash", 100, 800, 0.78515625], "38": ["vertical_and_slash", 1000, 6096, 0.7549774050712585], "39": ["vertical_and_slash", 1000, 6096, 0.9290085434913635], "40": ["vertical_and_slash", 1000, 6096, 0.8891969919204712], "41": ["vertical_and_slash", 1000, 6096, 0.96700119972229], "42": ["vertical_and_slash", 1000, 6096, 0.9606030583381653], "43": ["vertical_and_slash", 1000, 6096, 0.9600253701210022], "44": ["vertical_and_slash", 1000, 6096, 0.9765782952308655], "45": ["vertical_and_slash", 1000, 6096, 0.9731022715568542], "46": ["vertical_and_slash", 100, 800, 0.6953125], "47": ["vertical_and_slash", 1000, 6096, 0.9274213910102844], "48": ["vertical_and_slash", 30, 800, 0.9046525359153748], "49": ["vertical_and_slash", 30, 800, 0.9703969955444336], "50": ["vertical_and_slash", 30, 800, 0.9417571425437927], "51": ["vertical_and_slash", 30, 800, 0.963982343673706], "52": ["vertical_and_slash", 30, 800, 0.9459105134010315], "53": ["vertical_and_slash", 30, 800, 0.9592915177345276], "54": ["vertical_and_slash", 30, 800, 0.9595362544059753], "55": ["vertical_and_slash", 30, 800, 0.9740492105484009], "56": ["vertical_and_slash", 30, 800, 0.6957364678382874], "57": ["vertical_and_slash", 100, 800, 0.89453125], "58": ["vertical_and_slash", 30, 800, 0.9740748405456543], "59": ["vertical_and_slash", 30, 800, 0.9241670370101929], "60": ["vertical_and_slash", 500, 700, 0.9188547134399414], "61": ["vertical_and_slash", 30, 800, 0.9053714275360107], "62": ["vertical_and_slash", 1000, 6096, 0.958228588104248], "63": ["vertical_and_slash", 30, 800, 0.9557256102561951]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8570094704627991], "1": ["vertical_and_slash", 1000, 6096, 0.9893338680267334], "2": ["vertical_and_slash", 1000, 6096, 0.9449188113212585], "3": ["vertical_and_slash", 1000, 6096, 0.975352942943573], "4": ["vertical_and_slash", 1000, 6096, 0.9472012519836426], "5": ["vertical_and_slash", 1000, 6096, 0.989486813545227], "6": ["vertical_and_slash", 1000, 6096, 0.971174955368042], "7": ["vertical_and_slash", 1000, 6096, 0.8931871056556702], "8": ["vertical_and_slash", 30, 800, 0.9245004653930664], "9": ["vertical_and_slash", 100, 800, 0.859375], "10": ["vertical_and_slash", 100, 800, 0.859375], "11": ["vertical_and_slash", 30, 800, 0.8942111134529114], "12": ["vertical_and_slash", 3500, 100, 0.7404944896697998], "13": ["vertical_and_slash", 30, 800, 0.9478134512901306], "14": ["vertical_and_slash", 30, 800, 0.8190214037895203], "15": ["vertical_and_slash", 500, 700, 0.7766430974006653], "16": ["vertical_and_slash", 1000, 6096, 0.9349480867385864], "17": ["vertical_and_slash", 30, 800, 0.9102984070777893], "18": ["vertical_and_slash", 1000, 6096, 0.862310528755188], "19": ["vertical_and_slash", 500, 700, 0.91767817735672], "20": ["vertical_and_slash", 1000, 6096, 0.8369830250740051], "21": ["vertical_and_slash", 1000, 6096, 0.6888959407806396], "22": ["vertical_and_slash", 30, 800, 0.9091340899467468], "23": ["vertical_and_slash", 1000, 6096, 0.8453413248062134], "24": ["vertical_and_slash", 1000, 6096, 0.7894877195358276], "25": ["vertical_and_slash", 500, 700, 0.9467756748199463], "26": ["vertical_and_slash", 30, 800, 0.9523774981498718], "27": ["vertical_and_slash", 1000, 6096, 0.9306166768074036], "28": ["vertical_and_slash", 1000, 6096, 0.9391340017318726], "29": ["vertical_and_slash", 100, 800, 0.85546875], "30": ["vertical_and_slash", 1000, 6096, 0.92130446434021], "31": ["vertical_and_slash", 30, 800, 0.8674424290657043], "32": ["vertical_and_slash", 30, 800, 0.9739702939987183], "33": ["vertical_and_slash", 30, 800, 0.9720270037651062], "34": ["vertical_and_slash", 30, 800, 0.9013262987136841], "35": ["vertical_and_slash", 30, 800, 0.9723125100135803], "36": ["vertical_and_slash", 30, 800, 0.8971856832504272], "37": ["vertical_and_slash", 30, 800, 0.9473862051963806], "38": ["vertical_and_slash", 30, 800, 0.6448274254798889], "39": ["vertical_and_slash", 30, 800, 0.9578911066055298], "40": ["vertical_and_slash", 30, 800, 0.9355090260505676], "41": ["vertical_and_slash", 30, 800, 0.958286464214325], "42": ["vertical_and_slash", 30, 800, 0.93705153465271], "43": ["vertical_and_slash", 30, 800, 0.8799992203712463], "44": ["vertical_and_slash", 30, 800, 0.9685952663421631], "45": ["vertical_and_slash", 30, 800, 0.862156093120575], "46": ["vertical_and_slash", 30, 800, 0.9574906229972839], "47": ["vertical_and_slash", 30, 800, 0.9574345350265503], "48": ["vertical_and_slash", 1000, 6096, 0.8405346870422363], "49": ["vertical_and_slash", 30, 800, 0.8786194920539856], "50": ["vertical_and_slash", 1000, 6096, 0.8106364607810974], "51": ["vertical_and_slash", 30, 800, 0.929766833782196], "52": ["vertical_and_slash", 30, 800, 0.913433313369751], "53": ["vertical_and_slash", 30, 800, 0.7800456285476685], "54": ["vertical_and_slash", 30, 800, 0.8110105991363525], "55": ["vertical_and_slash", 30, 800, 0.8812217712402344], "56": ["vertical_and_slash", 1000, 6096, 0.8893566727638245], "57": ["vertical_and_slash", 1000, 6096, 0.8881974816322327], "58": ["vertical_and_slash", 1000, 6096, 0.9352526664733887], "59": ["vertical_and_slash", 1000, 6096, 0.8116486668586731], "60": ["vertical_and_slash", 1000, 6096, 0.9582773447036743], "61": ["vertical_and_slash", 1000, 6096, 0.7889048457145691], "62": ["vertical_and_slash", 1000, 6096, 0.8999911546707153], "63": ["vertical_and_slash", 1000, 6096, 0.9789445400238037]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9742184281349182], "1": ["vertical_and_slash", 1000, 6096, 0.9751096367835999], "2": ["vertical_and_slash", 1000, 6096, 0.660516619682312], "3": ["vertical_and_slash", 1000, 6096, 0.9875320196151733], "4": ["vertical_and_slash", 1000, 6096, 0.83885258436203], "5": ["vertical_and_slash", 1000, 6096, 0.8399583101272583], "6": ["vertical_and_slash", 1000, 6096, 0.8108904957771301], "7": ["vertical_and_slash", 1000, 6096, 0.5517553687095642], "8": ["vertical_and_slash", 1000, 6096, 0.7503235936164856], "9": ["vertical_and_slash", 1000, 6096, 0.9714740514755249], "10": ["vertical_and_slash", 1000, 6096, 0.8414002060890198], "11": ["vertical_and_slash", 30, 800, 0.9633986353874207], "12": ["vertical_and_slash", 1000, 6096, 0.9258506298065186], "13": ["vertical_and_slash", 1000, 6096, 0.9520205855369568], "14": ["vertical_and_slash", 3500, 100, 0.946774423122406], "15": ["vertical_and_slash", 30, 800, 0.9613867998123169], "16": ["vertical_and_slash", 30, 800, 0.623484194278717], "17": ["vertical_and_slash", 1000, 6096, 0.9434866309165955], "18": ["vertical_and_slash", 1000, 6096, 0.9629804491996765], "19": ["vertical_and_slash", 1000, 6096, 0.8463990092277527], "20": ["vertical_and_slash", 100, 800, 0.9375], "21": ["vertical_and_slash", 1000, 6096, 0.9062731862068176], "22": ["vertical_and_slash", 30, 800, 0.9750382900238037], "23": ["vertical_and_slash", 1000, 6096, 0.9480848908424377], "24": ["vertical_and_slash", 1000, 6096, 0.7576823830604553], "25": ["vertical_and_slash", 1000, 6096, 0.9742210507392883], "26": ["vertical_and_slash", 1000, 6096, 0.9860237240791321], "27": ["vertical_and_slash", 1000, 6096, 0.9676424860954285], "28": ["vertical_and_slash", 1000, 6096, 0.7657589316368103], "29": ["vertical_and_slash", 1000, 6096, 0.9561470150947571], "30": ["vertical_and_slash", 1000, 6096, 0.811066210269928], "31": ["vertical_and_slash", 1000, 6096, 0.9703875184059143], "32": ["vertical_and_slash", 30, 800, 0.9286283254623413], "33": ["vertical_and_slash", 30, 800, 0.8982791900634766], "34": ["vertical_and_slash", 500, 700, 0.9140600562095642], "35": ["vertical_and_slash", 30, 800, 0.9436793923377991], "36": ["vertical_and_slash", 30, 800, 0.9476034045219421], "37": ["vertical_and_slash", 30, 800, 0.8582310676574707], "38": ["vertical_and_slash", 30, 800, 0.9499164819717407], "39": ["vertical_and_slash", 30, 800, 0.9371611475944519], "40": ["vertical_and_slash", 30, 800, 0.9185857176780701], "41": ["vertical_and_slash", 1000, 6096, 0.8323348164558411], "42": ["vertical_and_slash", 30, 800, 0.9530657529830933], "43": ["vertical_and_slash", 1000, 6096, 0.8382992744445801], "44": ["vertical_and_slash", 1000, 6096, 0.9057599902153015], "45": ["vertical_and_slash", 1000, 6096, 0.7752314805984497], "46": ["vertical_and_slash", 30, 800, 0.9069003462791443], "47": ["vertical_and_slash", 1000, 6096, 0.7055371403694153], "48": ["vertical_and_slash", 1000, 6096, 0.8947019577026367], "49": ["vertical_and_slash", 3500, 100, 0.9098820090293884], "50": ["vertical_and_slash", 3500, 100, 0.8784118294715881], "51": ["vertical_and_slash", 30, 800, 0.7125860452651978], "52": ["vertical_and_slash", 3500, 100, 0.8325284719467163], "53": ["vertical_and_slash", 30, 800, 0.9111961126327515], "54": ["vertical_and_slash", 1000, 6096, 0.8651106953620911], "55": ["vertical_and_slash", 1000, 6096, 0.9234691262245178], "56": ["vertical_and_slash", 30, 800, 0.931998074054718], "57": ["vertical_and_slash", 100, 800, 0.9375], "58": ["vertical_and_slash", 3500, 100, 0.5599413514137268], "59": ["vertical_and_slash", 1000, 6096, 0.6640639305114746], "60": ["vertical_and_slash", 3500, 100, 0.856381893157959], "61": ["vertical_and_slash", 30, 800, 0.9680361151695251], "62": ["vertical_and_slash", 30, 800, 0.9638805985450745], "63": ["vertical_and_slash", 30, 800, 0.917905330657959]}, {"0": ["vertical_and_slash", 100, 800, 0.9296875], "1": ["vertical_and_slash", 30, 800, 0.7414695024490356], "2": ["vertical_and_slash", 1000, 6096, 0.9049969911575317], "3": ["vertical_and_slash", 30, 800, 0.9157560467720032], "4": ["vertical_and_slash", 1000, 6096, 0.7963104844093323], "5": ["vertical_and_slash", 30, 800, 0.9596757292747498], "6": ["vertical_and_slash", 1000, 6096, 0.811609148979187], "7": ["vertical_and_slash", 30, 800, 0.8984810709953308], "8": ["vertical_and_slash", 3500, 100, 0.9568498134613037], "9": ["vertical_and_slash", 500, 700, 0.938681960105896], "10": ["vertical_and_slash", 1000, 6096, 0.8480145335197449], "11": ["vertical_and_slash", 500, 700, 0.9550251364707947], "12": ["vertical_and_slash", 30, 800, 0.8775202631950378], "13": ["vertical_and_slash", 1000, 6096, 0.8311895728111267], "14": ["vertical_and_slash", 3500, 100, 0.7817966341972351], "15": ["vertical_and_slash", 1000, 6096, 0.8634781241416931], "16": ["vertical_and_slash", 1000, 6096, 0.973534882068634], "17": ["vertical_and_slash", 1000, 6096, 0.7367080450057983], "18": ["vertical_and_slash", 1000, 6096, 0.9810511469841003], "19": ["vertical_and_slash", 1000, 6096, 0.9637906551361084], "20": ["vertical_and_slash", 1000, 6096, 0.9951151609420776], "21": ["vertical_and_slash", 1000, 6096, 0.9066327214241028], "22": ["vertical_and_slash", 1000, 6096, 0.9962975978851318], "23": ["vertical_and_slash", 1000, 6096, 0.9916133880615234], "24": ["vertical_and_slash", 1000, 6096, 0.9506811499595642], "25": ["vertical_and_slash", 1000, 6096, 0.9717873930931091], "26": ["vertical_and_slash", 1000, 6096, 0.9603226184844971], "27": ["vertical_and_slash", 1000, 6096, 0.9728900790214539], "28": ["vertical_and_slash", 1000, 6096, 0.961214005947113], "29": ["vertical_and_slash", 1000, 6096, 0.8644313216209412], "30": ["vertical_and_slash", 1000, 6096, 0.9471644163131714], "31": ["vertical_and_slash", 1000, 6096, 0.9757005572319031], "32": ["vertical_and_slash", 1000, 6096, 0.9824827909469604], "33": ["vertical_and_slash", 1000, 6096, 0.9167914390563965], "34": ["vertical_and_slash", 1000, 6096, 0.9583839178085327], "35": ["vertical_and_slash", 1000, 6096, 0.9900406002998352], "36": ["vertical_and_slash", 1000, 6096, 0.80100017786026], "37": ["vertical_and_slash", 1000, 6096, 0.898510217666626], "38": ["vertical_and_slash", 1000, 6096, 0.9735856652259827], "39": ["vertical_and_slash", 1000, 6096, 0.8693722486495972], "40": ["vertical_and_slash", 30, 800, 0.9636111855506897], "41": ["vertical_and_slash", 3500, 100, 0.7425026297569275], "42": ["vertical_and_slash", 30, 800, 0.9030948877334595], "43": ["vertical_and_slash", 3500, 100, 0.6551296710968018], "44": ["vertical_and_slash", 100, 800, 0.87109375], "45": ["vertical_and_slash", 30, 800, 0.9544105529785156], "46": ["vertical_and_slash", 30, 800, 0.9498155117034912], "47": ["vertical_and_slash", 30, 800, 0.8914202451705933], "48": ["vertical_and_slash", 1000, 6096, 0.9824004769325256], "49": ["vertical_and_slash", 1000, 6096, 0.9972124695777893], "50": ["vertical_and_slash", 1000, 6096, 0.9074226021766663], "51": ["vertical_and_slash", 30, 800, 0.9646087288856506], "52": ["vertical_and_slash", 1000, 6096, 0.8756057024002075], "53": ["vertical_and_slash", 30, 800, 0.854377031326294], "54": ["vertical_and_slash", 1000, 6096, 0.9705929756164551], "55": ["vertical_and_slash", 1000, 6096, 0.9885392189025879], "56": ["vertical_and_slash", 1000, 6096, 0.9691387414932251], "57": ["vertical_and_slash", 100, 800, 0.91015625], "58": ["vertical_and_slash", 1000, 6096, 0.9891602396965027], "59": ["vertical_and_slash", 1000, 6096, 0.9547048807144165], "60": ["vertical_and_slash", 1000, 6096, 0.8162285089492798], "61": ["vertical_and_slash", 1000, 6096, 0.9620857238769531], "62": ["vertical_and_slash", 1000, 6096, 0.9821789264678955], "63": ["vertical_and_slash", 1000, 6096, 0.9610198140144348]}, {"0": ["vertical_and_slash", 3500, 100, 0.9789284467697144], "1": ["vertical_and_slash", 3500, 100, 0.9550416469573975], "2": ["vertical_and_slash", 3500, 100, 0.9365471005439758], "3": ["vertical_and_slash", 1000, 6096, 0.9786080718040466], "4": ["vertical_and_slash", 3500, 100, 0.9786985516548157], "5": ["vertical_and_slash", 3500, 100, 0.9676365256309509], "6": ["vertical_and_slash", 1000, 6096, 0.8523816466331482], "7": ["vertical_and_slash", 3500, 100, 0.951265811920166], "8": ["vertical_and_slash", 30, 800, 0.848352313041687], "9": ["vertical_and_slash", 1000, 6096, 0.9101062417030334], "10": ["vertical_and_slash", 500, 700, 0.9171656966209412], "11": ["vertical_and_slash", 1000, 6096, 0.930015504360199], "12": ["vertical_and_slash", 100, 800, 0.86328125], "13": ["vertical_and_slash", 3500, 100, 0.9597044587135315], "14": ["vertical_and_slash", 1000, 6096, 0.9388276934623718], "15": ["vertical_and_slash", 1000, 6096, 0.8079339265823364], "16": ["vertical_and_slash", 1000, 6096, 0.9974556565284729], "17": ["vertical_and_slash", 1000, 6096, 0.9940666556358337], "18": ["vertical_and_slash", 3500, 100, 0.9948021173477173], "19": ["vertical_and_slash", 1000, 6096, 0.9871312975883484], "20": ["vertical_and_slash", 1000, 6096, 0.9976180791854858], "21": ["vertical_and_slash", 1000, 6096, 0.9882305264472961], "22": ["vertical_and_slash", 1000, 6096, 0.9952536821365356], "23": ["vertical_and_slash", 1000, 6096, 0.9876596927642822], "24": ["vertical_and_slash", 1000, 6096, 0.9921155571937561], "25": ["vertical_and_slash", 1000, 6096, 0.994521975517273], "26": ["vertical_and_slash", 1000, 6096, 0.9950687289237976], "27": ["vertical_and_slash", 1000, 6096, 0.9919089674949646], "28": ["vertical_and_slash", 1000, 6096, 0.9920907616615295], "29": ["vertical_and_slash", 1000, 6096, 0.9823890924453735], "30": ["vertical_and_slash", 1000, 6096, 0.9705395698547363], "31": ["vertical_and_slash", 1000, 6096, 0.9829683899879456], "32": ["vertical_and_slash", 1000, 6096, 0.9926976561546326], "33": ["vertical_and_slash", 1000, 6096, 0.9905914664268494], "34": ["vertical_and_slash", 1000, 6096, 0.9878993630409241], "35": ["vertical_and_slash", 1000, 6096, 0.9976745247840881], "36": ["vertical_and_slash", 1000, 6096, 0.9332552552223206], "37": ["vertical_and_slash", 1000, 6096, 0.9908848404884338], "38": ["vertical_and_slash", 1000, 6096, 0.9747936129570007], "39": ["vertical_and_slash", 1000, 6096, 0.9599328637123108], "40": ["vertical_and_slash", 1000, 6096, 0.9857988953590393], "41": ["vertical_and_slash", 1000, 6096, 0.9953145384788513], "42": ["vertical_and_slash", 1000, 6096, 0.9863929152488708], "43": ["vertical_and_slash", 1000, 6096, 0.9918035864830017], "44": ["vertical_and_slash", 1000, 6096, 0.9913573861122131], "45": ["vertical_and_slash", 1000, 6096, 0.9950832724571228], "46": ["vertical_and_slash", 1000, 6096, 0.9930711984634399], "47": ["vertical_and_slash", 1000, 6096, 0.9958581328392029], "48": ["vertical_and_slash", 30, 800, 0.8624354600906372], "49": ["vertical_and_slash", 30, 800, 0.849473237991333], "50": ["vertical_and_slash", 30, 800, 0.9073121547698975], "51": ["vertical_and_slash", 100, 800, 0.8046875], "52": ["vertical_and_slash", 100, 800, 0.71484375], "53": ["vertical_and_slash", 30, 800, 0.8219325542449951], "54": ["vertical_and_slash", 30, 800, 0.8566333651542664], "55": ["vertical_and_slash", 30, 800, 0.7819647192955017], "56": ["vertical_and_slash", 1000, 6096, 0.840008020401001], "57": ["vertical_and_slash", 1000, 6096, 0.8735643029212952], "58": ["vertical_and_slash", 1000, 6096, 0.8833833336830139], "59": ["vertical_and_slash", 1000, 6096, 0.8271884322166443], "60": ["vertical_and_slash", 1000, 6096, 0.8406658172607422], "61": ["vertical_and_slash", 1000, 6096, 0.8429837226867676], "62": ["vertical_and_slash", 1000, 6096, 0.85932856798172], "63": ["vertical_and_slash", 1000, 6096, 0.8228768706321716]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9889729022979736], "1": ["vertical_and_slash", 1000, 6096, 0.9935887455940247], "2": ["vertical_and_slash", 1000, 6096, 0.9949935078620911], "3": ["vertical_and_slash", 1000, 6096, 0.984787106513977], "4": ["vertical_and_slash", 1000, 6096, 0.9928644895553589], "5": ["vertical_and_slash", 1000, 6096, 0.9918316006660461], "6": ["vertical_and_slash", 1000, 6096, 0.9962313771247864], "7": ["vertical_and_slash", 1000, 6096, 0.9900915026664734], "8": ["vertical_and_slash", 3500, 100, 0.9937083125114441], "9": ["vertical_and_slash", 3500, 100, 0.9974955916404724], "10": ["vertical_and_slash", 3500, 100, 0.9780661463737488], "11": ["vertical_and_slash", 1000, 6096, 0.9888462424278259], "12": ["vertical_and_slash", 3500, 100, 0.9700003862380981], "13": ["vertical_and_slash", 3500, 100, 0.990420401096344], "14": ["vertical_and_slash", 3500, 100, 0.9908552765846252], "15": ["vertical_and_slash", 3500, 100, 0.9826011061668396], "16": ["vertical_and_slash", 3500, 100, 0.9309127926826477], "17": ["vertical_and_slash", 3500, 100, 0.9210435748100281], "18": ["vertical_and_slash", 3500, 100, 0.9463978409767151], "19": ["vertical_and_slash", 500, 700, 0.854989230632782], "20": ["vertical_and_slash", 3500, 100, 0.9583246111869812], "21": ["vertical_and_slash", 3500, 100, 0.9506032466888428], "22": ["vertical_and_slash", 3500, 100, 0.951408326625824], "23": ["vertical_and_slash", 3500, 100, 0.9466915130615234], "24": ["vertical_and_slash", 100, 800, 0.74609375], "25": ["vertical_and_slash", 30, 800, 0.848252534866333], "26": ["vertical_and_slash", 1000, 6096, 0.8619548082351685], "27": ["vertical_and_slash", 500, 700, 0.8996201157569885], "28": ["vertical_and_slash", 500, 700, 0.8368968367576599], "29": ["vertical_and_slash", 3500, 100, 0.8350923657417297], "30": ["vertical_and_slash", 1000, 6096, 0.7345309853553772], "31": ["vertical_and_slash", 500, 700, 0.820966362953186], "32": ["vertical_and_slash", 1000, 6096, 0.9704338908195496], "33": ["vertical_and_slash", 1000, 6096, 0.964275598526001], "34": ["vertical_and_slash", 3500, 100, 0.5331658720970154], "35": ["vertical_and_slash", 1000, 6096, 0.5942737460136414], "36": ["vertical_and_slash", 1000, 6096, 0.698020875453949], "37": ["vertical_and_slash", 1000, 6096, 0.9432304501533508], "38": ["vertical_and_slash", 3500, 100, 0.5244500041007996], "39": ["vertical_and_slash", 1000, 6096, 0.6514526605606079], "40": ["vertical_and_slash", 3500, 100, 0.8472326993942261], "41": ["vertical_and_slash", 1000, 6096, 0.831987738609314], "42": ["vertical_and_slash", 500, 700, 0.951339840888977], "43": ["vertical_and_slash", 30, 800, 0.9635624289512634], "44": ["vertical_and_slash", 3500, 100, 0.9645004272460938], "45": ["vertical_and_slash", 1000, 6096, 0.9102885723114014], "46": ["vertical_and_slash", 1000, 6096, 0.9390802979469299], "47": ["vertical_and_slash", 3500, 100, 0.9381886720657349], "48": ["vertical_and_slash", 3500, 100, 0.8408583402633667], "49": ["vertical_and_slash", 1000, 6096, 0.94338458776474], "50": ["vertical_and_slash", 1000, 6096, 0.9321377873420715], "51": ["vertical_and_slash", 3500, 100, 0.9684271812438965], "52": ["vertical_and_slash", 3500, 100, 0.9686707258224487], "53": ["vertical_and_slash", 1000, 6096, 0.9831304550170898], "54": ["vertical_and_slash", 1000, 6096, 0.9404494762420654], "55": ["vertical_and_slash", 500, 700, 0.9219987392425537], "56": ["vertical_and_slash", 1000, 6096, 0.9741872549057007], "57": ["vertical_and_slash", 1000, 6096, 0.8037293553352356], "58": ["vertical_and_slash", 1000, 6096, 0.7988980412483215], "59": ["vertical_and_slash", 1000, 6096, 0.9072802662849426], "60": ["vertical_and_slash", 3500, 100, 0.7811290621757507], "61": ["vertical_and_slash", 1000, 6096, 0.5586568117141724], "62": ["vertical_and_slash", 1000, 6096, 0.9719745516777039], "63": ["vertical_and_slash", 1000, 6096, 0.9105277061462402]}, {"0": ["vertical_and_slash", 3500, 100, 0.989234447479248], "1": ["vertical_and_slash", 3500, 100, 0.9801751971244812], "2": ["vertical_and_slash", 3500, 100, 0.9867321252822876], "3": ["vertical_and_slash", 3500, 100, 0.9914985299110413], "4": ["vertical_and_slash", 3500, 100, 0.9943042993545532], "5": ["vertical_and_slash", 3500, 100, 0.9931344389915466], "6": ["vertical_and_slash", 3500, 100, 0.9948771595954895], "7": ["vertical_and_slash", 3500, 100, 0.9870005249977112], "8": ["vertical_and_slash", 1000, 6096, 0.9308728575706482], "9": ["vertical_and_slash", 1000, 6096, 0.9539312720298767], "10": ["vertical_and_slash", 1000, 6096, 0.9656527042388916], "11": ["vertical_and_slash", 1000, 6096, 0.7937611937522888], "12": ["vertical_and_slash", 1000, 6096, 0.9492560625076294], "13": ["vertical_and_slash", 1000, 6096, 0.9560684561729431], "14": ["vertical_and_slash", 1000, 6096, 0.8186072707176208], "15": ["vertical_and_slash", 1000, 6096, 0.9606759548187256], "16": ["vertical_and_slash", 1000, 6096, 0.99256432056427], "17": ["vertical_and_slash", 1000, 6096, 0.9975239038467407], "18": ["vertical_and_slash", 1000, 6096, 0.9977055191993713], "19": ["vertical_and_slash", 1000, 6096, 0.9983108639717102], "20": ["vertical_and_slash", 1000, 6096, 0.9941495060920715], "21": ["vertical_and_slash", 1000, 6096, 0.9990044236183167], "22": ["vertical_and_slash", 1000, 6096, 0.9975436329841614], "23": ["vertical_and_slash", 1000, 6096, 0.9985092878341675], "24": ["vertical_and_slash", 1000, 6096, 0.9655866026878357], "25": ["vertical_and_slash", 1000, 6096, 0.9192119836807251], "26": ["vertical_and_slash", 1000, 6096, 0.99366295337677], "27": ["vertical_and_slash", 1000, 6096, 0.9722167253494263], "28": ["vertical_and_slash", 1000, 6096, 0.8887385129928589], "29": ["vertical_and_slash", 1000, 6096, 0.9877192378044128], "30": ["vertical_and_slash", 1000, 6096, 0.9830015301704407], "31": ["vertical_and_slash", 1000, 6096, 0.9772177338600159], "32": ["vertical_and_slash", 30, 800, 0.8642340302467346], "33": ["vertical_and_slash", 3500, 100, 0.88213711977005], "34": ["vertical_and_slash", 3500, 100, 0.8150684237480164], "35": ["vertical_and_slash", 1000, 6096, 0.9975669384002686], "36": ["vertical_and_slash", 1000, 6096, 0.9877403974533081], "37": ["vertical_and_slash", 3500, 100, 0.988787829875946], "38": ["vertical_and_slash", 3500, 100, 0.8572766184806824], "39": ["vertical_and_slash", 100, 800, 0.98828125], "40": ["vertical_and_slash", 1000, 6096, 0.9752307534217834], "41": ["vertical_and_slash", 1000, 6096, 0.9954516291618347], "42": ["vertical_and_slash", 1000, 6096, 0.9968795776367188], "43": ["vertical_and_slash", 1000, 6096, 0.9926398992538452], "44": ["vertical_and_slash", 1000, 6096, 0.991080641746521], "45": ["vertical_and_slash", 1000, 6096, 0.9873877763748169], "46": ["vertical_and_slash", 1000, 6096, 0.9932673573493958], "47": ["vertical_and_slash", 1000, 6096, 0.9870055317878723], "48": ["vertical_and_slash", 30, 800, 0.9115626811981201], "49": ["vertical_and_slash", 100, 800, 0.76171875], "50": ["vertical_and_slash", 100, 800, 0.671875], "51": ["vertical_and_slash", 30, 800, 0.8820754289627075], "52": ["vertical_and_slash", 100, 800, 0.671875], "53": ["vertical_and_slash", 30, 800, 0.931029736995697], "54": ["vertical_and_slash", 100, 800, 0.84765625], "55": ["vertical_and_slash", 100, 800, 0.6640625], "56": ["vertical_and_slash", 3500, 100, 0.9925520420074463], "57": ["vertical_and_slash", 1000, 6096, 0.9964755177497864], "58": ["vertical_and_slash", 3500, 100, 0.9972584843635559], "59": ["vertical_and_slash", 1000, 6096, 0.9894404411315918], "60": ["vertical_and_slash", 3500, 100, 0.9976589679718018], "61": ["vertical_and_slash", 1000, 6096, 0.9972328543663025], "62": ["vertical_and_slash", 1000, 6096, 0.9339303970336914], "63": ["vertical_and_slash", 3500, 100, 0.9887061715126038]}, {"0": ["vertical_and_slash", 100, 800, 0.97265625], "1": ["vertical_and_slash", 100, 800, 0.98046875], "2": ["vertical_and_slash", 1000, 6096, 0.9939269423484802], "3": ["vertical_and_slash", 100, 800, 0.98828125], "4": ["vertical_and_slash", 100, 800, 0.9921875], "5": ["vertical_and_slash", 30, 800, 0.9312295317649841], "6": ["vertical_and_slash", 1000, 6096, 0.9910164475440979], "7": ["vertical_and_slash", 1000, 6096, 0.9979022741317749], "8": ["vertical_and_slash", 1000, 6096, 0.8934113383293152], "9": ["vertical_and_slash", 3500, 100, 0.8659201860427856], "10": ["vertical_and_slash", 500, 700, 0.731113076210022], "11": ["vertical_and_slash", 3500, 100, 0.763714611530304], "12": ["vertical_and_slash", 1000, 6096, 0.88387531042099], "13": ["vertical_and_slash", 3500, 100, 0.7041964530944824], "14": ["vertical_and_slash", 3500, 100, 0.8453197479248047], "15": ["vertical_and_slash", 3500, 100, 0.8526507616043091], "16": ["vertical_and_slash", 1000, 6096, 0.9191767573356628], "17": ["vertical_and_slash", 1000, 6096, 0.9891740083694458], "18": ["vertical_and_slash", 1000, 6096, 0.9941257834434509], "19": ["vertical_and_slash", 1000, 6096, 0.9543985724449158], "20": ["vertical_and_slash", 500, 700, 0.9246839880943298], "21": ["vertical_and_slash", 1000, 6096, 0.989008367061615], "22": ["vertical_and_slash", 1000, 6096, 0.9739415049552917], "23": ["vertical_and_slash", 3500, 100, 0.9061897397041321], "24": ["vertical_and_slash", 3500, 100, 0.9843717217445374], "25": ["vertical_and_slash", 3500, 100, 0.9899951219558716], "26": ["vertical_and_slash", 100, 800, 0.98828125], "27": ["vertical_and_slash", 3500, 100, 0.9727573394775391], "28": ["vertical_and_slash", 3500, 100, 0.9919485449790955], "29": ["vertical_and_slash", 3500, 100, 0.9911440014839172], "30": ["vertical_and_slash", 3500, 100, 0.9325807094573975], "31": ["vertical_and_slash", 3500, 100, 0.986329972743988], "32": ["vertical_and_slash", 3500, 100, 0.8217652440071106], "33": ["vertical_and_slash", 3500, 100, 0.9740002155303955], "34": ["vertical_and_slash", 3500, 100, 0.9271143674850464], "35": ["vertical_and_slash", 3500, 100, 0.9174104332923889], "36": ["vertical_and_slash", 3500, 100, 0.964229166507721], "37": ["vertical_and_slash", 3500, 100, 0.919661819934845], "38": ["vertical_and_slash", 3500, 100, 0.8317038416862488], "39": ["vertical_and_slash", 30, 800, 0.7808334231376648], "40": ["vertical_and_slash", 1000, 6096, 0.8558106422424316], "41": ["vertical_and_slash", 1000, 6096, 0.8482683897018433], "42": ["vertical_and_slash", 1000, 6096, 0.8039394617080688], "43": ["vertical_and_slash", 1000, 6096, 0.8606404662132263], "44": ["vertical_and_slash", 1000, 6096, 0.9154418706893921], "45": ["vertical_and_slash", 1000, 6096, 0.8999674320220947], "46": ["vertical_and_slash", 1000, 6096, 0.8572366833686829], "47": ["vertical_and_slash", 1000, 6096, 0.8897200226783752], "48": ["vertical_and_slash", 1000, 6096, 0.9732129573822021], "49": ["vertical_and_slash", 1000, 6096, 0.9888496398925781], "50": ["vertical_and_slash", 500, 700, 0.9322386384010315], "51": ["vertical_and_slash", 1000, 6096, 0.9691726565361023], "52": ["vertical_and_slash", 1000, 6096, 0.9766679406166077], "53": ["vertical_and_slash", 3500, 100, 0.9667726755142212], "54": ["vertical_and_slash", 3500, 100, 0.9779627323150635], "55": ["vertical_and_slash", 1000, 6096, 0.9741488099098206], "56": ["vertical_and_slash", 3500, 100, 0.9775235056877136], "57": ["vertical_and_slash", 3500, 100, 0.9945098757743835], "58": ["vertical_and_slash", 3500, 100, 0.9901373386383057], "59": ["vertical_and_slash", 3500, 100, 0.9897381067276001], "60": ["vertical_and_slash", 3500, 100, 0.9953264594078064], "61": ["vertical_and_slash", 3500, 100, 0.9959669709205627], "62": ["vertical_and_slash", 3500, 100, 0.9879618287086487], "63": ["vertical_and_slash", 3500, 100, 0.9891855716705322]}, {"0": ["vertical_and_slash", 3500, 100, 0.7835755348205566], "1": ["vertical_and_slash", 30, 800, 0.744536280632019], "2": ["vertical_and_slash", 500, 700, 0.7912058234214783], "3": ["vertical_and_slash", 100, 800, 0.79296875], "4": ["vertical_and_slash", 30, 800, 0.9743513464927673], "5": ["vertical_and_slash", 3500, 100, 0.6681130528450012], "6": ["vertical_and_slash", 30, 800, 0.9342909455299377], "7": ["vertical_and_slash", 30, 800, 0.815557062625885], "8": ["vertical_and_slash", 3500, 100, 0.9968534111976624], "9": ["vertical_and_slash", 100, 800, 0.99609375], "10": ["vertical_and_slash", 100, 800, 0.99609375], "11": ["vertical_and_slash", 3500, 100, 0.9708935618400574], "12": ["vertical_and_slash", 3500, 100, 0.9971928000450134], "13": ["vertical_and_slash", 3500, 100, 0.9575932621955872], "14": ["vertical_and_slash", 3500, 100, 0.9920603036880493], "15": ["vertical_and_slash", 3500, 100, 0.9983762502670288], "16": ["vertical_and_slash", 3500, 100, 0.9932623505592346], "17": ["vertical_and_slash", 3500, 100, 0.9907446503639221], "18": ["vertical_and_slash", 3500, 100, 0.9956991076469421], "19": ["vertical_and_slash", 3500, 100, 0.9965079426765442], "20": ["vertical_and_slash", 3500, 100, 0.9800114631652832], "21": ["vertical_and_slash", 3500, 100, 0.9965646862983704], "22": ["vertical_and_slash", 3500, 100, 0.9936301708221436], "23": ["vertical_and_slash", 100, 800, 0.99609375], "24": ["vertical_and_slash", 3500, 100, 0.994846761226654], "25": ["vertical_and_slash", 3500, 100, 0.9936004281044006], "26": ["vertical_and_slash", 3500, 100, 0.9875558614730835], "27": ["vertical_and_slash", 3500, 100, 0.994570255279541], "28": ["vertical_and_slash", 3500, 100, 0.9932993650436401], "29": ["vertical_and_slash", 3500, 100, 0.9847983121871948], "30": ["vertical_and_slash", 3500, 100, 0.9829658269882202], "31": ["vertical_and_slash", 3500, 100, 0.989538311958313], "32": ["vertical_and_slash", 1000, 6096, 0.8770189881324768], "33": ["vertical_and_slash", 100, 800, 0.859375], "34": ["vertical_and_slash", 3500, 100, 0.8551017642021179], "35": ["vertical_and_slash", 1000, 6096, 0.7229965925216675], "36": ["vertical_and_slash", 30, 800, 0.8999442458152771], "37": ["vertical_and_slash", 500, 700, 0.9420276284217834], "38": ["vertical_and_slash", 500, 700, 0.9297630190849304], "39": ["vertical_and_slash", 500, 700, 0.9168819189071655], "40": ["vertical_and_slash", 3500, 100, 0.9661527872085571], "41": ["vertical_and_slash", 3500, 100, 0.9704567790031433], "42": ["vertical_and_slash", 3500, 100, 0.9927958250045776], "43": ["vertical_and_slash", 3500, 100, 0.9797581434249878], "44": ["vertical_and_slash", 3500, 100, 0.978877604007721], "45": ["vertical_and_slash", 3500, 100, 0.9508986473083496], "46": ["vertical_and_slash", 3500, 100, 0.9823349714279175], "47": ["vertical_and_slash", 3500, 100, 0.9767053723335266], "48": ["vertical_and_slash", 500, 700, 0.9475476741790771], "49": ["vertical_and_slash", 3500, 100, 0.9753609895706177], "50": ["vertical_and_slash", 500, 700, 0.9288784861564636], "51": ["vertical_and_slash", 500, 700, 0.9526316523551941], "52": ["vertical_and_slash", 1000, 6096, 0.911916196346283], "53": ["vertical_and_slash", 100, 800, 0.98828125], "54": ["vertical_and_slash", 100, 800, 0.984375], "55": ["vertical_and_slash", 500, 700, 0.9779510498046875], "56": ["vertical_and_slash", 100, 800, 1.0], "57": ["vertical_and_slash", 1000, 6096, 0.9954347014427185], "58": ["vertical_and_slash", 1000, 6096, 0.9953975677490234], "59": ["vertical_and_slash", 1000, 6096, 0.998020350933075], "60": ["vertical_and_slash", 1000, 6096, 0.9883339405059814], "61": ["vertical_and_slash", 1000, 6096, 0.9883899092674255], "62": ["vertical_and_slash", 1000, 6096, 0.9978113174438477], "63": ["vertical_and_slash", 1000, 6096, 0.9961769580841064]}, {"0": ["vertical_and_slash", 3500, 100, 0.9907277822494507], "1": ["vertical_and_slash", 3500, 100, 0.993987500667572], "2": ["vertical_and_slash", 100, 800, 0.9921875], "3": ["vertical_and_slash", 100, 800, 0.9921875], "4": ["vertical_and_slash", 500, 700, 0.945554792881012], "5": ["vertical_and_slash", 3500, 100, 0.9937874674797058], "6": ["vertical_and_slash", 3500, 100, 0.9939504265785217], "7": ["vertical_and_slash", 100, 800, 0.9921875], "8": ["vertical_and_slash", 30, 800, 0.6786468029022217], "9": ["vertical_and_slash", 3500, 100, 0.986274242401123], "10": ["vertical_and_slash", 3500, 100, 0.9971485137939453], "11": ["vertical_and_slash", 3500, 100, 0.885717511177063], "12": ["vertical_and_slash", 500, 700, 0.9932376146316528], "13": ["vertical_and_slash", 3500, 100, 0.9916094541549683], "14": ["vertical_and_slash", 3500, 100, 0.7919186949729919], "15": ["vertical_and_slash", 3500, 100, 0.9175549149513245], "16": ["vertical_and_slash", 3500, 100, 0.9946252703666687], "17": ["vertical_and_slash", 3500, 100, 0.9971895217895508], "18": ["vertical_and_slash", 3500, 100, 0.9863329529762268], "19": ["vertical_and_slash", 3500, 100, 0.9967142939567566], "20": ["vertical_and_slash", 3500, 100, 0.9917566776275635], "21": ["vertical_and_slash", 3500, 100, 0.9929065704345703], "22": ["vertical_and_slash", 3500, 100, 0.9922209978103638], "23": ["vertical_and_slash", 3500, 100, 0.9922376871109009], "24": ["vertical_and_slash", 1000, 6096, 0.9966207146644592], "25": ["vertical_and_slash", 1000, 6096, 0.9971777200698853], "26": ["vertical_and_slash", 1000, 6096, 0.9975135922431946], "27": ["vertical_and_slash", 1000, 6096, 0.9897735118865967], "28": ["vertical_and_slash", 1000, 6096, 0.9943304657936096], "29": ["vertical_and_slash", 1000, 6096, 0.9955590963363647], "30": ["vertical_and_slash", 1000, 6096, 0.9963770508766174], "31": ["vertical_and_slash", 1000, 6096, 0.9903107285499573], "32": ["vertical_and_slash", 500, 700, 0.9399028420448303], "33": ["vertical_and_slash", 3500, 100, 0.9167885780334473], "34": ["vertical_and_slash", 500, 700, 0.9515976905822754], "35": ["vertical_and_slash", 500, 700, 0.9583256840705872], "36": ["vertical_and_slash", 500, 700, 0.9119857549667358], "37": ["vertical_and_slash", 100, 800, 0.8359375], "38": ["vertical_and_slash", 100, 800, 0.765625], "39": ["vertical_and_slash", 500, 700, 0.8745498657226562], "40": ["vertical_and_slash", 3500, 100, 0.9915809035301208], "41": ["vertical_and_slash", 3500, 100, 0.9915350675582886], "42": ["vertical_and_slash", 3500, 100, 0.9969307780265808], "43": ["vertical_and_slash", 3500, 100, 0.9906328320503235], "44": ["vertical_and_slash", 3500, 100, 0.9906835556030273], "45": ["vertical_and_slash", 3500, 100, 0.9881353378295898], "46": ["vertical_and_slash", 3500, 100, 0.9787862300872803], "47": ["vertical_and_slash", 3500, 100, 0.992257297039032], "48": ["vertical_and_slash", 3500, 100, 0.9877546429634094], "49": ["vertical_and_slash", 3500, 100, 0.9968995451927185], "50": ["vertical_and_slash", 3500, 100, 0.993320643901825], "51": ["vertical_and_slash", 3500, 100, 0.9516721963882446], "52": ["vertical_and_slash", 3500, 100, 0.9695364832878113], "53": ["vertical_and_slash", 3500, 100, 0.9981467127799988], "54": ["vertical_and_slash", 3500, 100, 0.9960101842880249], "55": ["vertical_and_slash", 3500, 100, 0.9977790713310242], "56": ["vertical_and_slash", 3500, 100, 0.9646185040473938], "57": ["vertical_and_slash", 3500, 100, 0.9305040836334229], "58": ["vertical_and_slash", 3500, 100, 0.9410441517829895], "59": ["vertical_and_slash", 500, 700, 0.870216965675354], "60": ["vertical_and_slash", 3500, 100, 0.9426347613334656], "61": ["vertical_and_slash", 3500, 100, 0.9567436575889587], "62": ["vertical_and_slash", 3500, 100, 0.9155870079994202], "63": ["vertical_and_slash", 3500, 100, 0.9737669825553894]}, {"0": ["vertical_and_slash", 3500, 100, 0.9568111300468445], "1": ["vertical_and_slash", 3500, 100, 0.9925446510314941], "2": ["vertical_and_slash", 3500, 100, 0.9890397191047668], "3": ["vertical_and_slash", 3500, 100, 0.997904360294342], "4": ["vertical_and_slash", 3500, 100, 0.987509548664093], "5": ["vertical_and_slash", 3500, 100, 0.9492571353912354], "6": ["vertical_and_slash", 100, 800, 0.99609375], "7": ["vertical_and_slash", 500, 700, 0.9470086097717285], "8": ["vertical_and_slash", 3500, 100, 0.9898065328598022], "9": ["vertical_and_slash", 3500, 100, 0.9906474947929382], "10": ["vertical_and_slash", 3500, 100, 0.9946801066398621], "11": ["vertical_and_slash", 3500, 100, 0.993366539478302], "12": ["vertical_and_slash", 3500, 100, 0.9938772916793823], "13": ["vertical_and_slash", 3500, 100, 0.971512496471405], "14": ["vertical_and_slash", 3500, 100, 0.9903855919837952], "15": ["vertical_and_slash", 3500, 100, 0.9967730045318604], "16": ["vertical_and_slash", 500, 700, 0.9719936847686768], "17": ["vertical_and_slash", 500, 700, 0.9771726727485657], "18": ["vertical_and_slash", 1000, 6096, 0.9917740821838379], "19": ["vertical_and_slash", 1000, 6096, 0.9802855253219604], "20": ["vertical_and_slash", 500, 700, 0.9933639168739319], "21": ["vertical_and_slash", 500, 700, 0.9938787221908569], "22": ["vertical_and_slash", 1000, 6096, 0.9823704361915588], "23": ["vertical_and_slash", 1000, 6096, 0.9905321598052979], "24": ["vertical_and_slash", 3500, 100, 0.844693660736084], "25": ["vertical_and_slash", 3500, 100, 0.9411169290542603], "26": ["vertical_and_slash", 3500, 100, 0.8705904483795166], "27": ["vertical_and_slash", 3500, 100, 0.9504412412643433], "28": ["vertical_and_slash", 3500, 100, 0.9115034937858582], "29": ["vertical_and_slash", 3500, 100, 0.920758843421936], "30": ["vertical_and_slash", 3500, 100, 0.8925650715827942], "31": ["vertical_and_slash", 3500, 100, 0.920913815498352], "32": ["vertical_and_slash", 30, 800, 0.9709855914115906], "33": ["vertical_and_slash", 30, 800, 0.9075706601142883], "34": ["vertical_and_slash", 500, 700, 0.9515993595123291], "35": ["vertical_and_slash", 100, 800, 0.78515625], "36": ["vertical_and_slash", 30, 800, 0.8037341237068176], "37": ["vertical_and_slash", 500, 700, 0.978553056716919], "38": ["vertical_and_slash", 30, 800, 0.9540588855743408], "39": ["vertical_and_slash", 500, 700, 0.9260756969451904], "40": ["vertical_and_slash", 3500, 100, 0.9576201438903809], "41": ["vertical_and_slash", 3500, 100, 0.9933168292045593], "42": ["vertical_and_slash", 500, 700, 0.9889325499534607], "43": ["vertical_and_slash", 3500, 100, 0.9841705560684204], "44": ["vertical_and_slash", 500, 700, 0.9863041639328003], "45": ["vertical_and_slash", 30, 800, 0.9396712183952332], "46": ["vertical_and_slash", 100, 800, 0.9921875], "47": ["vertical_and_slash", 500, 700, 0.9857985973358154], "48": ["vertical_and_slash", 500, 700, 0.9832188487052917], "49": ["vertical_and_slash", 3500, 100, 0.9919037222862244], "50": ["vertical_and_slash", 100, 800, 0.91796875], "51": ["vertical_and_slash", 500, 700, 0.9745897054672241], "52": ["vertical_and_slash", 3500, 100, 0.9779555797576904], "53": ["vertical_and_slash", 100, 800, 0.921875], "54": ["vertical_and_slash", 1000, 6096, 0.9891850352287292], "55": ["vertical_and_slash", 3500, 100, 0.9891913533210754], "56": ["vertical_and_slash", 100, 800, 0.98828125], "57": ["vertical_and_slash", 3500, 100, 0.9792505502700806], "58": ["vertical_and_slash", 3500, 100, 0.9927448630332947], "59": ["vertical_and_slash", 3500, 100, 0.9825442433357239], "60": ["vertical_and_slash", 100, 800, 0.98828125], "61": ["vertical_and_slash", 3500, 100, 0.9897831678390503], "62": ["vertical_and_slash", 500, 700, 0.9493371844291687], "63": ["vertical_and_slash", 100, 800, 0.9921875]}, {"0": ["vertical_and_slash", 3500, 100, 0.9876601696014404], "1": ["vertical_and_slash", 3500, 100, 0.9629367589950562], "2": ["vertical_and_slash", 500, 700, 0.9661287069320679], "3": ["vertical_and_slash", 100, 800, 0.96484375], "4": ["vertical_and_slash", 500, 700, 0.9671223163604736], "5": ["vertical_and_slash", 3500, 100, 0.9647591710090637], "6": ["vertical_and_slash", 3500, 100, 0.9863847494125366], "7": ["vertical_and_slash", 3500, 100, 0.9763343334197998], "8": ["vertical_and_slash", 500, 700, 0.9481918811798096], "9": ["vertical_and_slash", 500, 700, 0.9423896670341492], "10": ["vertical_and_slash", 30, 800, 0.9184979200363159], "11": ["vertical_and_slash", 30, 800, 0.9515105485916138], "12": ["vertical_and_slash", 500, 700, 0.9535124897956848], "13": ["vertical_and_slash", 100, 800, 0.85546875], "14": ["vertical_and_slash", 30, 800, 0.627446174621582], "15": ["vertical_and_slash", 500, 700, 0.893500566482544], "16": ["vertical_and_slash", 1000, 6096, 0.7488555908203125], "17": ["vertical_and_slash", 1000, 6096, 0.7726936936378479], "18": ["vertical_and_slash", 1000, 6096, 0.7700554132461548], "19": ["vertical_and_slash", 1000, 6096, 0.8397179245948792], "20": ["vertical_and_slash", 500, 700, 0.9368733167648315], "21": ["vertical_and_slash", 1000, 6096, 0.7806558012962341], "22": ["vertical_and_slash", 1000, 6096, 0.5591722130775452], "23": ["vertical_and_slash", 100, 800, 0.56640625], "24": ["vertical_and_slash", 3500, 100, 0.9060772061347961], "25": ["vertical_and_slash", 3500, 100, 0.926750123500824], "26": ["vertical_and_slash", 500, 700, 0.9414880275726318], "27": ["vertical_and_slash", 500, 700, 0.9601885676383972], "28": ["vertical_and_slash", 500, 700, 0.8730140328407288], "29": ["vertical_and_slash", 100, 800, 0.91796875], "30": ["vertical_and_slash", 100, 800, 0.9453125], "31": ["vertical_and_slash", 30, 800, 0.8037018179893494], "32": ["vertical_and_slash", 1000, 6096, 0.7970362305641174], "33": ["vertical_and_slash", 1000, 6096, 0.8430536389350891], "34": ["vertical_and_slash", 100, 800, 0.8359375], "35": ["vertical_and_slash", 1000, 6096, 0.5859518647193909], "36": ["vertical_and_slash", 1000, 6096, 0.8104005455970764], "37": ["vertical_and_slash", 100, 800, 0.890625], "38": ["vertical_and_slash", 3500, 100, 0.7313418984413147], "39": ["vertical_and_slash", 100, 800, 0.88671875], "40": ["vertical_and_slash", 100, 800, 0.9375], "41": ["vertical_and_slash", 500, 700, 0.9903108477592468], "42": ["vertical_and_slash", 100, 800, 0.97265625], "43": ["vertical_and_slash", 100, 800, 0.84765625], "44": ["vertical_and_slash", 100, 800, 0.8984375], "45": ["vertical_and_slash", 3500, 100, 0.9500426054000854], "46": ["vertical_and_slash", 3500, 100, 0.9478051662445068], "47": ["vertical_and_slash", 500, 700, 0.9887247085571289], "48": ["vertical_and_slash", 3500, 100, 0.9578651785850525], "49": ["vertical_and_slash", 500, 700, 0.8081033229827881], "50": ["vertical_and_slash", 3500, 100, 0.9292168021202087], "51": ["vertical_and_slash", 3500, 100, 0.9493986964225769], "52": ["vertical_and_slash", 3500, 100, 0.949333906173706], "53": ["vertical_and_slash", 3500, 100, 0.9516175389289856], "54": ["vertical_and_slash", 3500, 100, 0.9442576766014099], "55": ["vertical_and_slash", 3500, 100, 0.9327177405357361], "56": ["vertical_and_slash", 500, 700, 0.9517089128494263], "57": ["vertical_and_slash", 3500, 100, 0.8694164156913757], "58": ["vertical_and_slash", 1000, 6096, 0.9120225310325623], "59": ["vertical_and_slash", 3500, 100, 0.9636445045471191], "60": ["vertical_and_slash", 100, 800, 0.8984375], "61": ["vertical_and_slash", 3500, 100, 0.9777922034263611], "62": ["vertical_and_slash", 100, 800, 0.9453125], "63": ["vertical_and_slash", 500, 700, 0.9044644832611084]}, {"0": ["vertical_and_slash", 100, 800, 0.86328125], "1": ["vertical_and_slash", 100, 800, 0.86328125], "2": ["vertical_and_slash", 3500, 100, 0.5738235116004944], "3": ["vertical_and_slash", 100, 800, 0.78125], "4": ["vertical_and_slash", 100, 800, 0.875], "5": ["vertical_and_slash", 100, 800, 0.69921875], "6": ["vertical_and_slash", 100, 800, 0.58984375], "7": ["vertical_and_slash", 100, 800, 0.8359375], "8": ["vertical_and_slash", 3500, 100, 0.89006108045578], "9": ["vertical_and_slash", 500, 700, 0.8603876233100891], "10": ["vertical_and_slash", 500, 700, 0.7747532725334167], "11": ["vertical_and_slash", 3500, 100, 0.9188287854194641], "12": ["vertical_and_slash", 3500, 100, 0.8450748920440674], "13": ["vertical_and_slash", 500, 700, 0.9082717895507812], "14": ["vertical_and_slash", 3500, 100, 0.8648320436477661], "15": ["vertical_and_slash", 500, 700, 0.9189893007278442], "16": ["vertical_and_slash", 500, 700, 0.803281843662262], "17": ["vertical_and_slash", 3500, 100, 0.9215734004974365], "18": ["vertical_and_slash", 3500, 100, 0.8669242262840271], "19": ["vertical_and_slash", 3500, 100, 0.8105125427246094], "20": ["vertical_and_slash", 500, 700, 0.9334966540336609], "21": ["vertical_and_slash", 3500, 100, 0.8472580909729004], "22": ["vertical_and_slash", 3500, 100, 0.8667786121368408], "23": ["vertical_and_slash", 3500, 100, 0.8848408460617065], "24": ["vertical_and_slash", 3500, 100, 0.935181200504303], "25": ["vertical_and_slash", 3500, 100, 0.9758675694465637], "26": ["vertical_and_slash", 3500, 100, 0.9815950393676758], "27": ["vertical_and_slash", 500, 700, 0.9439741373062134], "28": ["vertical_and_slash", 3500, 100, 0.9789666533470154], "29": ["vertical_and_slash", 3500, 100, 0.9766592383384705], "30": ["vertical_and_slash", 3500, 100, 0.9569538831710815], "31": ["vertical_and_slash", 3500, 100, 0.9363120198249817], "32": ["vertical_and_slash", 3500, 100, 0.9861800074577332], "33": ["vertical_and_slash", 3500, 100, 0.9874187707901001], "34": ["vertical_and_slash", 3500, 100, 0.9916433095932007], "35": ["vertical_and_slash", 3500, 100, 0.975922703742981], "36": ["vertical_and_slash", 3500, 100, 0.9933940768241882], "37": ["vertical_and_slash", 3500, 100, 0.9902389049530029], "38": ["vertical_and_slash", 3500, 100, 0.9933968186378479], "39": ["vertical_and_slash", 3500, 100, 0.9895731806755066], "40": ["vertical_and_slash", 1000, 6096, 0.9342583417892456], "41": ["vertical_and_slash", 1000, 6096, 0.820548951625824], "42": ["vertical_and_slash", 1000, 6096, 0.8942907452583313], "43": ["vertical_and_slash", 100, 800, 0.84375], "44": ["vertical_and_slash", 1000, 6096, 0.8540371656417847], "45": ["vertical_and_slash", 1000, 6096, 0.689696729183197], "46": ["vertical_and_slash", 1000, 6096, 0.73775315284729], "47": ["vertical_and_slash", 100, 800, 0.84765625], "48": ["vertical_and_slash", 3500, 100, 0.9610375165939331], "49": ["vertical_and_slash", 1000, 6096, 0.9893020391464233], "50": ["vertical_and_slash", 1000, 6096, 0.9922006130218506], "51": ["vertical_and_slash", 3500, 100, 0.9898445010185242], "52": ["vertical_and_slash", 1000, 6096, 0.9856063723564148], "53": ["vertical_and_slash", 1000, 6096, 0.9913110733032227], "54": ["vertical_and_slash", 1000, 6096, 0.9843234419822693], "55": ["vertical_and_slash", 3500, 100, 0.9712828993797302], "56": ["vertical_and_slash", 3500, 100, 0.9731909036636353], "57": ["vertical_and_slash", 3500, 100, 0.9778128862380981], "58": ["vertical_and_slash", 3500, 100, 0.9945456385612488], "59": ["vertical_and_slash", 3500, 100, 0.9918779134750366], "60": ["vertical_and_slash", 3500, 100, 0.9928656220436096], "61": ["vertical_and_slash", 3500, 100, 0.9796142578125], "62": ["vertical_and_slash", 500, 700, 0.962554395198822], "63": ["vertical_and_slash", 3500, 100, 0.9633960127830505]}, {"0": ["vertical_and_slash", 100, 800, 0.99609375], "1": ["vertical_and_slash", 3500, 100, 0.9843919277191162], "2": ["vertical_and_slash", 3500, 100, 0.940421998500824], "3": ["vertical_and_slash", 500, 700, 0.9905564785003662], "4": ["vertical_and_slash", 3500, 100, 0.9895368218421936], "5": ["vertical_and_slash", 3500, 100, 0.9775332808494568], "6": ["vertical_and_slash", 3500, 100, 0.9897283315658569], "7": ["vertical_and_slash", 1000, 6096, 0.992956280708313], "8": ["vertical_and_slash", 100, 800, 0.85546875], "9": ["vertical_and_slash", 3500, 100, 0.8806954026222229], "10": ["vertical_and_slash", 3500, 100, 0.9191994667053223], "11": ["vertical_and_slash", 100, 800, 0.81640625], "12": ["vertical_and_slash", 3500, 100, 0.8784688711166382], "13": ["vertical_and_slash", 3500, 100, 0.9149487614631653], "14": ["vertical_and_slash", 3500, 100, 0.882009744644165], "15": ["vertical_and_slash", 3500, 100, 0.9057184457778931], "16": ["vertical_and_slash", 500, 700, 0.9917744994163513], "17": ["vertical_and_slash", 3500, 100, 0.9942914843559265], "18": ["vertical_and_slash", 500, 700, 0.931837797164917], "19": ["vertical_and_slash", 3500, 100, 0.9655508399009705], "20": ["vertical_and_slash", 3500, 100, 0.9865359663963318], "21": ["vertical_and_slash", 3500, 100, 0.9885656833648682], "22": ["vertical_and_slash", 3500, 100, 0.9907728433609009], "23": ["vertical_and_slash", 3500, 100, 0.9729240536689758], "24": ["vertical_and_slash", 100, 800, 0.77734375], "25": ["vertical_and_slash", 100, 800, 0.82421875], "26": ["vertical_and_slash", 100, 800, 0.82421875], "27": ["vertical_and_slash", 100, 800, 0.8671875], "28": ["vertical_and_slash", 3500, 100, 0.8764268755912781], "29": ["vertical_and_slash", 3500, 100, 0.9048717021942139], "30": ["vertical_and_slash", 100, 800, 0.640625], "31": ["vertical_and_slash", 3500, 100, 0.88221675157547], "32": ["vertical_and_slash", 1000, 6096, 0.979773759841919], "33": ["vertical_and_slash", 1000, 6096, 0.9915492534637451], "34": ["vertical_and_slash", 1000, 6096, 0.9881225228309631], "35": ["vertical_and_slash", 1000, 6096, 0.9293946027755737], "36": ["vertical_and_slash", 1000, 6096, 0.9852708578109741], "37": ["vertical_and_slash", 1000, 6096, 0.9631314277648926], "38": ["vertical_and_slash", 1000, 6096, 0.9800013303756714], "39": ["vertical_and_slash", 1000, 6096, 0.94340580701828], "40": ["vertical_and_slash", 3500, 100, 0.941611647605896], "41": ["vertical_and_slash", 3500, 100, 0.9937059879302979], "42": ["vertical_and_slash", 3500, 100, 0.9684086441993713], "43": ["vertical_and_slash", 3500, 100, 0.9831252098083496], "44": ["vertical_and_slash", 3500, 100, 0.9931964874267578], "45": ["vertical_and_slash", 3500, 100, 0.9851258397102356], "46": ["vertical_and_slash", 3500, 100, 0.9736831188201904], "47": ["vertical_and_slash", 3500, 100, 0.9556482434272766], "48": ["vertical_and_slash", 100, 800, 0.77734375], "49": ["vertical_and_slash", 500, 700, 0.9864726662635803], "50": ["vertical_and_slash", 1000, 6096, 0.9702827334403992], "51": ["vertical_and_slash", 100, 800, 0.859375], "52": ["vertical_and_slash", 500, 700, 0.9634029269218445], "53": ["vertical_and_slash", 3500, 100, 0.9040798544883728], "54": ["vertical_and_slash", 1000, 6096, 0.984891414642334], "55": ["vertical_and_slash", 3500, 100, 0.9025231003761292], "56": ["vertical_and_slash", 1000, 6096, 0.5801780819892883], "57": ["vertical_and_slash", 3500, 100, 0.6026175618171692], "58": ["vertical_and_slash", 100, 800, 0.87890625], "59": ["vertical_and_slash", 100, 800, 0.90625], "60": ["vertical_and_slash", 100, 800, 0.796875], "61": ["vertical_and_slash", 100, 800, 0.81640625], "62": ["vertical_and_slash", 100, 800, 0.69921875], "63": ["vertical_and_slash", 100, 800, 0.92578125]}, {"0": ["vertical_and_slash", 3500, 100, 0.8912773728370667], "1": ["vertical_and_slash", 100, 800, 0.73046875], "2": ["vertical_and_slash", 500, 700, 0.9705966114997864], "3": ["vertical_and_slash", 500, 700, 0.9534775614738464], "4": ["vertical_and_slash", 500, 700, 0.9602234959602356], "5": ["vertical_and_slash", 500, 700, 0.7640681266784668], "6": ["vertical_and_slash", 500, 700, 0.8960625529289246], "7": ["vertical_and_slash", 500, 700, 0.9381105303764343], "8": ["vertical_and_slash", 500, 700, 0.9239199757575989], "9": ["vertical_and_slash", 500, 700, 0.9690032005310059], "10": ["vertical_and_slash", 3500, 100, 0.9872485995292664], "11": ["vertical_and_slash", 3500, 100, 0.9736847877502441], "12": ["vertical_and_slash", 3500, 100, 0.8918418884277344], "13": ["vertical_and_slash", 3500, 100, 0.9853194355964661], "14": ["vertical_and_slash", 3500, 100, 0.9661954641342163], "15": ["vertical_and_slash", 3500, 100, 0.989924430847168], "16": ["vertical_and_slash", 3500, 100, 0.9828006625175476], "17": ["vertical_and_slash", 1000, 6096, 0.9904162883758545], "18": ["vertical_and_slash", 1000, 6096, 0.9779679775238037], "19": ["vertical_and_slash", 1000, 6096, 0.977623462677002], "20": ["vertical_and_slash", 3500, 100, 0.9842522144317627], "21": ["vertical_and_slash", 1000, 6096, 0.9913325905799866], "22": ["vertical_and_slash", 3500, 100, 0.9918047189712524], "23": ["vertical_and_slash", 1000, 6096, 0.9880527257919312], "24": ["vertical_and_slash", 500, 700, 0.9345956444740295], "25": ["vertical_and_slash", 500, 700, 0.9124588370323181], "26": ["vertical_and_slash", 100, 800, 0.54296875], "27": ["vertical_and_slash", 500, 700, 0.9536015391349792], "28": ["vertical_and_slash", 500, 700, 0.9425217509269714], "29": ["vertical_and_slash", 500, 700, 0.8839255571365356], "30": ["vertical_and_slash", 100, 800, 0.84765625], "31": ["vertical_and_slash", 100, 800, 0.55859375], "32": ["vertical_and_slash", 500, 700, 0.9849395751953125], "33": ["vertical_and_slash", 100, 800, 0.88671875], "34": ["vertical_and_slash", 3500, 100, 0.9827902913093567], "35": ["vertical_and_slash", 3500, 100, 0.9228121042251587], "36": ["vertical_and_slash", 3500, 100, 0.9638622403144836], "37": ["vertical_and_slash", 500, 700, 0.9598038196563721], "38": ["vertical_and_slash", 500, 700, 0.9752495288848877], "39": ["vertical_and_slash", 1000, 6096, 0.6491831541061401], "40": ["vertical_and_slash", 1000, 6096, 0.960233747959137], "41": ["vertical_and_slash", 1000, 6096, 0.980234682559967], "42": ["vertical_and_slash", 3500, 100, 0.7951210141181946], "43": ["vertical_and_slash", 1000, 6096, 0.9752976894378662], "44": ["vertical_and_slash", 1000, 6096, 0.9768164753913879], "45": ["vertical_and_slash", 1000, 6096, 0.96163010597229], "46": ["vertical_and_slash", 1000, 6096, 0.9755325317382812], "47": ["vertical_and_slash", 1000, 6096, 0.9402468800544739], "48": ["vertical_and_slash", 3500, 100, 0.6751712560653687], "49": ["vertical_and_slash", 3500, 100, 0.8446762561798096], "50": ["vertical_and_slash", 3500, 100, 0.8793857097625732], "51": ["vertical_and_slash", 3500, 100, 0.8784389495849609], "52": ["vertical_and_slash", 3500, 100, 0.8016602396965027], "53": ["vertical_and_slash", 100, 800, 0.828125], "54": ["vertical_and_slash", 100, 800, 0.76953125], "55": ["vertical_and_slash", 3500, 100, 0.8274632096290588], "56": ["vertical_and_slash", 3500, 100, 0.9881812334060669], "57": ["vertical_and_slash", 3500, 100, 0.9497858881950378], "58": ["vertical_and_slash", 3500, 100, 0.9523339867591858], "59": ["vertical_and_slash", 3500, 100, 0.9758387804031372], "60": ["vertical_and_slash", 3500, 100, 0.9715253710746765], "61": ["vertical_and_slash", 3500, 100, 0.9213313460350037], "62": ["vertical_and_slash", 3500, 100, 0.9587473273277283], "63": ["vertical_and_slash", 3500, 100, 0.9182716012001038]}, {"0": ["vertical_and_slash", 3500, 100, 0.9172759056091309], "1": ["vertical_and_slash", 3500, 100, 0.6655579209327698], "2": ["vertical_and_slash", 500, 700, 0.9658140540122986], "3": ["vertical_and_slash", 1000, 6096, 0.8348773717880249], "4": ["vertical_and_slash", 3500, 100, 0.9243808388710022], "5": ["vertical_and_slash", 1000, 6096, 0.7087244987487793], "6": ["vertical_and_slash", 3500, 100, 0.885430097579956], "7": ["vertical_and_slash", 1000, 6096, 0.7224352359771729], "8": ["vertical_and_slash", 500, 700, 0.9705525636672974], "9": ["vertical_and_slash", 500, 700, 0.9376171231269836], "10": ["vertical_and_slash", 500, 700, 0.9767099618911743], "11": ["vertical_and_slash", 500, 700, 0.9616074562072754], "12": ["vertical_and_slash", 3500, 100, 0.8123517036437988], "13": ["vertical_and_slash", 100, 800, 0.87890625], "14": ["vertical_and_slash", 500, 700, 0.9025720953941345], "15": ["vertical_and_slash", 500, 700, 0.9541915655136108], "16": ["vertical_and_slash", 100, 800, 0.5390625], "17": ["vertical_and_slash", 100, 800, 0.73046875], "18": ["vertical_and_slash", 3500, 100, 0.4263802766799927], "19": ["vertical_and_slash", 1000, 6096, 0.9795095324516296], "20": ["vertical_and_slash", 100, 800, 0.91015625], "21": ["vertical_and_slash", 100, 800, 0.56640625], "22": ["vertical_and_slash", 1000, 6096, 0.9578346610069275], "23": ["vertical_and_slash", 100, 800, 0.92578125], "24": ["vertical_and_slash", 500, 700, 0.9528711438179016], "25": ["vertical_and_slash", 100, 800, 0.83984375], "26": ["vertical_and_slash", 100, 800, 0.7734375], "27": ["vertical_and_slash", 100, 800, 0.8984375], "28": ["vertical_and_slash", 500, 700, 0.9102920293807983], "29": ["vertical_and_slash", 500, 700, 0.976436197757721], "30": ["vertical_and_slash", 100, 800, 0.81640625], "31": ["vertical_and_slash", 3500, 100, 0.6377772092819214], "32": ["vertical_and_slash", 1000, 6096, 0.7069229483604431], "33": ["vertical_and_slash", 3500, 100, 0.6545475721359253], "34": ["vertical_and_slash", 1000, 6096, 0.674875795841217], "35": ["vertical_and_slash", 1000, 6096, 0.8407121896743774], "36": ["vertical_and_slash", 3500, 100, 0.7341983318328857], "37": ["vertical_and_slash", 3500, 100, 0.6213680505752563], "38": ["vertical_and_slash", 1000, 6096, 0.6595532298088074], "39": ["vertical_and_slash", 100, 800, 0.765625], "40": ["vertical_and_slash", 500, 700, 0.8786453008651733], "41": ["vertical_and_slash", 3500, 100, 0.7305452227592468], "42": ["vertical_and_slash", 3500, 100, 0.709612250328064], "43": ["vertical_and_slash", 3500, 100, 0.6395569443702698], "44": ["vertical_and_slash", 500, 700, 0.7893043756484985], "45": ["vertical_and_slash", 3500, 100, 0.6667971611022949], "46": ["vertical_and_slash", 100, 800, 0.7265625], "47": ["vertical_and_slash", 500, 700, 0.8869481086730957], "48": ["vertical_and_slash", 100, 800, 0.96484375], "49": ["vertical_and_slash", 100, 800, 0.94921875], "50": ["vertical_and_slash", 100, 800, 0.8203125], "51": ["vertical_and_slash", 100, 800, 0.953125], "52": ["vertical_and_slash", 100, 800, 0.9609375], "53": ["vertical_and_slash", 500, 700, 0.8488200902938843], "54": ["vertical_and_slash", 500, 700, 0.9650416970252991], "55": ["vertical_and_slash", 500, 700, 0.9758148193359375], "56": ["vertical_and_slash", 500, 700, 0.9511939287185669], "57": ["vertical_and_slash", 500, 700, 0.9830299019813538], "58": ["vertical_and_slash", 30, 800, 0.8381481766700745], "59": ["vertical_and_slash", 500, 700, 0.9488144516944885], "60": ["vertical_and_slash", 500, 700, 0.9706761240959167], "61": ["vertical_and_slash", 500, 700, 0.9084353446960449], "62": ["vertical_and_slash", 500, 700, 0.9478020071983337], "63": ["vertical_and_slash", 500, 700, 0.9546762108802795]}, {"0": ["vertical_and_slash", 100, 800, 0.6796875], "1": ["vertical_and_slash", 100, 800, 0.703125], "2": ["vertical_and_slash", 500, 700, 0.9602418541908264], "3": ["vertical_and_slash", 500, 700, 0.9170849323272705], "4": ["vertical_and_slash", 3500, 100, 0.9286561608314514], "5": ["vertical_and_slash", 3500, 100, 0.889431893825531], "6": ["vertical_and_slash", 500, 700, 0.9509568810462952], "7": ["vertical_and_slash", 500, 700, 0.9338938593864441], "8": ["vertical_and_slash", 3500, 100, 0.9657849073410034], "9": ["vertical_and_slash", 3500, 100, 0.9743061661720276], "10": ["vertical_and_slash", 3500, 100, 0.9789344668388367], "11": ["vertical_and_slash", 3500, 100, 0.9909844398498535], "12": ["vertical_and_slash", 3500, 100, 0.9827134013175964], "13": ["vertical_and_slash", 3500, 100, 0.9883438944816589], "14": ["vertical_and_slash", 3500, 100, 0.9891793727874756], "15": ["vertical_and_slash", 3500, 100, 0.9776980876922607], "16": ["vertical_and_slash", 500, 700, 0.7636292576789856], "17": ["vertical_and_slash", 500, 700, 0.8237202763557434], "18": ["vertical_and_slash", 500, 700, 0.8932206630706787], "19": ["vertical_and_slash", 1000, 6096, 0.7205848693847656], "20": ["vertical_and_slash", 3500, 100, 0.844261109828949], "21": ["vertical_and_slash", 500, 700, 0.7703114748001099], "22": ["vertical_and_slash", 3500, 100, 0.7517996430397034], "23": ["vertical_and_slash", 100, 800, 0.80078125], "24": ["vertical_and_slash", 3500, 100, 0.9880732297897339], "25": ["vertical_and_slash", 3500, 100, 0.9986861348152161], "26": ["vertical_and_slash", 1000, 6096, 0.9695584774017334], "27": ["vertical_and_slash", 3500, 100, 0.9974360466003418], "28": ["vertical_and_slash", 3500, 100, 0.9641760587692261], "29": ["vertical_and_slash", 1000, 6096, 0.995729923248291], "30": ["vertical_and_slash", 3500, 100, 0.9987977743148804], "31": ["vertical_and_slash", 3500, 100, 0.9926090240478516], "32": ["vertical_and_slash", 3500, 100, 0.9844653010368347], "33": ["vertical_and_slash", 3500, 100, 0.9889568090438843], "34": ["vertical_and_slash", 3500, 100, 0.9854952692985535], "35": ["vertical_and_slash", 3500, 100, 0.991062343120575], "36": ["vertical_and_slash", 3500, 100, 0.9899463057518005], "37": ["vertical_and_slash", 3500, 100, 0.9653140902519226], "38": ["vertical_and_slash", 3500, 100, 0.9510097503662109], "39": ["vertical_and_slash", 3500, 100, 0.9889203906059265], "40": ["vertical_and_slash", 3500, 100, 0.9164236187934875], "41": ["vertical_and_slash", 3500, 100, 0.9390261769294739], "42": ["vertical_and_slash", 3500, 100, 0.9891646504402161], "43": ["vertical_and_slash", 3500, 100, 0.7476050853729248], "44": ["vertical_and_slash", 3500, 100, 0.9039555788040161], "45": ["vertical_and_slash", 500, 700, 0.7526301741600037], "46": ["vertical_and_slash", 3500, 100, 0.9415871500968933], "47": ["vertical_and_slash", 3500, 100, 0.9802533388137817], "48": ["vertical_and_slash", 100, 800, 0.90234375], "49": ["vertical_and_slash", 100, 800, 0.96484375], "50": ["vertical_and_slash", 3500, 100, 0.9594748616218567], "51": ["vertical_and_slash", 500, 700, 0.9137744307518005], "52": ["vertical_and_slash", 3500, 100, 0.965060293674469], "53": ["vertical_and_slash", 500, 700, 0.850739598274231], "54": ["vertical_and_slash", 500, 700, 0.918722927570343], "55": ["vertical_and_slash", 3500, 100, 0.9696455597877502], "56": ["vertical_and_slash", 500, 700, 0.9771435856819153], "57": ["vertical_and_slash", 500, 700, 0.9490928053855896], "58": ["vertical_and_slash", 500, 700, 0.9604542255401611], "59": ["vertical_and_slash", 500, 700, 0.9582706689834595], "60": ["vertical_and_slash", 500, 700, 0.8896958827972412], "61": ["vertical_and_slash", 500, 700, 0.9331803917884827], "62": ["vertical_and_slash", 500, 700, 0.9472953081130981], "63": ["vertical_and_slash", 500, 700, 0.9622774124145508]}, {"0": ["vertical_and_slash", 500, 700, 0.9235531687736511], "1": ["vertical_and_slash", 3500, 100, 0.938729465007782], "2": ["vertical_and_slash", 500, 700, 0.9392189383506775], "3": ["vertical_and_slash", 500, 700, 0.8618362545967102], "4": ["vertical_and_slash", 1000, 6096, 0.8950901031494141], "5": ["vertical_and_slash", 500, 700, 0.8307701945304871], "6": ["vertical_and_slash", 3500, 100, 0.9725077748298645], "7": ["vertical_and_slash", 3500, 100, 0.9374240636825562], "8": ["vertical_and_slash", 100, 800, 0.87890625], "9": ["vertical_and_slash", 3500, 100, 0.660778820514679], "10": ["vertical_and_slash", 100, 800, 0.7109375], "11": ["vertical_and_slash", 3500, 100, 0.8093603253364563], "12": ["vertical_and_slash", 100, 800, 0.90234375], "13": ["vertical_and_slash", 500, 700, 0.9434791803359985], "14": ["vertical_and_slash", 3500, 100, 0.8032044768333435], "15": ["vertical_and_slash", 100, 800, 0.75390625], "16": ["vertical_and_slash", 1000, 6096, 0.725911021232605], "17": ["vertical_and_slash", 100, 800, 0.8515625], "18": ["vertical_and_slash", 3500, 100, 0.5619454979896545], "19": ["vertical_and_slash", 100, 800, 0.62890625], "20": ["vertical_and_slash", 500, 700, 0.9541657567024231], "21": ["vertical_and_slash", 1000, 6096, 0.41424816846847534], "22": ["vertical_and_slash", 3500, 100, 0.4934057295322418], "23": ["vertical_and_slash", 3500, 100, 0.5511322021484375], "24": ["vertical_and_slash", 500, 700, 0.8567164540290833], "25": ["vertical_and_slash", 500, 700, 0.8812995553016663], "26": ["vertical_and_slash", 3500, 100, 0.9403984546661377], "27": ["vertical_and_slash", 3500, 100, 0.9631072878837585], "28": ["vertical_and_slash", 3500, 100, 0.7775975465774536], "29": ["vertical_and_slash", 500, 700, 0.8957552313804626], "30": ["vertical_and_slash", 3500, 100, 0.7852135300636292], "31": ["vertical_and_slash", 500, 700, 0.8961012363433838], "32": ["vertical_and_slash", 1000, 6096, 0.5874782800674438], "33": ["vertical_and_slash", 3500, 100, 0.9084096550941467], "34": ["vertical_and_slash", 3500, 100, 0.6444074511528015], "35": ["vertical_and_slash", 100, 800, 0.75], "36": ["vertical_and_slash", 100, 800, 0.9296875], "37": ["vertical_and_slash", 500, 700, 0.9378244876861572], "38": ["vertical_and_slash", 500, 700, 0.7127028703689575], "39": ["vertical_and_slash", 500, 700, 0.7804259061813354], "40": ["vertical_and_slash", 3500, 100, 0.7858994603157043], "41": ["vertical_and_slash", 3500, 100, 0.6669465899467468], "42": ["vertical_and_slash", 3500, 100, 0.6791957020759583], "43": ["vertical_and_slash", 1000, 6096, 0.6457493901252747], "44": ["vertical_and_slash", 3500, 100, 0.7083989977836609], "45": ["vertical_and_slash", 1000, 6096, 0.5428474545478821], "46": ["vertical_and_slash", 500, 700, 0.8241465091705322], "47": ["vertical_and_slash", 1000, 6096, 0.6828351616859436], "48": ["vertical_and_slash", 500, 700, 0.9361491799354553], "49": ["vertical_and_slash", 100, 800, 0.8828125], "50": ["vertical_and_slash", 500, 700, 0.9272996783256531], "51": ["vertical_and_slash", 500, 700, 0.8529561758041382], "52": ["vertical_and_slash", 500, 700, 0.9525046944618225], "53": ["vertical_and_slash", 500, 700, 0.9412909150123596], "54": ["vertical_and_slash", 500, 700, 0.8498014807701111], "55": ["vertical_and_slash", 500, 700, 0.8889865875244141], "56": ["vertical_and_slash", 500, 700, 0.9671611189842224], "57": ["vertical_and_slash", 500, 700, 0.8608015179634094], "58": ["vertical_and_slash", 500, 700, 0.9474720358848572], "59": ["vertical_and_slash", 500, 700, 0.944932222366333], "60": ["vertical_and_slash", 500, 700, 0.9733250737190247], "61": ["vertical_and_slash", 500, 700, 0.9545419216156006], "62": ["vertical_and_slash", 500, 700, 0.9667558670043945], "63": ["vertical_and_slash", 500, 700, 0.9631653428077698]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7496695518493652], "1": ["vertical_and_slash", 3500, 100, 0.8596221804618835], "2": ["vertical_and_slash", 100, 800, 0.8671875], "3": ["vertical_and_slash", 1000, 6096, 0.8617557883262634], "4": ["vertical_and_slash", 3500, 100, 0.7287901639938354], "5": ["vertical_and_slash", 100, 800, 0.921875], "6": ["vertical_and_slash", 1000, 6096, 0.7381325364112854], "7": ["vertical_and_slash", 100, 800, 0.9140625], "8": ["vertical_and_slash", 1000, 6096, 0.9559597969055176], "9": ["vertical_and_slash", 3500, 100, 0.9879756569862366], "10": ["vertical_and_slash", 500, 700, 0.9510224461555481], "11": ["vertical_and_slash", 100, 800, 0.9296875], "12": ["vertical_and_slash", 500, 700, 0.9005899429321289], "13": ["vertical_and_slash", 1000, 6096, 0.9675498604774475], "14": ["vertical_and_slash", 1000, 6096, 0.9871469736099243], "15": ["vertical_and_slash", 3500, 100, 0.9716113805770874], "16": ["vertical_and_slash", 500, 700, 0.9559484124183655], "17": ["vertical_and_slash", 500, 700, 0.8463518619537354], "18": ["vertical_and_slash", 500, 700, 0.9214984178543091], "19": ["vertical_and_slash", 500, 700, 0.902097761631012], "20": ["vertical_and_slash", 500, 700, 0.936926007270813], "21": ["vertical_and_slash", 500, 700, 0.9576201438903809], "22": ["vertical_and_slash", 500, 700, 0.9675182700157166], "23": ["vertical_and_slash", 500, 700, 0.9140583872795105], "24": ["vertical_and_slash", 1000, 6096, 0.673351526260376], "25": ["vertical_and_slash", 1000, 6096, 0.5240767598152161], "26": ["vertical_and_slash", 3500, 100, 0.6004130840301514], "27": ["vertical_and_slash", 100, 800, 0.76171875], "28": ["vertical_and_slash", 100, 800, 0.93359375], "29": ["vertical_and_slash", 1000, 6096, 0.6656946539878845], "30": ["vertical_and_slash", 500, 700, 0.9567519426345825], "31": ["vertical_and_slash", 1000, 6096, 0.4885537028312683], "32": ["vertical_and_slash", 3500, 100, 0.9298751354217529], "33": ["vertical_and_slash", 3500, 100, 0.9406963586807251], "34": ["vertical_and_slash", 3500, 100, 0.8715591430664062], "35": ["vertical_and_slash", 3500, 100, 0.9440723657608032], "36": ["vertical_and_slash", 3500, 100, 0.9538466930389404], "37": ["vertical_and_slash", 3500, 100, 0.9129367470741272], "38": ["vertical_and_slash", 3500, 100, 0.9450848698616028], "39": ["vertical_and_slash", 3500, 100, 0.9099283218383789], "40": ["vertical_and_slash", 500, 700, 0.9198766350746155], "41": ["vertical_and_slash", 100, 800, 0.91015625], "42": ["vertical_and_slash", 1000, 6096, 0.6102993488311768], "43": ["vertical_and_slash", 500, 700, 0.9464604258537292], "44": ["vertical_and_slash", 100, 800, 0.5859375], "45": ["vertical_and_slash", 3500, 100, 0.6477950811386108], "46": ["vertical_and_slash", 500, 700, 0.9437583684921265], "47": ["vertical_and_slash", 500, 700, 0.8746417164802551], "48": ["vertical_and_slash", 100, 800, 0.88671875], "49": ["vertical_and_slash", 500, 700, 0.902003288269043], "50": ["vertical_and_slash", 1000, 6096, 0.772221028804779], "51": ["vertical_and_slash", 3500, 100, 0.8579270243644714], "52": ["vertical_and_slash", 500, 700, 0.9757109880447388], "53": ["vertical_and_slash", 500, 700, 0.9344776272773743], "54": ["vertical_and_slash", 500, 700, 0.9817276000976562], "55": ["vertical_and_slash", 500, 700, 0.8612959384918213], "56": ["vertical_and_slash", 3500, 100, 0.9692418575286865], "57": ["vertical_and_slash", 500, 700, 0.8868080377578735], "58": ["vertical_and_slash", 3500, 100, 0.9568169713020325], "59": ["vertical_and_slash", 500, 700, 0.9639961123466492], "60": ["vertical_and_slash", 3500, 100, 0.9795114398002625], "61": ["vertical_and_slash", 3500, 100, 0.9567192792892456], "62": ["vertical_and_slash", 3500, 100, 0.9630458354949951], "63": ["vertical_and_slash", 3500, 100, 0.9566053748130798]}, {"0": ["vertical_and_slash", 3500, 100, 0.9153379201889038], "1": ["vertical_and_slash", 3500, 100, 0.87638258934021], "2": ["vertical_and_slash", 3500, 100, 0.8938676714897156], "3": ["vertical_and_slash", 3500, 100, 0.9136584997177124], "4": ["vertical_and_slash", 3500, 100, 0.8962718844413757], "5": ["vertical_and_slash", 1000, 6096, 0.6956083178520203], "6": ["vertical_and_slash", 3500, 100, 0.8113754391670227], "7": ["vertical_and_slash", 1000, 6096, 0.710088849067688], "8": ["vertical_and_slash", 3500, 100, 0.9681419730186462], "9": ["vertical_and_slash", 3500, 100, 0.9801700711250305], "10": ["vertical_and_slash", 3500, 100, 0.9807161688804626], "11": ["vertical_and_slash", 3500, 100, 0.8965005278587341], "12": ["vertical_and_slash", 500, 700, 0.8242300152778625], "13": ["vertical_and_slash", 500, 700, 0.9477909803390503], "14": ["vertical_and_slash", 3500, 100, 0.9457080960273743], "15": ["vertical_and_slash", 1000, 6096, 0.9652479290962219], "16": ["vertical_and_slash", 3500, 100, 0.6683395504951477], "17": ["vertical_and_slash", 1000, 6096, 0.6481978297233582], "18": ["vertical_and_slash", 100, 800, 0.859375], "19": ["vertical_and_slash", 3500, 100, 0.6751890182495117], "20": ["vertical_and_slash", 3500, 100, 0.8317847847938538], "21": ["vertical_and_slash", 1000, 6096, 0.8239542245864868], "22": ["vertical_and_slash", 3500, 100, 0.662051260471344], "23": ["vertical_and_slash", 1000, 6096, 0.8980678915977478], "24": ["vertical_and_slash", 3500, 100, 0.8442667126655579], "25": ["vertical_and_slash", 3500, 100, 0.9205355048179626], "26": ["vertical_and_slash", 1000, 6096, 0.6343604922294617], "27": ["vertical_and_slash", 1000, 6096, 0.7298882007598877], "28": ["vertical_and_slash", 3500, 100, 0.9014506340026855], "29": ["vertical_and_slash", 100, 800, 0.77734375], "30": ["vertical_and_slash", 3500, 100, 0.9501014351844788], "31": ["vertical_and_slash", 100, 800, 0.95703125], "32": ["vertical_and_slash", 3500, 100, 0.8876692056655884], "33": ["vertical_and_slash", 100, 800, 0.78125], "34": ["vertical_and_slash", 100, 800, 0.67578125], "35": ["vertical_and_slash", 100, 800, 0.83984375], "36": ["vertical_and_slash", 100, 800, 0.81640625], "37": ["vertical_and_slash", 100, 800, 0.79296875], "38": ["vertical_and_slash", 3500, 100, 0.818053662776947], "39": ["vertical_and_slash", 3500, 100, 0.6810324788093567], "40": ["vertical_and_slash", 1000, 6096, 0.8810566663742065], "41": ["vertical_and_slash", 500, 700, 0.9502764344215393], "42": ["vertical_and_slash", 1000, 6096, 0.563878059387207], "43": ["vertical_and_slash", 1000, 6096, 0.7614567875862122], "44": ["vertical_and_slash", 1000, 6096, 0.8440126776695251], "45": ["vertical_and_slash", 1000, 6096, 0.5969464778900146], "46": ["vertical_and_slash", 1000, 6096, 0.8249104022979736], "47": ["vertical_and_slash", 1000, 6096, 0.6035175919532776], "48": ["vertical_and_slash", 100, 800, 0.921875], "49": ["vertical_and_slash", 500, 700, 0.788329541683197], "50": ["vertical_and_slash", 3500, 100, 0.8162152767181396], "51": ["vertical_and_slash", 500, 700, 0.9258014559745789], "52": ["vertical_and_slash", 3500, 100, 0.9572008848190308], "53": ["vertical_and_slash", 3500, 100, 0.8982542753219604], "54": ["vertical_and_slash", 100, 800, 0.90234375], "55": ["vertical_and_slash", 100, 800, 0.87890625], "56": ["vertical_and_slash", 1000, 6096, 0.9749533534049988], "57": ["vertical_and_slash", 100, 800, 0.84375], "58": ["vertical_and_slash", 100, 800, 0.828125], "59": ["vertical_and_slash", 1000, 6096, 0.9437289237976074], "60": ["vertical_and_slash", 1000, 6096, 0.971001148223877], "61": ["vertical_and_slash", 1000, 6096, 0.9878829121589661], "62": ["vertical_and_slash", 500, 700, 0.9077582359313965], "63": ["vertical_and_slash", 1000, 6096, 0.9547158479690552]}, {"0": ["vertical_and_slash", 500, 700, 0.9352920055389404], "1": ["vertical_and_slash", 500, 700, 0.948700487613678], "2": ["vertical_and_slash", 500, 700, 0.8576359152793884], "3": ["vertical_and_slash", 500, 700, 0.9658151268959045], "4": ["vertical_and_slash", 100, 800, 0.70703125], "5": ["vertical_and_slash", 100, 800, 0.734375], "6": ["vertical_and_slash", 500, 700, 0.9520484805107117], "7": ["vertical_and_slash", 500, 700, 0.9372457265853882], "8": ["vertical_and_slash", 3500, 100, 0.9784584641456604], "9": ["vertical_and_slash", 3500, 100, 0.9799786806106567], "10": ["vertical_and_slash", 3500, 100, 0.9834732413291931], "11": ["vertical_and_slash", 3500, 100, 0.7602543830871582], "12": ["vertical_and_slash", 3500, 100, 0.9541900157928467], "13": ["vertical_and_slash", 3500, 100, 0.9800408482551575], "14": ["vertical_and_slash", 1000, 6096, 0.9277165532112122], "15": ["vertical_and_slash", 1000, 6096, 0.933063805103302], "16": ["vertical_and_slash", 3500, 100, 0.8796552419662476], "17": ["vertical_and_slash", 1000, 6096, 0.8781232833862305], "18": ["vertical_and_slash", 3500, 100, 0.9360027313232422], "19": ["vertical_and_slash", 3500, 100, 0.957278847694397], "20": ["vertical_and_slash", 500, 700, 0.853188693523407], "21": ["vertical_and_slash", 3500, 100, 0.903051495552063], "22": ["vertical_and_slash", 3500, 100, 0.9258144497871399], "23": ["vertical_and_slash", 3500, 100, 0.9301741123199463], "24": ["vertical_and_slash", 3500, 100, 0.9913139939308167], "25": ["vertical_and_slash", 500, 700, 0.9256521463394165], "26": ["vertical_and_slash", 3500, 100, 0.983381450176239], "27": ["vertical_and_slash", 3500, 100, 0.9755795001983643], "28": ["vertical_and_slash", 3500, 100, 0.9933006763458252], "29": ["vertical_and_slash", 500, 700, 0.9814509749412537], "30": ["vertical_and_slash", 3500, 100, 0.9872087240219116], "31": ["vertical_and_slash", 3500, 100, 0.9820801019668579], "32": ["vertical_and_slash", 500, 700, 0.9364616274833679], "33": ["vertical_and_slash", 500, 700, 0.9520294666290283], "34": ["vertical_and_slash", 3500, 100, 0.8861096501350403], "35": ["vertical_and_slash", 3500, 100, 0.9379492402076721], "36": ["vertical_and_slash", 3500, 100, 0.9138891100883484], "37": ["vertical_and_slash", 3500, 100, 0.9530428051948547], "38": ["vertical_and_slash", 500, 700, 0.806837797164917], "39": ["vertical_and_slash", 3500, 100, 0.9335740804672241], "40": ["vertical_and_slash", 3500, 100, 0.7274299263954163], "41": ["vertical_and_slash", 3500, 100, 0.9369240999221802], "42": ["vertical_and_slash", 3500, 100, 0.9785604476928711], "43": ["vertical_and_slash", 3500, 100, 0.9354209303855896], "44": ["vertical_and_slash", 3500, 100, 0.9060103893280029], "45": ["vertical_and_slash", 1000, 6096, 0.9668133854866028], "46": ["vertical_and_slash", 3500, 100, 0.915110170841217], "47": ["vertical_and_slash", 3500, 100, 0.7563911080360413], "48": ["vertical_and_slash", 1000, 6096, 0.806316077709198], "49": ["vertical_and_slash", 3500, 100, 0.8748717904090881], "50": ["vertical_and_slash", 3500, 100, 0.9596402645111084], "51": ["vertical_and_slash", 3500, 100, 0.9528685808181763], "52": ["vertical_and_slash", 1000, 6096, 0.7452870607376099], "53": ["vertical_and_slash", 3500, 100, 0.9261314868927002], "54": ["vertical_and_slash", 3500, 100, 0.9404175281524658], "55": ["vertical_and_slash", 3500, 100, 0.9476838707923889], "56": ["vertical_and_slash", 3500, 100, 0.9164759516716003], "57": ["vertical_and_slash", 3500, 100, 0.9316505789756775], "58": ["vertical_and_slash", 3500, 100, 0.9816410541534424], "59": ["vertical_and_slash", 3500, 100, 0.9707901477813721], "60": ["vertical_and_slash", 3500, 100, 0.9735865592956543], "61": ["vertical_and_slash", 3500, 100, 0.9566919803619385], "62": ["vertical_and_slash", 3500, 100, 0.9782949686050415], "63": ["vertical_and_slash", 3500, 100, 0.9302451014518738]}, {"0": ["vertical_and_slash", 100, 800, 0.703125], "1": ["vertical_and_slash", 500, 700, 0.9259166717529297], "2": ["vertical_and_slash", 500, 700, 0.898754894733429], "3": ["vertical_and_slash", 500, 700, 0.8751533031463623], "4": ["vertical_and_slash", 500, 700, 0.8819247484207153], "5": ["vertical_and_slash", 500, 700, 0.9383598566055298], "6": ["vertical_and_slash", 500, 700, 0.8888697028160095], "7": ["vertical_and_slash", 500, 700, 0.9166496396064758], "8": ["vertical_and_slash", 3500, 100, 0.9523483514785767], "9": ["vertical_and_slash", 3500, 100, 0.9332773685455322], "10": ["vertical_and_slash", 3500, 100, 0.9380015134811401], "11": ["vertical_and_slash", 3500, 100, 0.9744426608085632], "12": ["vertical_and_slash", 3500, 100, 0.9765921235084534], "13": ["vertical_and_slash", 1000, 6096, 0.8886577486991882], "14": ["vertical_and_slash", 1000, 6096, 0.848466694355011], "15": ["vertical_and_slash", 3500, 100, 0.9449490904808044], "16": ["vertical_and_slash", 3500, 100, 0.9678999781608582], "17": ["vertical_and_slash", 3500, 100, 0.9480109214782715], "18": ["vertical_and_slash", 3500, 100, 0.9748749732971191], "19": ["vertical_and_slash", 3500, 100, 0.9631155729293823], "20": ["vertical_and_slash", 3500, 100, 0.9898808002471924], "21": ["vertical_and_slash", 3500, 100, 0.9752532839775085], "22": ["vertical_and_slash", 3500, 100, 0.9900683760643005], "23": ["vertical_and_slash", 3500, 100, 0.9149385690689087], "24": ["vertical_and_slash", 3500, 100, 0.8955833315849304], "25": ["vertical_and_slash", 3500, 100, 0.8899248242378235], "26": ["vertical_and_slash", 3500, 100, 0.9320335388183594], "27": ["vertical_and_slash", 3500, 100, 0.9765962362289429], "28": ["vertical_and_slash", 3500, 100, 0.9450191855430603], "29": ["vertical_and_slash", 3500, 100, 0.8273606896400452], "30": ["vertical_and_slash", 3500, 100, 0.782917857170105], "31": ["vertical_and_slash", 3500, 100, 0.9653434753417969], "32": ["vertical_and_slash", 3500, 100, 0.9754748940467834], "33": ["vertical_and_slash", 3500, 100, 0.9643625020980835], "34": ["vertical_and_slash", 3500, 100, 0.9740322828292847], "35": ["vertical_and_slash", 3500, 100, 0.9322476387023926], "36": ["vertical_and_slash", 3500, 100, 0.9177095293998718], "37": ["vertical_and_slash", 3500, 100, 0.7941846251487732], "38": ["vertical_and_slash", 3500, 100, 0.966869592666626], "39": ["vertical_and_slash", 3500, 100, 0.9647049307823181], "40": ["vertical_and_slash", 3500, 100, 0.9851105809211731], "41": ["vertical_and_slash", 3500, 100, 0.9630828499794006], "42": ["vertical_and_slash", 3500, 100, 0.9843681454658508], "43": ["vertical_and_slash", 3500, 100, 0.9261260628700256], "44": ["vertical_and_slash", 3500, 100, 0.9736993312835693], "45": ["vertical_and_slash", 3500, 100, 0.9744687676429749], "46": ["vertical_and_slash", 3500, 100, 0.9898709058761597], "47": ["vertical_and_slash", 3500, 100, 0.9602358341217041], "48": ["vertical_and_slash", 3500, 100, 0.9775543212890625], "49": ["vertical_and_slash", 3500, 100, 0.9529302716255188], "50": ["vertical_and_slash", 3500, 100, 0.9666684865951538], "51": ["vertical_and_slash", 3500, 100, 0.9736175537109375], "52": ["vertical_and_slash", 3500, 100, 0.9860132336616516], "53": ["vertical_and_slash", 3500, 100, 0.9540337920188904], "54": ["vertical_and_slash", 3500, 100, 0.9931196570396423], "55": ["vertical_and_slash", 3500, 100, 0.9757741093635559], "56": ["vertical_and_slash", 3500, 100, 0.9801194071769714], "57": ["vertical_and_slash", 3500, 100, 0.977241575717926], "58": ["vertical_and_slash", 3500, 100, 0.9919672012329102], "59": ["vertical_and_slash", 3500, 100, 0.9765520691871643], "60": ["vertical_and_slash", 1000, 6096, 0.937942624092102], "61": ["vertical_and_slash", 3500, 100, 0.9480440020561218], "62": ["vertical_and_slash", 3500, 100, 0.9801198840141296], "63": ["vertical_and_slash", 3500, 100, 0.9502633810043335]}, {"0": ["vertical_and_slash", 3500, 100, 0.9934690594673157], "1": ["vertical_and_slash", 500, 700, 0.977031409740448], "2": ["vertical_and_slash", 3500, 100, 0.9896852970123291], "3": ["vertical_and_slash", 3500, 100, 0.9852878451347351], "4": ["vertical_and_slash", 3500, 100, 0.9847687482833862], "5": ["vertical_and_slash", 3500, 100, 0.9897464513778687], "6": ["vertical_and_slash", 3500, 100, 0.9909552931785583], "7": ["vertical_and_slash", 500, 700, 0.9815772175788879], "8": ["vertical_and_slash", 500, 700, 0.9338489174842834], "9": ["vertical_and_slash", 3500, 100, 0.9900406002998352], "10": ["vertical_and_slash", 3500, 100, 0.9912142753601074], "11": ["vertical_and_slash", 3500, 100, 0.9818570017814636], "12": ["vertical_and_slash", 3500, 100, 0.9807837009429932], "13": ["vertical_and_slash", 3500, 100, 0.9874989986419678], "14": ["vertical_and_slash", 3500, 100, 0.9747102856636047], "15": ["vertical_and_slash", 3500, 100, 0.9813843369483948], "16": ["vertical_and_slash", 3500, 100, 0.8108372092247009], "17": ["vertical_and_slash", 500, 700, 0.9419264197349548], "18": ["vertical_and_slash", 3500, 100, 0.893466591835022], "19": ["vertical_and_slash", 500, 700, 0.9048703908920288], "20": ["vertical_and_slash", 3500, 100, 0.8463205099105835], "21": ["vertical_and_slash", 100, 800, 0.83203125], "22": ["vertical_and_slash", 3500, 100, 0.7748790979385376], "23": ["vertical_and_slash", 3500, 100, 0.7705393433570862], "24": ["vertical_and_slash", 100, 800, 0.94921875], "25": ["vertical_and_slash", 500, 700, 0.9470537304878235], "26": ["vertical_and_slash", 500, 700, 0.965074896812439], "27": ["vertical_and_slash", 500, 700, 0.9763477444648743], "28": ["vertical_and_slash", 100, 800, 0.796875], "29": ["vertical_and_slash", 500, 700, 0.9595227241516113], "30": ["vertical_and_slash", 100, 800, 0.828125], "31": ["vertical_and_slash", 500, 700, 0.9627025723457336], "32": ["vertical_and_slash", 3500, 100, 0.9902749061584473], "33": ["vertical_and_slash", 3500, 100, 0.9794493913650513], "34": ["vertical_and_slash", 3500, 100, 0.97124844789505], "35": ["vertical_and_slash", 3500, 100, 0.9872910976409912], "36": ["vertical_and_slash", 3500, 100, 0.9833245873451233], "37": ["vertical_and_slash", 3500, 100, 0.9702378511428833], "38": ["vertical_and_slash", 500, 700, 0.9443055391311646], "39": ["vertical_and_slash", 500, 700, 0.9601110219955444], "40": ["vertical_and_slash", 3500, 100, 0.9897782206535339], "41": ["vertical_and_slash", 3500, 100, 0.9835505485534668], "42": ["vertical_and_slash", 3500, 100, 0.9760687351226807], "43": ["vertical_and_slash", 3500, 100, 0.9825303554534912], "44": ["vertical_and_slash", 3500, 100, 0.9784859418869019], "45": ["vertical_and_slash", 3500, 100, 0.9468154311180115], "46": ["vertical_and_slash", 3500, 100, 0.9830037355422974], "47": ["vertical_and_slash", 3500, 100, 0.9833465814590454], "48": ["vertical_and_slash", 1000, 6096, 0.7487910985946655], "49": ["vertical_and_slash", 500, 700, 0.9148843884468079], "50": ["vertical_and_slash", 100, 800, 0.84765625], "51": ["vertical_and_slash", 100, 800, 0.89453125], "52": ["vertical_and_slash", 100, 800, 0.921875], "53": ["vertical_and_slash", 100, 800, 0.9375], "54": ["vertical_and_slash", 100, 800, 0.59375], "55": ["vertical_and_slash", 1000, 6096, 0.5525405406951904], "56": ["vertical_and_slash", 3500, 100, 0.9353383183479309], "57": ["vertical_and_slash", 3500, 100, 0.9773064851760864], "58": ["vertical_and_slash", 3500, 100, 0.9426538944244385], "59": ["vertical_and_slash", 500, 700, 0.9257447719573975], "60": ["vertical_and_slash", 3500, 100, 0.9396028518676758], "61": ["vertical_and_slash", 1000, 6096, 0.7888521552085876], "62": ["vertical_and_slash", 500, 700, 0.9019811749458313], "63": ["vertical_and_slash", 100, 800, 0.66796875]}, {"0": ["vertical_and_slash", 3500, 100, 0.9745257496833801], "1": ["vertical_and_slash", 3500, 100, 0.969102680683136], "2": ["vertical_and_slash", 3500, 100, 0.9710516929626465], "3": ["vertical_and_slash", 3500, 100, 0.9740806818008423], "4": ["vertical_and_slash", 3500, 100, 0.9713349938392639], "5": ["vertical_and_slash", 3500, 100, 0.9826104640960693], "6": ["vertical_and_slash", 3500, 100, 0.910589337348938], "7": ["vertical_and_slash", 3500, 100, 0.9639050960540771], "8": ["vertical_and_slash", 3500, 100, 0.9458454251289368], "9": ["vertical_and_slash", 3500, 100, 0.9027787446975708], "10": ["vertical_and_slash", 1000, 6096, 0.9464290142059326], "11": ["vertical_and_slash", 3500, 100, 0.8324559330940247], "12": ["vertical_and_slash", 3500, 100, 0.973550021648407], "13": ["vertical_and_slash", 1000, 6096, 0.8459972739219666], "14": ["vertical_and_slash", 1000, 6096, 0.7554816603660583], "15": ["vertical_and_slash", 100, 800, 0.89453125], "16": ["vertical_and_slash", 3500, 100, 0.9826359748840332], "17": ["vertical_and_slash", 3500, 100, 0.9592951536178589], "18": ["vertical_and_slash", 3500, 100, 0.9707708358764648], "19": ["vertical_and_slash", 3500, 100, 0.9894759058952332], "20": ["vertical_and_slash", 3500, 100, 0.9818575382232666], "21": ["vertical_and_slash", 3500, 100, 0.9790334701538086], "22": ["vertical_and_slash", 3500, 100, 0.9825162291526794], "23": ["vertical_and_slash", 3500, 100, 0.9846310019493103], "24": ["vertical_and_slash", 3500, 100, 0.9840637445449829], "25": ["vertical_and_slash", 3500, 100, 0.9851915836334229], "26": ["vertical_and_slash", 3500, 100, 0.9955214858055115], "27": ["vertical_and_slash", 3500, 100, 0.9925698041915894], "28": ["vertical_and_slash", 3500, 100, 0.9759266972541809], "29": ["vertical_and_slash", 3500, 100, 0.9711252450942993], "30": ["vertical_and_slash", 3500, 100, 0.9900737404823303], "31": ["vertical_and_slash", 3500, 100, 0.9612108469009399], "32": ["vertical_and_slash", 3500, 100, 0.944320559501648], "33": ["vertical_and_slash", 500, 700, 0.8272921442985535], "34": ["vertical_and_slash", 3500, 100, 0.9549692869186401], "35": ["vertical_and_slash", 3500, 100, 0.8344492316246033], "36": ["vertical_and_slash", 500, 700, 0.9452566504478455], "37": ["vertical_and_slash", 3500, 100, 0.9484704732894897], "38": ["vertical_and_slash", 3500, 100, 0.9613643288612366], "39": ["vertical_and_slash", 3500, 100, 0.8643355965614319], "40": ["vertical_and_slash", 100, 800, 0.85546875], "41": ["vertical_and_slash", 3500, 100, 0.7134413719177246], "42": ["vertical_and_slash", 100, 800, 0.66015625], "43": ["vertical_and_slash", 1000, 6096, 0.5391620993614197], "44": ["vertical_and_slash", 100, 800, 0.86328125], "45": ["vertical_and_slash", 3500, 100, 0.7891409993171692], "46": ["vertical_and_slash", 3500, 100, 0.870570182800293], "47": ["vertical_and_slash", 100, 800, 0.921875], "48": ["vertical_and_slash", 3500, 100, 0.9779072999954224], "49": ["vertical_and_slash", 3500, 100, 0.986322820186615], "50": ["vertical_and_slash", 3500, 100, 0.9394063949584961], "51": ["vertical_and_slash", 3500, 100, 0.9818272590637207], "52": ["vertical_and_slash", 3500, 100, 0.9892240166664124], "53": ["vertical_and_slash", 500, 700, 0.9575015306472778], "54": ["vertical_and_slash", 3500, 100, 0.9739760756492615], "55": ["vertical_and_slash", 3500, 100, 0.9925129413604736], "56": ["vertical_and_slash", 3500, 100, 0.9584961533546448], "57": ["vertical_and_slash", 3500, 100, 0.9801770448684692], "58": ["vertical_and_slash", 500, 700, 0.9663845896720886], "59": ["vertical_and_slash", 3500, 100, 0.9879083037376404], "60": ["vertical_and_slash", 500, 700, 0.9807072281837463], "61": ["vertical_and_slash", 3500, 100, 0.9742472171783447], "62": ["vertical_and_slash", 3500, 100, 0.9716566205024719], "63": ["vertical_and_slash", 500, 700, 0.9001070261001587]}, {"0": ["vertical_and_slash", 3500, 100, 0.9935299158096313], "1": ["vertical_and_slash", 3500, 100, 0.9741522073745728], "2": ["vertical_and_slash", 3500, 100, 0.9885899424552917], "3": ["vertical_and_slash", 3500, 100, 0.9880567789077759], "4": ["vertical_and_slash", 3500, 100, 0.9715696573257446], "5": ["vertical_and_slash", 3500, 100, 0.9552604556083679], "6": ["vertical_and_slash", 3500, 100, 0.9864245653152466], "7": ["vertical_and_slash", 3500, 100, 0.984980046749115], "8": ["vertical_and_slash", 100, 800, 0.91015625], "9": ["vertical_and_slash", 500, 700, 0.8580556511878967], "10": ["vertical_and_slash", 100, 800, 0.7734375], "11": ["vertical_and_slash", 3500, 100, 0.842710018157959], "12": ["vertical_and_slash", 100, 800, 0.78125], "13": ["vertical_and_slash", 3500, 100, 0.9513072967529297], "14": ["vertical_and_slash", 100, 800, 0.67578125], "15": ["vertical_and_slash", 500, 700, 0.8684537410736084], "16": ["vertical_and_slash", 3500, 100, 0.9953277111053467], "17": ["vertical_and_slash", 3500, 100, 0.9935365915298462], "18": ["vertical_and_slash", 3500, 100, 0.9909681677818298], "19": ["vertical_and_slash", 3500, 100, 0.99338299036026], "20": ["vertical_and_slash", 3500, 100, 0.9857259392738342], "21": ["vertical_and_slash", 3500, 100, 0.9918326139450073], "22": ["vertical_and_slash", 3500, 100, 0.9856480956077576], "23": ["vertical_and_slash", 3500, 100, 0.9808263778686523], "24": ["vertical_and_slash", 3500, 100, 0.9967392086982727], "25": ["vertical_and_slash", 3500, 100, 0.9926450848579407], "26": ["vertical_and_slash", 3500, 100, 0.9949225783348083], "27": ["vertical_and_slash", 3500, 100, 0.9974654912948608], "28": ["vertical_and_slash", 3500, 100, 0.9973580837249756], "29": ["vertical_and_slash", 3500, 100, 0.9927332997322083], "30": ["vertical_and_slash", 3500, 100, 0.9952042102813721], "31": ["vertical_and_slash", 3500, 100, 0.9948554635047913], "32": ["vertical_and_slash", 3500, 100, 0.8797567486763], "33": ["vertical_and_slash", 3500, 100, 0.9925482273101807], "34": ["vertical_and_slash", 100, 800, 0.89453125], "35": ["vertical_and_slash", 100, 800, 0.94921875], "36": ["vertical_and_slash", 3500, 100, 0.9622527360916138], "37": ["vertical_and_slash", 3500, 100, 0.957059919834137], "38": ["vertical_and_slash", 3500, 100, 0.9871107935905457], "39": ["vertical_and_slash", 100, 800, 0.8125], "40": ["vertical_and_slash", 3500, 100, 0.9916707873344421], "41": ["vertical_and_slash", 3500, 100, 0.9805683493614197], "42": ["vertical_and_slash", 3500, 100, 0.9954753518104553], "43": ["vertical_and_slash", 3500, 100, 0.9904047250747681], "44": ["vertical_and_slash", 3500, 100, 0.9873695969581604], "45": ["vertical_and_slash", 3500, 100, 0.9931991696357727], "46": ["vertical_and_slash", 3500, 100, 0.9964242577552795], "47": ["vertical_and_slash", 3500, 100, 0.995173454284668], "48": ["vertical_and_slash", 3500, 100, 0.9120794534683228], "49": ["vertical_and_slash", 500, 700, 0.9393628239631653], "50": ["vertical_and_slash", 500, 700, 0.964061975479126], "51": ["vertical_and_slash", 500, 700, 0.9630538821220398], "52": ["vertical_and_slash", 500, 700, 0.913459300994873], "53": ["vertical_and_slash", 3500, 100, 0.9621788859367371], "54": ["vertical_and_slash", 500, 700, 0.9659594893455505], "55": ["vertical_and_slash", 500, 700, 0.9328891634941101], "56": ["vertical_and_slash", 3500, 100, 0.9869107604026794], "57": ["vertical_and_slash", 3500, 100, 0.9847235083580017], "58": ["vertical_and_slash", 3500, 100, 0.9923204183578491], "59": ["vertical_and_slash", 3500, 100, 0.9966049790382385], "60": ["vertical_and_slash", 3500, 100, 0.993318498134613], "61": ["vertical_and_slash", 3500, 100, 0.9799386262893677], "62": ["vertical_and_slash", 3500, 100, 0.9924190640449524], "63": ["vertical_and_slash", 3500, 100, 0.9961683750152588]}, {"0": ["vertical_and_slash", 3500, 100, 0.9834356307983398], "1": ["vertical_and_slash", 3500, 100, 0.9815507531166077], "2": ["vertical_and_slash", 3500, 100, 0.9913938641548157], "3": ["vertical_and_slash", 500, 700, 0.9356886744499207], "4": ["vertical_and_slash", 3500, 100, 0.9716065526008606], "5": ["vertical_and_slash", 3500, 100, 0.9788938760757446], "6": ["vertical_and_slash", 3500, 100, 0.984035313129425], "7": ["vertical_and_slash", 3500, 100, 0.9798508882522583], "8": ["vertical_and_slash", 3500, 100, 0.9920605421066284], "9": ["vertical_and_slash", 3500, 100, 0.9939397573471069], "10": ["vertical_and_slash", 3500, 100, 0.9874736070632935], "11": ["vertical_and_slash", 3500, 100, 0.9942572712898254], "12": ["vertical_and_slash", 3500, 100, 0.9905843734741211], "13": ["vertical_and_slash", 3500, 100, 0.9906211495399475], "14": ["vertical_and_slash", 3500, 100, 0.9898806214332581], "15": ["vertical_and_slash", 3500, 100, 0.98800128698349], "16": ["vertical_and_slash", 500, 700, 0.811134397983551], "17": ["vertical_and_slash", 3500, 100, 0.9599683880805969], "18": ["vertical_and_slash", 3500, 100, 0.9574970602989197], "19": ["vertical_and_slash", 3500, 100, 0.8907107710838318], "20": ["vertical_and_slash", 3500, 100, 0.9689314365386963], "21": ["vertical_and_slash", 3500, 100, 0.9797499179840088], "22": ["vertical_and_slash", 500, 700, 0.940491795539856], "23": ["vertical_and_slash", 500, 700, 0.9730205535888672], "24": ["vertical_and_slash", 500, 700, 0.9629628658294678], "25": ["vertical_and_slash", 500, 700, 0.962997555732727], "26": ["vertical_and_slash", 500, 700, 0.9754246473312378], "27": ["vertical_and_slash", 500, 700, 0.9294710755348206], "28": ["vertical_and_slash", 500, 700, 0.967545211315155], "29": ["vertical_and_slash", 500, 700, 0.9709930419921875], "30": ["vertical_and_slash", 500, 700, 0.9758651256561279], "31": ["vertical_and_slash", 500, 700, 0.978417158126831], "32": ["vertical_and_slash", 3500, 100, 0.9873017072677612], "33": ["vertical_and_slash", 3500, 100, 0.9952558875083923], "34": ["vertical_and_slash", 3500, 100, 0.9853914976119995], "35": ["vertical_and_slash", 3500, 100, 0.9931790232658386], "36": ["vertical_and_slash", 3500, 100, 0.9892403483390808], "37": ["vertical_and_slash", 3500, 100, 0.9933624863624573], "38": ["vertical_and_slash", 3500, 100, 0.9948121905326843], "39": ["vertical_and_slash", 3500, 100, 0.988161027431488], "40": ["vertical_and_slash", 3500, 100, 0.9537132382392883], "41": ["vertical_and_slash", 3500, 100, 0.9472135901451111], "42": ["vertical_and_slash", 3500, 100, 0.9118943214416504], "43": ["vertical_and_slash", 3500, 100, 0.9868097901344299], "44": ["vertical_and_slash", 3500, 100, 0.9778892993927002], "45": ["vertical_and_slash", 3500, 100, 0.9888788461685181], "46": ["vertical_and_slash", 3500, 100, 0.9746713638305664], "47": ["vertical_and_slash", 3500, 100, 0.9856075644493103], "48": ["vertical_and_slash", 3500, 100, 0.9772860407829285], "49": ["vertical_and_slash", 3500, 100, 0.9804226756095886], "50": ["vertical_and_slash", 3500, 100, 0.9969807267189026], "51": ["vertical_and_slash", 3500, 100, 0.9949020743370056], "52": ["vertical_and_slash", 3500, 100, 0.9787827730178833], "53": ["vertical_and_slash", 3500, 100, 0.9909366369247437], "54": ["vertical_and_slash", 3500, 100, 0.9919124245643616], "55": ["vertical_and_slash", 3500, 100, 0.9955978989601135], "56": ["vertical_and_slash", 3500, 100, 0.8990845680236816], "57": ["vertical_and_slash", 3500, 100, 0.9309099316596985], "58": ["vertical_and_slash", 100, 800, 0.625], "59": ["vertical_and_slash", 3500, 100, 0.96012943983078], "60": ["vertical_and_slash", 3500, 100, 0.9819222688674927], "61": ["vertical_and_slash", 100, 800, 0.82421875], "62": ["vertical_and_slash", 3500, 100, 0.7997073531150818], "63": ["vertical_and_slash", 3500, 100, 0.8978152275085449]}, {"0": ["vertical_and_slash", 3500, 100, 0.9860507249832153], "1": ["vertical_and_slash", 3500, 100, 0.9693625569343567], "2": ["vertical_and_slash", 3500, 100, 0.9881510138511658], "3": ["vertical_and_slash", 500, 700, 0.9455029368400574], "4": ["vertical_and_slash", 3500, 100, 0.9909356236457825], "5": ["vertical_and_slash", 3500, 100, 0.9874776005744934], "6": ["vertical_and_slash", 3500, 100, 0.9801960587501526], "7": ["vertical_and_slash", 3500, 100, 0.9825074672698975], "8": ["vertical_and_slash", 3500, 100, 0.9693286418914795], "9": ["vertical_and_slash", 3500, 100, 0.9832403063774109], "10": ["vertical_and_slash", 500, 700, 0.9666092991828918], "11": ["vertical_and_slash", 500, 700, 0.9436779022216797], "12": ["vertical_and_slash", 500, 700, 0.9733625650405884], "13": ["vertical_and_slash", 500, 700, 0.9608787894248962], "14": ["vertical_and_slash", 500, 700, 0.9602070450782776], "15": ["vertical_and_slash", 3500, 100, 0.9707927107810974], "16": ["vertical_and_slash", 500, 700, 0.9479504823684692], "17": ["vertical_and_slash", 3500, 100, 0.9939893484115601], "18": ["vertical_and_slash", 3500, 100, 0.9864140152931213], "19": ["vertical_and_slash", 3500, 100, 0.9816831946372986], "20": ["vertical_and_slash", 3500, 100, 0.9704391956329346], "21": ["vertical_and_slash", 3500, 100, 0.989727795124054], "22": ["vertical_and_slash", 3500, 100, 0.9797508120536804], "23": ["vertical_and_slash", 3500, 100, 0.9825446605682373], "24": ["vertical_and_slash", 3500, 100, 0.9675376415252686], "25": ["vertical_and_slash", 100, 800, 0.9140625], "26": ["vertical_and_slash", 100, 800, 0.83984375], "27": ["vertical_and_slash", 3500, 100, 0.9225685000419617], "28": ["vertical_and_slash", 100, 800, 0.765625], "29": ["vertical_and_slash", 3500, 100, 0.980011522769928], "30": ["vertical_and_slash", 100, 800, 0.859375], "31": ["vertical_and_slash", 100, 800, 0.84375], "32": ["vertical_and_slash", 3500, 100, 0.9910426735877991], "33": ["vertical_and_slash", 3500, 100, 0.9954135417938232], "34": ["vertical_and_slash", 3500, 100, 0.9950270056724548], "35": ["vertical_and_slash", 3500, 100, 0.994208812713623], "36": ["vertical_and_slash", 3500, 100, 0.9957065582275391], "37": ["vertical_and_slash", 3500, 100, 0.9899305105209351], "38": ["vertical_and_slash", 3500, 100, 0.9971266984939575], "39": ["vertical_and_slash", 3500, 100, 0.9940054416656494], "40": ["vertical_and_slash", 3500, 100, 0.9949353933334351], "41": ["vertical_and_slash", 3500, 100, 0.9925987720489502], "42": ["vertical_and_slash", 3500, 100, 0.9946073293685913], "43": ["vertical_and_slash", 3500, 100, 0.9900208115577698], "44": ["vertical_and_slash", 3500, 100, 0.9938176870346069], "45": ["vertical_and_slash", 3500, 100, 0.9908909201622009], "46": ["vertical_and_slash", 3500, 100, 0.9901870489120483], "47": ["vertical_and_slash", 3500, 100, 0.9860982298851013], "48": ["vertical_and_slash", 3500, 100, 0.9902120232582092], "49": ["vertical_and_slash", 3500, 100, 0.9837114214897156], "50": ["vertical_and_slash", 3500, 100, 0.9894671440124512], "51": ["vertical_and_slash", 3500, 100, 0.9886361956596375], "52": ["vertical_and_slash", 500, 700, 0.8847241401672363], "53": ["vertical_and_slash", 3500, 100, 0.9899260997772217], "54": ["vertical_and_slash", 3500, 100, 0.9812637567520142], "55": ["vertical_and_slash", 3500, 100, 0.9770070910453796], "56": ["vertical_and_slash", 3500, 100, 0.9782301783561707], "57": ["vertical_and_slash", 500, 700, 0.9272943139076233], "58": ["vertical_and_slash", 3500, 100, 0.9888419508934021], "59": ["vertical_and_slash", 3500, 100, 0.993484616279602], "60": ["vertical_and_slash", 3500, 100, 0.9835225343704224], "61": ["vertical_and_slash", 3500, 100, 0.9899589419364929], "62": ["vertical_and_slash", 3500, 100, 0.9904535412788391], "63": ["vertical_and_slash", 3500, 100, 0.9910566210746765]}, {"0": ["vertical_and_slash", 30, 800, 0.9816769361495972], "1": ["vertical_and_slash", 500, 700, 0.9688439965248108], "2": ["vertical_and_slash", 1000, 6096, 0.7536623477935791], "3": ["vertical_and_slash", 30, 800, 0.8840263485908508], "4": ["vertical_and_slash", 30, 800, 0.8360317945480347], "5": ["vertical_and_slash", 30, 800, 0.9631949067115784], "6": ["vertical_and_slash", 500, 700, 0.9829679131507874], "7": ["vertical_and_slash", 30, 800, 0.9669715762138367], "8": ["vertical_and_slash", 3500, 100, 0.9929385185241699], "9": ["vertical_and_slash", 3500, 100, 0.9509998559951782], "10": ["vertical_and_slash", 1000, 6096, 0.9403776526451111], "11": ["vertical_and_slash", 3500, 100, 0.9891856908798218], "12": ["vertical_and_slash", 3500, 100, 0.9785104990005493], "13": ["vertical_and_slash", 3500, 100, 0.9891200661659241], "14": ["vertical_and_slash", 3500, 100, 0.983830988407135], "15": ["vertical_and_slash", 3500, 100, 0.9903037548065186], "16": ["vertical_and_slash", 3500, 100, 0.9970307946205139], "17": ["vertical_and_slash", 3500, 100, 0.9875391721725464], "18": ["vertical_and_slash", 3500, 100, 0.9916441440582275], "19": ["vertical_and_slash", 3500, 100, 0.9846616387367249], "20": ["vertical_and_slash", 3500, 100, 0.9970266819000244], "21": ["vertical_and_slash", 3500, 100, 0.9977745413780212], "22": ["vertical_and_slash", 3500, 100, 0.9950437545776367], "23": ["vertical_and_slash", 3500, 100, 0.9896721243858337], "24": ["vertical_and_slash", 3500, 100, 0.9948613047599792], "25": ["vertical_and_slash", 3500, 100, 0.9882466197013855], "26": ["vertical_and_slash", 3500, 100, 0.9844822883605957], "27": ["vertical_and_slash", 3500, 100, 0.9926570653915405], "28": ["vertical_and_slash", 3500, 100, 0.9934661984443665], "29": ["vertical_and_slash", 3500, 100, 0.9907051920890808], "30": ["vertical_and_slash", 3500, 100, 0.9959378838539124], "31": ["vertical_and_slash", 3500, 100, 0.9873634576797485], "32": ["vertical_and_slash", 3500, 100, 0.9973869919776917], "33": ["vertical_and_slash", 3500, 100, 0.9969012141227722], "34": ["vertical_and_slash", 3500, 100, 0.99776291847229], "35": ["vertical_and_slash", 3500, 100, 0.9987270832061768], "36": ["vertical_and_slash", 3500, 100, 0.9965413212776184], "37": ["vertical_and_slash", 3500, 100, 0.9973158836364746], "38": ["vertical_and_slash", 3500, 100, 0.9952020645141602], "39": ["vertical_and_slash", 3500, 100, 0.9979771375656128], "40": ["vertical_and_slash", 3500, 100, 0.9957018494606018], "41": ["vertical_and_slash", 3500, 100, 0.9917395114898682], "42": ["vertical_and_slash", 3500, 100, 0.9981345534324646], "43": ["vertical_and_slash", 3500, 100, 0.9967067241668701], "44": ["vertical_and_slash", 3500, 100, 0.9985161423683167], "45": ["vertical_and_slash", 3500, 100, 0.9912196397781372], "46": ["vertical_and_slash", 3500, 100, 0.998462975025177], "47": ["vertical_and_slash", 3500, 100, 0.993273913860321], "48": ["vertical_and_slash", 3500, 100, 0.9966496825218201], "49": ["vertical_and_slash", 3500, 100, 0.9972293376922607], "50": ["vertical_and_slash", 3500, 100, 0.995913028717041], "51": ["vertical_and_slash", 3500, 100, 0.9969287514686584], "52": ["vertical_and_slash", 3500, 100, 0.9956841468811035], "53": ["vertical_and_slash", 3500, 100, 0.9974303245544434], "54": ["vertical_and_slash", 3500, 100, 0.9977185130119324], "55": ["vertical_and_slash", 3500, 100, 0.9959943294525146], "56": ["vertical_and_slash", 3500, 100, 0.9770470261573792], "57": ["vertical_and_slash", 3500, 100, 0.9812949895858765], "58": ["vertical_and_slash", 3500, 100, 0.9618099927902222], "59": ["vertical_and_slash", 3500, 100, 0.9889312982559204], "60": ["vertical_and_slash", 3500, 100, 0.9852684736251831], "61": ["vertical_and_slash", 500, 700, 0.9630783796310425], "62": ["vertical_and_slash", 3500, 100, 0.988717257976532], "63": ["vertical_and_slash", 3500, 100, 0.9677683115005493]}, {"0": ["vertical_and_slash", 100, 800, 0.85546875], "1": ["vertical_and_slash", 100, 800, 0.64453125], "2": ["vertical_and_slash", 3500, 100, 0.8751624226570129], "3": ["vertical_and_slash", 3500, 100, 0.8419883251190186], "4": ["vertical_and_slash", 3500, 100, 0.9655344486236572], "5": ["vertical_and_slash", 1000, 6096, 0.9812319278717041], "6": ["vertical_and_slash", 3500, 100, 0.9319475889205933], "7": ["vertical_and_slash", 100, 800, 0.859375], "8": ["vertical_and_slash", 3500, 100, 0.9865992069244385], "9": ["vertical_and_slash", 3500, 100, 0.9928697347640991], "10": ["vertical_and_slash", 3500, 100, 0.993285596370697], "11": ["vertical_and_slash", 3500, 100, 0.9941803812980652], "12": ["vertical_and_slash", 3500, 100, 0.9953170418739319], "13": ["vertical_and_slash", 3500, 100, 0.9915094971656799], "14": ["vertical_and_slash", 3500, 100, 0.9939022064208984], "15": ["vertical_and_slash", 3500, 100, 0.9941838979721069], "16": ["vertical_and_slash", 3500, 100, 0.9970489144325256], "17": ["vertical_and_slash", 3500, 100, 0.9907398223876953], "18": ["vertical_and_slash", 3500, 100, 0.9953538775444031], "19": ["vertical_and_slash", 3500, 100, 0.9974677562713623], "20": ["vertical_and_slash", 3500, 100, 0.9967694878578186], "21": ["vertical_and_slash", 3500, 100, 0.9965317249298096], "22": ["vertical_and_slash", 3500, 100, 0.9895948171615601], "23": ["vertical_and_slash", 3500, 100, 0.9925476312637329], "24": ["vertical_and_slash", 3500, 100, 0.9974475502967834], "25": ["vertical_and_slash", 3500, 100, 0.9792030453681946], "26": ["vertical_and_slash", 3500, 100, 0.9945592284202576], "27": ["vertical_and_slash", 3500, 100, 0.9957191944122314], "28": ["vertical_and_slash", 3500, 100, 0.9986262321472168], "29": ["vertical_and_slash", 3500, 100, 0.991987407207489], "30": ["vertical_and_slash", 3500, 100, 0.9950934052467346], "31": ["vertical_and_slash", 3500, 100, 0.9955022931098938], "32": ["vertical_and_slash", 3500, 100, 0.9095386862754822], "33": ["vertical_and_slash", 500, 700, 0.9351682066917419], "34": ["vertical_and_slash", 500, 700, 0.9671248197555542], "35": ["vertical_and_slash", 500, 700, 0.9519439339637756], "36": ["vertical_and_slash", 3500, 100, 0.9619711637496948], "37": ["vertical_and_slash", 500, 700, 0.9298387765884399], "38": ["vertical_and_slash", 100, 800, 0.8671875], "39": ["vertical_and_slash", 3500, 100, 0.9285886287689209], "40": ["vertical_and_slash", 1000, 6096, 0.9405380487442017], "41": ["vertical_and_slash", 100, 800, 0.85546875], "42": ["vertical_and_slash", 1000, 6096, 0.9839639663696289], "43": ["vertical_and_slash", 1000, 6096, 0.9902369379997253], "44": ["vertical_and_slash", 100, 800, 0.78125], "45": ["vertical_and_slash", 100, 800, 0.97265625], "46": ["vertical_and_slash", 1000, 6096, 0.9582470655441284], "47": ["vertical_and_slash", 3500, 100, 0.9837421774864197], "48": ["vertical_and_slash", 500, 700, 0.8284077048301697], "49": ["vertical_and_slash", 3500, 100, 0.985299289226532], "50": ["vertical_and_slash", 3500, 100, 0.9616895318031311], "51": ["vertical_and_slash", 3500, 100, 0.9924027919769287], "52": ["vertical_and_slash", 3500, 100, 0.957042932510376], "53": ["vertical_and_slash", 3500, 100, 0.963946521282196], "54": ["vertical_and_slash", 3500, 100, 0.8480279445648193], "55": ["vertical_and_slash", 3500, 100, 0.9630613327026367], "56": ["vertical_and_slash", 500, 700, 0.9712311029434204], "57": ["vertical_and_slash", 500, 700, 0.9818218350410461], "58": ["vertical_and_slash", 500, 700, 0.9877148270606995], "59": ["vertical_and_slash", 3500, 100, 0.9862560033798218], "60": ["vertical_and_slash", 3500, 100, 0.9899246096611023], "61": ["vertical_and_slash", 3500, 100, 0.9860796928405762], "62": ["vertical_and_slash", 500, 700, 0.9458436965942383], "63": ["vertical_and_slash", 3500, 100, 0.9892349243164062]}, {"0": ["vertical_and_slash", 500, 700, 0.9821614027023315], "1": ["vertical_and_slash", 1000, 6096, 0.5930508971214294], "2": ["vertical_and_slash", 500, 700, 0.8632054328918457], "3": ["vertical_and_slash", 100, 750, 0.859388530254364], "4": ["vertical_and_slash", 500, 700, 0.9629453420639038], "5": ["vertical_and_slash", 500, 700, 0.9819124341011047], "6": ["vertical_and_slash", 500, 700, 0.9654081463813782], "7": ["vertical_and_slash", 3500, 100, 0.984787106513977], "8": ["vertical_and_slash", 3500, 100, 0.998702883720398], "9": ["vertical_and_slash", 3500, 100, 0.9960705041885376], "10": ["vertical_and_slash", 3500, 100, 0.9961130023002625], "11": ["vertical_and_slash", 3500, 100, 0.9905411005020142], "12": ["vertical_and_slash", 3500, 100, 0.9990281462669373], "13": ["vertical_and_slash", 3500, 100, 0.9958359003067017], "14": ["vertical_and_slash", 3500, 100, 0.9961923360824585], "15": ["vertical_and_slash", 3500, 100, 0.9961417317390442], "16": ["vertical_and_slash", 3500, 100, 0.9958252906799316], "17": ["vertical_and_slash", 3500, 100, 0.9841646552085876], "18": ["vertical_and_slash", 3500, 100, 0.9953019618988037], "19": ["vertical_and_slash", 3500, 100, 0.9967029690742493], "20": ["vertical_and_slash", 3500, 100, 0.997063159942627], "21": ["vertical_and_slash", 3500, 100, 0.9988467693328857], "22": ["vertical_and_slash", 3500, 100, 0.9900544881820679], "23": ["vertical_and_slash", 3500, 100, 0.9954469203948975], "24": ["vertical_and_slash", 3500, 100, 0.984004020690918], "25": ["vertical_and_slash", 3500, 100, 0.989708662033081], "26": ["vertical_and_slash", 3500, 100, 0.9933953881263733], "27": ["vertical_and_slash", 3500, 100, 0.9962702393531799], "28": ["vertical_and_slash", 3500, 100, 0.9941362142562866], "29": ["vertical_and_slash", 3500, 100, 0.9726771712303162], "30": ["vertical_and_slash", 500, 700, 0.9883086085319519], "31": ["vertical_and_slash", 3500, 100, 0.9945148825645447], "32": ["vertical_and_slash", 3500, 100, 0.9973352551460266], "33": ["vertical_and_slash", 3500, 100, 0.9988137483596802], "34": ["vertical_and_slash", 3500, 100, 0.9975069165229797], "35": ["vertical_and_slash", 3500, 100, 0.9940058588981628], "36": ["vertical_and_slash", 3500, 100, 0.9924278259277344], "37": ["vertical_and_slash", 3500, 100, 0.9980859160423279], "38": ["vertical_and_slash", 3500, 100, 0.9933356046676636], "39": ["vertical_and_slash", 3500, 100, 0.9971199035644531], "40": ["vertical_and_slash", 3500, 100, 0.9950771927833557], "41": ["vertical_and_slash", 3500, 100, 0.9930292963981628], "42": ["vertical_and_slash", 3500, 100, 0.9716687798500061], "43": ["vertical_and_slash", 3500, 100, 0.9920291900634766], "44": ["vertical_and_slash", 3500, 100, 0.992365837097168], "45": ["vertical_and_slash", 500, 700, 0.9190462827682495], "46": ["vertical_and_slash", 3500, 100, 0.9944526553153992], "47": ["vertical_and_slash", 3500, 100, 0.9952993988990784], "48": ["vertical_and_slash", 100, 800, 0.8203125], "49": ["vertical_and_slash", 3500, 100, 0.9714290499687195], "50": ["vertical_and_slash", 500, 700, 0.9782711267471313], "51": ["vertical_and_slash", 500, 700, 0.9202145934104919], "52": ["vertical_and_slash", 3500, 100, 0.9753976464271545], "53": ["vertical_and_slash", 3500, 100, 0.9307593703269958], "54": ["vertical_and_slash", 3500, 100, 0.983466386795044], "55": ["vertical_and_slash", 3500, 100, 0.9713626503944397], "56": ["vertical_and_slash", 3500, 100, 0.9970031976699829], "57": ["vertical_and_slash", 3500, 100, 0.9987886548042297], "58": ["vertical_and_slash", 3500, 100, 0.9979972243309021], "59": ["vertical_and_slash", 3500, 100, 0.9940574169158936], "60": ["vertical_and_slash", 3500, 100, 0.9967668652534485], "61": ["vertical_and_slash", 3500, 100, 0.9889846444129944], "62": ["vertical_and_slash", 3500, 100, 0.9984423518180847], "63": ["vertical_and_slash", 3500, 100, 0.9178576469421387]}, {"0": ["vertical_and_slash", 3500, 100, 0.994283139705658], "1": ["vertical_and_slash", 3500, 100, 0.9870986342430115], "2": ["vertical_and_slash", 3500, 100, 0.9937490820884705], "3": ["vertical_and_slash", 3500, 100, 0.9814319014549255], "4": ["vertical_and_slash", 3500, 100, 0.9965795278549194], "5": ["vertical_and_slash", 3500, 100, 0.9901399612426758], "6": ["vertical_and_slash", 3500, 100, 0.9920057058334351], "7": ["vertical_and_slash", 3500, 100, 0.9932083487510681], "8": ["vertical_and_slash", 3500, 100, 0.9983678460121155], "9": ["vertical_and_slash", 3500, 100, 0.9952306747436523], "10": ["vertical_and_slash", 3500, 100, 0.9971240162849426], "11": ["vertical_and_slash", 3500, 100, 0.996796190738678], "12": ["vertical_and_slash", 3500, 100, 0.9980627298355103], "13": ["vertical_and_slash", 3500, 100, 0.9980696439743042], "14": ["vertical_and_slash", 3500, 100, 0.9957737922668457], "15": ["vertical_and_slash", 3500, 100, 0.9942795038223267], "16": ["vertical_and_slash", 3500, 100, 0.995792031288147], "17": ["vertical_and_slash", 500, 700, 0.9892464876174927], "18": ["vertical_and_slash", 3500, 100, 0.9937695264816284], "19": ["vertical_and_slash", 3500, 100, 0.9939377307891846], "20": ["vertical_and_slash", 3500, 100, 0.9971799850463867], "21": ["vertical_and_slash", 3500, 100, 0.9859187602996826], "22": ["vertical_and_slash", 3500, 100, 0.9923368692398071], "23": ["vertical_and_slash", 3500, 100, 0.9946647882461548], "24": ["vertical_and_slash", 3500, 100, 0.9926993250846863], "25": ["vertical_and_slash", 3500, 100, 0.9839783906936646], "26": ["vertical_and_slash", 3500, 100, 0.9957607984542847], "27": ["vertical_and_slash", 3500, 100, 0.9946277141571045], "28": ["vertical_and_slash", 3500, 100, 0.9937255382537842], "29": ["vertical_and_slash", 3500, 100, 0.9968739748001099], "30": ["vertical_and_slash", 3500, 100, 0.9940816164016724], "31": ["vertical_and_slash", 3500, 100, 0.9936948418617249], "32": ["vertical_and_slash", 3500, 100, 0.995246410369873], "33": ["vertical_and_slash", 3500, 100, 0.9893045425415039], "34": ["vertical_and_slash", 3500, 100, 0.9925962090492249], "35": ["vertical_and_slash", 3500, 100, 0.9973395466804504], "36": ["vertical_and_slash", 3500, 100, 0.9932698011398315], "37": ["vertical_and_slash", 3500, 100, 0.978549599647522], "38": ["vertical_and_slash", 3500, 100, 0.9971792697906494], "39": ["vertical_and_slash", 3500, 100, 0.9948020577430725], "40": ["vertical_and_slash", 3500, 100, 0.9974785447120667], "41": ["vertical_and_slash", 3500, 100, 0.9915496706962585], "42": ["vertical_and_slash", 3500, 100, 0.9960010647773743], "43": ["vertical_and_slash", 3500, 100, 0.991628885269165], "44": ["vertical_and_slash", 3500, 100, 0.9899886250495911], "45": ["vertical_and_slash", 3500, 100, 0.986937940120697], "46": ["vertical_and_slash", 3500, 100, 0.9980350136756897], "47": ["vertical_and_slash", 3500, 100, 0.9934993982315063], "48": ["vertical_and_slash", 3500, 100, 0.9980757832527161], "49": ["vertical_and_slash", 3500, 100, 0.9857726693153381], "50": ["vertical_and_slash", 3500, 100, 0.9936213493347168], "51": ["vertical_and_slash", 500, 700, 0.9929655194282532], "52": ["vertical_and_slash", 3500, 100, 0.9974775910377502], "53": ["vertical_and_slash", 3500, 100, 0.9962310791015625], "54": ["vertical_and_slash", 3500, 100, 0.9881971478462219], "55": ["vertical_and_slash", 3500, 100, 0.9941026568412781], "56": ["vertical_and_slash", 3500, 100, 0.9926061034202576], "57": ["vertical_and_slash", 3500, 100, 0.9954981207847595], "58": ["vertical_and_slash", 3500, 100, 0.985428512096405], "59": ["vertical_and_slash", 3500, 100, 0.9956464767456055], "60": ["vertical_and_slash", 3500, 100, 0.9979294538497925], "61": ["vertical_and_slash", 3500, 100, 0.9962117075920105], "62": ["vertical_and_slash", 3500, 100, 0.992692768573761], "63": ["vertical_and_slash", 3500, 100, 0.9846969246864319]}, {"0": ["vertical_and_slash", 3500, 100, 0.9965339303016663], "1": ["vertical_and_slash", 3500, 100, 0.9963349103927612], "2": ["vertical_and_slash", 3500, 100, 0.9940310716629028], "3": ["vertical_and_slash", 3500, 100, 0.9894582629203796], "4": ["vertical_and_slash", 3500, 100, 0.9937266111373901], "5": ["vertical_and_slash", 3500, 100, 0.9971659183502197], "6": ["vertical_and_slash", 3500, 100, 0.9907942414283752], "7": ["vertical_and_slash", 3500, 100, 0.9948437809944153], "8": ["vertical_and_slash", 3500, 100, 0.995910108089447], "9": ["vertical_and_slash", 3500, 100, 0.9886879920959473], "10": ["vertical_and_slash", 3500, 100, 0.9833970069885254], "11": ["vertical_and_slash", 3500, 100, 0.9980002045631409], "12": ["vertical_and_slash", 3500, 100, 0.9932418465614319], "13": ["vertical_and_slash", 3500, 100, 0.9934983849525452], "14": ["vertical_and_slash", 3500, 100, 0.9921889305114746], "15": ["vertical_and_slash", 3500, 100, 0.9972502589225769], "16": ["vertical_and_slash", 1000, 6096, 0.8982585072517395], "17": ["vertical_and_slash", 3500, 100, 0.9044343829154968], "18": ["vertical_and_slash", 3500, 100, 0.7550644278526306], "19": ["vertical_and_slash", 1000, 6096, 0.9826245903968811], "20": ["vertical_and_slash", 1000, 6096, 0.8757326602935791], "21": ["vertical_and_slash", 3500, 100, 0.9962483048439026], "22": ["vertical_and_slash", 1000, 6096, 0.7055431008338928], "23": ["vertical_and_slash", 3500, 100, 0.9754273891448975], "24": ["vertical_and_slash", 3500, 100, 0.9979358911514282], "25": ["vertical_and_slash", 3500, 100, 0.9970053434371948], "26": ["vertical_and_slash", 3500, 100, 0.99580979347229], "27": ["vertical_and_slash", 3500, 100, 0.9930216073989868], "28": ["vertical_and_slash", 3500, 100, 0.9983382821083069], "29": ["vertical_and_slash", 3500, 100, 0.9939486980438232], "30": ["vertical_and_slash", 3500, 100, 0.9972152709960938], "31": ["vertical_and_slash", 3500, 100, 0.9985945820808411], "32": ["vertical_and_slash", 3500, 100, 0.9881453514099121], "33": ["vertical_and_slash", 3500, 100, 0.9974541068077087], "34": ["vertical_and_slash", 3500, 100, 0.9954564571380615], "35": ["vertical_and_slash", 3500, 100, 0.9956981539726257], "36": ["vertical_and_slash", 3500, 100, 0.9970194697380066], "37": ["vertical_and_slash", 3500, 100, 0.9940815567970276], "38": ["vertical_and_slash", 3500, 100, 0.9957315921783447], "39": ["vertical_and_slash", 3500, 100, 0.9891277551651001], "40": ["vertical_and_slash", 3500, 100, 0.9942750930786133], "41": ["vertical_and_slash", 100, 750, 0.9808269143104553], "42": ["vertical_and_slash", 3500, 100, 0.9985926151275635], "43": ["vertical_and_slash", 3500, 100, 0.9938944578170776], "44": ["vertical_and_slash", 3500, 100, 0.9751749038696289], "45": ["vertical_and_slash", 3500, 100, 0.9940256476402283], "46": ["vertical_and_slash", 3500, 100, 0.9984766840934753], "47": ["vertical_and_slash", 3500, 100, 0.985347330570221], "48": ["vertical_and_slash", 3500, 100, 0.995014488697052], "49": ["vertical_and_slash", 3500, 100, 0.998054563999176], "50": ["vertical_and_slash", 3500, 100, 0.9970176219940186], "51": ["vertical_and_slash", 3500, 100, 0.9970603585243225], "52": ["vertical_and_slash", 3500, 100, 0.9987606406211853], "53": ["vertical_and_slash", 3500, 100, 0.995428740978241], "54": ["vertical_and_slash", 3500, 100, 0.9970635175704956], "55": ["vertical_and_slash", 3500, 100, 0.9983212351799011], "56": ["vertical_and_slash", 3500, 100, 0.9937512278556824], "57": ["vertical_and_slash", 3500, 100, 0.9957647919654846], "58": ["vertical_and_slash", 3500, 100, 0.9770709276199341], "59": ["vertical_and_slash", 3500, 100, 0.9930875301361084], "60": ["vertical_and_slash", 3500, 100, 0.9609254002571106], "61": ["vertical_and_slash", 3500, 100, 0.9635921120643616], "62": ["vertical_and_slash", 3500, 100, 0.9659416675567627], "63": ["vertical_and_slash", 3500, 100, 0.9923191070556641]}, {"0": ["vertical_and_slash", 3500, 100, 0.9940589070320129], "1": ["vertical_and_slash", 3500, 100, 0.9953606724739075], "2": ["vertical_and_slash", 3500, 100, 0.9982944130897522], "3": ["vertical_and_slash", 3500, 100, 0.9960057139396667], "4": ["vertical_and_slash", 3500, 100, 0.9919970035552979], "5": ["vertical_and_slash", 3500, 100, 0.9968030452728271], "6": ["vertical_and_slash", 3500, 100, 0.9942511916160583], "7": ["vertical_and_slash", 3500, 100, 0.9968409538269043], "8": ["vertical_and_slash", 3500, 100, 0.9931003451347351], "9": ["vertical_and_slash", 3500, 100, 0.9946414232254028], "10": ["vertical_and_slash", 3500, 100, 0.9785905480384827], "11": ["vertical_and_slash", 3500, 100, 0.9929383993148804], "12": ["vertical_and_slash", 3500, 100, 0.9939600229263306], "13": ["vertical_and_slash", 3500, 100, 0.9977303147315979], "14": ["vertical_and_slash", 3500, 100, 0.9969484806060791], "15": ["vertical_and_slash", 3500, 100, 0.993472695350647], "16": ["vertical_and_slash", 3500, 100, 0.9944469928741455], "17": ["vertical_and_slash", 3500, 100, 0.9937347769737244], "18": ["vertical_and_slash", 3500, 100, 0.9924288392066956], "19": ["vertical_and_slash", 3500, 100, 0.9938857555389404], "20": ["vertical_and_slash", 3500, 100, 0.9924522638320923], "21": ["vertical_and_slash", 3500, 100, 0.983268141746521], "22": ["vertical_and_slash", 3500, 100, 0.9917649626731873], "23": ["vertical_and_slash", 3500, 100, 0.9979719519615173], "24": ["vertical_and_slash", 3500, 100, 0.9928731322288513], "25": ["vertical_and_slash", 3500, 100, 0.9911989569664001], "26": ["vertical_and_slash", 3500, 100, 0.9921943545341492], "27": ["vertical_and_slash", 3500, 100, 0.9939458966255188], "28": ["vertical_and_slash", 3500, 100, 0.9928938746452332], "29": ["vertical_and_slash", 3500, 100, 0.9953910112380981], "30": ["vertical_and_slash", 3500, 100, 0.9914188981056213], "31": ["vertical_and_slash", 3500, 100, 0.9963929057121277], "32": ["vertical_and_slash", 3500, 100, 0.9610835313796997], "33": ["vertical_and_slash", 3500, 100, 0.9311060905456543], "34": ["vertical_and_slash", 3500, 100, 0.9828798770904541], "35": ["vertical_and_slash", 3500, 100, 0.9846517443656921], "36": ["vertical_and_slash", 3500, 100, 0.9766188859939575], "37": ["vertical_and_slash", 3500, 100, 0.9925597906112671], "38": ["vertical_and_slash", 3500, 100, 0.9806798696517944], "39": ["vertical_and_slash", 3500, 100, 0.9844620227813721], "40": ["vertical_and_slash", 3500, 100, 0.9892520308494568], "41": ["vertical_and_slash", 3500, 100, 0.9943017363548279], "42": ["vertical_and_slash", 3500, 100, 0.9947935938835144], "43": ["vertical_and_slash", 3500, 100, 0.996826708316803], "44": ["vertical_and_slash", 3500, 100, 0.9948967695236206], "45": ["vertical_and_slash", 3500, 100, 0.9946262836456299], "46": ["vertical_and_slash", 3500, 100, 0.9970338344573975], "47": ["vertical_and_slash", 3500, 100, 0.9972960948944092], "48": ["vertical_and_slash", 500, 700, 0.9648767709732056], "49": ["vertical_and_slash", 3500, 100, 0.9891539216041565], "50": ["vertical_and_slash", 3500, 100, 0.9845811128616333], "51": ["vertical_and_slash", 500, 700, 0.9547650814056396], "52": ["vertical_and_slash", 3500, 100, 0.8830257654190063], "53": ["vertical_and_slash", 3500, 100, 0.8833856582641602], "54": ["vertical_and_slash", 500, 700, 0.880495011806488], "55": ["vertical_and_slash", 3500, 100, 0.9961242079734802], "56": ["vertical_and_slash", 1000, 6096, 0.6288138628005981], "57": ["vertical_and_slash", 1000, 6096, 0.9587975144386292], "58": ["vertical_and_slash", 100, 800, 0.75390625], "59": ["vertical_and_slash", 100, 800, 0.8125], "60": ["vertical_and_slash", 100, 800, 0.77734375], "61": ["vertical_and_slash", 500, 700, 0.8634182810783386], "62": ["vertical_and_slash", 500, 700, 0.954070508480072], "63": ["vertical_and_slash", 100, 800, 0.8359375]}, {"0": ["vertical_and_slash", 3500, 100, 0.926410973072052], "1": ["vertical_and_slash", 500, 700, 0.9038847088813782], "2": ["vertical_and_slash", 3500, 100, 0.8084004521369934], "3": ["vertical_and_slash", 3500, 100, 0.8985547423362732], "4": ["vertical_and_slash", 3500, 100, 0.8292893767356873], "5": ["vertical_and_slash", 3500, 100, 0.877776563167572], "6": ["vertical_and_slash", 500, 700, 0.9220594167709351], "7": ["vertical_and_slash", 500, 700, 0.8458794355392456], "8": ["vertical_and_slash", 500, 700, 0.9653024077415466], "9": ["vertical_and_slash", 500, 700, 0.9477277994155884], "10": ["vertical_and_slash", 3500, 100, 0.9791892170906067], "11": ["vertical_and_slash", 3500, 100, 0.9959557056427002], "12": ["vertical_and_slash", 3500, 100, 0.9955291152000427], "13": ["vertical_and_slash", 3500, 100, 0.9955999851226807], "14": ["vertical_and_slash", 3500, 100, 0.9893903732299805], "15": ["vertical_and_slash", 3500, 100, 0.9947712421417236], "16": ["vertical_and_slash", 3500, 100, 0.9952905774116516], "17": ["vertical_and_slash", 3500, 100, 0.9872400164604187], "18": ["vertical_and_slash", 3500, 100, 0.9920352101325989], "19": ["vertical_and_slash", 3500, 100, 0.9842081665992737], "20": ["vertical_and_slash", 3500, 100, 0.9937992691993713], "21": ["vertical_and_slash", 3500, 100, 0.993089497089386], "22": ["vertical_and_slash", 3500, 100, 0.9877390265464783], "23": ["vertical_and_slash", 3500, 100, 0.9937437176704407], "24": ["vertical_and_slash", 3500, 100, 0.9988968372344971], "25": ["vertical_and_slash", 3500, 100, 0.9977028965950012], "26": ["vertical_and_slash", 3500, 100, 0.9926161766052246], "27": ["vertical_and_slash", 3500, 100, 0.9969656467437744], "28": ["vertical_and_slash", 3500, 100, 0.9970042705535889], "29": ["vertical_and_slash", 3500, 100, 0.9965638518333435], "30": ["vertical_and_slash", 3500, 100, 0.9983795881271362], "31": ["vertical_and_slash", 3500, 100, 0.9983185529708862], "32": ["vertical_and_slash", 3500, 100, 0.9948440790176392], "33": ["vertical_and_slash", 3500, 100, 0.9884504675865173], "34": ["vertical_and_slash", 3500, 100, 0.9932392835617065], "35": ["vertical_and_slash", 3500, 100, 0.9937313795089722], "36": ["vertical_and_slash", 3500, 100, 0.995603084564209], "37": ["vertical_and_slash", 3500, 100, 0.9904265403747559], "38": ["vertical_and_slash", 3500, 100, 0.9942371845245361], "39": ["vertical_and_slash", 3500, 100, 0.9889329075813293], "40": ["vertical_and_slash", 3500, 100, 0.9978731870651245], "41": ["vertical_and_slash", 3500, 100, 0.9969605803489685], "42": ["vertical_and_slash", 3500, 100, 0.9954173564910889], "43": ["vertical_and_slash", 3500, 100, 0.9983218908309937], "44": ["vertical_and_slash", 3500, 100, 0.9987098574638367], "45": ["vertical_and_slash", 3500, 100, 0.9976196885108948], "46": ["vertical_and_slash", 3500, 100, 0.9967032074928284], "47": ["vertical_and_slash", 3500, 100, 0.999004602432251], "48": ["vertical_and_slash", 3500, 100, 0.9769386053085327], "49": ["vertical_and_slash", 3500, 100, 0.9921500086784363], "50": ["vertical_and_slash", 3500, 100, 0.9865761399269104], "51": ["vertical_and_slash", 3500, 100, 0.9960470795631409], "52": ["vertical_and_slash", 3500, 100, 0.989920437335968], "53": ["vertical_and_slash", 3500, 100, 0.9823832511901855], "54": ["vertical_and_slash", 3500, 100, 0.9895340204238892], "55": ["vertical_and_slash", 3500, 100, 0.993971586227417], "56": ["vertical_and_slash", 500, 700, 0.8515392541885376], "57": ["vertical_and_slash", 3500, 100, 0.9856919050216675], "58": ["vertical_and_slash", 3500, 100, 0.9814953804016113], "59": ["vertical_and_slash", 3500, 100, 0.9759620428085327], "60": ["vertical_and_slash", 3500, 100, 0.9683717489242554], "61": ["vertical_and_slash", 3500, 100, 0.9599020481109619], "62": ["vertical_and_slash", 500, 700, 0.981633722782135], "63": ["vertical_and_slash", 500, 700, 0.962720513343811]}, {"0": ["vertical_and_slash", 500, 700, 0.9760509729385376], "1": ["vertical_and_slash", 100, 800, 0.9140625], "2": ["vertical_and_slash", 3500, 100, 0.9803258776664734], "3": ["vertical_and_slash", 3500, 100, 0.9945576190948486], "4": ["vertical_and_slash", 3500, 100, 0.9919577240943909], "5": ["vertical_and_slash", 500, 700, 0.9638975262641907], "6": ["vertical_and_slash", 3500, 100, 0.9863293766975403], "7": ["vertical_and_slash", 3500, 100, 0.9873749017715454], "8": ["vertical_and_slash", 3500, 100, 0.9962354898452759], "9": ["vertical_and_slash", 3500, 100, 0.9980999827384949], "10": ["vertical_and_slash", 3500, 100, 0.9986320734024048], "11": ["vertical_and_slash", 3500, 100, 0.9971606135368347], "12": ["vertical_and_slash", 3500, 100, 0.9924837946891785], "13": ["vertical_and_slash", 3500, 100, 0.9986775517463684], "14": ["vertical_and_slash", 3500, 100, 0.9965189695358276], "15": ["vertical_and_slash", 3500, 100, 0.9981424808502197], "16": ["vertical_and_slash", 3500, 100, 0.9960253834724426], "17": ["vertical_and_slash", 3500, 100, 0.9927490949630737], "18": ["vertical_and_slash", 3500, 100, 0.9969534873962402], "19": ["vertical_and_slash", 3500, 100, 0.9950013756752014], "20": ["vertical_and_slash", 3500, 100, 0.997197687625885], "21": ["vertical_and_slash", 3500, 100, 0.9960795044898987], "22": ["vertical_and_slash", 3500, 100, 0.9981025457382202], "23": ["vertical_and_slash", 3500, 100, 0.9835009574890137], "24": ["vertical_and_slash", 3500, 100, 0.9957877993583679], "25": ["vertical_and_slash", 3500, 100, 0.9961029887199402], "26": ["vertical_and_slash", 3500, 100, 0.9972015023231506], "27": ["vertical_and_slash", 3500, 100, 0.9978209137916565], "28": ["vertical_and_slash", 3500, 100, 0.9982097148895264], "29": ["vertical_and_slash", 3500, 100, 0.9966439008712769], "30": ["vertical_and_slash", 3500, 100, 0.9976173639297485], "31": ["vertical_and_slash", 3500, 100, 0.996286928653717], "32": ["vertical_and_slash", 3500, 100, 0.9608461260795593], "33": ["vertical_and_slash", 3500, 100, 0.9700272679328918], "34": ["vertical_and_slash", 3500, 100, 0.9571788907051086], "35": ["vertical_and_slash", 3500, 100, 0.9793148040771484], "36": ["vertical_and_slash", 3500, 100, 0.971154510974884], "37": ["vertical_and_slash", 3500, 100, 0.9744235873222351], "38": ["vertical_and_slash", 3500, 100, 0.9425231218338013], "39": ["vertical_and_slash", 3500, 100, 0.9525507092475891], "40": ["vertical_and_slash", 3500, 100, 0.9913869500160217], "41": ["vertical_and_slash", 3500, 100, 0.9919694066047668], "42": ["vertical_and_slash", 3500, 100, 0.9947969913482666], "43": ["vertical_and_slash", 3500, 100, 0.9970285892486572], "44": ["vertical_and_slash", 3500, 100, 0.9961259961128235], "45": ["vertical_and_slash", 3500, 100, 0.9963821768760681], "46": ["vertical_and_slash", 3500, 100, 0.9963488578796387], "47": ["vertical_and_slash", 3500, 100, 0.9960923194885254], "48": ["vertical_and_slash", 3500, 100, 0.9648990035057068], "49": ["vertical_and_slash", 3500, 100, 0.9795606732368469], "50": ["vertical_and_slash", 500, 700, 0.9871711730957031], "51": ["vertical_and_slash", 3500, 100, 0.9599869847297668], "52": ["vertical_and_slash", 3500, 100, 0.953072190284729], "53": ["vertical_and_slash", 3500, 100, 0.9827073812484741], "54": ["vertical_and_slash", 3500, 100, 0.9802069067955017], "55": ["vertical_and_slash", 3500, 100, 0.989281177520752], "56": ["vertical_and_slash", 3500, 100, 0.9912860989570618], "57": ["vertical_and_slash", 3500, 100, 0.9951843619346619], "58": ["vertical_and_slash", 3500, 100, 0.9977142810821533], "59": ["vertical_and_slash", 3500, 100, 0.9952535629272461], "60": ["vertical_and_slash", 3500, 100, 0.9775490164756775], "61": ["vertical_and_slash", 3500, 100, 0.9915897846221924], "62": ["vertical_and_slash", 3500, 100, 0.9967782497406006], "63": ["vertical_and_slash", 3500, 100, 0.9949506521224976]}, {"0": ["vertical_and_slash", 100, 750, 0.899582028388977], "1": ["vertical_and_slash", 100, 750, 0.822944700717926], "2": ["vertical_and_slash", 500, 700, 0.9583812355995178], "3": ["vertical_and_slash", 100, 750, 0.7988901734352112], "4": ["vertical_and_slash", 100, 800, 0.7265625], "5": ["vertical_and_slash", 100, 800, 0.66796875], "6": ["vertical_and_slash", 500, 700, 0.8682586550712585], "7": ["vertical_and_slash", 500, 700, 0.9809157848358154], "8": ["vertical_and_slash", 3500, 100, 0.9959195256233215], "9": ["vertical_and_slash", 3500, 100, 0.9976243376731873], "10": ["vertical_and_slash", 3500, 100, 0.9950311183929443], "11": ["vertical_and_slash", 3500, 100, 0.9928670525550842], "12": ["vertical_and_slash", 3500, 100, 0.9880411028862], "13": ["vertical_and_slash", 3500, 100, 0.9934726357460022], "14": ["vertical_and_slash", 3500, 100, 0.9978863000869751], "15": ["vertical_and_slash", 3500, 100, 0.9970613718032837], "16": ["vertical_and_slash", 100, 750, 0.9189301133155823], "17": ["vertical_and_slash", 3500, 100, 0.9888627529144287], "18": ["vertical_and_slash", 3500, 100, 0.9846489429473877], "19": ["vertical_and_slash", 3500, 100, 0.991469144821167], "20": ["vertical_and_slash", 3500, 100, 0.9848438501358032], "21": ["vertical_and_slash", 3500, 100, 0.9861941933631897], "22": ["vertical_and_slash", 500, 700, 0.989849865436554], "23": ["vertical_and_slash", 3500, 100, 0.993848979473114], "24": ["vertical_and_slash", 3500, 100, 0.9696059823036194], "25": ["vertical_and_slash", 3500, 100, 0.9628701210021973], "26": ["vertical_and_slash", 3500, 100, 0.9468668699264526], "27": ["vertical_and_slash", 3500, 100, 0.9666162133216858], "28": ["vertical_and_slash", 3500, 100, 0.9182108044624329], "29": ["vertical_and_slash", 3500, 100, 0.9490826725959778], "30": ["vertical_and_slash", 3500, 100, 0.9458636045455933], "31": ["vertical_and_slash", 3500, 100, 0.9182499647140503], "32": ["vertical_and_slash", 3500, 100, 0.996895432472229], "33": ["vertical_and_slash", 3500, 100, 0.9838885068893433], "34": ["vertical_and_slash", 3500, 100, 0.9968536496162415], "35": ["vertical_and_slash", 3500, 100, 0.9938068985939026], "36": ["vertical_and_slash", 3500, 100, 0.9981303215026855], "37": ["vertical_and_slash", 3500, 100, 0.9987578988075256], "38": ["vertical_and_slash", 3500, 100, 0.9957799315452576], "39": ["vertical_and_slash", 3500, 100, 0.9975001215934753], "40": ["vertical_and_slash", 3500, 100, 0.9744022488594055], "41": ["vertical_and_slash", 3500, 100, 0.9962021708488464], "42": ["vertical_and_slash", 3500, 100, 0.9911454916000366], "43": ["vertical_and_slash", 3500, 100, 0.9872938394546509], "44": ["vertical_and_slash", 3500, 100, 0.9959802031517029], "45": ["vertical_and_slash", 3500, 100, 0.9935950636863708], "46": ["vertical_and_slash", 3500, 100, 0.9928322434425354], "47": ["vertical_and_slash", 3500, 100, 0.9829630851745605], "48": ["vertical_and_slash", 3500, 100, 0.9885193109512329], "49": ["vertical_and_slash", 3500, 100, 0.9965454339981079], "50": ["vertical_and_slash", 3500, 100, 0.9968684315681458], "51": ["vertical_and_slash", 3500, 100, 0.9974493980407715], "52": ["vertical_and_slash", 3500, 100, 0.9963216781616211], "53": ["vertical_and_slash", 3500, 100, 0.9973174929618835], "54": ["vertical_and_slash", 3500, 100, 0.9947272539138794], "55": ["vertical_and_slash", 3500, 100, 0.9965949654579163], "56": ["vertical_and_slash", 3500, 100, 0.9846914410591125], "57": ["vertical_and_slash", 3500, 100, 0.9953823685646057], "58": ["vertical_and_slash", 3500, 100, 0.9926524758338928], "59": ["vertical_and_slash", 3500, 100, 0.9822392463684082], "60": ["vertical_and_slash", 3500, 100, 0.9701448082923889], "61": ["vertical_and_slash", 3500, 100, 0.9886531829833984], "62": ["vertical_and_slash", 3500, 100, 0.9747079610824585], "63": ["vertical_and_slash", 3500, 100, 0.9967947006225586]}, {"0": ["vertical_and_slash", 3500, 100, 0.9728377461433411], "1": ["vertical_and_slash", 3500, 100, 0.9736676812171936], "2": ["vertical_and_slash", 500, 700, 0.9334978461265564], "3": ["vertical_and_slash", 3500, 100, 0.9716323018074036], "4": ["vertical_and_slash", 3500, 100, 0.9865375757217407], "5": ["vertical_and_slash", 3500, 100, 0.9747716188430786], "6": ["vertical_and_slash", 3500, 100, 0.9609605669975281], "7": ["vertical_and_slash", 3500, 100, 0.9628746509552002], "8": ["vertical_and_slash", 3500, 100, 0.9968886971473694], "9": ["vertical_and_slash", 3500, 100, 0.9928338527679443], "10": ["vertical_and_slash", 3500, 100, 0.9972191452980042], "11": ["vertical_and_slash", 3500, 100, 0.996264636516571], "12": ["vertical_and_slash", 3500, 100, 0.9906057715415955], "13": ["vertical_and_slash", 3500, 100, 0.9926996231079102], "14": ["vertical_and_slash", 3500, 100, 0.9975998401641846], "15": ["vertical_and_slash", 3500, 100, 0.9963307976722717], "16": ["vertical_and_slash", 500, 700, 0.7711327075958252], "17": ["vertical_and_slash", 1000, 6096, 0.9118080735206604], "18": ["vertical_and_slash", 3500, 100, 0.9166032671928406], "19": ["vertical_and_slash", 3500, 100, 0.8764269948005676], "20": ["vertical_and_slash", 3500, 100, 0.9672272205352783], "21": ["vertical_and_slash", 1000, 6096, 0.8808735609054565], "22": ["vertical_and_slash", 100, 800, 0.703125], "23": ["vertical_and_slash", 3500, 100, 0.8352665305137634], "24": ["vertical_and_slash", 3500, 100, 0.9643692374229431], "25": ["vertical_and_slash", 3500, 100, 0.9837585687637329], "26": ["vertical_and_slash", 3500, 100, 0.9584507346153259], "27": ["vertical_and_slash", 100, 800, 0.890625], "28": ["vertical_and_slash", 3500, 100, 0.9443742036819458], "29": ["vertical_and_slash", 3500, 100, 0.9804176092147827], "30": ["vertical_and_slash", 3500, 100, 0.9282007813453674], "31": ["vertical_and_slash", 3500, 100, 0.8864558935165405], "32": ["vertical_and_slash", 3500, 100, 0.9975972771644592], "33": ["vertical_and_slash", 3500, 100, 0.9970207214355469], "34": ["vertical_and_slash", 500, 700, 0.9745737314224243], "35": ["vertical_and_slash", 3500, 100, 0.998170793056488], "36": ["vertical_and_slash", 3500, 100, 0.9043524265289307], "37": ["vertical_and_slash", 3500, 100, 0.9786716103553772], "38": ["vertical_and_slash", 100, 750, 0.912280797958374], "39": ["vertical_and_slash", 3500, 100, 0.9981688857078552], "40": ["vertical_and_slash", 3500, 100, 0.995818555355072], "41": ["vertical_and_slash", 3500, 100, 0.992000162601471], "42": ["vertical_and_slash", 3500, 100, 0.9969509840011597], "43": ["vertical_and_slash", 3500, 100, 0.9964844584465027], "44": ["vertical_and_slash", 3500, 100, 0.9948069453239441], "45": ["vertical_and_slash", 3500, 100, 0.9963444471359253], "46": ["vertical_and_slash", 3500, 100, 0.991661787033081], "47": ["vertical_and_slash", 3500, 100, 0.9956120252609253], "48": ["vertical_and_slash", 100, 800, 0.515625], "49": ["vertical_and_slash", 500, 700, 0.9011071920394897], "50": ["vertical_and_slash", 3500, 100, 0.945164144039154], "51": ["vertical_and_slash", 3500, 100, 0.9766556024551392], "52": ["vertical_and_slash", 500, 700, 0.9776272177696228], "53": ["vertical_and_slash", 1000, 6096, 0.9820273518562317], "54": ["vertical_and_slash", 1000, 6096, 0.947547197341919], "55": ["vertical_and_slash", 100, 800, 0.91015625], "56": ["vertical_and_slash", 3500, 100, 0.9015623927116394], "57": ["vertical_and_slash", 3500, 100, 0.9808081984519958], "58": ["vertical_and_slash", 3500, 100, 0.9024316668510437], "59": ["vertical_and_slash", 3500, 100, 0.9785370826721191], "60": ["vertical_and_slash", 3500, 100, 0.9932002425193787], "61": ["vertical_and_slash", 3500, 100, 0.9785913825035095], "62": ["vertical_and_slash", 3500, 100, 0.9931173324584961], "63": ["vertical_and_slash", 3500, 100, 0.9908445477485657]}, {"0": ["vertical_and_slash", 1000, 6096, 0.6268668174743652], "1": ["vertical_and_slash", 1000, 6096, 0.8411374092102051], "2": ["vertical_and_slash", 3500, 100, 0.7453959584236145], "3": ["vertical_and_slash", 3500, 100, 0.7174959778785706], "4": ["vertical_and_slash", 3500, 100, 0.742283821105957], "5": ["vertical_and_slash", 3500, 100, 0.7954835891723633], "6": ["vertical_and_slash", 1000, 6096, 0.6621241569519043], "7": ["vertical_and_slash", 3500, 100, 0.7451500296592712], "8": ["vertical_and_slash", 3500, 100, 0.9803186655044556], "9": ["vertical_and_slash", 3500, 100, 0.9945070147514343], "10": ["vertical_and_slash", 3500, 100, 0.9860700368881226], "11": ["vertical_and_slash", 3500, 100, 0.9914740324020386], "12": ["vertical_and_slash", 3500, 100, 0.989849865436554], "13": ["vertical_and_slash", 3500, 100, 0.9915969371795654], "14": ["vertical_and_slash", 3500, 100, 0.9933056831359863], "15": ["vertical_and_slash", 3500, 100, 0.9939157962799072], "16": ["vertical_and_slash", 3500, 100, 0.9522590041160583], "17": ["vertical_and_slash", 3500, 100, 0.9068601727485657], "18": ["vertical_and_slash", 3500, 100, 0.9808191657066345], "19": ["vertical_and_slash", 3500, 100, 0.9876607060432434], "20": ["vertical_and_slash", 3500, 100, 0.9426457285881042], "21": ["vertical_and_slash", 3500, 100, 0.9046946167945862], "22": ["vertical_and_slash", 3500, 100, 0.9942885637283325], "23": ["vertical_and_slash", 3500, 100, 0.8879977464675903], "24": ["vertical_and_slash", 3500, 100, 0.9925447702407837], "25": ["vertical_and_slash", 3500, 100, 0.9867292642593384], "26": ["vertical_and_slash", 3500, 100, 0.9944289326667786], "27": ["vertical_and_slash", 3500, 100, 0.9955700635910034], "28": ["vertical_and_slash", 3500, 100, 0.9959954619407654], "29": ["vertical_and_slash", 3500, 100, 0.9938628077507019], "30": ["vertical_and_slash", 3500, 100, 0.985379695892334], "31": ["vertical_and_slash", 3500, 100, 0.9918847680091858], "32": ["vertical_and_slash", 3500, 100, 0.9693474173545837], "33": ["vertical_and_slash", 3500, 100, 0.9866985082626343], "34": ["vertical_and_slash", 3500, 100, 0.9269212484359741], "35": ["vertical_and_slash", 3500, 100, 0.9846547245979309], "36": ["vertical_and_slash", 3500, 100, 0.9793662428855896], "37": ["vertical_and_slash", 3500, 100, 0.9619736671447754], "38": ["vertical_and_slash", 3500, 100, 0.9895050525665283], "39": ["vertical_and_slash", 3500, 100, 0.977032482624054], "40": ["vertical_and_slash", 3500, 100, 0.9929519295692444], "41": ["vertical_and_slash", 3500, 100, 0.991627037525177], "42": ["vertical_and_slash", 3500, 100, 0.989172637462616], "43": ["vertical_and_slash", 3500, 100, 0.9848465919494629], "44": ["vertical_and_slash", 3500, 100, 0.9654366374015808], "45": ["vertical_and_slash", 3500, 100, 0.9883670210838318], "46": ["vertical_and_slash", 3500, 100, 0.9511902928352356], "47": ["vertical_and_slash", 3500, 100, 0.980022132396698], "48": ["vertical_and_slash", 3500, 100, 0.9908267855644226], "49": ["vertical_and_slash", 3500, 100, 0.9934496879577637], "50": ["vertical_and_slash", 3500, 100, 0.99526447057724], "51": ["vertical_and_slash", 3500, 100, 0.9941709637641907], "52": ["vertical_and_slash", 3500, 100, 0.996648907661438], "53": ["vertical_and_slash", 3500, 100, 0.9944847822189331], "54": ["vertical_and_slash", 3500, 100, 0.9933546185493469], "55": ["vertical_and_slash", 3500, 100, 0.9852567911148071], "56": ["vertical_and_slash", 3500, 100, 0.9966784119606018], "57": ["vertical_and_slash", 3500, 100, 0.9957041144371033], "58": ["vertical_and_slash", 3500, 100, 0.9892695546150208], "59": ["vertical_and_slash", 3500, 100, 0.9928306937217712], "60": ["vertical_and_slash", 3500, 100, 0.9973081350326538], "61": ["vertical_and_slash", 3500, 100, 0.9908182621002197], "62": ["vertical_and_slash", 3500, 100, 0.996945858001709], "63": ["vertical_and_slash", 3500, 100, 0.9910781383514404]}, {"0": ["vertical_and_slash", 3500, 100, 0.9935545325279236], "1": ["vertical_and_slash", 3500, 100, 0.9960451126098633], "2": ["vertical_and_slash", 3500, 100, 0.9969857335090637], "3": ["vertical_and_slash", 3500, 100, 0.9953396916389465], "4": ["vertical_and_slash", 3500, 100, 0.9892334342002869], "5": ["vertical_and_slash", 3500, 100, 0.9809786677360535], "6": ["vertical_and_slash", 3500, 100, 0.9932035207748413], "7": ["vertical_and_slash", 3500, 100, 0.9959555268287659], "8": ["vertical_and_slash", 1000, 6096, 0.9467699527740479], "9": ["vertical_and_slash", 3500, 100, 0.8757457733154297], "10": ["vertical_and_slash", 500, 700, 0.9670417308807373], "11": ["vertical_and_slash", 1000, 6096, 0.9034522175788879], "12": ["vertical_and_slash", 1000, 6096, 0.9458696246147156], "13": ["vertical_and_slash", 1000, 6096, 0.8914185166358948], "14": ["vertical_and_slash", 3500, 100, 0.9820428490638733], "15": ["vertical_and_slash", 3500, 100, 0.9477982521057129], "16": ["vertical_and_slash", 3500, 100, 0.9979928731918335], "17": ["vertical_and_slash", 3500, 100, 0.9969666004180908], "18": ["vertical_and_slash", 3500, 100, 0.9974613785743713], "19": ["vertical_and_slash", 3500, 100, 0.9955276846885681], "20": ["vertical_and_slash", 3500, 100, 0.9975286722183228], "21": ["vertical_and_slash", 3500, 100, 0.9960598349571228], "22": ["vertical_and_slash", 3500, 100, 0.9973285794258118], "23": ["vertical_and_slash", 3500, 100, 0.998725414276123], "24": ["vertical_and_slash", 3500, 100, 0.9944144487380981], "25": ["vertical_and_slash", 3500, 100, 0.9958868622779846], "26": ["vertical_and_slash", 3500, 100, 0.9934322834014893], "27": ["vertical_and_slash", 3500, 100, 0.9980337619781494], "28": ["vertical_and_slash", 3500, 100, 0.9945402145385742], "29": ["vertical_and_slash", 3500, 100, 0.9982663989067078], "30": ["vertical_and_slash", 3500, 100, 0.9983693957328796], "31": ["vertical_and_slash", 3500, 100, 0.9953178763389587], "32": ["vertical_and_slash", 3500, 100, 0.9971832633018494], "33": ["vertical_and_slash", 3500, 100, 0.9983299970626831], "34": ["vertical_and_slash", 3500, 100, 0.9981076717376709], "35": ["vertical_and_slash", 3500, 100, 0.9944111704826355], "36": ["vertical_and_slash", 3500, 100, 0.998265266418457], "37": ["vertical_and_slash", 3500, 100, 0.9981406927108765], "38": ["vertical_and_slash", 3500, 100, 0.9879467487335205], "39": ["vertical_and_slash", 3500, 100, 0.9888224601745605], "40": ["vertical_and_slash", 3500, 100, 0.9954575896263123], "41": ["vertical_and_slash", 3500, 100, 0.9934769868850708], "42": ["vertical_and_slash", 3500, 100, 0.9951595067977905], "43": ["vertical_and_slash", 3500, 100, 0.9935469031333923], "44": ["vertical_and_slash", 3500, 100, 0.9946380257606506], "45": ["vertical_and_slash", 3500, 100, 0.991359293460846], "46": ["vertical_and_slash", 3500, 100, 0.9957219362258911], "47": ["vertical_and_slash", 3500, 100, 0.9830321669578552], "48": ["vertical_and_slash", 3500, 100, 0.9948733448982239], "49": ["vertical_and_slash", 3500, 100, 0.9907969832420349], "50": ["vertical_and_slash", 3500, 100, 0.9939855933189392], "51": ["vertical_and_slash", 3500, 100, 0.9945279359817505], "52": ["vertical_and_slash", 3500, 100, 0.9918983578681946], "53": ["vertical_and_slash", 3500, 100, 0.9602131843566895], "54": ["vertical_and_slash", 3500, 100, 0.9875707030296326], "55": ["vertical_and_slash", 3500, 100, 0.9924103021621704], "56": ["vertical_and_slash", 3500, 100, 0.9986662864685059], "57": ["vertical_and_slash", 3500, 100, 0.992368757724762], "58": ["vertical_and_slash", 3500, 100, 0.9941779375076294], "59": ["vertical_and_slash", 3500, 100, 0.9963560104370117], "60": ["vertical_and_slash", 3500, 100, 0.9958046674728394], "61": ["vertical_and_slash", 3500, 100, 0.9957137107849121], "62": ["vertical_and_slash", 3500, 100, 0.9955122470855713], "63": ["vertical_and_slash", 3500, 100, 0.9972467422485352]}, {"0": ["vertical_and_slash", 3500, 100, 0.9478133320808411], "1": ["vertical_and_slash", 3500, 100, 0.9602490067481995], "2": ["vertical_and_slash", 500, 700, 0.9512388110160828], "3": ["vertical_and_slash", 100, 800, 0.87109375], "4": ["vertical_and_slash", 3500, 100, 0.9932677745819092], "5": ["vertical_and_slash", 3500, 100, 0.9749032855033875], "6": ["vertical_and_slash", 3500, 100, 0.9859316349029541], "7": ["vertical_and_slash", 3500, 100, 0.9241750836372375], "8": ["vertical_and_slash", 500, 700, 0.9831571578979492], "9": ["vertical_and_slash", 3500, 100, 0.9944893717765808], "10": ["vertical_and_slash", 500, 700, 0.9887070059776306], "11": ["vertical_and_slash", 500, 700, 0.9102075099945068], "12": ["vertical_and_slash", 500, 700, 0.9669085741043091], "13": ["vertical_and_slash", 3500, 100, 0.9880101680755615], "14": ["vertical_and_slash", 500, 700, 0.9807811379432678], "15": ["vertical_and_slash", 500, 700, 0.9880161881446838], "16": ["vertical_and_slash", 3500, 100, 0.9785226583480835], "17": ["vertical_and_slash", 100, 750, 0.93854159116745], "18": ["vertical_and_slash", 100, 750, 0.8131067156791687], "19": ["vertical_and_slash", 500, 700, 0.9704875946044922], "20": ["vertical_and_slash", 500, 700, 0.9429653286933899], "21": ["vertical_and_slash", 500, 700, 0.8169287443161011], "22": ["vertical_and_slash", 3500, 100, 0.6945810914039612], "23": ["vertical_and_slash", 3500, 100, 0.9935157895088196], "24": ["vertical_and_slash", 3500, 100, 0.9913562536239624], "25": ["vertical_and_slash", 3500, 100, 0.9853289723396301], "26": ["vertical_and_slash", 3500, 100, 0.9887819886207581], "27": ["vertical_and_slash", 100, 750, 0.9481726884841919], "28": ["vertical_and_slash", 3500, 100, 0.996806263923645], "29": ["vertical_and_slash", 3500, 100, 0.9895129203796387], "30": ["vertical_and_slash", 3500, 100, 0.9952664971351624], "31": ["vertical_and_slash", 500, 700, 0.9578965306282043], "32": ["vertical_and_slash", 3500, 100, 0.9955458641052246], "33": ["vertical_and_slash", 3500, 100, 0.9956406354904175], "34": ["vertical_and_slash", 3500, 100, 0.9978598356246948], "35": ["vertical_and_slash", 3500, 100, 0.9969398379325867], "36": ["vertical_and_slash", 3500, 100, 0.9981594085693359], "37": ["vertical_and_slash", 3500, 100, 0.9928349256515503], "38": ["vertical_and_slash", 3500, 100, 0.9964509606361389], "39": ["vertical_and_slash", 3500, 100, 0.9984942078590393], "40": ["vertical_and_slash", 3500, 100, 0.9981864094734192], "41": ["vertical_and_slash", 3500, 100, 0.9969301819801331], "42": ["vertical_and_slash", 3500, 100, 0.9859491586685181], "43": ["vertical_and_slash", 3500, 100, 0.9932251572608948], "44": ["vertical_and_slash", 3500, 100, 0.9959598183631897], "45": ["vertical_and_slash", 3500, 100, 0.9915273189544678], "46": ["vertical_and_slash", 3500, 100, 0.9989070296287537], "47": ["vertical_and_slash", 3500, 100, 0.9986959099769592], "48": ["vertical_and_slash", 3500, 100, 0.9960612654685974], "49": ["vertical_and_slash", 3500, 100, 0.997186005115509], "50": ["vertical_and_slash", 3500, 100, 0.9859063625335693], "51": ["vertical_and_slash", 3500, 100, 0.9978888630867004], "52": ["vertical_and_slash", 3500, 100, 0.996772289276123], "53": ["vertical_and_slash", 3500, 100, 0.9976909756660461], "54": ["vertical_and_slash", 3500, 100, 0.9942771792411804], "55": ["vertical_and_slash", 3500, 100, 0.9965730905532837], "56": ["vertical_and_slash", 3500, 100, 0.9426637887954712], "57": ["vertical_and_slash", 500, 700, 0.9077122211456299], "58": ["vertical_and_slash", 500, 700, 0.9164279699325562], "59": ["vertical_and_slash", 3500, 100, 0.9920472502708435], "60": ["vertical_and_slash", 100, 800, 0.90234375], "61": ["vertical_and_slash", 500, 700, 0.8094764947891235], "62": ["vertical_and_slash", 3500, 100, 0.8005000352859497], "63": ["vertical_and_slash", 500, 700, 0.8705887198448181]}, {"0": ["vertical_and_slash", 3500, 100, 0.9626390933990479], "1": ["vertical_and_slash", 100, 800, 0.79296875], "2": ["vertical_and_slash", 3500, 100, 0.765951931476593], "3": ["vertical_and_slash", 3500, 100, 0.7800429463386536], "4": ["vertical_and_slash", 3500, 100, 0.9740782976150513], "5": ["vertical_and_slash", 3500, 100, 0.871109127998352], "6": ["vertical_and_slash", 3500, 100, 0.9563307762145996], "7": ["vertical_and_slash", 3500, 100, 0.9101066589355469], "8": ["vertical_and_slash", 3500, 100, 0.9655263423919678], "9": ["vertical_and_slash", 500, 700, 0.9610968232154846], "10": ["vertical_and_slash", 500, 700, 0.9515025019645691], "11": ["vertical_and_slash", 500, 700, 0.9089682698249817], "12": ["vertical_and_slash", 3500, 100, 0.9623213410377502], "13": ["vertical_and_slash", 3500, 100, 0.965064287185669], "14": ["vertical_and_slash", 3500, 100, 0.9520805478096008], "15": ["vertical_and_slash", 3500, 100, 0.9472141265869141], "16": ["vertical_and_slash", 3500, 100, 0.9234274625778198], "17": ["vertical_and_slash", 3500, 100, 0.9304496645927429], "18": ["vertical_and_slash", 3500, 100, 0.9556105732917786], "19": ["vertical_and_slash", 3500, 100, 0.9216629266738892], "20": ["vertical_and_slash", 3500, 100, 0.8034037947654724], "21": ["vertical_and_slash", 3500, 100, 0.9095696210861206], "22": ["vertical_and_slash", 1000, 6096, 0.8272226452827454], "23": ["vertical_and_slash", 3500, 100, 0.7490889430046082], "24": ["vertical_and_slash", 3500, 100, 0.9982590079307556], "25": ["vertical_and_slash", 3500, 100, 0.9935034513473511], "26": ["vertical_and_slash", 3500, 100, 0.937999427318573], "27": ["vertical_and_slash", 3500, 100, 0.9777122735977173], "28": ["vertical_and_slash", 3500, 100, 0.9811050891876221], "29": ["vertical_and_slash", 3500, 100, 0.936195969581604], "30": ["vertical_and_slash", 3500, 100, 0.9556421637535095], "31": ["vertical_and_slash", 3500, 100, 0.9502548575401306], "32": ["vertical_and_slash", 3500, 100, 0.9802871346473694], "33": ["vertical_and_slash", 100, 750, 0.8931972980499268], "34": ["vertical_and_slash", 3500, 100, 0.9764273166656494], "35": ["vertical_and_slash", 3500, 100, 0.9800068736076355], "36": ["vertical_and_slash", 3500, 100, 0.9855831861495972], "37": ["vertical_and_slash", 3500, 100, 0.993865430355072], "38": ["vertical_and_slash", 500, 700, 0.9662306308746338], "39": ["vertical_and_slash", 3500, 100, 0.9748856425285339], "40": ["vertical_and_slash", 3500, 100, 0.8975186347961426], "41": ["vertical_and_slash", 1000, 6096, 0.8399168252944946], "42": ["vertical_and_slash", 1000, 6096, 0.7780442237854004], "43": ["vertical_and_slash", 1000, 6096, 0.8229435682296753], "44": ["vertical_and_slash", 1000, 6096, 0.832515299320221], "45": ["vertical_and_slash", 1000, 6096, 0.8131482005119324], "46": ["vertical_and_slash", 3500, 100, 0.9084001183509827], "47": ["vertical_and_slash", 3500, 100, 0.9354747533798218], "48": ["vertical_and_slash", 3500, 100, 0.9949605464935303], "49": ["vertical_and_slash", 3500, 100, 0.9924782514572144], "50": ["vertical_and_slash", 3500, 100, 0.9827143549919128], "51": ["vertical_and_slash", 3500, 100, 0.9957362413406372], "52": ["vertical_and_slash", 3500, 100, 0.9914284944534302], "53": ["vertical_and_slash", 3500, 100, 0.9801662564277649], "54": ["vertical_and_slash", 3500, 100, 0.9892519116401672], "55": ["vertical_and_slash", 3500, 100, 0.9885778427124023], "56": ["vertical_and_slash", 3500, 100, 0.9931341409683228], "57": ["vertical_and_slash", 3500, 100, 0.9903366565704346], "58": ["vertical_and_slash", 3500, 100, 0.9976650476455688], "59": ["vertical_and_slash", 3500, 100, 0.99544358253479], "60": ["vertical_and_slash", 3500, 100, 0.9932839870452881], "61": ["vertical_and_slash", 3500, 100, 0.9966924786567688], "62": ["vertical_and_slash", 3500, 100, 0.994101881980896], "63": ["vertical_and_slash", 3500, 100, 0.9973053336143494]}, {"0": ["vertical_and_slash", 100, 750, 0.9707180261611938], "1": ["vertical_and_slash", 3500, 100, 0.9802829623222351], "2": ["vertical_and_slash", 3500, 100, 0.9623221755027771], "3": ["vertical_and_slash", 100, 750, 0.9743637442588806], "4": ["vertical_and_slash", 3500, 100, 0.9685831069946289], "5": ["vertical_and_slash", 500, 700, 0.93028324842453], "6": ["vertical_and_slash", 3500, 100, 0.9637101888656616], "7": ["vertical_and_slash", 100, 750, 0.9697932004928589], "8": ["vertical_and_slash", 3500, 100, 0.9498746395111084], "9": ["vertical_and_slash", 3500, 100, 0.9920356869697571], "10": ["vertical_and_slash", 3500, 100, 0.9711196422576904], "11": ["vertical_and_slash", 3500, 100, 0.9681365489959717], "12": ["vertical_and_slash", 3500, 100, 0.9170305728912354], "13": ["vertical_and_slash", 3500, 100, 0.9401235580444336], "14": ["vertical_and_slash", 3500, 100, 0.9972625970840454], "15": ["vertical_and_slash", 3500, 100, 0.9656456708908081], "16": ["vertical_and_slash", 3500, 100, 0.9961220622062683], "17": ["vertical_and_slash", 3500, 100, 0.9914001822471619], "18": ["vertical_and_slash", 3500, 100, 0.9651716351509094], "19": ["vertical_and_slash", 3500, 100, 0.9970184564590454], "20": ["vertical_and_slash", 3500, 100, 0.9865334033966064], "21": ["vertical_and_slash", 3500, 100, 0.9935438632965088], "22": ["vertical_and_slash", 3500, 100, 0.9958479404449463], "23": ["vertical_and_slash", 3500, 100, 0.9920876026153564], "24": ["vertical_and_slash", 3500, 100, 0.8919223546981812], "25": ["vertical_and_slash", 100, 800, 0.73828125], "26": ["vertical_and_slash", 3500, 100, 0.9040138125419617], "27": ["vertical_and_slash", 3500, 100, 0.9220767021179199], "28": ["vertical_and_slash", 500, 700, 0.8633096218109131], "29": ["vertical_and_slash", 3500, 100, 0.948582649230957], "30": ["vertical_and_slash", 100, 800, 0.8125], "31": ["vertical_and_slash", 3500, 100, 0.99616938829422], "32": ["vertical_and_slash", 100, 800, 0.5], "33": ["vertical_and_slash", 500, 700, 0.9654490351676941], "34": ["vertical_and_slash", 500, 700, 0.9709374904632568], "35": ["vertical_and_slash", 500, 700, 0.9791161417961121], "36": ["vertical_and_slash", 500, 700, 0.9559553861618042], "37": ["vertical_and_slash", 500, 700, 0.9669602513313293], "38": ["vertical_and_slash", 500, 700, 0.9747380018234253], "39": ["vertical_and_slash", 500, 700, 0.9789488315582275], "40": ["vertical_and_slash", 3500, 100, 0.9945711493492126], "41": ["vertical_and_slash", 3500, 100, 0.9974036812782288], "42": ["vertical_and_slash", 3500, 100, 0.9964024424552917], "43": ["vertical_and_slash", 3500, 100, 0.9955281615257263], "44": ["vertical_and_slash", 3500, 100, 0.9976097345352173], "45": ["vertical_and_slash", 3500, 100, 0.9984892010688782], "46": ["vertical_and_slash", 3500, 100, 0.995108962059021], "47": ["vertical_and_slash", 3500, 100, 0.9982434511184692], "48": ["vertical_and_slash", 3500, 100, 0.9913103580474854], "49": ["vertical_and_slash", 3500, 100, 0.9962610006332397], "50": ["vertical_and_slash", 3500, 100, 0.9940928816795349], "51": ["vertical_and_slash", 3500, 100, 0.9958260655403137], "52": ["vertical_and_slash", 3500, 100, 0.9973545074462891], "53": ["vertical_and_slash", 3500, 100, 0.9937942028045654], "54": ["vertical_and_slash", 3500, 100, 0.9873053431510925], "55": ["vertical_and_slash", 3500, 100, 0.9921920299530029], "56": ["vertical_and_slash", 3500, 100, 0.9979903101921082], "57": ["vertical_and_slash", 3500, 100, 0.9958221316337585], "58": ["vertical_and_slash", 3500, 100, 0.9948664307594299], "59": ["vertical_and_slash", 3500, 100, 0.9957322478294373], "60": ["vertical_and_slash", 3500, 100, 0.9975112080574036], "61": ["vertical_and_slash", 3500, 100, 0.9946689009666443], "62": ["vertical_and_slash", 3500, 100, 0.9948416352272034], "63": ["vertical_and_slash", 3500, 100, 0.9934567809104919]}, {"0": ["vertical_and_slash", 3500, 100, 0.9763519167900085], "1": ["vertical_and_slash", 3500, 100, 0.9602594971656799], "2": ["vertical_and_slash", 3500, 100, 0.9450265765190125], "3": ["vertical_and_slash", 3500, 100, 0.9791201949119568], "4": ["vertical_and_slash", 3500, 100, 0.9681959748268127], "5": ["vertical_and_slash", 3500, 100, 0.9905064702033997], "6": ["vertical_and_slash", 3500, 100, 0.961053192615509], "7": ["vertical_and_slash", 3500, 100, 0.9644065499305725], "8": ["vertical_and_slash", 3500, 100, 0.9973450899124146], "9": ["vertical_and_slash", 3500, 100, 0.9966305494308472], "10": ["vertical_and_slash", 3500, 100, 0.9969980716705322], "11": ["vertical_and_slash", 3500, 100, 0.9958729147911072], "12": ["vertical_and_slash", 3500, 100, 0.9980596899986267], "13": ["vertical_and_slash", 3500, 100, 0.9981325268745422], "14": ["vertical_and_slash", 3500, 100, 0.9981429576873779], "15": ["vertical_and_slash", 3500, 100, 0.9960127472877502], "16": ["vertical_and_slash", 3500, 100, 0.9978178143501282], "17": ["vertical_and_slash", 3500, 100, 0.9917479753494263], "18": ["vertical_and_slash", 1000, 6096, 0.9681281447410583], "19": ["vertical_and_slash", 1000, 6096, 0.9794018864631653], "20": ["vertical_and_slash", 3500, 100, 0.9896470308303833], "21": ["vertical_and_slash", 3500, 100, 0.9913672208786011], "22": ["vertical_and_slash", 1000, 6096, 0.9689056277275085], "23": ["vertical_and_slash", 3500, 100, 0.9948355555534363], "24": ["vertical_and_slash", 500, 700, 0.9196532368659973], "25": ["vertical_and_slash", 100, 750, 0.9150657057762146], "26": ["vertical_and_slash", 100, 750, 0.9301779270172119], "27": ["vertical_and_slash", 100, 750, 0.918380081653595], "28": ["vertical_and_slash", 100, 750, 0.9087221026420593], "29": ["vertical_and_slash", 100, 750, 0.9338465929031372], "30": ["vertical_and_slash", 100, 750, 0.9184107184410095], "31": ["vertical_and_slash", 100, 750, 0.8370739817619324], "32": ["vertical_and_slash", 3500, 100, 0.9951123595237732], "33": ["vertical_and_slash", 3500, 100, 0.9915667176246643], "34": ["vertical_and_slash", 3500, 100, 0.9906138181686401], "35": ["vertical_and_slash", 3500, 100, 0.9957793354988098], "36": ["vertical_and_slash", 3500, 100, 0.9973171949386597], "37": ["vertical_and_slash", 3500, 100, 0.9956631064414978], "38": ["vertical_and_slash", 3500, 100, 0.9926979541778564], "39": ["vertical_and_slash", 3500, 100, 0.993641197681427], "40": ["vertical_and_slash", 3500, 100, 0.9922868013381958], "41": ["vertical_and_slash", 3500, 100, 0.9928418397903442], "42": ["vertical_and_slash", 3500, 100, 0.9798092842102051], "43": ["vertical_and_slash", 3500, 100, 0.996331512928009], "44": ["vertical_and_slash", 3500, 100, 0.9936864376068115], "45": ["vertical_and_slash", 3500, 100, 0.9904117584228516], "46": ["vertical_and_slash", 3500, 100, 0.9941045641899109], "47": ["vertical_and_slash", 3500, 100, 0.9959853291511536], "48": ["vertical_and_slash", 3500, 100, 0.997940182685852], "49": ["vertical_and_slash", 3500, 100, 0.9966842532157898], "50": ["vertical_and_slash", 3500, 100, 0.9953781366348267], "51": ["vertical_and_slash", 3500, 100, 0.9986039996147156], "52": ["vertical_and_slash", 3500, 100, 0.9950948357582092], "53": ["vertical_and_slash", 3500, 100, 0.9977318644523621], "54": ["vertical_and_slash", 3500, 100, 0.9981207847595215], "55": ["vertical_and_slash", 3500, 100, 0.9979984760284424], "56": ["vertical_and_slash", 3500, 100, 0.9888167977333069], "57": ["vertical_and_slash", 3500, 100, 0.9910814166069031], "58": ["vertical_and_slash", 500, 700, 0.974578320980072], "59": ["vertical_and_slash", 3500, 100, 0.9893765449523926], "60": ["vertical_and_slash", 3500, 100, 0.9975110292434692], "61": ["vertical_and_slash", 3500, 100, 0.9845767021179199], "62": ["vertical_and_slash", 3500, 100, 0.9974707365036011], "63": ["vertical_and_slash", 3500, 100, 0.9581850171089172]}, {"0": ["vertical_and_slash", 3500, 100, 0.9939310550689697], "1": ["vertical_and_slash", 3500, 100, 0.981820821762085], "2": ["vertical_and_slash", 3500, 100, 0.9946637153625488], "3": ["vertical_and_slash", 3500, 100, 0.9952095150947571], "4": ["vertical_and_slash", 3500, 100, 0.9968380928039551], "5": ["vertical_and_slash", 3500, 100, 0.997765064239502], "6": ["vertical_and_slash", 3500, 100, 0.9971145987510681], "7": ["vertical_and_slash", 3500, 100, 0.9777239561080933], "8": ["vertical_and_slash", 3500, 100, 0.9850772023200989], "9": ["vertical_and_slash", 3500, 100, 0.9684692025184631], "10": ["vertical_and_slash", 3500, 100, 0.9772451519966125], "11": ["vertical_and_slash", 3500, 100, 0.9910671710968018], "12": ["vertical_and_slash", 3500, 100, 0.9002072811126709], "13": ["vertical_and_slash", 3500, 100, 0.9904926419258118], "14": ["vertical_and_slash", 500, 700, 0.9429045915603638], "15": ["vertical_and_slash", 3500, 100, 0.9260460734367371], "16": ["vertical_and_slash", 3500, 100, 0.9821746945381165], "17": ["vertical_and_slash", 3500, 100, 0.9844980835914612], "18": ["vertical_and_slash", 3500, 100, 0.9762932658195496], "19": ["vertical_and_slash", 3500, 100, 0.9928142428398132], "20": ["vertical_and_slash", 3500, 100, 0.995028018951416], "21": ["vertical_and_slash", 3500, 100, 0.9890473484992981], "22": ["vertical_and_slash", 500, 700, 0.7722472548484802], "23": ["vertical_and_slash", 3500, 100, 0.9640660285949707], "24": ["vertical_and_slash", 3500, 100, 0.9729565382003784], "25": ["vertical_and_slash", 100, 750, 0.931086540222168], "26": ["vertical_and_slash", 3500, 100, 0.9809194803237915], "27": ["vertical_and_slash", 3500, 100, 0.9906665682792664], "28": ["vertical_and_slash", 500, 700, 0.9581154584884644], "29": ["vertical_and_slash", 3500, 100, 0.9938306212425232], "30": ["vertical_and_slash", 500, 700, 0.929822564125061], "31": ["vertical_and_slash", 3500, 100, 0.9699233174324036], "32": ["vertical_and_slash", 3500, 100, 0.9534093141555786], "33": ["vertical_and_slash", 3500, 100, 0.9903049468994141], "34": ["vertical_and_slash", 3500, 100, 0.9883601665496826], "35": ["vertical_and_slash", 3500, 100, 0.9778764843940735], "36": ["vertical_and_slash", 3500, 100, 0.9875636696815491], "37": ["vertical_and_slash", 3500, 100, 0.9839208126068115], "38": ["vertical_and_slash", 500, 700, 0.8747838139533997], "39": ["vertical_and_slash", 3500, 100, 0.976969301700592], "40": ["vertical_and_slash", 3500, 100, 0.9834225177764893], "41": ["vertical_and_slash", 3500, 100, 0.9954813122749329], "42": ["vertical_and_slash", 3500, 100, 0.9797183275222778], "43": ["vertical_and_slash", 3500, 100, 0.991348385810852], "44": ["vertical_and_slash", 3500, 100, 0.986585259437561], "45": ["vertical_and_slash", 3500, 100, 0.9958811402320862], "46": ["vertical_and_slash", 3500, 100, 0.9862039685249329], "47": ["vertical_and_slash", 3500, 100, 0.9809949398040771], "48": ["vertical_and_slash", 3500, 100, 0.9795269966125488], "49": ["vertical_and_slash", 3500, 100, 0.9873529076576233], "50": ["vertical_and_slash", 3500, 100, 0.9765223860740662], "51": ["vertical_and_slash", 3500, 100, 0.9890322685241699], "52": ["vertical_and_slash", 100, 750, 0.986261785030365], "53": ["vertical_and_slash", 3500, 100, 0.9945112466812134], "54": ["vertical_and_slash", 3500, 100, 0.9666581153869629], "55": ["vertical_and_slash", 100, 750, 0.9453997611999512], "56": ["vertical_and_slash", 3500, 100, 0.9379056692123413], "57": ["vertical_and_slash", 3500, 100, 0.936591625213623], "58": ["vertical_and_slash", 3500, 100, 0.9757940769195557], "59": ["vertical_and_slash", 3500, 100, 0.9384666681289673], "60": ["vertical_and_slash", 3500, 100, 0.9212881922721863], "61": ["vertical_and_slash", 3500, 100, 0.9868927001953125], "62": ["vertical_and_slash", 3500, 100, 0.9385613203048706], "63": ["vertical_and_slash", 3500, 100, 0.9679697155952454]}, {"0": ["vertical_and_slash", 3500, 100, 0.9857991337776184], "1": ["vertical_and_slash", 3500, 100, 0.9878666996955872], "2": ["vertical_and_slash", 3500, 100, 0.9775307774543762], "3": ["vertical_and_slash", 3500, 100, 0.994657039642334], "4": ["vertical_and_slash", 3500, 100, 0.9946602582931519], "5": ["vertical_and_slash", 500, 700, 0.9754164814949036], "6": ["vertical_and_slash", 3500, 100, 0.9944545030593872], "7": ["vertical_and_slash", 3500, 100, 0.9900602698326111], "8": ["vertical_and_slash", 1000, 6096, 0.6416069269180298], "9": ["vertical_and_slash", 3500, 100, 0.9421439170837402], "10": ["vertical_and_slash", 500, 700, 0.9324153661727905], "11": ["vertical_and_slash", 3500, 100, 0.9190346002578735], "12": ["vertical_and_slash", 500, 700, 0.9025391936302185], "13": ["vertical_and_slash", 500, 700, 0.9041240811347961], "14": ["vertical_and_slash", 100, 800, 0.69921875], "15": ["vertical_and_slash", 100, 800, 0.7421875], "16": ["vertical_and_slash", 3500, 100, 0.9649198651313782], "17": ["vertical_and_slash", 3500, 100, 0.9472401142120361], "18": ["vertical_and_slash", 3500, 100, 0.9678804874420166], "19": ["vertical_and_slash", 3500, 100, 0.954724133014679], "20": ["vertical_and_slash", 3500, 100, 0.7711028456687927], "21": ["vertical_and_slash", 3500, 100, 0.9690527319908142], "22": ["vertical_and_slash", 3500, 100, 0.9663679003715515], "23": ["vertical_and_slash", 3500, 100, 0.9744104146957397], "24": ["vertical_and_slash", 1000, 6096, 0.9765480160713196], "25": ["vertical_and_slash", 100, 800, 0.734375], "26": ["vertical_and_slash", 3500, 100, 0.8478866219520569], "27": ["vertical_and_slash", 3500, 100, 0.9340740442276001], "28": ["vertical_and_slash", 3500, 100, 0.6601127982139587], "29": ["vertical_and_slash", 3500, 100, 0.8969439268112183], "30": ["vertical_and_slash", 100, 800, 0.74609375], "31": ["vertical_and_slash", 3500, 100, 0.9611671566963196], "32": ["vertical_and_slash", 3500, 100, 0.9977543950080872], "33": ["vertical_and_slash", 3500, 100, 0.994455873966217], "34": ["vertical_and_slash", 3500, 100, 0.9939024448394775], "35": ["vertical_and_slash", 3500, 100, 0.9955153465270996], "36": ["vertical_and_slash", 3500, 100, 0.9961793422698975], "37": ["vertical_and_slash", 3500, 100, 0.9984307885169983], "38": ["vertical_and_slash", 3500, 100, 0.9958702325820923], "39": ["vertical_and_slash", 3500, 100, 0.9913764595985413], "40": ["vertical_and_slash", 3500, 100, 0.9992827773094177], "41": ["vertical_and_slash", 3500, 100, 0.997836172580719], "42": ["vertical_and_slash", 3500, 100, 0.9983772039413452], "43": ["vertical_and_slash", 3500, 100, 0.9978077411651611], "44": ["vertical_and_slash", 3500, 100, 0.9986857175827026], "45": ["vertical_and_slash", 3500, 100, 0.998812735080719], "46": ["vertical_and_slash", 3500, 100, 0.9981910586357117], "47": ["vertical_and_slash", 3500, 100, 0.998163104057312], "48": ["vertical_and_slash", 100, 800, 0.91015625], "49": ["vertical_and_slash", 100, 750, 0.945033073425293], "50": ["vertical_and_slash", 500, 700, 0.9252634048461914], "51": ["vertical_and_slash", 500, 700, 0.9584423303604126], "52": ["vertical_and_slash", 500, 700, 0.9508960843086243], "53": ["vertical_and_slash", 500, 700, 0.9548971652984619], "54": ["vertical_and_slash", 500, 700, 0.9278125166893005], "55": ["vertical_and_slash", 100, 800, 0.76953125], "56": ["vertical_and_slash", 1000, 6096, 0.971391499042511], "57": ["vertical_and_slash", 3500, 100, 0.9630081653594971], "58": ["vertical_and_slash", 3500, 100, 0.976312518119812], "59": ["vertical_and_slash", 3500, 100, 0.9746572375297546], "60": ["vertical_and_slash", 3500, 100, 0.9210913777351379], "61": ["vertical_and_slash", 1000, 6096, 0.9915266633033752], "62": ["vertical_and_slash", 3500, 100, 0.9726375937461853], "63": ["vertical_and_slash", 3500, 100, 0.9879733324050903]}, {"0": ["vertical_and_slash", 3500, 100, 0.9826613664627075], "1": ["vertical_and_slash", 3500, 100, 0.9777262210845947], "2": ["vertical_and_slash", 3500, 100, 0.974787712097168], "3": ["vertical_and_slash", 3500, 100, 0.9725432991981506], "4": ["vertical_and_slash", 3500, 100, 0.9656158685684204], "5": ["vertical_and_slash", 3500, 100, 0.9768548011779785], "6": ["vertical_and_slash", 3500, 100, 0.963869571685791], "7": ["vertical_and_slash", 3500, 100, 0.9605903625488281], "8": ["vertical_and_slash", 3500, 100, 0.9866607785224915], "9": ["vertical_and_slash", 3500, 100, 0.9941683411598206], "10": ["vertical_and_slash", 3500, 100, 0.9872555136680603], "11": ["vertical_and_slash", 3500, 100, 0.9980606436729431], "12": ["vertical_and_slash", 3500, 100, 0.9972106218338013], "13": ["vertical_and_slash", 3500, 100, 0.9974410533905029], "14": ["vertical_and_slash", 3500, 100, 0.993687093257904], "15": ["vertical_and_slash", 3500, 100, 0.9915313124656677], "16": ["vertical_and_slash", 3500, 100, 0.9973343014717102], "17": ["vertical_and_slash", 3500, 100, 0.997478723526001], "18": ["vertical_and_slash", 3500, 100, 0.9940242767333984], "19": ["vertical_and_slash", 3500, 100, 0.9940628409385681], "20": ["vertical_and_slash", 3500, 100, 0.9982221126556396], "21": ["vertical_and_slash", 3500, 100, 0.9903855323791504], "22": ["vertical_and_slash", 3500, 100, 0.9959637522697449], "23": ["vertical_and_slash", 3500, 100, 0.996314525604248], "24": ["vertical_and_slash", 3500, 100, 0.9939233064651489], "25": ["vertical_and_slash", 3500, 100, 0.9885157346725464], "26": ["vertical_and_slash", 3500, 100, 0.9876769185066223], "27": ["vertical_and_slash", 3500, 100, 0.9890137910842896], "28": ["vertical_and_slash", 3500, 100, 0.9949268698692322], "29": ["vertical_and_slash", 3500, 100, 0.9897232055664062], "30": ["vertical_and_slash", 3500, 100, 0.990836501121521], "31": ["vertical_and_slash", 3500, 100, 0.9844744205474854], "32": ["vertical_and_slash", 3500, 100, 0.9956731796264648], "33": ["vertical_and_slash", 3500, 100, 0.9969680905342102], "34": ["vertical_and_slash", 3500, 100, 0.9976813793182373], "35": ["vertical_and_slash", 3500, 100, 0.9974281787872314], "36": ["vertical_and_slash", 3500, 100, 0.9973292946815491], "37": ["vertical_and_slash", 3500, 100, 0.9956596493721008], "38": ["vertical_and_slash", 3500, 100, 0.9958901405334473], "39": ["vertical_and_slash", 3500, 100, 0.9971887469291687], "40": ["vertical_and_slash", 3500, 100, 0.996138334274292], "41": ["vertical_and_slash", 3500, 100, 0.9970946907997131], "42": ["vertical_and_slash", 3500, 100, 0.9967288374900818], "43": ["vertical_and_slash", 3500, 100, 0.9946870803833008], "44": ["vertical_and_slash", 3500, 100, 0.9970371723175049], "45": ["vertical_and_slash", 3500, 100, 0.99556565284729], "46": ["vertical_and_slash", 3500, 100, 0.9917713403701782], "47": ["vertical_and_slash", 3500, 100, 0.994647204875946], "48": ["vertical_and_slash", 3500, 100, 0.9949663281440735], "49": ["vertical_and_slash", 3500, 100, 0.9563922882080078], "50": ["vertical_and_slash", 3500, 100, 0.9974775910377502], "51": ["vertical_and_slash", 3500, 100, 0.9938712120056152], "52": ["vertical_and_slash", 3500, 100, 0.9782556891441345], "53": ["vertical_and_slash", 3500, 100, 0.9957535266876221], "54": ["vertical_and_slash", 3500, 100, 0.9919800758361816], "55": ["vertical_and_slash", 3500, 100, 0.9974851608276367], "56": ["vertical_and_slash", 3500, 100, 0.992744505405426], "57": ["vertical_and_slash", 3500, 100, 0.994763195514679], "58": ["vertical_and_slash", 3500, 100, 0.9963898658752441], "59": ["vertical_and_slash", 3500, 100, 0.9968500733375549], "60": ["vertical_and_slash", 3500, 100, 0.995318591594696], "61": ["vertical_and_slash", 3500, 100, 0.9944504499435425], "62": ["vertical_and_slash", 3500, 100, 0.9969882369041443], "63": ["vertical_and_slash", 3500, 100, 0.9961622357368469]}, {"0": ["vertical_and_slash", 3500, 100, 0.9971364736557007], "1": ["vertical_and_slash", 3500, 100, 0.9968711733818054], "2": ["vertical_and_slash", 3500, 100, 0.9933744668960571], "3": ["vertical_and_slash", 3500, 100, 0.9943379163742065], "4": ["vertical_and_slash", 3500, 100, 0.9983575940132141], "5": ["vertical_and_slash", 3500, 100, 0.9898219704627991], "6": ["vertical_and_slash", 3500, 100, 0.9962285161018372], "7": ["vertical_and_slash", 3500, 100, 0.9890733361244202], "8": ["vertical_and_slash", 3500, 100, 0.9909411072731018], "9": ["vertical_and_slash", 3500, 100, 0.9963884353637695], "10": ["vertical_and_slash", 3500, 100, 0.9890854358673096], "11": ["vertical_and_slash", 3500, 100, 0.9975678324699402], "12": ["vertical_and_slash", 3500, 100, 0.9941215515136719], "13": ["vertical_and_slash", 3500, 100, 0.9977753162384033], "14": ["vertical_and_slash", 3500, 100, 0.9976856708526611], "15": ["vertical_and_slash", 3500, 100, 0.980667769908905], "16": ["vertical_and_slash", 500, 700, 0.9551181197166443], "17": ["vertical_and_slash", 500, 700, 0.953357994556427], "18": ["vertical_and_slash", 500, 700, 0.9387120604515076], "19": ["vertical_and_slash", 3500, 100, 0.9702189564704895], "20": ["vertical_and_slash", 500, 700, 0.9639173150062561], "21": ["vertical_and_slash", 500, 700, 0.9836777448654175], "22": ["vertical_and_slash", 500, 700, 0.9743127822875977], "23": ["vertical_and_slash", 500, 700, 0.973580539226532], "24": ["vertical_and_slash", 3500, 100, 0.9914095997810364], "25": ["vertical_and_slash", 3500, 100, 0.987567663192749], "26": ["vertical_and_slash", 3500, 100, 0.9944868087768555], "27": ["vertical_and_slash", 3500, 100, 0.9752585291862488], "28": ["vertical_and_slash", 500, 700, 0.9873413443565369], "29": ["vertical_and_slash", 500, 700, 0.9854512810707092], "30": ["vertical_and_slash", 500, 700, 0.9673793315887451], "31": ["vertical_and_slash", 3500, 100, 0.9938398003578186], "32": ["vertical_and_slash", 3500, 100, 0.9946873784065247], "33": ["vertical_and_slash", 3500, 100, 0.9730810523033142], "34": ["vertical_and_slash", 3500, 100, 0.9924407005310059], "35": ["vertical_and_slash", 3500, 100, 0.9992765188217163], "36": ["vertical_and_slash", 3500, 100, 0.9977195858955383], "37": ["vertical_and_slash", 3500, 100, 0.9851970076560974], "38": ["vertical_and_slash", 3500, 100, 0.9986899495124817], "39": ["vertical_and_slash", 3500, 100, 0.9926126003265381], "40": ["vertical_and_slash", 3500, 100, 0.9937347769737244], "41": ["vertical_and_slash", 3500, 100, 0.9872777462005615], "42": ["vertical_and_slash", 3500, 100, 0.9972860813140869], "43": ["vertical_and_slash", 3500, 100, 0.9980745911598206], "44": ["vertical_and_slash", 3500, 100, 0.9940880537033081], "45": ["vertical_and_slash", 3500, 100, 0.9978346824645996], "46": ["vertical_and_slash", 3500, 100, 0.9959419965744019], "47": ["vertical_and_slash", 3500, 100, 0.9977437853813171], "48": ["vertical_and_slash", 3500, 100, 0.9931797981262207], "49": ["vertical_and_slash", 3500, 100, 0.9974952340126038], "50": ["vertical_and_slash", 3500, 100, 0.9970213174819946], "51": ["vertical_and_slash", 3500, 100, 0.998236894607544], "52": ["vertical_and_slash", 3500, 100, 0.9977062940597534], "53": ["vertical_and_slash", 3500, 100, 0.9975285530090332], "54": ["vertical_and_slash", 3500, 100, 0.9949022531509399], "55": ["vertical_and_slash", 3500, 100, 0.9980334043502808], "56": ["vertical_and_slash", 3500, 100, 0.997947633266449], "57": ["vertical_and_slash", 3500, 100, 0.9934724569320679], "58": ["vertical_and_slash", 100, 750, 0.9808768630027771], "59": ["vertical_and_slash", 3500, 100, 0.9974403977394104], "60": ["vertical_and_slash", 3500, 100, 0.995011031627655], "61": ["vertical_and_slash", 3500, 100, 0.9965994954109192], "62": ["vertical_and_slash", 3500, 100, 0.9971248507499695], "63": ["vertical_and_slash", 3500, 100, 0.9918531179428101]}, {"0": ["vertical_and_slash", 3500, 100, 0.9871485233306885], "1": ["vertical_and_slash", 3500, 100, 0.9727885127067566], "2": ["vertical_and_slash", 3500, 100, 0.9779970049858093], "3": ["vertical_and_slash", 3500, 100, 0.9843007922172546], "4": ["vertical_and_slash", 3500, 100, 0.977283775806427], "5": ["vertical_and_slash", 3500, 100, 0.9808070659637451], "6": ["vertical_and_slash", 3500, 100, 0.994341254234314], "7": ["vertical_and_slash", 500, 700, 0.7427809834480286], "8": ["vertical_and_slash", 3500, 100, 0.9963023066520691], "9": ["vertical_and_slash", 3500, 100, 0.9973962306976318], "10": ["vertical_and_slash", 3500, 100, 0.9943841099739075], "11": ["vertical_and_slash", 3500, 100, 0.995815634727478], "12": ["vertical_and_slash", 3500, 100, 0.9987611174583435], "13": ["vertical_and_slash", 3500, 100, 0.9917445778846741], "14": ["vertical_and_slash", 3500, 100, 0.9983685612678528], "15": ["vertical_and_slash", 100, 750, 0.9709450006484985], "16": ["vertical_and_slash", 3500, 100, 0.9988679885864258], "17": ["vertical_and_slash", 3500, 100, 0.9915663003921509], "18": ["vertical_and_slash", 3500, 100, 0.9971009492874146], "19": ["vertical_and_slash", 3500, 100, 0.9894904494285583], "20": ["vertical_and_slash", 3500, 100, 0.9964075684547424], "21": ["vertical_and_slash", 3500, 100, 0.975359320640564], "22": ["vertical_and_slash", 3500, 100, 0.993469774723053], "23": ["vertical_and_slash", 3500, 100, 0.9989268183708191], "24": ["vertical_and_slash", 3500, 100, 0.9951943755149841], "25": ["vertical_and_slash", 3500, 100, 0.9944730997085571], "26": ["vertical_and_slash", 3500, 100, 0.9917603135108948], "27": ["vertical_and_slash", 500, 700, 0.9678224325180054], "28": ["vertical_and_slash", 3500, 100, 0.9940803647041321], "29": ["vertical_and_slash", 3500, 100, 0.9974925518035889], "30": ["vertical_and_slash", 3500, 100, 0.9981379508972168], "31": ["vertical_and_slash", 3500, 100, 0.9967208504676819], "32": ["vertical_and_slash", 100, 800, 0.8203125], "33": ["vertical_and_slash", 3500, 100, 0.9417598247528076], "34": ["vertical_and_slash", 500, 700, 0.9549974799156189], "35": ["vertical_and_slash", 100, 750, 0.9597418308258057], "36": ["vertical_and_slash", 100, 800, 0.85546875], "37": ["vertical_and_slash", 100, 800, 0.64453125], "38": ["vertical_and_slash", 3500, 100, 0.9333158135414124], "39": ["vertical_and_slash", 100, 800, 0.7734375], "40": ["vertical_and_slash", 3500, 100, 0.9959282279014587], "41": ["vertical_and_slash", 3500, 100, 0.999439001083374], "42": ["vertical_and_slash", 3500, 100, 0.9969329833984375], "43": ["vertical_and_slash", 3500, 100, 0.9993851184844971], "44": ["vertical_and_slash", 3500, 100, 0.9983884692192078], "45": ["vertical_and_slash", 3500, 100, 0.9966574311256409], "46": ["vertical_and_slash", 3500, 100, 0.9987480044364929], "47": ["vertical_and_slash", 3500, 100, 0.997458815574646], "48": ["vertical_and_slash", 3500, 100, 0.997226357460022], "49": ["vertical_and_slash", 3500, 100, 0.9981943964958191], "50": ["vertical_and_slash", 3500, 100, 0.9891274571418762], "51": ["vertical_and_slash", 3500, 100, 0.998871922492981], "52": ["vertical_and_slash", 3500, 100, 0.940406858921051], "53": ["vertical_and_slash", 3500, 100, 0.9977985620498657], "54": ["vertical_and_slash", 3500, 100, 0.9986429214477539], "55": ["vertical_and_slash", 3500, 100, 0.9982568621635437], "56": ["vertical_and_slash", 100, 800, 0.86328125], "57": ["vertical_and_slash", 1000, 6096, 0.9822483658790588], "58": ["vertical_and_slash", 1000, 6096, 0.984887421131134], "59": ["vertical_and_slash", 3500, 100, 0.9811902046203613], "60": ["vertical_and_slash", 1000, 6096, 0.9147958159446716], "61": ["vertical_and_slash", 1000, 6096, 0.975057065486908], "62": ["vertical_and_slash", 3500, 100, 0.958748996257782], "63": ["vertical_and_slash", 1000, 6096, 0.9642420411109924]}, {"0": ["vertical_and_slash", 3500, 100, 0.9980002045631409], "1": ["vertical_and_slash", 3500, 100, 0.9973200559616089], "2": ["vertical_and_slash", 3500, 100, 0.9977452754974365], "3": ["vertical_and_slash", 3500, 100, 0.9958822131156921], "4": ["vertical_and_slash", 3500, 100, 0.996937096118927], "5": ["vertical_and_slash", 3500, 100, 0.9925647974014282], "6": ["vertical_and_slash", 3500, 100, 0.9944379925727844], "7": ["vertical_and_slash", 3500, 100, 0.9990218877792358], "8": ["vertical_and_slash", 3500, 100, 0.9934820532798767], "9": ["vertical_and_slash", 3500, 100, 0.9807037711143494], "10": ["vertical_and_slash", 3500, 100, 0.9946115612983704], "11": ["vertical_and_slash", 3500, 100, 0.994213342666626], "12": ["vertical_and_slash", 500, 700, 0.9141184687614441], "13": ["vertical_and_slash", 3500, 100, 0.9875326752662659], "14": ["vertical_and_slash", 3500, 100, 0.9958921670913696], "15": ["vertical_and_slash", 3500, 100, 0.9836806654930115], "16": ["vertical_and_slash", 3500, 100, 0.9843685626983643], "17": ["vertical_and_slash", 3500, 100, 0.9905728697776794], "18": ["vertical_and_slash", 3500, 100, 0.9800220727920532], "19": ["vertical_and_slash", 3500, 100, 0.9947136044502258], "20": ["vertical_and_slash", 3500, 100, 0.9902245402336121], "21": ["vertical_and_slash", 3500, 100, 0.9951120018959045], "22": ["vertical_and_slash", 3500, 100, 0.9865958094596863], "23": ["vertical_and_slash", 3500, 100, 0.993402361869812], "24": ["vertical_and_slash", 3500, 100, 0.9710633158683777], "25": ["vertical_and_slash", 3500, 100, 0.889805257320404], "26": ["vertical_and_slash", 3500, 100, 0.914910078048706], "27": ["vertical_and_slash", 3500, 100, 0.8409860134124756], "28": ["vertical_and_slash", 3500, 100, 0.8705323934555054], "29": ["vertical_and_slash", 100, 800, 0.859375], "30": ["vertical_and_slash", 3500, 100, 0.9011846780776978], "31": ["vertical_and_slash", 3500, 100, 0.9464913010597229], "32": ["vertical_and_slash", 3500, 100, 0.9962992668151855], "33": ["vertical_and_slash", 3500, 100, 0.9923105239868164], "34": ["vertical_and_slash", 3500, 100, 0.9983095526695251], "35": ["vertical_and_slash", 3500, 100, 0.9963307976722717], "36": ["vertical_and_slash", 3500, 100, 0.9856176972389221], "37": ["vertical_and_slash", 3500, 100, 0.992557168006897], "38": ["vertical_and_slash", 3500, 100, 0.9970160126686096], "39": ["vertical_and_slash", 3500, 100, 0.9972580671310425], "40": ["vertical_and_slash", 3500, 100, 0.9616989493370056], "41": ["vertical_and_slash", 3500, 100, 0.9652915000915527], "42": ["vertical_and_slash", 500, 700, 0.985378623008728], "43": ["vertical_and_slash", 100, 800, 0.71875], "44": ["vertical_and_slash", 500, 700, 0.9421245455741882], "45": ["vertical_and_slash", 500, 700, 0.9347689151763916], "46": ["vertical_and_slash", 3500, 100, 0.9622902274131775], "47": ["vertical_and_slash", 500, 700, 0.9431679248809814], "48": ["vertical_and_slash", 3500, 100, 0.9709579944610596], "49": ["vertical_and_slash", 3500, 100, 0.9617305397987366], "50": ["vertical_and_slash", 3500, 100, 0.9729613065719604], "51": ["vertical_and_slash", 3500, 100, 0.9401161074638367], "52": ["vertical_and_slash", 3500, 100, 0.9718809723854065], "53": ["vertical_and_slash", 3500, 100, 0.9688206315040588], "54": ["vertical_and_slash", 3500, 100, 0.9684852957725525], "55": ["vertical_and_slash", 3500, 100, 0.9712363481521606], "56": ["vertical_and_slash", 3500, 100, 0.9864313006401062], "57": ["vertical_and_slash", 3500, 100, 0.9912204742431641], "58": ["vertical_and_slash", 500, 700, 0.9903832077980042], "59": ["vertical_and_slash", 500, 700, 0.992457389831543], "60": ["vertical_and_slash", 3500, 100, 0.951326847076416], "61": ["vertical_and_slash", 3500, 100, 0.9837555885314941], "62": ["vertical_and_slash", 500, 700, 0.987200140953064], "63": ["vertical_and_slash", 500, 700, 0.9875743389129639]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7227118015289307], "1": ["vertical_and_slash", 100, 750, 0.7391621470451355], "2": ["vertical_and_slash", 1000, 6096, 0.7268348932266235], "3": ["vertical_and_slash", 1000, 6096, 0.7554751634597778], "4": ["vertical_and_slash", 1000, 6096, 0.8326123356819153], "5": ["vertical_and_slash", 3500, 100, 0.99610835313797], "6": ["vertical_and_slash", 1000, 6096, 0.7299988865852356], "7": ["vertical_and_slash", 1000, 6096, 0.7451336979866028], "8": ["vertical_and_slash", 3500, 100, 0.9947753548622131], "9": ["vertical_and_slash", 3500, 100, 0.9975543022155762], "10": ["vertical_and_slash", 3500, 100, 0.9983291029930115], "11": ["vertical_and_slash", 3500, 100, 0.9942117929458618], "12": ["vertical_and_slash", 3500, 100, 0.9973145723342896], "13": ["vertical_and_slash", 3500, 100, 0.9973160028457642], "14": ["vertical_and_slash", 3500, 100, 0.9960935711860657], "15": ["vertical_and_slash", 3500, 100, 0.9959337711334229], "16": ["vertical_and_slash", 3500, 100, 0.9844751358032227], "17": ["vertical_and_slash", 3500, 100, 0.9961165189743042], "18": ["vertical_and_slash", 3500, 100, 0.9964843392372131], "19": ["vertical_and_slash", 3500, 100, 0.9933270215988159], "20": ["vertical_and_slash", 3500, 100, 0.9908462166786194], "21": ["vertical_and_slash", 3500, 100, 0.9935579299926758], "22": ["vertical_and_slash", 3500, 100, 0.9938632249832153], "23": ["vertical_and_slash", 3500, 100, 0.9536958336830139], "24": ["vertical_and_slash", 500, 700, 0.9922996163368225], "25": ["vertical_and_slash", 3500, 100, 0.9894166588783264], "26": ["vertical_and_slash", 100, 750, 0.9852467179298401], "27": ["vertical_and_slash", 3500, 100, 0.9912678599357605], "28": ["vertical_and_slash", 3500, 100, 0.9855595231056213], "29": ["vertical_and_slash", 3500, 100, 0.9844669699668884], "30": ["vertical_and_slash", 3500, 100, 0.9905785322189331], "31": ["vertical_and_slash", 3500, 100, 0.9939610958099365], "32": ["vertical_and_slash", 3500, 100, 0.9817345142364502], "33": ["vertical_and_slash", 100, 750, 0.9810519218444824], "34": ["vertical_and_slash", 100, 750, 0.9749069809913635], "35": ["vertical_and_slash", 100, 750, 0.986911416053772], "36": ["vertical_and_slash", 100, 750, 0.8704575300216675], "37": ["vertical_and_slash", 100, 750, 0.9620357155799866], "38": ["vertical_and_slash", 500, 700, 0.974058210849762], "39": ["vertical_and_slash", 100, 750, 0.958741307258606], "40": ["vertical_and_slash", 3500, 100, 0.9898220896720886], "41": ["vertical_and_slash", 500, 700, 0.9814236760139465], "42": ["vertical_and_slash", 500, 700, 0.9858487844467163], "43": ["vertical_and_slash", 3500, 100, 0.9738571643829346], "44": ["vertical_and_slash", 3500, 100, 0.9721683263778687], "45": ["vertical_and_slash", 3500, 100, 0.9966035485267639], "46": ["vertical_and_slash", 500, 700, 0.980961799621582], "47": ["vertical_and_slash", 100, 750, 0.9434794783592224], "48": ["vertical_and_slash", 3500, 100, 0.9818546175956726], "49": ["vertical_and_slash", 3500, 100, 0.9961957931518555], "50": ["vertical_and_slash", 3500, 100, 0.9940114617347717], "51": ["vertical_and_slash", 100, 750, 0.9589138627052307], "52": ["vertical_and_slash", 3500, 100, 0.9909583330154419], "53": ["vertical_and_slash", 3500, 100, 0.9979774951934814], "54": ["vertical_and_slash", 3500, 100, 0.9928249716758728], "55": ["vertical_and_slash", 3500, 100, 0.9921364784240723], "56": ["vertical_and_slash", 3500, 100, 0.9786327481269836], "57": ["vertical_and_slash", 3500, 100, 0.9872820377349854], "58": ["vertical_and_slash", 3500, 100, 0.9927536249160767], "59": ["vertical_and_slash", 1000, 6096, 0.9288374185562134], "60": ["vertical_and_slash", 1000, 6096, 0.9698650240898132], "61": ["vertical_and_slash", 3500, 100, 0.9832680821418762], "62": ["vertical_and_slash", 3500, 100, 0.9806138873100281], "63": ["vertical_and_slash", 1000, 6096, 0.9627635478973389]}, {"0": ["vertical_and_slash", 500, 700, 0.9787808656692505], "1": ["vertical_and_slash", 100, 800, 0.66015625], "2": ["vertical_and_slash", 500, 700, 0.8723381757736206], "3": ["vertical_and_slash", 100, 800, 0.88671875], "4": ["vertical_and_slash", 3500, 100, 0.9557732939720154], "5": ["vertical_and_slash", 500, 700, 0.9627295136451721], "6": ["vertical_and_slash", 500, 700, 0.9496796131134033], "7": ["vertical_and_slash", 500, 700, 0.9828166961669922], "8": ["vertical_and_slash", 3500, 100, 0.9963545799255371], "9": ["vertical_and_slash", 3500, 100, 0.9986056685447693], "10": ["vertical_and_slash", 3500, 100, 0.9962592720985413], "11": ["vertical_and_slash", 3500, 100, 0.9865131974220276], "12": ["vertical_and_slash", 3500, 100, 0.9940925240516663], "13": ["vertical_and_slash", 3500, 100, 0.9912338852882385], "14": ["vertical_and_slash", 3500, 100, 0.9981741309165955], "15": ["vertical_and_slash", 3500, 100, 0.9941041469573975], "16": ["vertical_and_slash", 3500, 100, 0.9871466755867004], "17": ["vertical_and_slash", 3500, 100, 0.9908421635627747], "18": ["vertical_and_slash", 3500, 100, 0.990251362323761], "19": ["vertical_and_slash", 3500, 100, 0.9936344027519226], "20": ["vertical_and_slash", 500, 700, 0.9796211123466492], "21": ["vertical_and_slash", 3500, 100, 0.9832311868667603], "22": ["vertical_and_slash", 3500, 100, 0.9957398772239685], "23": ["vertical_and_slash", 3500, 100, 0.9946489334106445], "24": ["vertical_and_slash", 3500, 100, 0.9716681838035583], "25": ["vertical_and_slash", 3500, 100, 0.9702328443527222], "26": ["vertical_and_slash", 3500, 100, 0.9949590563774109], "27": ["vertical_and_slash", 3500, 100, 0.9783271551132202], "28": ["vertical_and_slash", 3500, 100, 0.9844025373458862], "29": ["vertical_and_slash", 500, 700, 0.9863194227218628], "30": ["vertical_and_slash", 500, 700, 0.9835267066955566], "31": ["vertical_and_slash", 3500, 100, 0.9758065938949585], "32": ["vertical_and_slash", 3500, 100, 0.955339252948761], "33": ["vertical_and_slash", 500, 700, 0.9459304809570312], "34": ["vertical_and_slash", 3500, 100, 0.9771563410758972], "35": ["vertical_and_slash", 500, 700, 0.9529558420181274], "36": ["vertical_and_slash", 500, 700, 0.7338439226150513], "37": ["vertical_and_slash", 3500, 100, 0.968394935131073], "38": ["vertical_and_slash", 3500, 100, 0.9634709358215332], "39": ["vertical_and_slash", 3500, 100, 0.9329047799110413], "40": ["vertical_and_slash", 3500, 100, 0.9950606226921082], "41": ["vertical_and_slash", 3500, 100, 0.9939326047897339], "42": ["vertical_and_slash", 3500, 100, 0.9901408553123474], "43": ["vertical_and_slash", 100, 750, 0.884402334690094], "44": ["vertical_and_slash", 3500, 100, 0.9971237182617188], "45": ["vertical_and_slash", 3500, 100, 0.9957478642463684], "46": ["vertical_and_slash", 3500, 100, 0.9983047246932983], "47": ["vertical_and_slash", 3500, 100, 0.996533989906311], "48": ["vertical_and_slash", 3500, 100, 0.9810707569122314], "49": ["vertical_and_slash", 500, 700, 0.9978175163269043], "50": ["vertical_and_slash", 3500, 100, 0.9756927490234375], "51": ["vertical_and_slash", 500, 700, 0.9738315939903259], "52": ["vertical_and_slash", 3500, 100, 0.9802463054656982], "53": ["vertical_and_slash", 500, 700, 0.9943123459815979], "54": ["vertical_and_slash", 500, 700, 0.9835458993911743], "55": ["vertical_and_slash", 3500, 100, 0.9813981652259827], "56": ["vertical_and_slash", 3500, 100, 0.9803796410560608], "57": ["vertical_and_slash", 500, 700, 0.9780364632606506], "58": ["vertical_and_slash", 100, 750, 0.9646508097648621], "59": ["vertical_and_slash", 500, 700, 0.9892991185188293], "60": ["vertical_and_slash", 100, 750, 0.9560303092002869], "61": ["vertical_and_slash", 100, 750, 0.908588171005249], "62": ["vertical_and_slash", 100, 750, 0.9796624183654785], "63": ["vertical_and_slash", 100, 750, 0.9553687572479248]}, {"0": ["vertical_and_slash", 3500, 100, 0.9963559508323669], "1": ["vertical_and_slash", 3500, 100, 0.9934741854667664], "2": ["vertical_and_slash", 500, 700, 0.992142379283905], "3": ["vertical_and_slash", 100, 750, 0.9861782193183899], "4": ["vertical_and_slash", 3500, 100, 0.9953817129135132], "5": ["vertical_and_slash", 100, 750, 0.9802891612052917], "6": ["vertical_and_slash", 500, 700, 0.9870567917823792], "7": ["vertical_and_slash", 100, 750, 0.9866126775741577], "8": ["vertical_and_slash", 3500, 100, 0.9949246644973755], "9": ["vertical_and_slash", 3500, 100, 0.9917769432067871], "10": ["vertical_and_slash", 100, 750, 0.9803691506385803], "11": ["vertical_and_slash", 3500, 100, 0.9942638278007507], "12": ["vertical_and_slash", 3500, 100, 0.9775207042694092], "13": ["vertical_and_slash", 3500, 100, 0.982894241809845], "14": ["vertical_and_slash", 3500, 100, 0.9773128032684326], "15": ["vertical_and_slash", 3500, 100, 0.974577009677887], "16": ["vertical_and_slash", 3500, 100, 0.990338921546936], "17": ["vertical_and_slash", 3500, 100, 0.9968396425247192], "18": ["vertical_and_slash", 3500, 100, 0.9910425543785095], "19": ["vertical_and_slash", 500, 700, 0.9404577612876892], "20": ["vertical_and_slash", 3500, 100, 0.9908447265625], "21": ["vertical_and_slash", 3500, 100, 0.9930487871170044], "22": ["vertical_and_slash", 3500, 100, 0.9946773052215576], "23": ["vertical_and_slash", 3500, 100, 0.984718382358551], "24": ["vertical_and_slash", 500, 700, 0.9755082726478577], "25": ["vertical_and_slash", 100, 750, 0.8470464944839478], "26": ["vertical_and_slash", 500, 700, 0.9474022388458252], "27": ["vertical_and_slash", 500, 700, 0.9661937355995178], "28": ["vertical_and_slash", 500, 700, 0.9311940670013428], "29": ["vertical_and_slash", 500, 700, 0.9578240513801575], "30": ["vertical_and_slash", 500, 700, 0.9391536712646484], "31": ["vertical_and_slash", 3500, 100, 0.9304941892623901], "32": ["vertical_and_slash", 3500, 100, 0.9912314414978027], "33": ["vertical_and_slash", 500, 700, 0.9729458689689636], "34": ["vertical_and_slash", 3500, 100, 0.9954802989959717], "35": ["vertical_and_slash", 3500, 100, 0.9947611093521118], "36": ["vertical_and_slash", 500, 700, 0.9610899090766907], "37": ["vertical_and_slash", 3500, 100, 0.98304283618927], "38": ["vertical_and_slash", 3500, 100, 0.9990308284759521], "39": ["vertical_and_slash", 3500, 100, 0.9833992123603821], "40": ["vertical_and_slash", 3500, 100, 0.9983760118484497], "41": ["vertical_and_slash", 3500, 100, 0.99656742811203], "42": ["vertical_and_slash", 3500, 100, 0.996590793132782], "43": ["vertical_and_slash", 3500, 100, 0.9978159666061401], "44": ["vertical_and_slash", 3500, 100, 0.9950272440910339], "45": ["vertical_and_slash", 3500, 100, 0.9880179762840271], "46": ["vertical_and_slash", 3500, 100, 0.9968637824058533], "47": ["vertical_and_slash", 3500, 100, 0.9909763932228088], "48": ["vertical_and_slash", 500, 700, 0.966448962688446], "49": ["vertical_and_slash", 3500, 100, 0.9927787780761719], "50": ["vertical_and_slash", 3500, 100, 0.9902259707450867], "51": ["vertical_and_slash", 3500, 100, 0.9950190186500549], "52": ["vertical_and_slash", 3500, 100, 0.9828783869743347], "53": ["vertical_and_slash", 500, 700, 0.995979368686676], "54": ["vertical_and_slash", 3500, 100, 0.9967334866523743], "55": ["vertical_and_slash", 3500, 100, 0.9955008029937744], "56": ["vertical_and_slash", 3500, 100, 0.8949801325798035], "57": ["vertical_and_slash", 3500, 100, 0.9654185771942139], "58": ["vertical_and_slash", 3500, 100, 0.9676187038421631], "59": ["vertical_and_slash", 3500, 100, 0.9322010278701782], "60": ["vertical_and_slash", 3500, 100, 0.9424247741699219], "61": ["vertical_and_slash", 3500, 100, 0.9574859738349915], "62": ["vertical_and_slash", 3500, 100, 0.970115065574646], "63": ["vertical_and_slash", 3500, 100, 0.9150857329368591]}, {"0": ["vertical_and_slash", 3500, 100, 0.9941188097000122], "1": ["vertical_and_slash", 3500, 100, 0.9950684309005737], "2": ["vertical_and_slash", 3500, 100, 0.9851258993148804], "3": ["vertical_and_slash", 3500, 100, 0.9947274923324585], "4": ["vertical_and_slash", 100, 750, 0.9862303733825684], "5": ["vertical_and_slash", 3500, 100, 0.9886133670806885], "6": ["vertical_and_slash", 3500, 100, 0.9916771054267883], "7": ["vertical_and_slash", 3500, 100, 0.9892164468765259], "8": ["vertical_and_slash", 100, 750, 0.9075641632080078], "9": ["vertical_and_slash", 100, 800, 0.76953125], "10": ["vertical_and_slash", 100, 800, 0.96484375], "11": ["vertical_and_slash", 3500, 100, 0.9671242237091064], "12": ["vertical_and_slash", 100, 800, 0.75], "13": ["vertical_and_slash", 500, 700, 0.830882728099823], "14": ["vertical_and_slash", 3500, 100, 0.9426618218421936], "15": ["vertical_and_slash", 100, 800, 0.85546875], "16": ["vertical_and_slash", 3500, 100, 0.9136013388633728], "17": ["vertical_and_slash", 3500, 100, 0.9380723834037781], "18": ["vertical_and_slash", 3500, 100, 0.9515624642372131], "19": ["vertical_and_slash", 3500, 100, 0.9589139819145203], "20": ["vertical_and_slash", 3500, 100, 0.9687409996986389], "21": ["vertical_and_slash", 3500, 100, 0.922919750213623], "22": ["vertical_and_slash", 3500, 100, 0.9327558875083923], "23": ["vertical_and_slash", 3500, 100, 0.914228618144989], "24": ["vertical_and_slash", 3500, 100, 0.9788975715637207], "25": ["vertical_and_slash", 3500, 100, 0.973120391368866], "26": ["vertical_and_slash", 3500, 100, 0.9799728393554688], "27": ["vertical_and_slash", 3500, 100, 0.9671465158462524], "28": ["vertical_and_slash", 3500, 100, 0.9865114092826843], "29": ["vertical_and_slash", 3500, 100, 0.9811680912971497], "30": ["vertical_and_slash", 500, 700, 0.9616523385047913], "31": ["vertical_and_slash", 3500, 100, 0.9882767796516418], "32": ["vertical_and_slash", 3500, 100, 0.9889876246452332], "33": ["vertical_and_slash", 500, 700, 0.9776366353034973], "34": ["vertical_and_slash", 500, 700, 0.9695335030555725], "35": ["vertical_and_slash", 3500, 100, 0.985745370388031], "36": ["vertical_and_slash", 3500, 100, 0.9937112331390381], "37": ["vertical_and_slash", 3500, 100, 0.9865038394927979], "38": ["vertical_and_slash", 3500, 100, 0.9909293055534363], "39": ["vertical_and_slash", 3500, 100, 0.978290319442749], "40": ["vertical_and_slash", 3500, 100, 0.9981828331947327], "41": ["vertical_and_slash", 500, 700, 0.978047788143158], "42": ["vertical_and_slash", 500, 700, 0.944717526435852], "43": ["vertical_and_slash", 3500, 100, 0.9866669774055481], "44": ["vertical_and_slash", 3500, 100, 0.9940350651741028], "45": ["vertical_and_slash", 3500, 100, 0.9977557063102722], "46": ["vertical_and_slash", 100, 750, 0.8469586372375488], "47": ["vertical_and_slash", 3500, 100, 0.9833143949508667], "48": ["vertical_and_slash", 3500, 100, 0.9749195575714111], "49": ["vertical_and_slash", 3500, 100, 0.9002892374992371], "50": ["vertical_and_slash", 3500, 100, 0.9544698596000671], "51": ["vertical_and_slash", 3500, 100, 0.9774844646453857], "52": ["vertical_and_slash", 3500, 100, 0.9585850238800049], "53": ["vertical_and_slash", 3500, 100, 0.9462693333625793], "54": ["vertical_and_slash", 100, 800, 0.86328125], "55": ["vertical_and_slash", 3500, 100, 0.9716248512268066], "56": ["vertical_and_slash", 3500, 100, 0.9922828078269958], "57": ["vertical_and_slash", 100, 750, 0.9326270222663879], "58": ["vertical_and_slash", 3500, 100, 0.9844812750816345], "59": ["vertical_and_slash", 3500, 100, 0.9896049499511719], "60": ["vertical_and_slash", 100, 800, 0.94921875], "61": ["vertical_and_slash", 3500, 100, 0.9914078712463379], "62": ["vertical_and_slash", 3500, 100, 0.9749128818511963], "63": ["vertical_and_slash", 3500, 100, 0.9843443036079407]}, {"0": ["vertical_and_slash", 500, 700, 0.9883238077163696], "1": ["vertical_and_slash", 500, 700, 0.9885198473930359], "2": ["vertical_and_slash", 500, 700, 0.9770777821540833], "3": ["vertical_and_slash", 100, 750, 0.9742189049720764], "4": ["vertical_and_slash", 3500, 100, 0.9874204397201538], "5": ["vertical_and_slash", 3500, 100, 0.986889123916626], "6": ["vertical_and_slash", 100, 750, 0.9890825748443604], "7": ["vertical_and_slash", 100, 750, 0.9754319190979004], "8": ["vertical_and_slash", 100, 750, 0.9670644998550415], "9": ["vertical_and_slash", 3500, 100, 0.989413857460022], "10": ["vertical_and_slash", 100, 750, 0.9873670339584351], "11": ["vertical_and_slash", 100, 750, 0.9393261671066284], "12": ["vertical_and_slash", 100, 750, 0.9834675192832947], "13": ["vertical_and_slash", 100, 750, 0.9739778637886047], "14": ["vertical_and_slash", 100, 750, 0.9598566293716431], "15": ["vertical_and_slash", 100, 750, 0.9717019200325012], "16": ["vertical_and_slash", 500, 700, 0.9166276454925537], "17": ["vertical_and_slash", 3500, 100, 0.9907848834991455], "18": ["vertical_and_slash", 3500, 100, 0.9903581142425537], "19": ["vertical_and_slash", 500, 700, 0.9580197930335999], "20": ["vertical_and_slash", 500, 700, 0.9701711535453796], "21": ["vertical_and_slash", 100, 750, 0.9470651745796204], "22": ["vertical_and_slash", 500, 700, 0.9775917530059814], "23": ["vertical_and_slash", 100, 750, 0.8181455135345459], "24": ["vertical_and_slash", 3500, 100, 0.9458271861076355], "25": ["vertical_and_slash", 3500, 100, 0.9725922346115112], "26": ["vertical_and_slash", 3500, 100, 0.9434858560562134], "27": ["vertical_and_slash", 3500, 100, 0.9659452438354492], "28": ["vertical_and_slash", 1000, 6096, 0.914588451385498], "29": ["vertical_and_slash", 3500, 100, 0.9404117465019226], "30": ["vertical_and_slash", 3500, 100, 0.9620093107223511], "31": ["vertical_and_slash", 3500, 100, 0.968653678894043], "32": ["vertical_and_slash", 3500, 100, 0.914693295955658], "33": ["vertical_and_slash", 100, 800, 0.88671875], "34": ["vertical_and_slash", 100, 800, 0.77734375], "35": ["vertical_and_slash", 3500, 100, 0.9856390953063965], "36": ["vertical_and_slash", 3500, 100, 0.9856829643249512], "37": ["vertical_and_slash", 100, 800, 0.75390625], "38": ["vertical_and_slash", 100, 800, 0.921875], "39": ["vertical_and_slash", 1000, 6096, 0.8207946419715881], "40": ["vertical_and_slash", 500, 700, 0.9026387929916382], "41": ["vertical_and_slash", 500, 700, 0.9312556982040405], "42": ["vertical_and_slash", 3500, 100, 0.9854640960693359], "43": ["vertical_and_slash", 500, 700, 0.972061038017273], "44": ["vertical_and_slash", 100, 750, 0.8406323790550232], "45": ["vertical_and_slash", 3500, 100, 0.9868512153625488], "46": ["vertical_and_slash", 3500, 100, 0.9698919057846069], "47": ["vertical_and_slash", 3500, 100, 0.987265944480896], "48": ["vertical_and_slash", 3500, 100, 0.9817126393318176], "49": ["vertical_and_slash", 100, 750, 0.9822719693183899], "50": ["vertical_and_slash", 3500, 100, 0.9964709877967834], "51": ["vertical_and_slash", 500, 700, 0.9929688572883606], "52": ["vertical_and_slash", 500, 700, 0.9930868148803711], "53": ["vertical_and_slash", 500, 700, 0.9907029271125793], "54": ["vertical_and_slash", 500, 700, 0.9718948006629944], "55": ["vertical_and_slash", 500, 700, 0.9932291507720947], "56": ["vertical_and_slash", 3500, 100, 0.9957431554794312], "57": ["vertical_and_slash", 3500, 100, 0.9949696660041809], "58": ["vertical_and_slash", 3500, 100, 0.9981205463409424], "59": ["vertical_and_slash", 3500, 100, 0.9846609234809875], "60": ["vertical_and_slash", 3500, 100, 0.9880010485649109], "61": ["vertical_and_slash", 3500, 100, 0.992313027381897], "62": ["vertical_and_slash", 3500, 100, 0.9756492376327515], "63": ["vertical_and_slash", 3500, 100, 0.9890857934951782]}, {"0": ["vertical_and_slash", 100, 750, 0.9937179088592529], "1": ["vertical_and_slash", 100, 750, 0.9961262345314026], "2": ["vertical_and_slash", 100, 750, 0.9859903454780579], "3": ["vertical_and_slash", 100, 750, 0.9948785305023193], "4": ["vertical_and_slash", 100, 750, 0.9955084919929504], "5": ["vertical_and_slash", 100, 750, 0.9884966611862183], "6": ["vertical_and_slash", 100, 750, 0.9956446886062622], "7": ["vertical_and_slash", 100, 750, 0.9822808504104614], "8": ["vertical_and_slash", 3500, 100, 0.8363969922065735], "9": ["vertical_and_slash", 500, 700, 0.9815676808357239], "10": ["vertical_and_slash", 500, 700, 0.9584004282951355], "11": ["vertical_and_slash", 100, 800, 0.73828125], "12": ["vertical_and_slash", 3500, 100, 0.9506569504737854], "13": ["vertical_and_slash", 100, 800, 0.9375], "14": ["vertical_and_slash", 500, 700, 0.7002615332603455], "15": ["vertical_and_slash", 100, 750, 0.9017624258995056], "16": ["vertical_and_slash", 3500, 100, 0.9923032522201538], "17": ["vertical_and_slash", 500, 700, 0.9814305305480957], "18": ["vertical_and_slash", 3500, 100, 0.98540198802948], "19": ["vertical_and_slash", 3500, 100, 0.986023485660553], "20": ["vertical_and_slash", 500, 700, 0.9875867366790771], "21": ["vertical_and_slash", 100, 750, 0.9838369488716125], "22": ["vertical_and_slash", 3500, 100, 0.9956971406936646], "23": ["vertical_and_slash", 3500, 100, 0.9976233839988708], "24": ["vertical_and_slash", 500, 700, 0.8618735671043396], "25": ["vertical_and_slash", 3500, 100, 0.9840959310531616], "26": ["vertical_and_slash", 500, 700, 0.9827197790145874], "27": ["vertical_and_slash", 500, 700, 0.9858628511428833], "28": ["vertical_and_slash", 500, 700, 0.9800659418106079], "29": ["vertical_and_slash", 500, 700, 0.9538211822509766], "30": ["vertical_and_slash", 100, 800, 0.9140625], "31": ["vertical_and_slash", 3500, 100, 0.9556357860565186], "32": ["vertical_and_slash", 100, 800, 0.8671875], "33": ["vertical_and_slash", 500, 700, 0.9856330752372742], "34": ["vertical_and_slash", 3500, 100, 0.9896810054779053], "35": ["vertical_and_slash", 500, 700, 0.9692833423614502], "36": ["vertical_and_slash", 500, 700, 0.9923194050788879], "37": ["vertical_and_slash", 500, 700, 0.9944474697113037], "38": ["vertical_and_slash", 100, 750, 0.9460062980651855], "39": ["vertical_and_slash", 500, 700, 0.9938809275627136], "40": ["vertical_and_slash", 500, 700, 0.9738017320632935], "41": ["vertical_and_slash", 500, 700, 0.967767596244812], "42": ["vertical_and_slash", 500, 700, 0.9650017023086548], "43": ["vertical_and_slash", 100, 750, 0.9838941097259521], "44": ["vertical_and_slash", 500, 700, 0.9523524045944214], "45": ["vertical_and_slash", 500, 700, 0.8137993812561035], "46": ["vertical_and_slash", 100, 750, 0.9829829931259155], "47": ["vertical_and_slash", 100, 750, 0.6799432039260864], "48": ["vertical_and_slash", 500, 700, 0.9641908407211304], "49": ["vertical_and_slash", 3500, 100, 0.9937065243721008], "50": ["vertical_and_slash", 3500, 100, 0.9807449579238892], "51": ["vertical_and_slash", 500, 700, 0.9871459603309631], "52": ["vertical_and_slash", 100, 750, 0.9192060828208923], "53": ["vertical_and_slash", 3500, 100, 0.9936875104904175], "54": ["vertical_and_slash", 3500, 100, 0.9922778606414795], "55": ["vertical_and_slash", 3500, 100, 0.9856830835342407], "56": ["vertical_and_slash", 3500, 100, 0.9978744983673096], "57": ["vertical_and_slash", 3500, 100, 0.9983111023902893], "58": ["vertical_and_slash", 3500, 100, 0.988058865070343], "59": ["vertical_and_slash", 100, 750, 0.8913553953170776], "60": ["vertical_and_slash", 3500, 100, 0.9966568350791931], "61": ["vertical_and_slash", 3500, 100, 0.9955816864967346], "62": ["vertical_and_slash", 3500, 100, 0.9777510166168213], "63": ["vertical_and_slash", 3500, 100, 0.9943772554397583]}, {"0": ["vertical_and_slash", 500, 700, 0.8400236964225769], "1": ["vertical_and_slash", 3500, 100, 0.937382161617279], "2": ["vertical_and_slash", 100, 800, 0.8125], "3": ["vertical_and_slash", 100, 800, 0.7890625], "4": ["vertical_and_slash", 3500, 100, 0.9364286661148071], "5": ["vertical_and_slash", 100, 800, 0.765625], "6": ["vertical_and_slash", 3500, 100, 0.8367471694946289], "7": ["vertical_and_slash", 1000, 6096, 0.5601140856742859], "8": ["vertical_and_slash", 3500, 100, 0.9873181581497192], "9": ["vertical_and_slash", 100, 750, 0.7951456904411316], "10": ["vertical_and_slash", 3500, 100, 0.9935299754142761], "11": ["vertical_and_slash", 3500, 100, 0.9930294156074524], "12": ["vertical_and_slash", 3500, 100, 0.9902819991111755], "13": ["vertical_and_slash", 500, 700, 0.9217315316200256], "14": ["vertical_and_slash", 500, 700, 0.9862986207008362], "15": ["vertical_and_slash", 3500, 100, 0.9438391923904419], "16": ["vertical_and_slash", 500, 700, 0.9866422414779663], "17": ["vertical_and_slash", 100, 750, 0.8493568301200867], "18": ["vertical_and_slash", 500, 700, 0.9795662760734558], "19": ["vertical_and_slash", 100, 750, 0.9594753980636597], "20": ["vertical_and_slash", 500, 700, 0.9772626161575317], "21": ["vertical_and_slash", 100, 750, 0.963122546672821], "22": ["vertical_and_slash", 500, 700, 0.9398162364959717], "23": ["vertical_and_slash", 500, 700, 0.9591696262359619], "24": ["vertical_and_slash", 100, 800, 0.921875], "25": ["vertical_and_slash", 500, 700, 0.8860001564025879], "26": ["vertical_and_slash", 500, 700, 0.959342360496521], "27": ["vertical_and_slash", 500, 700, 0.9509842991828918], "28": ["vertical_and_slash", 500, 700, 0.9226822257041931], "29": ["vertical_and_slash", 3500, 100, 0.9692535996437073], "30": ["vertical_and_slash", 500, 700, 0.904583215713501], "31": ["vertical_and_slash", 500, 700, 0.9863643050193787], "32": ["vertical_and_slash", 3500, 100, 0.9781808257102966], "33": ["vertical_and_slash", 3500, 100, 0.9805306196212769], "34": ["vertical_and_slash", 3500, 100, 0.9784607887268066], "35": ["vertical_and_slash", 3500, 100, 0.9556692838668823], "36": ["vertical_and_slash", 3500, 100, 0.9743000864982605], "37": ["vertical_and_slash", 3500, 100, 0.9888134598731995], "38": ["vertical_and_slash", 3500, 100, 0.9777820110321045], "39": ["vertical_and_slash", 500, 700, 0.8528692722320557], "40": ["vertical_and_slash", 500, 700, 0.9581615924835205], "41": ["vertical_and_slash", 500, 700, 0.9887604117393494], "42": ["vertical_and_slash", 500, 700, 0.9756917953491211], "43": ["vertical_and_slash", 500, 700, 0.9835035800933838], "44": ["vertical_and_slash", 500, 700, 0.971200168132782], "45": ["vertical_and_slash", 500, 700, 0.9854931831359863], "46": ["vertical_and_slash", 500, 700, 0.9610379338264465], "47": ["vertical_and_slash", 500, 700, 0.9782274961471558], "48": ["vertical_and_slash", 3500, 100, 0.9817398190498352], "49": ["vertical_and_slash", 3500, 100, 0.9916533827781677], "50": ["vertical_and_slash", 500, 700, 0.9833522439002991], "51": ["vertical_and_slash", 3500, 100, 0.9750106334686279], "52": ["vertical_and_slash", 3500, 100, 0.9811976552009583], "53": ["vertical_and_slash", 3500, 100, 0.9829654097557068], "54": ["vertical_and_slash", 3500, 100, 0.9546154737472534], "55": ["vertical_and_slash", 3500, 100, 0.9843735694885254], "56": ["vertical_and_slash", 100, 750, 0.9714645743370056], "57": ["vertical_and_slash", 500, 700, 0.972328245639801], "58": ["vertical_and_slash", 100, 750, 0.9217227697372437], "59": ["vertical_and_slash", 3500, 100, 0.9911988377571106], "60": ["vertical_and_slash", 500, 700, 0.9786014556884766], "61": ["vertical_and_slash", 100, 750, 0.845099925994873], "62": ["vertical_and_slash", 3500, 100, 0.992199718952179], "63": ["vertical_and_slash", 100, 800, 0.96875]}, {"0": ["vertical_and_slash", 3500, 100, 0.980701744556427], "1": ["vertical_and_slash", 3500, 100, 0.9627414345741272], "2": ["vertical_and_slash", 3500, 100, 0.9871503114700317], "3": ["vertical_and_slash", 3500, 100, 0.9905191659927368], "4": ["vertical_and_slash", 3500, 100, 0.9812014102935791], "5": ["vertical_and_slash", 500, 700, 0.8816037178039551], "6": ["vertical_and_slash", 3500, 100, 0.9912583231925964], "7": ["vertical_and_slash", 3500, 100, 0.962593674659729], "8": ["vertical_and_slash", 3500, 100, 0.9859939813613892], "9": ["vertical_and_slash", 500, 700, 0.972049355506897], "10": ["vertical_and_slash", 3500, 100, 0.9851554036140442], "11": ["vertical_and_slash", 3500, 100, 0.9912015795707703], "12": ["vertical_and_slash", 3500, 100, 0.9888198375701904], "13": ["vertical_and_slash", 3500, 100, 0.9882067441940308], "14": ["vertical_and_slash", 500, 700, 0.9614861011505127], "15": ["vertical_and_slash", 3500, 100, 0.9851382970809937], "16": ["vertical_and_slash", 500, 700, 0.9830257296562195], "17": ["vertical_and_slash", 500, 700, 0.9887971878051758], "18": ["vertical_and_slash", 500, 700, 0.8849800229072571], "19": ["vertical_and_slash", 500, 700, 0.9880145788192749], "20": ["vertical_and_slash", 500, 700, 0.971123218536377], "21": ["vertical_and_slash", 500, 700, 0.9207893013954163], "22": ["vertical_and_slash", 500, 700, 0.9813519716262817], "23": ["vertical_and_slash", 500, 700, 0.9839648604393005], "24": ["vertical_and_slash", 500, 700, 0.9482806324958801], "25": ["vertical_and_slash", 3500, 100, 0.993384838104248], "26": ["vertical_and_slash", 3500, 100, 0.9927131533622742], "27": ["vertical_and_slash", 500, 700, 0.9783619046211243], "28": ["vertical_and_slash", 500, 700, 0.9894798994064331], "29": ["vertical_and_slash", 500, 700, 0.9704422950744629], "30": ["vertical_and_slash", 500, 700, 0.9720821380615234], "31": ["vertical_and_slash", 3500, 100, 0.9836769104003906], "32": ["vertical_and_slash", 3500, 100, 0.9564710855484009], "33": ["vertical_and_slash", 3500, 100, 0.968597412109375], "34": ["vertical_and_slash", 3500, 100, 0.9376870393753052], "35": ["vertical_and_slash", 3500, 100, 0.9626307487487793], "36": ["vertical_and_slash", 3500, 100, 0.9407920837402344], "37": ["vertical_and_slash", 3500, 100, 0.9304219484329224], "38": ["vertical_and_slash", 3500, 100, 0.961587131023407], "39": ["vertical_and_slash", 3500, 100, 0.9728710651397705], "40": ["vertical_and_slash", 500, 700, 0.9650471210479736], "41": ["vertical_and_slash", 500, 700, 0.9734203815460205], "42": ["vertical_and_slash", 3500, 100, 0.9918591380119324], "43": ["vertical_and_slash", 500, 700, 0.9831333160400391], "44": ["vertical_and_slash", 500, 700, 0.9677253365516663], "45": ["vertical_and_slash", 500, 700, 0.981693685054779], "46": ["vertical_and_slash", 500, 700, 0.9793500304222107], "47": ["vertical_and_slash", 500, 700, 0.9817217588424683], "48": ["vertical_and_slash", 500, 700, 0.9936875700950623], "49": ["vertical_and_slash", 500, 700, 0.9737863540649414], "50": ["vertical_and_slash", 500, 700, 0.9292354583740234], "51": ["vertical_and_slash", 3500, 100, 0.9888380169868469], "52": ["vertical_and_slash", 100, 800, 0.9375], "53": ["vertical_and_slash", 500, 700, 0.9883688688278198], "54": ["vertical_and_slash", 3500, 100, 0.9852707982063293], "55": ["vertical_and_slash", 100, 800, 0.87890625], "56": ["vertical_and_slash", 3500, 100, 0.9926839470863342], "57": ["vertical_and_slash", 3500, 100, 0.9344926476478577], "58": ["vertical_and_slash", 500, 700, 0.9736244082450867], "59": ["vertical_and_slash", 3500, 100, 0.990509569644928], "60": ["vertical_and_slash", 500, 700, 0.9819197654724121], "61": ["vertical_and_slash", 3500, 100, 0.9444815516471863], "62": ["vertical_and_slash", 500, 700, 0.9567142128944397], "63": ["vertical_and_slash", 3500, 100, 0.9461817741394043]}, {"0": ["vertical_and_slash", 3500, 100, 0.9671319127082825], "1": ["vertical_and_slash", 500, 700, 0.9446508884429932], "2": ["vertical_and_slash", 3500, 100, 0.9076815843582153], "3": ["vertical_and_slash", 3500, 100, 0.9542126655578613], "4": ["vertical_and_slash", 3500, 100, 0.9364123344421387], "5": ["vertical_and_slash", 3500, 100, 0.9577588438987732], "6": ["vertical_and_slash", 3500, 100, 0.9493610858917236], "7": ["vertical_and_slash", 3500, 100, 0.9249312281608582], "8": ["vertical_and_slash", 3500, 100, 0.8926833868026733], "9": ["vertical_and_slash", 1000, 6096, 0.7126339077949524], "10": ["vertical_and_slash", 1000, 6096, 0.7891420125961304], "11": ["vertical_and_slash", 100, 800, 0.578125], "12": ["vertical_and_slash", 500, 700, 0.9463756084442139], "13": ["vertical_and_slash", 3500, 100, 0.9257423281669617], "14": ["vertical_and_slash", 100, 800, 0.6484375], "15": ["vertical_and_slash", 1000, 6096, 0.6477133631706238], "16": ["vertical_and_slash", 100, 800, 0.79296875], "17": ["vertical_and_slash", 500, 700, 0.9443665742874146], "18": ["vertical_and_slash", 500, 700, 0.9708283543586731], "19": ["vertical_and_slash", 500, 700, 0.9665108323097229], "20": ["vertical_and_slash", 100, 750, 0.9237576723098755], "21": ["vertical_and_slash", 100, 750, 0.9333072304725647], "22": ["vertical_and_slash", 500, 700, 0.9795149564743042], "23": ["vertical_and_slash", 3500, 100, 0.8304385542869568], "24": ["vertical_and_slash", 3500, 100, 0.968262255191803], "25": ["vertical_and_slash", 3500, 100, 0.9573091864585876], "26": ["vertical_and_slash", 3500, 100, 0.9541662335395813], "27": ["vertical_and_slash", 500, 700, 0.9183540940284729], "28": ["vertical_and_slash", 3500, 100, 0.8797848224639893], "29": ["vertical_and_slash", 3500, 100, 0.9426365494728088], "30": ["vertical_and_slash", 3500, 100, 0.9267366528511047], "31": ["vertical_and_slash", 3500, 100, 0.9433652758598328], "32": ["vertical_and_slash", 3500, 100, 0.9836041927337646], "33": ["vertical_and_slash", 3500, 100, 0.9505170583724976], "34": ["vertical_and_slash", 3500, 100, 0.9555638432502747], "35": ["vertical_and_slash", 3500, 100, 0.972544252872467], "36": ["vertical_and_slash", 3500, 100, 0.9421955943107605], "37": ["vertical_and_slash", 3500, 100, 0.9666458368301392], "38": ["vertical_and_slash", 3500, 100, 0.9665914177894592], "39": ["vertical_and_slash", 3500, 100, 0.9484471678733826], "40": ["vertical_and_slash", 500, 700, 0.9659901857376099], "41": ["vertical_and_slash", 500, 700, 0.9689428806304932], "42": ["vertical_and_slash", 100, 800, 0.91796875], "43": ["vertical_and_slash", 500, 700, 0.9658474922180176], "44": ["vertical_and_slash", 500, 700, 0.946921169757843], "45": ["vertical_and_slash", 100, 750, 0.9594663381576538], "46": ["vertical_and_slash", 500, 700, 0.9605693817138672], "47": ["vertical_and_slash", 500, 700, 0.9493117928504944], "48": ["vertical_and_slash", 100, 800, 0.609375], "49": ["vertical_and_slash", 100, 750, 0.9482254385948181], "50": ["vertical_and_slash", 100, 800, 0.83203125], "51": ["vertical_and_slash", 100, 750, 0.8058269619941711], "52": ["vertical_and_slash", 100, 750, 0.9104240536689758], "53": ["vertical_and_slash", 100, 750, 0.9566232562065125], "54": ["vertical_and_slash", 500, 700, 0.964683473110199], "55": ["vertical_and_slash", 1000, 6096, 0.6569716930389404], "56": ["vertical_and_slash", 100, 800, 0.90625], "57": ["vertical_and_slash", 100, 750, 0.8216474652290344], "58": ["vertical_and_slash", 100, 750, 0.9466930031776428], "59": ["vertical_and_slash", 3500, 100, 0.9039538502693176], "60": ["vertical_and_slash", 3500, 100, 0.9235866665840149], "61": ["vertical_and_slash", 100, 750, 0.8954473733901978], "62": ["vertical_and_slash", 100, 800, 0.85546875], "63": ["vertical_and_slash", 100, 800, 0.86328125]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9214634299278259], "1": ["vertical_and_slash", 100, 800, 0.9375], "2": ["vertical_and_slash", 100, 800, 0.87109375], "3": ["vertical_and_slash", 1000, 6096, 0.7854868173599243], "4": ["vertical_and_slash", 3500, 100, 0.9759478569030762], "5": ["vertical_and_slash", 500, 700, 0.9772587418556213], "6": ["vertical_and_slash", 100, 800, 0.8046875], "7": ["vertical_and_slash", 100, 800, 0.84375], "8": ["vertical_and_slash", 100, 750, 0.9246247410774231], "9": ["vertical_and_slash", 100, 800, 0.84765625], "10": ["vertical_and_slash", 500, 700, 0.8876572847366333], "11": ["vertical_and_slash", 500, 700, 0.9559313654899597], "12": ["vertical_and_slash", 100, 800, 0.62890625], "13": ["vertical_and_slash", 100, 800, 0.63671875], "14": ["vertical_and_slash", 500, 700, 0.9640429615974426], "15": ["vertical_and_slash", 500, 700, 0.9336685538291931], "16": ["vertical_and_slash", 1000, 6096, 0.9711282253265381], "17": ["vertical_and_slash", 500, 700, 0.9497082829475403], "18": ["vertical_and_slash", 3500, 100, 0.9864729046821594], "19": ["vertical_and_slash", 3500, 100, 0.9781782627105713], "20": ["vertical_and_slash", 1000, 6096, 0.9761489033699036], "21": ["vertical_and_slash", 100, 800, 0.94140625], "22": ["vertical_and_slash", 500, 700, 0.9040678143501282], "23": ["vertical_and_slash", 1000, 6096, 0.9788802862167358], "24": ["vertical_and_slash", 100, 800, 0.72265625], "25": ["vertical_and_slash", 500, 700, 0.8939054012298584], "26": ["vertical_and_slash", 500, 700, 0.9338541030883789], "27": ["vertical_and_slash", 100, 800, 0.87109375], "28": ["vertical_and_slash", 100, 800, 0.8828125], "29": ["vertical_and_slash", 3500, 100, 0.5045566558837891], "30": ["vertical_and_slash", 100, 800, 0.86328125], "31": ["vertical_and_slash", 100, 800, 0.828125], "32": ["vertical_and_slash", 500, 700, 0.9943075776100159], "33": ["vertical_and_slash", 500, 700, 0.9849719405174255], "34": ["vertical_and_slash", 500, 700, 0.989171028137207], "35": ["vertical_and_slash", 3500, 100, 0.9934377074241638], "36": ["vertical_and_slash", 500, 700, 0.9855682253837585], "37": ["vertical_and_slash", 500, 700, 0.9935199022293091], "38": ["vertical_and_slash", 500, 700, 0.9854236245155334], "39": ["vertical_and_slash", 500, 700, 0.9942601323127747], "40": ["vertical_and_slash", 3500, 100, 0.8491178750991821], "41": ["vertical_and_slash", 500, 700, 0.9425336122512817], "42": ["vertical_and_slash", 500, 700, 0.9344311356544495], "43": ["vertical_and_slash", 3500, 100, 0.7699980139732361], "44": ["vertical_and_slash", 100, 800, 0.90234375], "45": ["vertical_and_slash", 100, 800, 0.71875], "46": ["vertical_and_slash", 3500, 100, 0.9604572653770447], "47": ["vertical_and_slash", 100, 800, 0.90234375], "48": ["vertical_and_slash", 100, 750, 0.924805223941803], "49": ["vertical_and_slash", 500, 700, 0.9849457144737244], "50": ["vertical_and_slash", 500, 700, 0.9512607455253601], "51": ["vertical_and_slash", 500, 700, 0.9814988970756531], "52": ["vertical_and_slash", 500, 700, 0.9930247068405151], "53": ["vertical_and_slash", 500, 700, 0.9756758809089661], "54": ["vertical_and_slash", 500, 700, 0.970357358455658], "55": ["vertical_and_slash", 500, 700, 0.9817487597465515], "56": ["vertical_and_slash", 100, 750, 0.9836804270744324], "57": ["vertical_and_slash", 3500, 100, 0.9943484663963318], "58": ["vertical_and_slash", 3500, 100, 0.9876598119735718], "59": ["vertical_and_slash", 500, 700, 0.9890317916870117], "60": ["vertical_and_slash", 500, 700, 0.9829592108726501], "61": ["vertical_and_slash", 1000, 6096, 0.9715815782546997], "62": ["vertical_and_slash", 500, 700, 0.9867123365402222], "63": ["vertical_and_slash", 500, 700, 0.9939300417900085]}, {"0": ["vertical_and_slash", 100, 800, 0.9765625], "1": ["vertical_and_slash", 500, 700, 0.9885557889938354], "2": ["vertical_and_slash", 3500, 100, 0.9854679703712463], "3": ["vertical_and_slash", 500, 700, 0.986272394657135], "4": ["vertical_and_slash", 100, 800, 0.984375], "5": ["vertical_and_slash", 3500, 100, 0.9564793109893799], "6": ["vertical_and_slash", 100, 800, 0.8359375], "7": ["vertical_and_slash", 3500, 100, 0.9817067980766296], "8": ["vertical_and_slash", 100, 800, 0.921875], "9": ["vertical_and_slash", 500, 700, 0.9802719950675964], "10": ["vertical_and_slash", 500, 700, 0.838211715221405], "11": ["vertical_and_slash", 500, 700, 0.9765668511390686], "12": ["vertical_and_slash", 500, 700, 0.986148476600647], "13": ["vertical_and_slash", 100, 800, 0.890625], "14": ["vertical_and_slash", 500, 700, 0.958724319934845], "15": ["vertical_and_slash", 500, 700, 0.9736776947975159], "16": ["vertical_and_slash", 100, 800, 0.86328125], "17": ["vertical_and_slash", 100, 800, 0.796875], "18": ["vertical_and_slash", 500, 700, 0.9487460851669312], "19": ["vertical_and_slash", 500, 700, 0.9184947609901428], "20": ["vertical_and_slash", 100, 800, 0.82421875], "21": ["vertical_and_slash", 3500, 100, 0.8936122059822083], "22": ["vertical_and_slash", 100, 800, 0.70703125], "23": ["vertical_and_slash", 3500, 100, 0.8715357780456543], "24": ["vertical_and_slash", 500, 700, 0.8960596919059753], "25": ["vertical_and_slash", 100, 800, 0.921875], "26": ["vertical_and_slash", 100, 800, 0.96484375], "27": ["vertical_and_slash", 100, 800, 0.91015625], "28": ["vertical_and_slash", 100, 800, 0.86328125], "29": ["vertical_and_slash", 100, 800, 0.875], "30": ["vertical_and_slash", 500, 700, 0.9446543455123901], "31": ["vertical_and_slash", 100, 800, 0.76171875], "32": ["vertical_and_slash", 500, 700, 0.965939998626709], "33": ["vertical_and_slash", 3500, 100, 0.954131543636322], "34": ["vertical_and_slash", 3500, 100, 0.9465757012367249], "35": ["vertical_and_slash", 3500, 100, 0.9854691624641418], "36": ["vertical_and_slash", 500, 700, 0.9794289469718933], "37": ["vertical_and_slash", 100, 800, 0.91015625], "38": ["vertical_and_slash", 3500, 100, 0.8847824335098267], "39": ["vertical_and_slash", 100, 800, 0.8984375], "40": ["vertical_and_slash", 3500, 100, 0.9872503876686096], "41": ["vertical_and_slash", 500, 700, 0.9668098092079163], "42": ["vertical_and_slash", 3500, 100, 0.9836851954460144], "43": ["vertical_and_slash", 3500, 100, 0.9824649691581726], "44": ["vertical_and_slash", 500, 700, 0.8599885702133179], "45": ["vertical_and_slash", 3500, 100, 0.9949566125869751], "46": ["vertical_and_slash", 3500, 100, 0.9806324243545532], "47": ["vertical_and_slash", 100, 800, 0.8828125], "48": ["vertical_and_slash", 500, 700, 0.6984035968780518], "49": ["vertical_and_slash", 30, 800, 0.9816009998321533], "50": ["vertical_and_slash", 30, 800, 0.8915724754333496], "51": ["vertical_and_slash", 500, 700, 0.9726131558418274], "52": ["vertical_and_slash", 30, 800, 0.7166405320167542], "53": ["vertical_and_slash", 30, 800, 0.9959474802017212], "54": ["vertical_and_slash", 100, 750, 0.9925654530525208], "55": ["vertical_and_slash", 500, 700, 0.923687219619751], "56": ["vertical_and_slash", 500, 700, 0.9310286641120911], "57": ["vertical_and_slash", 100, 800, 0.9609375], "58": ["vertical_and_slash", 500, 700, 0.9916408658027649], "59": ["vertical_and_slash", 100, 800, 0.91796875], "60": ["vertical_and_slash", 500, 700, 0.9793931245803833], "61": ["vertical_and_slash", 500, 700, 0.9769693613052368], "62": ["vertical_and_slash", 100, 800, 0.953125], "63": ["vertical_and_slash", 100, 800, 0.96484375]}, {"0": ["vertical_and_slash", 500, 700, 0.9470762610435486], "1": ["vertical_and_slash", 500, 700, 0.9149185419082642], "2": ["vertical_and_slash", 500, 700, 0.9547353386878967], "3": ["vertical_and_slash", 500, 700, 0.9693796038627625], "4": ["vertical_and_slash", 500, 700, 0.9130157232284546], "5": ["vertical_and_slash", 500, 700, 0.9774608612060547], "6": ["vertical_and_slash", 500, 700, 0.8968120217323303], "7": ["vertical_and_slash", 500, 700, 0.8880744576454163], "8": ["vertical_and_slash", 3500, 100, 0.945666491985321], "9": ["vertical_and_slash", 100, 800, 0.9609375], "10": ["vertical_and_slash", 1000, 6096, 0.6013244390487671], "11": ["vertical_and_slash", 1000, 6096, 0.6867892742156982], "12": ["vertical_and_slash", 30, 800, 0.8970251679420471], "13": ["vertical_and_slash", 100, 800, 0.64453125], "14": ["vertical_and_slash", 100, 800, 0.8515625], "15": ["vertical_and_slash", 1000, 6096, 0.9843730926513672], "16": ["vertical_and_slash", 500, 700, 0.9810933470726013], "17": ["vertical_and_slash", 500, 700, 0.9393362998962402], "18": ["vertical_and_slash", 500, 700, 0.9735823273658752], "19": ["vertical_and_slash", 500, 700, 0.9621881246566772], "20": ["vertical_and_slash", 500, 700, 0.9647457003593445], "21": ["vertical_and_slash", 500, 700, 0.9732133150100708], "22": ["vertical_and_slash", 500, 700, 0.9574955105781555], "23": ["vertical_and_slash", 500, 700, 0.945225715637207], "24": ["vertical_and_slash", 100, 800, 0.91015625], "25": ["vertical_and_slash", 500, 700, 0.8241280913352966], "26": ["vertical_and_slash", 500, 700, 0.9614141583442688], "27": ["vertical_and_slash", 100, 800, 0.875], "28": ["vertical_and_slash", 100, 800, 0.93359375], "29": ["vertical_and_slash", 3500, 100, 0.9607701897621155], "30": ["vertical_and_slash", 100, 800, 0.9375], "31": ["vertical_and_slash", 500, 700, 0.8861958980560303], "32": ["vertical_and_slash", 3500, 100, 0.9662611484527588], "33": ["vertical_and_slash", 500, 700, 0.9712302088737488], "34": ["vertical_and_slash", 500, 700, 0.8344451785087585], "35": ["vertical_and_slash", 500, 700, 0.9655841588973999], "36": ["vertical_and_slash", 500, 700, 0.9080643653869629], "37": ["vertical_and_slash", 500, 700, 0.960566520690918], "38": ["vertical_and_slash", 100, 800, 0.94140625], "39": ["vertical_and_slash", 3500, 100, 0.9718257784843445], "40": ["vertical_and_slash", 1000, 6096, 0.8896823525428772], "41": ["vertical_and_slash", 500, 700, 0.9354812502861023], "42": ["vertical_and_slash", 1000, 6096, 0.9713326692581177], "43": ["vertical_and_slash", 1000, 6096, 0.9047271013259888], "44": ["vertical_and_slash", 1000, 6096, 0.9463189244270325], "45": ["vertical_and_slash", 100, 800, 0.8515625], "46": ["vertical_and_slash", 100, 800, 0.828125], "47": ["vertical_and_slash", 1000, 6096, 0.9778203964233398], "48": ["vertical_and_slash", 3500, 100, 0.979678213596344], "49": ["vertical_and_slash", 100, 800, 0.91796875], "50": ["vertical_and_slash", 3500, 100, 0.9598188400268555], "51": ["vertical_and_slash", 3500, 100, 0.972526490688324], "52": ["vertical_and_slash", 500, 700, 0.9608749747276306], "53": ["vertical_and_slash", 500, 700, 0.9321617484092712], "54": ["vertical_and_slash", 500, 700, 0.8401045799255371], "55": ["vertical_and_slash", 3500, 100, 0.9916496872901917], "56": ["vertical_and_slash", 100, 800, 0.6171875], "57": ["vertical_and_slash", 1000, 6096, 0.8058498501777649], "58": ["vertical_and_slash", 1000, 6096, 0.7182456254959106], "59": ["vertical_and_slash", 100, 800, 0.82421875], "60": ["vertical_and_slash", 1000, 6096, 0.6521841287612915], "61": ["vertical_and_slash", 1000, 6096, 0.9360571503639221], "62": ["vertical_and_slash", 100, 800, 0.9609375], "63": ["vertical_and_slash", 1000, 6096, 0.6815474629402161]}, {"0": ["vertical_and_slash", 100, 800, 0.921875], "1": ["vertical_and_slash", 500, 700, 0.9814538359642029], "2": ["vertical_and_slash", 3500, 100, 0.9492993950843811], "3": ["vertical_and_slash", 100, 800, 0.9609375], "4": ["vertical_and_slash", 3500, 100, 0.9296199679374695], "5": ["vertical_and_slash", 100, 800, 0.9296875], "6": ["vertical_and_slash", 100, 800, 0.90234375], "7": ["vertical_and_slash", 3500, 100, 0.923884391784668], "8": ["vertical_and_slash", 500, 700, 0.9124513864517212], "9": ["vertical_and_slash", 100, 800, 0.87109375], "10": ["vertical_and_slash", 100, 800, 0.91796875], "11": ["vertical_and_slash", 100, 800, 0.90234375], "12": ["vertical_and_slash", 100, 800, 0.8984375], "13": ["vertical_and_slash", 100, 750, 0.9267962574958801], "14": ["vertical_and_slash", 100, 800, 0.87890625], "15": ["vertical_and_slash", 500, 700, 0.8267379403114319], "16": ["vertical_and_slash", 500, 700, 0.9811784625053406], "17": ["vertical_and_slash", 500, 700, 0.8441471457481384], "18": ["vertical_and_slash", 3500, 100, 0.9741179943084717], "19": ["vertical_and_slash", 3500, 100, 0.9646090865135193], "20": ["vertical_and_slash", 3500, 100, 0.9738689064979553], "21": ["vertical_and_slash", 500, 700, 0.9567211270332336], "22": ["vertical_and_slash", 500, 700, 0.9401639699935913], "23": ["vertical_and_slash", 500, 700, 0.8797400593757629], "24": ["vertical_and_slash", 500, 700, 0.8600099086761475], "25": ["vertical_and_slash", 500, 700, 0.9367791414260864], "26": ["vertical_and_slash", 1000, 6096, 0.8031479120254517], "27": ["vertical_and_slash", 500, 700, 0.9161806106567383], "28": ["vertical_and_slash", 1000, 6096, 0.7752811312675476], "29": ["vertical_and_slash", 500, 700, 0.8955652117729187], "30": ["vertical_and_slash", 1000, 6096, 0.7252691984176636], "31": ["vertical_and_slash", 500, 700, 0.9335694313049316], "32": ["vertical_and_slash", 3500, 100, 0.9847425222396851], "33": ["vertical_and_slash", 1000, 6096, 0.9584336876869202], "34": ["vertical_and_slash", 3500, 100, 0.966917097568512], "35": ["vertical_and_slash", 100, 800, 0.94921875], "36": ["vertical_and_slash", 100, 800, 0.9609375], "37": ["vertical_and_slash", 100, 800, 0.953125], "38": ["vertical_and_slash", 500, 700, 0.9724488258361816], "39": ["vertical_and_slash", 100, 750, 0.8776999711990356], "40": ["vertical_and_slash", 100, 800, 0.7578125], "41": ["vertical_and_slash", 500, 700, 0.9448313117027283], "42": ["vertical_and_slash", 100, 800, 0.71484375], "43": ["vertical_and_slash", 3500, 100, 0.9549469947814941], "44": ["vertical_and_slash", 100, 800, 0.91015625], "45": ["vertical_and_slash", 100, 800, 0.75390625], "46": ["vertical_and_slash", 3500, 100, 0.9456886649131775], "47": ["vertical_and_slash", 100, 800, 0.92578125], "48": ["vertical_and_slash", 100, 800, 0.93359375], "49": ["vertical_and_slash", 100, 800, 0.7109375], "50": ["vertical_and_slash", 3500, 100, 0.8324546217918396], "51": ["vertical_and_slash", 1000, 6096, 0.8339883685112], "52": ["vertical_and_slash", 1000, 6096, 0.8346136808395386], "53": ["vertical_and_slash", 1000, 6096, 0.7437535524368286], "54": ["vertical_and_slash", 3500, 100, 0.8286469578742981], "55": ["vertical_and_slash", 1000, 6096, 0.7687234878540039], "56": ["vertical_and_slash", 100, 800, 0.7421875], "57": ["vertical_and_slash", 500, 700, 0.8950325846672058], "58": ["vertical_and_slash", 100, 800, 0.9296875], "59": ["vertical_and_slash", 500, 700, 0.9507440328598022], "60": ["vertical_and_slash", 100, 750, 0.8726681470870972], "61": ["vertical_and_slash", 100, 800, 0.87109375], "62": ["vertical_and_slash", 3500, 100, 0.8373590707778931], "63": ["vertical_and_slash", 1000, 6096, 0.9734319448471069]}, {"0": ["vertical_and_slash", 3500, 100, 0.8739125728607178], "1": ["vertical_and_slash", 3500, 100, 0.9198444485664368], "2": ["vertical_and_slash", 100, 800, 0.87890625], "3": ["vertical_and_slash", 30, 800, 0.9311037659645081], "4": ["vertical_and_slash", 100, 800, 0.73828125], "5": ["vertical_and_slash", 3500, 100, 0.9306046366691589], "6": ["vertical_and_slash", 500, 700, 0.8945528864860535], "7": ["vertical_and_slash", 3500, 100, 0.9126960039138794], "8": ["vertical_and_slash", 500, 700, 0.8016852736473083], "9": ["vertical_and_slash", 100, 800, 0.88671875], "10": ["vertical_and_slash", 500, 700, 0.8650792241096497], "11": ["vertical_and_slash", 500, 700, 0.8418971300125122], "12": ["vertical_and_slash", 100, 750, 0.7922348380088806], "13": ["vertical_and_slash", 500, 700, 0.7339740991592407], "14": ["vertical_and_slash", 500, 700, 0.7798085808753967], "15": ["vertical_and_slash", 100, 800, 0.671875], "16": ["vertical_and_slash", 3500, 100, 0.9255445003509521], "17": ["vertical_and_slash", 500, 700, 0.9652724266052246], "18": ["vertical_and_slash", 500, 700, 0.9371723532676697], "19": ["vertical_and_slash", 500, 700, 0.8736885786056519], "20": ["vertical_and_slash", 30, 800, 0.7195981740951538], "21": ["vertical_and_slash", 3500, 100, 0.9808559417724609], "22": ["vertical_and_slash", 3500, 100, 0.9545947313308716], "23": ["vertical_and_slash", 500, 700, 0.9859827756881714], "24": ["vertical_and_slash", 3500, 100, 0.9898564219474792], "25": ["vertical_and_slash", 3500, 100, 0.9735372066497803], "26": ["vertical_and_slash", 3500, 100, 0.9804157614707947], "27": ["vertical_and_slash", 500, 700, 0.9758102893829346], "28": ["vertical_and_slash", 500, 700, 0.7896785140037537], "29": ["vertical_and_slash", 3500, 100, 0.9795226454734802], "30": ["vertical_and_slash", 3500, 100, 0.9745214581489563], "31": ["vertical_and_slash", 500, 700, 0.9783229827880859], "32": ["vertical_and_slash", 100, 800, 0.7578125], "33": ["vertical_and_slash", 30, 800, 0.8892262578010559], "34": ["vertical_and_slash", 500, 700, 0.928130567073822], "35": ["vertical_and_slash", 500, 700, 0.9262915253639221], "36": ["vertical_and_slash", 3500, 100, 0.9587469696998596], "37": ["vertical_and_slash", 500, 700, 0.9210153222084045], "38": ["vertical_and_slash", 3500, 100, 0.9420099854469299], "39": ["vertical_and_slash", 500, 700, 0.9520406723022461], "40": ["vertical_and_slash", 500, 700, 0.9572274684906006], "41": ["vertical_and_slash", 500, 700, 0.9336647391319275], "42": ["vertical_and_slash", 500, 700, 0.9509835243225098], "43": ["vertical_and_slash", 100, 800, 0.8828125], "44": ["vertical_and_slash", 500, 700, 0.9103185534477234], "45": ["vertical_and_slash", 500, 700, 0.9191944003105164], "46": ["vertical_and_slash", 500, 700, 0.8257020115852356], "47": ["vertical_and_slash", 500, 700, 0.9661493301391602], "48": ["vertical_and_slash", 500, 700, 0.9450398683547974], "49": ["vertical_and_slash", 500, 700, 0.8951587677001953], "50": ["vertical_and_slash", 1000, 6096, 0.7518162131309509], "51": ["vertical_and_slash", 500, 700, 0.9749075174331665], "52": ["vertical_and_slash", 500, 700, 0.9674242734909058], "53": ["vertical_and_slash", 500, 700, 0.8985055685043335], "54": ["vertical_and_slash", 100, 800, 0.859375], "55": ["vertical_and_slash", 500, 700, 0.9133719801902771], "56": ["vertical_and_slash", 500, 700, 0.9643236994743347], "57": ["vertical_and_slash", 3500, 100, 0.968787670135498], "58": ["vertical_and_slash", 3500, 100, 0.9359950423240662], "59": ["vertical_and_slash", 3500, 100, 0.9486719965934753], "60": ["vertical_and_slash", 500, 700, 0.9676948189735413], "61": ["vertical_and_slash", 500, 700, 0.904003381729126], "62": ["vertical_and_slash", 3500, 100, 0.9407141804695129], "63": ["vertical_and_slash", 3500, 100, 0.9361897706985474]}, {"0": ["vertical_and_slash", 100, 800, 0.86328125], "1": ["vertical_and_slash", 30, 800, 0.982953667640686], "2": ["vertical_and_slash", 30, 800, 0.8530048727989197], "3": ["vertical_and_slash", 100, 800, 0.8828125], "4": ["vertical_and_slash", 30, 800, 0.9648202657699585], "5": ["vertical_and_slash", 30, 800, 0.9443895220756531], "6": ["vertical_and_slash", 30, 800, 0.8393809795379639], "7": ["vertical_and_slash", 30, 800, 0.9389616250991821], "8": ["vertical_and_slash", 30, 800, 0.7104648351669312], "9": ["vertical_and_slash", 30, 800, 0.8663231134414673], "10": ["vertical_and_slash", 30, 800, 0.978753924369812], "11": ["vertical_and_slash", 1000, 6096, 0.7122654914855957], "12": ["vertical_and_slash", 1000, 6096, 0.6662688851356506], "13": ["vertical_and_slash", 100, 800, 0.97265625], "14": ["vertical_and_slash", 1000, 6096, 0.6950275897979736], "15": ["vertical_and_slash", 30, 800, 0.800190269947052], "16": ["vertical_and_slash", 500, 700, 0.9352073669433594], "17": ["vertical_and_slash", 30, 800, 0.9207461476325989], "18": ["vertical_and_slash", 100, 750, 0.9696995615959167], "19": ["vertical_and_slash", 500, 700, 0.9767144918441772], "20": ["vertical_and_slash", 100, 750, 0.8907751441001892], "21": ["vertical_and_slash", 100, 750, 0.9050142765045166], "22": ["vertical_and_slash", 500, 700, 0.9598274230957031], "23": ["vertical_and_slash", 100, 750, 0.7951990962028503], "24": ["vertical_and_slash", 500, 700, 0.9581713676452637], "25": ["vertical_and_slash", 500, 700, 0.9746555089950562], "26": ["vertical_and_slash", 1000, 6096, 0.8876649737358093], "27": ["vertical_and_slash", 3500, 100, 0.9737825393676758], "28": ["vertical_and_slash", 500, 700, 0.9511380791664124], "29": ["vertical_and_slash", 500, 700, 0.9568895697593689], "30": ["vertical_and_slash", 30, 800, 0.970806896686554], "31": ["vertical_and_slash", 30, 800, 0.9834022521972656], "32": ["vertical_and_slash", 30, 800, 0.9670526385307312], "33": ["vertical_and_slash", 30, 800, 0.8988778591156006], "34": ["vertical_and_slash", 30, 800, 0.9750128388404846], "35": ["vertical_and_slash", 30, 800, 0.9334320425987244], "36": ["vertical_and_slash", 30, 800, 0.9819129705429077], "37": ["vertical_and_slash", 30, 800, 0.8711813688278198], "38": ["vertical_and_slash", 30, 800, 0.9427667260169983], "39": ["vertical_and_slash", 30, 800, 0.9266768097877502], "40": ["vertical_and_slash", 500, 700, 0.9899290800094604], "41": ["vertical_and_slash", 500, 700, 0.9782542586326599], "42": ["vertical_and_slash", 500, 700, 0.9447435140609741], "43": ["vertical_and_slash", 500, 700, 0.9703333377838135], "44": ["vertical_and_slash", 500, 700, 0.9844834208488464], "45": ["vertical_and_slash", 500, 700, 0.9826511144638062], "46": ["vertical_and_slash", 500, 700, 0.9715730547904968], "47": ["vertical_and_slash", 500, 700, 0.9734385013580322], "48": ["vertical_and_slash", 500, 700, 0.836034893989563], "49": ["vertical_and_slash", 100, 750, 0.898857831954956], "50": ["vertical_and_slash", 100, 800, 0.87109375], "51": ["vertical_and_slash", 100, 750, 0.8986822366714478], "52": ["vertical_and_slash", 100, 800, 0.9375], "53": ["vertical_and_slash", 100, 750, 0.8337450623512268], "54": ["vertical_and_slash", 500, 700, 0.9467383623123169], "55": ["vertical_and_slash", 500, 700, 0.9352756142616272], "56": ["vertical_and_slash", 3500, 100, 0.9662188291549683], "57": ["vertical_and_slash", 500, 700, 0.9239659905433655], "58": ["vertical_and_slash", 500, 700, 0.9502314329147339], "59": ["vertical_and_slash", 500, 700, 0.9479988813400269], "60": ["vertical_and_slash", 500, 700, 0.8582519888877869], "61": ["vertical_and_slash", 500, 700, 0.9503857493400574], "62": ["vertical_and_slash", 500, 700, 0.9299347996711731], "63": ["vertical_and_slash", 500, 700, 0.9766162037849426]}]
diff --git a/minference/configs/Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json b/minference/configs/Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json
index bf840765..2237283a 100644
--- a/minference/configs/Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json
+++ b/minference/configs/Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json
@@ -1 +1 @@
-[{"0": ["vertical_and_slash", 1000, 6096, 0.9645951390266418], "1": ["vertical_and_slash", 1000, 6096, 0.9643118977546692], "2": ["vertical_and_slash", 1000, 6096, 0.9868560433387756], "3": ["vertical_and_slash", 1000, 6096, 0.9066882133483887], "4": ["vertical_and_slash", 1000, 6096, 0.8384770750999451], "5": ["vertical_and_slash", 1000, 6096, 0.835468590259552], "6": ["vertical_and_slash", 1000, 6096, 0.8871183395385742], "7": ["vertical_and_slash", 1000, 6096, 0.8453579545021057], "8": ["vertical_and_slash", 1000, 6096, 0.8092553615570068], "9": ["vertical_and_slash", 1000, 6096, 0.8697246313095093], "10": ["vertical_and_slash", 1000, 6096, 0.9496452212333679], "11": ["vertical_and_slash", 1000, 6096, 0.8842584490776062], "12": ["vertical_and_slash", 1000, 6096, 0.613018274307251], "13": ["vertical_and_slash", 1000, 6096, 0.5473001599311829], "14": ["vertical_and_slash", 1000, 6096, 0.6623730659484863], "15": ["vertical_and_slash", 1000, 6096, 0.6256281137466431], "16": ["vertical_and_slash", 1000, 6096, 0.6286638975143433], "17": ["vertical_and_slash", 1000, 6096, 0.6928349733352661], "18": ["vertical_and_slash", 1000, 6096, 0.6307621598243713], "19": ["vertical_and_slash", 1000, 6096, 0.6331143975257874], "20": ["vertical_and_slash", 1000, 6096, 0.5811723470687866], "21": ["vertical_and_slash", 1000, 6096, 0.5251738429069519], "22": ["vertical_and_slash", 1000, 6096, 0.4680206775665283], "23": ["vertical_and_slash", 1000, 6096, 0.6884945631027222], "24": ["vertical_and_slash", 1000, 6096, 0.9140691161155701], "25": ["vertical_and_slash", 1000, 6096, 0.854979932308197], "26": ["vertical_and_slash", 1000, 6096, 0.8844645619392395], "27": ["vertical_and_slash", 1000, 6096, 0.8390430212020874], "28": ["vertical_and_slash", 1000, 6096, 0.40988317131996155], "29": ["vertical_and_slash", 3500, 100, 0.81468665599823], "30": ["vertical_and_slash", 3500, 100, 0.5836060643196106], "31": ["vertical_and_slash", 3500, 100, 0.9285933375358582]}, {"0": ["vertical_and_slash", 30, 800, 0.9606457948684692], "1": ["vertical_and_slash", 500, 700, 0.9559118747711182], "2": ["vertical_and_slash", 1000, 6096, 0.8758323192596436], "3": ["vertical_and_slash", 500, 700, 0.9542838931083679], "4": ["vertical_and_slash", 30, 800, 0.9751565456390381], "5": ["vertical_and_slash", 30, 800, 0.9853302836418152], "6": ["vertical_and_slash", 30, 800, 0.9257169365882874], "7": ["vertical_and_slash", 30, 800, 0.9427204132080078], "8": ["vertical_and_slash", 30, 800, 0.9230360388755798], "9": ["vertical_and_slash", 1000, 6096, 0.587185800075531], "10": ["vertical_and_slash", 30, 800, 0.952581524848938], "11": ["vertical_and_slash", 500, 700, 0.9271975755691528], "12": ["vertical_and_slash", 1000, 6096, 0.9231844544410706], "13": ["vertical_and_slash", 30, 800, 0.9554020166397095], "14": ["vertical_and_slash", 30, 800, 0.8323891162872314], "15": ["vertical_and_slash", 30, 800, 0.930496335029602], "16": ["vertical_and_slash", 30, 800, 0.9835872054100037], "17": ["vertical_and_slash", 30, 800, 0.9782426953315735], "18": ["vertical_and_slash", 30, 800, 0.9922719597816467], "19": ["vertical_and_slash", 30, 800, 0.954961359500885], "20": ["vertical_and_slash", 30, 800, 0.9950350522994995], "21": ["vertical_and_slash", 30, 800, 0.9947351813316345], "22": ["vertical_and_slash", 30, 800, 0.8034972548484802], "23": ["vertical_and_slash", 1000, 6096, 0.7825947403907776], "24": ["vertical_and_slash", 30, 800, 0.9483550190925598], "25": ["vertical_and_slash", 500, 700, 0.8846591114997864], "26": ["vertical_and_slash", 30, 800, 0.9582298398017883], "27": ["vertical_and_slash", 30, 800, 0.9463901519775391], "28": ["vertical_and_slash", 1000, 6096, 0.8778873682022095], "29": ["vertical_and_slash", 1000, 6096, 0.9215229153633118], "30": ["vertical_and_slash", 30, 800, 0.9545977115631104], "31": ["vertical_and_slash", 30, 800, 0.8700023889541626]}, {"0": ["vertical_and_slash", 30, 800, 0.9917542934417725], "1": ["vertical_and_slash", 30, 800, 0.9637826681137085], "2": ["vertical_and_slash", 30, 800, 0.9905248284339905], "3": ["vertical_and_slash", 1000, 6096, 0.9387498497962952], "4": ["vertical_and_slash", 30, 800, 0.9815536737442017], "5": ["vertical_and_slash", 30, 800, 0.9495874047279358], "6": ["vertical_and_slash", 100, 800, 0.86328125], "7": ["vertical_and_slash", 30, 800, 0.9903317093849182], "8": ["vertical_and_slash", 30, 800, 0.9950467944145203], "9": ["vertical_and_slash", 30, 800, 0.993314266204834], "10": ["vertical_and_slash", 30, 800, 0.9852835536003113], "11": ["vertical_and_slash", 30, 800, 0.9859053492546082], "12": ["vertical_and_slash", 30, 800, 0.994464099407196], "13": ["vertical_and_slash", 30, 800, 0.9855010509490967], "14": ["vertical_and_slash", 100, 800, 0.62890625], "15": ["vertical_and_slash", 1000, 6096, 0.8527955412864685], "16": ["vertical_and_slash", 30, 800, 0.9897146821022034], "17": ["vertical_and_slash", 30, 800, 0.9801600575447083], "18": ["vertical_and_slash", 30, 800, 0.9936752915382385], "19": ["vertical_and_slash", 1000, 6096, 0.9319088459014893], "20": ["vertical_and_slash", 1000, 6096, 0.9181358218193054], "21": ["vertical_and_slash", 1000, 6096, 0.9484037756919861], "22": ["vertical_and_slash", 1000, 6096, 0.9803578853607178], "23": ["vertical_and_slash", 1000, 6096, 0.9990358948707581], "24": ["vertical_and_slash", 30, 800, 0.984993577003479], "25": ["vertical_and_slash", 100, 800, 0.99609375], "26": ["vertical_and_slash", 100, 800, 0.99609375], "27": ["vertical_and_slash", 30, 800, 0.9951220750808716], "28": ["vertical_and_slash", 30, 800, 0.9924372434616089], "29": ["vertical_and_slash", 30, 800, 0.9929786324501038], "30": ["vertical_and_slash", 30, 800, 0.9946975111961365], "31": ["vertical_and_slash", 30, 800, 0.9946795105934143]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9770999550819397], "1": ["vertical_and_slash", 1000, 6096, 0.9795235395431519], "2": ["vertical_and_slash", 100, 800, 1.0], "3": ["vertical_and_slash", 1000, 6096, 0.9940649271011353], "4": ["vertical_and_slash", 30, 800, 0.9976175427436829], "5": ["vertical_and_slash", 30, 800, 0.9964977502822876], "6": ["vertical_and_slash", 30, 800, 0.9852505922317505], "7": ["vertical_and_slash", 30, 800, 0.9912457466125488], "8": ["vertical_and_slash", 30, 800, 0.9946132302284241], "9": ["vertical_and_slash", 30, 800, 0.9836555123329163], "10": ["vertical_and_slash", 30, 800, 0.9907234907150269], "11": ["vertical_and_slash", 30, 800, 0.9940478801727295], "12": ["vertical_and_slash", 100, 800, 0.90625], "13": ["vertical_and_slash", 30, 800, 0.9403843283653259], "14": ["vertical_and_slash", 500, 700, 0.9940536618232727], "15": ["vertical_and_slash", 30, 800, 0.9930430054664612], "16": ["vertical_and_slash", 30, 800, 0.990331768989563], "17": ["vertical_and_slash", 30, 800, 0.9973565340042114], "18": ["vertical_and_slash", 30, 800, 0.9962788820266724], "19": ["vertical_and_slash", 30, 800, 0.9755663871765137], "20": ["vertical_and_slash", 500, 700, 0.9660491943359375], "21": ["vertical_and_slash", 100, 800, 0.9609375], "22": ["vertical_and_slash", 1000, 6096, 0.9093130826950073], "23": ["vertical_and_slash", 30, 800, 0.9103626608848572], "24": ["vertical_and_slash", 30, 800, 0.9940206408500671], "25": ["vertical_and_slash", 1000, 6096, 0.9898346066474915], "26": ["vertical_and_slash", 30, 800, 0.9925925135612488], "27": ["vertical_and_slash", 100, 800, 0.74609375], "28": ["vertical_and_slash", 30, 800, 0.984257161617279], "29": ["vertical_and_slash", 100, 800, 0.97265625], "30": ["vertical_and_slash", 30, 800, 0.9798495173454285], "31": ["vertical_and_slash", 30, 800, 0.9899864792823792]}, {"0": ["vertical_and_slash", 100, 800, 0.91015625], "1": ["vertical_and_slash", 30, 800, 0.9502514600753784], "2": ["vertical_and_slash", 30, 800, 0.9773570895195007], "3": ["vertical_and_slash", 30, 800, 0.9736469388008118], "4": ["vertical_and_slash", 30, 800, 0.9908788800239563], "5": ["vertical_and_slash", 30, 800, 0.8939412236213684], "6": ["vertical_and_slash", 1000, 6096, 0.8417163491249084], "7": ["vertical_and_slash", 30, 800, 0.9903419017791748], "8": ["vertical_and_slash", 100, 800, 0.9140625], "9": ["vertical_and_slash", 30, 800, 0.9721925854682922], "10": ["vertical_and_slash", 30, 800, 0.8961870670318604], "11": ["vertical_and_slash", 30, 800, 0.9606669545173645], "12": ["vertical_and_slash", 30, 800, 0.9913156032562256], "13": ["vertical_and_slash", 30, 800, 0.985747218132019], "14": ["vertical_and_slash", 30, 800, 0.9929512143135071], "15": ["vertical_and_slash", 100, 800, 0.9375], "16": ["vertical_and_slash", 1000, 6096, 0.6194612383842468], "17": ["vertical_and_slash", 1000, 6096, 0.9939616918563843], "18": ["vertical_and_slash", 1000, 6096, 0.9973123073577881], "19": ["vertical_and_slash", 1000, 6096, 0.5516155958175659], "20": ["vertical_and_slash", 100, 800, 0.97265625], "21": ["vertical_and_slash", 1000, 6096, 0.9808182120323181], "22": ["vertical_and_slash", 30, 800, 0.9840195178985596], "23": ["vertical_and_slash", 1000, 6096, 0.9757125973701477], "24": ["vertical_and_slash", 30, 800, 0.9877440929412842], "25": ["vertical_and_slash", 30, 800, 0.9822482466697693], "26": ["vertical_and_slash", 30, 800, 0.8953061103820801], "27": ["vertical_and_slash", 30, 800, 0.9795226454734802], "28": ["vertical_and_slash", 1000, 6096, 0.9304757118225098], "29": ["vertical_and_slash", 100, 800, 0.765625], "30": ["vertical_and_slash", 30, 800, 0.973455548286438], "31": ["vertical_and_slash", 30, 800, 0.9915329813957214]}, {"0": ["vertical_and_slash", 30, 800, 0.9787898063659668], "1": ["vertical_and_slash", 1000, 6096, 0.9787205457687378], "2": ["vertical_and_slash", 30, 800, 0.9839415550231934], "3": ["vertical_and_slash", 30, 800, 0.9879974722862244], "4": ["vertical_and_slash", 30, 800, 0.9873318076133728], "5": ["vertical_and_slash", 100, 800, 0.8203125], "6": ["vertical_and_slash", 30, 800, 0.9408391118049622], "7": ["vertical_and_slash", 100, 800, 0.80078125], "8": ["vertical_and_slash", 1000, 6096, 0.8980521559715271], "9": ["vertical_and_slash", 1000, 6096, 0.9289278984069824], "10": ["vertical_and_slash", 1000, 6096, 0.8854653239250183], "11": ["vertical_and_slash", 1000, 6096, 0.9555475115776062], "12": ["vertical_and_slash", 30, 800, 0.9772993326187134], "13": ["vertical_and_slash", 30, 800, 0.9894649386405945], "14": ["vertical_and_slash", 30, 800, 0.9916768074035645], "15": ["vertical_and_slash", 30, 800, 0.9852606654167175], "16": ["vertical_and_slash", 100, 800, 0.98828125], "17": ["vertical_and_slash", 1000, 6096, 0.9101518392562866], "18": ["vertical_and_slash", 1000, 6096, 0.9234060645103455], "19": ["vertical_and_slash", 30, 800, 0.935745120048523], "20": ["vertical_and_slash", 100, 800, 0.9765625], "21": ["vertical_and_slash", 1000, 6096, 0.9155886769294739], "22": ["vertical_and_slash", 100, 800, 0.85546875], "23": ["vertical_and_slash", 100, 800, 0.94921875], "24": ["vertical_and_slash", 1000, 6096, 0.9800405502319336], "25": ["vertical_and_slash", 30, 800, 0.9936150908470154], "26": ["vertical_and_slash", 30, 800, 0.9960456490516663], "27": ["vertical_and_slash", 1000, 6096, 0.9885473251342773], "28": ["vertical_and_slash", 30, 800, 0.9735875725746155], "29": ["vertical_and_slash", 30, 800, 0.9771297574043274], "30": ["vertical_and_slash", 30, 800, 0.979208767414093], "31": ["vertical_and_slash", 30, 800, 0.9677841663360596]}, {"0": ["vertical_and_slash", 30, 800, 0.9898972511291504], "1": ["vertical_and_slash", 30, 800, 0.9888962507247925], "2": ["vertical_and_slash", 100, 800, 0.8515625], "3": ["vertical_and_slash", 100, 800, 0.83203125], "4": ["vertical_and_slash", 30, 800, 0.9903795719146729], "5": ["vertical_and_slash", 30, 800, 0.9518011212348938], "6": ["vertical_and_slash", 100, 800, 0.91015625], "7": ["vertical_and_slash", 30, 800, 0.9933650493621826], "8": ["vertical_and_slash", 30, 800, 0.9954792857170105], "9": ["vertical_and_slash", 30, 800, 0.9556100368499756], "10": ["vertical_and_slash", 30, 800, 0.9929815530776978], "11": ["vertical_and_slash", 30, 800, 0.9911448359489441], "12": ["vertical_and_slash", 30, 800, 0.9894125461578369], "13": ["vertical_and_slash", 30, 800, 0.9900504946708679], "14": ["vertical_and_slash", 30, 800, 0.9373334050178528], "15": ["vertical_and_slash", 30, 800, 0.9928350448608398], "16": ["vertical_and_slash", 30, 800, 0.9924395084381104], "17": ["vertical_and_slash", 30, 800, 0.9719960689544678], "18": ["vertical_and_slash", 30, 800, 0.959073007106781], "19": ["vertical_and_slash", 30, 800, 0.9884441494941711], "20": ["vertical_and_slash", 100, 800, 0.921875], "21": ["vertical_and_slash", 1000, 6096, 0.6786824464797974], "22": ["vertical_and_slash", 100, 800, 0.91796875], "23": ["vertical_and_slash", 30, 800, 0.968828558921814], "24": ["vertical_and_slash", 100, 800, 0.78515625], "25": ["vertical_and_slash", 1000, 6096, 0.8599422574043274], "26": ["vertical_and_slash", 1000, 6096, 0.7635548710823059], "27": ["vertical_and_slash", 100, 800, 0.95703125], "28": ["vertical_and_slash", 100, 800, 0.6953125], "29": ["vertical_and_slash", 100, 800, 0.98046875], "30": ["vertical_and_slash", 100, 800, 1.0], "31": ["vertical_and_slash", 100, 800, 0.88671875]}, {"0": ["vertical_and_slash", 30, 800, 0.9594353437423706], "1": ["vertical_and_slash", 30, 800, 0.9896920323371887], "2": ["vertical_and_slash", 30, 800, 0.9922080039978027], "3": ["vertical_and_slash", 100, 800, 0.44140625], "4": ["vertical_and_slash", 30, 800, 0.9757390022277832], "5": ["vertical_and_slash", 30, 800, 0.9919450283050537], "6": ["vertical_and_slash", 1000, 6096, 0.8012803196907043], "7": ["vertical_and_slash", 1000, 6096, 0.820481538772583], "8": ["vertical_and_slash", 30, 800, 0.9347220659255981], "9": ["vertical_and_slash", 30, 800, 0.9701992273330688], "10": ["vertical_and_slash", 30, 800, 0.8196504712104797], "11": ["vertical_and_slash", 30, 800, 0.9785101413726807], "12": ["vertical_and_slash", 30, 800, 0.9739136695861816], "13": ["vertical_and_slash", 30, 800, 0.8736869692802429], "14": ["vertical_and_slash", 30, 800, 0.9743740558624268], "15": ["vertical_and_slash", 100, 800, 0.9375], "16": ["vertical_and_slash", 100, 800, 0.93359375], "17": ["vertical_and_slash", 30, 800, 0.9722828269004822], "18": ["vertical_and_slash", 100, 800, 0.94140625], "19": ["vertical_and_slash", 30, 800, 0.9577817320823669], "20": ["vertical_and_slash", 100, 800, 0.97265625], "21": ["vertical_and_slash", 3500, 100, 0.7349414825439453], "22": ["vertical_and_slash", 1000, 6096, 0.7663673162460327], "23": ["vertical_and_slash", 100, 800, 0.66796875], "24": ["vertical_and_slash", 100, 800, 0.94140625], "25": ["vertical_and_slash", 1000, 6096, 0.687431812286377], "26": ["vertical_and_slash", 100, 800, 0.84375], "27": ["vertical_and_slash", 100, 800, 0.9453125], "28": ["vertical_and_slash", 30, 800, 0.9529873728752136], "29": ["vertical_and_slash", 30, 800, 0.9674840569496155], "30": ["vertical_and_slash", 30, 800, 0.9221987128257751], "31": ["vertical_and_slash", 30, 800, 0.9387267231941223]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9813447594642639], "1": ["vertical_and_slash", 1000, 6096, 0.7434703707695007], "2": ["vertical_and_slash", 100, 800, 0.8984375], "3": ["vertical_and_slash", 100, 800, 0.69921875], "4": ["vertical_and_slash", 30, 800, 0.9847737550735474], "5": ["vertical_and_slash", 1000, 6096, 0.9276951551437378], "6": ["vertical_and_slash", 30, 800, 0.9802067875862122], "7": ["vertical_and_slash", 100, 800, 0.94921875], "8": ["vertical_and_slash", 1000, 6096, 0.8924590945243835], "9": ["vertical_and_slash", 1000, 6096, 0.48737046122550964], "10": ["vertical_and_slash", 100, 800, 0.81640625], "11": ["vertical_and_slash", 1000, 6096, 0.5120941996574402], "12": ["vertical_and_slash", 100, 800, 0.94140625], "13": ["vertical_and_slash", 30, 800, 0.9506304860115051], "14": ["vertical_and_slash", 100, 800, 0.92578125], "15": ["vertical_and_slash", 100, 800, 0.9765625], "16": ["vertical_and_slash", 30, 800, 0.8123543858528137], "17": ["vertical_and_slash", 1000, 6096, 0.9536045789718628], "18": ["vertical_and_slash", 1000, 6096, 0.9222833514213562], "19": ["vertical_and_slash", 1000, 6096, 0.9397181272506714], "20": ["vertical_and_slash", 30, 800, 0.9818680286407471], "21": ["vertical_and_slash", 30, 800, 0.9743596911430359], "22": ["vertical_and_slash", 30, 800, 0.9852566123008728], "23": ["vertical_and_slash", 30, 800, 0.9250922799110413], "24": ["vertical_and_slash", 100, 800, 0.95703125], "25": ["vertical_and_slash", 100, 800, 0.9765625], "26": ["vertical_and_slash", 1000, 6096, 0.9655635356903076], "27": ["vertical_and_slash", 1000, 6096, 0.9585272669792175], "28": ["vertical_and_slash", 100, 800, 0.91796875], "29": ["vertical_and_slash", 1000, 6096, 0.8941929936408997], "30": ["vertical_and_slash", 1000, 6096, 0.9122419357299805], "31": ["vertical_and_slash", 1000, 6096, 0.8547379970550537]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7614610195159912], "1": ["vertical_and_slash", 1000, 6096, 0.8187241554260254], "2": ["vertical_and_slash", 1000, 6096, 0.7961371541023254], "3": ["vertical_and_slash", 1000, 6096, 0.8252145648002625], "4": ["vertical_and_slash", 1000, 6096, 0.9270076155662537], "5": ["vertical_and_slash", 1000, 6096, 0.8690700531005859], "6": ["vertical_and_slash", 1000, 6096, 0.9444302320480347], "7": ["vertical_and_slash", 1000, 6096, 0.9755919575691223], "8": ["vertical_and_slash", 500, 700, 0.9716585874557495], "9": ["vertical_and_slash", 30, 800, 0.9762371778488159], "10": ["vertical_and_slash", 30, 800, 0.9724705219268799], "11": ["vertical_and_slash", 500, 700, 0.99483323097229], "12": ["vertical_and_slash", 100, 800, 0.67578125], "13": ["vertical_and_slash", 100, 800, 0.70703125], "14": ["vertical_and_slash", 100, 800, 0.796875], "15": ["vertical_and_slash", 1000, 6096, 0.8170258402824402], "16": ["vertical_and_slash", 100, 800, 0.90625], "17": ["vertical_and_slash", 3500, 100, 0.8228636980056763], "18": ["vertical_and_slash", 30, 800, 0.856238603591919], "19": ["vertical_and_slash", 100, 800, 0.7734375], "20": ["vertical_and_slash", 1000, 6096, 0.8687199354171753], "21": ["vertical_and_slash", 1000, 6096, 0.9029819965362549], "22": ["vertical_and_slash", 1000, 6096, 0.9524213075637817], "23": ["vertical_and_slash", 100, 800, 0.89453125], "24": ["vertical_and_slash", 500, 700, 0.9886078238487244], "25": ["vertical_and_slash", 1000, 6096, 0.7378285527229309], "26": ["vertical_and_slash", 100, 800, 0.9921875], "27": ["vertical_and_slash", 1000, 6096, 0.817205011844635], "28": ["vertical_and_slash", 1000, 6096, 0.6564486622810364], "29": ["vertical_and_slash", 1000, 6096, 0.8640334606170654], "30": ["vertical_and_slash", 100, 800, 0.953125], "31": ["vertical_and_slash", 1000, 6096, 0.6869896054267883]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9531474113464355], "1": ["vertical_and_slash", 1000, 6096, 0.8817346096038818], "2": ["vertical_and_slash", 1000, 6096, 0.7733508348464966], "3": ["vertical_and_slash", 3500, 100, 0.5279473066329956], "4": ["vertical_and_slash", 500, 700, 0.9626384377479553], "5": ["vertical_and_slash", 1000, 6096, 0.8951384425163269], "6": ["vertical_and_slash", 1000, 6096, 0.7266367077827454], "7": ["vertical_and_slash", 1000, 6096, 0.7165513038635254], "8": ["vertical_and_slash", 100, 800, 0.89453125], "9": ["vertical_and_slash", 30, 800, 0.972051739692688], "10": ["vertical_and_slash", 30, 800, 0.86252760887146], "11": ["vertical_and_slash", 30, 800, 0.9809393882751465], "12": ["vertical_and_slash", 1000, 6096, 0.6574413776397705], "13": ["vertical_and_slash", 1000, 6096, 0.6483432054519653], "14": ["vertical_and_slash", 1000, 6096, 0.827599048614502], "15": ["vertical_and_slash", 30, 800, 0.9594753980636597], "16": ["vertical_and_slash", 500, 700, 0.9684355854988098], "17": ["vertical_and_slash", 100, 800, 0.82421875], "18": ["vertical_and_slash", 3500, 100, 0.7781898975372314], "19": ["vertical_and_slash", 1000, 6096, 0.6947312951087952], "20": ["vertical_and_slash", 500, 700, 0.9240661859512329], "21": ["vertical_and_slash", 500, 700, 0.8862026333808899], "22": ["vertical_and_slash", 500, 700, 0.9065015912055969], "23": ["vertical_and_slash", 500, 700, 0.9163179993629456], "24": ["vertical_and_slash", 1000, 6096, 0.8770501017570496], "25": ["vertical_and_slash", 100, 800, 0.92578125], "26": ["vertical_and_slash", 1000, 6096, 0.848781943321228], "27": ["vertical_and_slash", 100, 800, 0.94921875], "28": ["vertical_and_slash", 100, 800, 0.67578125], "29": ["vertical_and_slash", 100, 800, 0.84765625], "30": ["vertical_and_slash", 500, 700, 0.9309013485908508], "31": ["vertical_and_slash", 1000, 6096, 0.7004151940345764]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9423537254333496], "1": ["vertical_and_slash", 1000, 6096, 0.8992472290992737], "2": ["vertical_and_slash", 1000, 6096, 0.88288414478302], "3": ["vertical_and_slash", 1000, 6096, 0.9055640697479248], "4": ["vertical_and_slash", 100, 800, 0.9765625], "5": ["vertical_and_slash", 100, 800, 0.58984375], "6": ["vertical_and_slash", 100, 800, 0.515625], "7": ["vertical_and_slash", 100, 800, 0.6328125], "8": ["vertical_and_slash", 100, 800, 0.91015625], "9": ["vertical_and_slash", 500, 700, 0.9147842526435852], "10": ["vertical_and_slash", 500, 700, 0.9749142527580261], "11": ["vertical_and_slash", 500, 700, 0.9328352808952332], "12": ["vertical_and_slash", 1000, 6096, 0.7294385433197021], "13": ["vertical_and_slash", 100, 800, 0.83984375], "14": ["vertical_and_slash", 100, 800, 0.88671875], "15": ["vertical_and_slash", 3500, 100, 0.8809608817100525], "16": ["vertical_and_slash", 500, 700, 0.9740177989006042], "17": ["vertical_and_slash", 500, 700, 0.9707237482070923], "18": ["vertical_and_slash", 500, 700, 0.7931056618690491], "19": ["vertical_and_slash", 500, 700, 0.9593538045883179], "20": ["vertical_and_slash", 100, 800, 0.734375], "21": ["vertical_and_slash", 100, 800, 0.921875], "22": ["vertical_and_slash", 1000, 6096, 0.627185583114624], "23": ["vertical_and_slash", 100, 800, 0.73828125], "24": ["vertical_and_slash", 500, 700, 0.8667991161346436], "25": ["vertical_and_slash", 500, 700, 0.9207716584205627], "26": ["vertical_and_slash", 500, 700, 0.87169349193573], "27": ["vertical_and_slash", 500, 700, 0.9095858931541443], "28": ["vertical_and_slash", 3500, 100, 0.7628918886184692], "29": ["vertical_and_slash", 3500, 100, 0.7131507396697998], "30": ["vertical_and_slash", 100, 800, 0.65625], "31": ["vertical_and_slash", 1000, 6096, 0.9907655715942383]}, {"0": ["vertical_and_slash", 500, 700, 0.9588789343833923], "1": ["vertical_and_slash", 1000, 6096, 0.8120258450508118], "2": ["vertical_and_slash", 500, 700, 0.9652808904647827], "3": ["vertical_and_slash", 100, 800, 0.96875], "4": ["vertical_and_slash", 100, 800, 0.953125], "5": ["vertical_and_slash", 100, 800, 0.94140625], "6": ["vertical_and_slash", 100, 800, 0.90234375], "7": ["vertical_and_slash", 100, 800, 0.94921875], "8": ["vertical_and_slash", 500, 700, 0.8896629214286804], "9": ["vertical_and_slash", 500, 700, 0.9433374404907227], "10": ["vertical_and_slash", 100, 800, 0.80078125], "11": ["vertical_and_slash", 100, 800, 0.921875], "12": ["vertical_and_slash", 500, 700, 0.9875270128250122], "13": ["vertical_and_slash", 500, 700, 0.9760839343070984], "14": ["vertical_and_slash", 100, 800, 0.6953125], "15": ["vertical_and_slash", 100, 800, 0.95703125], "16": ["vertical_and_slash", 500, 700, 0.9754391312599182], "17": ["vertical_and_slash", 3500, 100, 0.9772301316261292], "18": ["vertical_and_slash", 500, 700, 0.9719763994216919], "19": ["vertical_and_slash", 500, 700, 0.9928269386291504], "20": ["vertical_and_slash", 30, 800, 0.8755784034729004], "21": ["vertical_and_slash", 100, 800, 0.97265625], "22": ["vertical_and_slash", 500, 700, 0.9814659357070923], "23": ["vertical_and_slash", 1000, 6096, 0.9750019311904907], "24": ["vertical_and_slash", 500, 700, 0.9711320996284485], "25": ["vertical_and_slash", 500, 700, 0.9722046852111816], "26": ["vertical_and_slash", 500, 700, 0.9684996604919434], "27": ["vertical_and_slash", 500, 700, 0.9434762597084045], "28": ["vertical_and_slash", 500, 700, 0.9834445118904114], "29": ["vertical_and_slash", 100, 800, 0.9609375], "30": ["vertical_and_slash", 30, 800, 0.9223980903625488], "31": ["vertical_and_slash", 100, 800, 0.9609375]}, {"0": ["vertical_and_slash", 3500, 100, 0.6316103339195251], "1": ["vertical_and_slash", 100, 800, 0.796875], "2": ["vertical_and_slash", 1000, 6096, 0.6698543429374695], "3": ["vertical_and_slash", 1000, 6096, 0.6502960324287415], "4": ["vertical_and_slash", 1000, 6096, 0.9311055541038513], "5": ["vertical_and_slash", 1000, 6096, 0.899242639541626], "6": ["vertical_and_slash", 1000, 6096, 0.8542839884757996], "7": ["vertical_and_slash", 100, 800, 0.9609375], "8": ["vertical_and_slash", 1000, 6096, 0.6151388883590698], "9": ["vertical_and_slash", 100, 800, 0.8515625], "10": ["vertical_and_slash", 100, 800, 0.671875], "11": ["vertical_and_slash", 100, 800, 0.85546875], "12": ["vertical_and_slash", 100, 800, 0.8828125], "13": ["vertical_and_slash", 100, 800, 0.8828125], "14": ["vertical_and_slash", 100, 800, 0.88671875], "15": ["vertical_and_slash", 100, 800, 0.91015625], "16": ["vertical_and_slash", 1000, 6096, 0.6039607524871826], "17": ["vertical_and_slash", 3500, 100, 0.6183164715766907], "18": ["vertical_and_slash", 3500, 100, 0.760739803314209], "19": ["vertical_and_slash", 100, 800, 0.59375], "20": ["vertical_and_slash", 3500, 100, 0.8643622994422913], "21": ["vertical_and_slash", 3500, 100, 0.90613853931427], "22": ["vertical_and_slash", 100, 800, 0.9453125], "23": ["vertical_and_slash", 3500, 100, 0.885551393032074], "24": ["vertical_and_slash", 100, 800, 0.9609375], "25": ["vertical_and_slash", 100, 800, 0.90625], "26": ["vertical_and_slash", 100, 800, 0.69921875], "27": ["vertical_and_slash", 100, 800, 0.53125], "28": ["vertical_and_slash", 30, 800, 0.8135810494422913], "29": ["vertical_and_slash", 500, 700, 0.9743149280548096], "30": ["vertical_and_slash", 1000, 6096, 0.7735022306442261], "31": ["vertical_and_slash", 1000, 6096, 0.6747613549232483]}, {"0": ["vertical_and_slash", 3500, 100, 0.9415304660797119], "1": ["vertical_and_slash", 100, 800, 0.95703125], "2": ["vertical_and_slash", 100, 800, 0.96484375], "3": ["vertical_and_slash", 3500, 100, 0.9686872363090515], "4": ["vertical_and_slash", 100, 800, 0.91015625], "5": ["vertical_and_slash", 3500, 100, 0.9223687648773193], "6": ["vertical_and_slash", 100, 800, 0.953125], "7": ["vertical_and_slash", 500, 700, 0.9196726679801941], "8": ["vertical_and_slash", 500, 700, 0.9857269525527954], "9": ["vertical_and_slash", 500, 700, 0.9771939516067505], "10": ["vertical_and_slash", 500, 700, 0.969268262386322], "11": ["vertical_and_slash", 500, 700, 0.9814961552619934], "12": ["vertical_and_slash", 100, 800, 0.80078125], "13": ["vertical_and_slash", 1000, 6096, 0.7912960052490234], "14": ["vertical_and_slash", 100, 800, 0.80078125], "15": ["vertical_and_slash", 100, 800, 0.94140625], "16": ["vertical_and_slash", 100, 800, 0.6875], "17": ["vertical_and_slash", 100, 800, 0.90625], "18": ["vertical_and_slash", 100, 800, 0.88671875], "19": ["vertical_and_slash", 1000, 6096, 0.858091413974762], "20": ["vertical_and_slash", 1000, 6096, 0.8563435673713684], "21": ["vertical_and_slash", 100, 800, 0.9453125], "22": ["vertical_and_slash", 100, 800, 0.87109375], "23": ["vertical_and_slash", 100, 800, 0.9453125], "24": ["vertical_and_slash", 1000, 6096, 0.9440521001815796], "25": ["vertical_and_slash", 1000, 6096, 0.97614985704422], "26": ["vertical_and_slash", 30, 800, 0.9994377493858337], "27": ["vertical_and_slash", 100, 800, 0.91796875], "28": ["vertical_and_slash", 100, 800, 0.87890625], "29": ["vertical_and_slash", 1000, 6096, 0.8056965470314026], "30": ["vertical_and_slash", 1000, 6096, 0.891052782535553], "31": ["vertical_and_slash", 1000, 6096, 0.9028414487838745]}, {"0": ["vertical_and_slash", 30, 800, 0.9854811429977417], "1": ["vertical_and_slash", 1000, 6096, 0.6822168231010437], "2": ["vertical_and_slash", 1000, 6096, 0.6856634616851807], "3": ["vertical_and_slash", 3500, 100, 0.9271772503852844], "4": ["vertical_and_slash", 100, 800, 0.93359375], "5": ["vertical_and_slash", 1000, 6096, 0.7962403297424316], "6": ["vertical_and_slash", 500, 700, 0.9195069074630737], "7": ["vertical_and_slash", 500, 700, 0.9743179082870483], "8": ["vertical_and_slash", 500, 700, 0.9383020401000977], "9": ["vertical_and_slash", 500, 700, 0.8858011364936829], "10": ["vertical_and_slash", 100, 800, 0.79296875], "11": ["vertical_and_slash", 100, 800, 0.8203125], "12": ["vertical_and_slash", 500, 700, 0.9806155562400818], "13": ["vertical_and_slash", 100, 800, 0.9453125], "14": ["vertical_and_slash", 100, 800, 0.97265625], "15": ["vertical_and_slash", 100, 800, 0.9296875], "16": ["vertical_and_slash", 100, 800, 0.7890625], "17": ["vertical_and_slash", 1000, 6096, 0.48270800709724426], "18": ["vertical_and_slash", 1000, 6096, 0.5019736289978027], "19": ["vertical_and_slash", 100, 800, 0.91015625], "20": ["vertical_and_slash", 100, 800, 0.90625], "21": ["vertical_and_slash", 100, 800, 0.75], "22": ["vertical_and_slash", 100, 800, 0.921875], "23": ["vertical_and_slash", 100, 800, 0.89453125], "24": ["vertical_and_slash", 100, 800, 0.84765625], "25": ["vertical_and_slash", 100, 800, 0.71875], "26": ["vertical_and_slash", 100, 800, 0.87890625], "27": ["vertical_and_slash", 100, 800, 0.8984375], "28": ["vertical_and_slash", 1000, 6096, 0.6697965264320374], "29": ["vertical_and_slash", 1000, 6096, 0.6754442453384399], "30": ["vertical_and_slash", 1000, 6096, 0.89468914270401], "31": ["vertical_and_slash", 30, 800, 0.9896609783172607]}, {"0": ["vertical_and_slash", 3500, 100, 0.8044015765190125], "1": ["vertical_and_slash", 1000, 6096, 0.8649858832359314], "2": ["vertical_and_slash", 100, 800, 0.84765625], "3": ["vertical_and_slash", 1000, 6096, 0.9512100219726562], "4": ["vertical_and_slash", 3500, 100, 0.9771698713302612], "5": ["vertical_and_slash", 1000, 6096, 0.8880830407142639], "6": ["vertical_and_slash", 3500, 100, 0.9925544857978821], "7": ["vertical_and_slash", 1000, 6096, 0.9684502482414246], "8": ["vertical_and_slash", 3500, 100, 0.8654065728187561], "9": ["vertical_and_slash", 1000, 6096, 0.9729434847831726], "10": ["vertical_and_slash", 3500, 100, 0.9787231087684631], "11": ["vertical_and_slash", 100, 800, 0.9765625], "12": ["vertical_and_slash", 500, 700, 0.9800076484680176], "13": ["vertical_and_slash", 500, 700, 0.9781122803688049], "14": ["vertical_and_slash", 500, 700, 0.980514407157898], "15": ["vertical_and_slash", 500, 700, 0.9572811126708984], "16": ["vertical_and_slash", 100, 800, 0.8359375], "17": ["vertical_and_slash", 500, 700, 0.961165189743042], "18": ["vertical_and_slash", 3500, 100, 0.9327417612075806], "19": ["vertical_and_slash", 100, 800, 0.921875], "20": ["vertical_and_slash", 1000, 6096, 0.9528481364250183], "21": ["vertical_and_slash", 100, 800, 0.90625], "22": ["vertical_and_slash", 100, 800, 0.9921875], "23": ["vertical_and_slash", 1000, 6096, 0.9193229675292969], "24": ["vertical_and_slash", 100, 800, 0.97265625], "25": ["vertical_and_slash", 100, 800, 0.83984375], "26": ["vertical_and_slash", 100, 800, 0.96484375], "27": ["vertical_and_slash", 3500, 100, 0.9813305139541626], "28": ["vertical_and_slash", 500, 700, 0.9507763385772705], "29": ["vertical_and_slash", 500, 700, 0.9917595982551575], "30": ["vertical_and_slash", 30, 800, 0.9635836482048035], "31": ["vertical_and_slash", 500, 700, 0.9638065099716187]}, {"0": ["vertical_and_slash", 500, 700, 0.9572822451591492], "1": ["vertical_and_slash", 500, 700, 0.9417194724082947], "2": ["vertical_and_slash", 100, 800, 0.8984375], "3": ["vertical_and_slash", 500, 700, 0.9779292941093445], "4": ["vertical_and_slash", 500, 700, 0.9675058126449585], "5": ["vertical_and_slash", 500, 700, 0.9766573309898376], "6": ["vertical_and_slash", 100, 800, 0.84765625], "7": ["vertical_and_slash", 500, 700, 0.9693805575370789], "8": ["vertical_and_slash", 1000, 6096, 0.6297406554222107], "9": ["vertical_and_slash", 100, 800, 0.859375], "10": ["vertical_and_slash", 3500, 100, 0.8125107884407043], "11": ["vertical_and_slash", 500, 700, 0.9787761569023132], "12": ["vertical_and_slash", 500, 700, 0.9456966519355774], "13": ["vertical_and_slash", 3500, 100, 0.9644347429275513], "14": ["vertical_and_slash", 100, 800, 0.921875], "15": ["vertical_and_slash", 1000, 6096, 0.9385132789611816], "16": ["vertical_and_slash", 1000, 6096, 0.9825614094734192], "17": ["vertical_and_slash", 1000, 6096, 0.995094895362854], "18": ["vertical_and_slash", 1000, 6096, 0.9585238099098206], "19": ["vertical_and_slash", 1000, 6096, 0.9814403057098389], "20": ["vertical_and_slash", 500, 700, 0.9920734167098999], "21": ["vertical_and_slash", 1000, 6096, 0.9164177775382996], "22": ["vertical_and_slash", 100, 800, 0.9375], "23": ["vertical_and_slash", 100, 800, 0.93359375], "24": ["vertical_and_slash", 100, 800, 0.8828125], "25": ["vertical_and_slash", 100, 800, 0.86328125], "26": ["vertical_and_slash", 3500, 100, 0.9445839524269104], "27": ["vertical_and_slash", 100, 800, 0.8515625], "28": ["vertical_and_slash", 1000, 6096, 0.9946870803833008], "29": ["vertical_and_slash", 1000, 6096, 0.980959415435791], "30": ["vertical_and_slash", 500, 700, 0.994160532951355], "31": ["vertical_and_slash", 3500, 100, 0.9895169138908386]}, {"0": ["vertical_and_slash", 3500, 100, 0.9912729263305664], "1": ["vertical_and_slash", 100, 800, 0.96484375], "2": ["vertical_and_slash", 3500, 100, 0.9686254858970642], "3": ["vertical_and_slash", 100, 800, 0.95703125], "4": ["vertical_and_slash", 3500, 100, 0.9111233353614807], "5": ["vertical_and_slash", 100, 800, 0.93359375], "6": ["vertical_and_slash", 3500, 100, 0.9810769557952881], "7": ["vertical_and_slash", 500, 700, 0.98154217004776], "8": ["vertical_and_slash", 3500, 100, 0.973520815372467], "9": ["vertical_and_slash", 3500, 100, 0.9850409030914307], "10": ["vertical_and_slash", 500, 700, 0.971245527267456], "11": ["vertical_and_slash", 500, 700, 0.9817143082618713], "12": ["vertical_and_slash", 100, 800, 0.9921875], "13": ["vertical_and_slash", 500, 700, 0.9767857193946838], "14": ["vertical_and_slash", 100, 800, 0.9296875], "15": ["vertical_and_slash", 500, 700, 0.9780569076538086], "16": ["vertical_and_slash", 100, 800, 0.94140625], "17": ["vertical_and_slash", 100, 800, 0.953125], "18": ["vertical_and_slash", 1000, 6096, 0.9731783270835876], "19": ["vertical_and_slash", 1000, 6096, 0.9841699004173279], "20": ["vertical_and_slash", 3500, 100, 0.7990682125091553], "21": ["vertical_and_slash", 100, 800, 0.96875], "22": ["vertical_and_slash", 100, 800, 0.86328125], "23": ["vertical_and_slash", 100, 800, 0.9296875], "24": ["vertical_and_slash", 1000, 6096, 0.9868585467338562], "25": ["vertical_and_slash", 100, 800, 0.9921875], "26": ["vertical_and_slash", 500, 700, 0.9919374585151672], "27": ["vertical_and_slash", 100, 800, 0.98828125], "28": ["vertical_and_slash", 100, 800, 0.890625], "29": ["vertical_and_slash", 100, 800, 0.91015625], "30": ["vertical_and_slash", 100, 800, 0.875], "31": ["vertical_and_slash", 100, 800, 0.90625]}, {"0": ["vertical_and_slash", 100, 800, 0.56640625], "1": ["vertical_and_slash", 100, 800, 0.90234375], "2": ["vertical_and_slash", 1000, 6096, 0.89035964012146], "3": ["vertical_and_slash", 1000, 6096, 0.9361557960510254], "4": ["vertical_and_slash", 1000, 6096, 0.9623773694038391], "5": ["vertical_and_slash", 1000, 6096, 0.9804529547691345], "6": ["vertical_and_slash", 1000, 6096, 0.9724429845809937], "7": ["vertical_and_slash", 1000, 6096, 0.9953715205192566], "8": ["vertical_and_slash", 500, 700, 0.9900636076927185], "9": ["vertical_and_slash", 100, 800, 0.8671875], "10": ["vertical_and_slash", 500, 700, 0.9860572814941406], "11": ["vertical_and_slash", 30, 800, 0.9200822710990906], "12": ["vertical_and_slash", 100, 800, 0.7890625], "13": ["vertical_and_slash", 100, 800, 0.71875], "14": ["vertical_and_slash", 100, 800, 0.88671875], "15": ["vertical_and_slash", 3500, 100, 0.8809337615966797], "16": ["vertical_and_slash", 100, 800, 0.9609375], "17": ["vertical_and_slash", 500, 700, 0.977523684501648], "18": ["vertical_and_slash", 500, 700, 0.9648466110229492], "19": ["vertical_and_slash", 100, 800, 0.92578125], "20": ["vertical_and_slash", 100, 800, 0.8984375], "21": ["vertical_and_slash", 100, 800, 0.98046875], "22": ["vertical_and_slash", 100, 800, 0.94921875], "23": ["vertical_and_slash", 100, 800, 0.890625], "24": ["vertical_and_slash", 500, 700, 0.9612880349159241], "25": ["vertical_and_slash", 100, 800, 0.96875], "26": ["vertical_and_slash", 1000, 6096, 0.9904219508171082], "27": ["vertical_and_slash", 100, 800, 0.98046875], "28": ["vertical_and_slash", 500, 700, 0.9728029370307922], "29": ["vertical_and_slash", 500, 700, 0.9610400795936584], "30": ["vertical_and_slash", 500, 700, 0.9590328335762024], "31": ["vertical_and_slash", 500, 700, 0.9732471108436584]}, {"0": ["vertical_and_slash", 100, 800, 0.8359375], "1": ["vertical_and_slash", 100, 800, 0.81640625], "2": ["vertical_and_slash", 100, 800, 0.6640625], "3": ["vertical_and_slash", 100, 800, 0.953125], "4": ["vertical_and_slash", 500, 700, 0.9603883624076843], "5": ["vertical_and_slash", 1000, 6096, 0.9699860215187073], "6": ["vertical_and_slash", 1000, 6096, 0.9892503023147583], "7": ["vertical_and_slash", 1000, 6096, 0.9920991659164429], "8": ["vertical_and_slash", 500, 700, 0.9887894988059998], "9": ["vertical_and_slash", 100, 800, 0.9609375], "10": ["vertical_and_slash", 100, 800, 0.98046875], "11": ["vertical_and_slash", 100, 800, 0.8515625], "12": ["vertical_and_slash", 100, 800, 0.84765625], "13": ["vertical_and_slash", 3500, 100, 0.9337024688720703], "14": ["vertical_and_slash", 100, 800, 0.88671875], "15": ["vertical_and_slash", 100, 800, 0.91015625], "16": ["vertical_and_slash", 1000, 6096, 0.9892725348472595], "17": ["vertical_and_slash", 1000, 6096, 0.9876350164413452], "18": ["vertical_and_slash", 1000, 6096, 0.9888733625411987], "19": ["vertical_and_slash", 1000, 6096, 0.9845653176307678], "20": ["vertical_and_slash", 1000, 6096, 0.9609242677688599], "21": ["vertical_and_slash", 100, 800, 0.98828125], "22": ["vertical_and_slash", 500, 700, 0.9696791172027588], "23": ["vertical_and_slash", 3500, 100, 0.8103806376457214], "24": ["vertical_and_slash", 1000, 6096, 0.9616240859031677], "25": ["vertical_and_slash", 100, 800, 0.87890625], "26": ["vertical_and_slash", 1000, 6096, 0.9824818968772888], "27": ["vertical_and_slash", 100, 800, 0.94140625], "28": ["vertical_and_slash", 500, 700, 0.996360719203949], "29": ["vertical_and_slash", 100, 800, 0.99609375], "30": ["vertical_and_slash", 100, 800, 0.9921875], "31": ["vertical_and_slash", 100, 800, 0.98828125]}, {"0": ["vertical_and_slash", 100, 800, 0.99609375], "1": ["vertical_and_slash", 100, 800, 0.93359375], "2": ["vertical_and_slash", 100, 800, 0.83203125], "3": ["vertical_and_slash", 100, 800, 0.734375], "4": ["vertical_and_slash", 30, 800, 0.9953799247741699], "5": ["vertical_and_slash", 500, 700, 0.9949384927749634], "6": ["vertical_and_slash", 500, 700, 0.987656831741333], "7": ["vertical_and_slash", 30, 800, 0.9914078116416931], "8": ["vertical_and_slash", 100, 800, 0.92578125], "9": ["vertical_and_slash", 500, 700, 0.9965551495552063], "10": ["vertical_and_slash", 500, 700, 0.9983668327331543], "11": ["vertical_and_slash", 1000, 6096, 0.9965943098068237], "12": ["vertical_and_slash", 3500, 100, 0.9983046054840088], "13": ["vertical_and_slash", 3500, 100, 0.9971177577972412], "14": ["vertical_and_slash", 3500, 100, 0.9972085356712341], "15": ["vertical_and_slash", 100, 800, 1.0], "16": ["vertical_and_slash", 100, 800, 0.99609375], "17": ["vertical_and_slash", 500, 700, 0.9965201616287231], "18": ["vertical_and_slash", 100, 800, 0.9765625], "19": ["vertical_and_slash", 500, 700, 0.9975128173828125], "20": ["vertical_and_slash", 1000, 6096, 0.9829357862472534], "21": ["vertical_and_slash", 100, 800, 0.93359375], "22": ["vertical_and_slash", 3500, 100, 0.9525547623634338], "23": ["vertical_and_slash", 3500, 100, 0.9518142342567444], "24": ["vertical_and_slash", 3500, 100, 0.9944693446159363], "25": ["vertical_and_slash", 100, 800, 0.98828125], "26": ["vertical_and_slash", 3500, 100, 0.9936266541481018], "27": ["vertical_and_slash", 100, 800, 0.98046875], "28": ["vertical_and_slash", 1000, 6096, 0.9858771562576294], "29": ["vertical_and_slash", 3500, 100, 0.9896631836891174], "30": ["vertical_and_slash", 3500, 100, 0.9762122631072998], "31": ["vertical_and_slash", 3500, 100, 0.9886800646781921]}, {"0": ["vertical_and_slash", 3500, 100, 0.9691346883773804], "1": ["vertical_and_slash", 3500, 100, 0.9351215362548828], "2": ["vertical_and_slash", 3500, 100, 0.9609357118606567], "3": ["vertical_and_slash", 100, 800, 0.97265625], "4": ["vertical_and_slash", 100, 800, 0.94140625], "5": ["vertical_and_slash", 500, 700, 0.9768573641777039], "6": ["vertical_and_slash", 3500, 100, 0.9625533819198608], "7": ["vertical_and_slash", 3500, 100, 0.9791122078895569], "8": ["vertical_and_slash", 100, 800, 0.98828125], "9": ["vertical_and_slash", 100, 800, 0.9765625], "10": ["vertical_and_slash", 100, 800, 0.99609375], "11": ["vertical_and_slash", 100, 800, 0.984375], "12": ["vertical_and_slash", 1000, 6096, 0.8757453560829163], "13": ["vertical_and_slash", 100, 800, 0.78515625], "14": ["vertical_and_slash", 100, 800, 0.83984375], "15": ["vertical_and_slash", 100, 800, 0.8359375], "16": ["vertical_and_slash", 500, 700, 0.9865701794624329], "17": ["vertical_and_slash", 500, 700, 0.9824267029762268], "18": ["vertical_and_slash", 500, 700, 0.9177895188331604], "19": ["vertical_and_slash", 500, 700, 0.9836146831512451], "20": ["vertical_and_slash", 500, 700, 0.971502423286438], "21": ["vertical_and_slash", 500, 700, 0.9808719158172607], "22": ["vertical_and_slash", 500, 700, 0.9893147945404053], "23": ["vertical_and_slash", 100, 800, 0.99609375], "24": ["vertical_and_slash", 100, 800, 0.97265625], "25": ["vertical_and_slash", 500, 700, 0.9651071429252625], "26": ["vertical_and_slash", 100, 800, 0.94921875], "27": ["vertical_and_slash", 500, 700, 0.9888646602630615], "28": ["vertical_and_slash", 3500, 100, 0.9266110062599182], "29": ["vertical_and_slash", 100, 800, 0.9140625], "30": ["vertical_and_slash", 100, 800, 0.890625], "31": ["vertical_and_slash", 3500, 100, 0.9608930945396423]}, {"0": ["vertical_and_slash", 100, 800, 0.97265625], "1": ["vertical_and_slash", 100, 800, 0.98828125], "2": ["vertical_and_slash", 100, 800, 0.98046875], "3": ["vertical_and_slash", 100, 800, 0.98046875], "4": ["vertical_and_slash", 3500, 100, 0.9496995806694031], "5": ["vertical_and_slash", 3500, 100, 0.9672608375549316], "6": ["vertical_and_slash", 3500, 100, 0.968944251537323], "7": ["vertical_and_slash", 100, 800, 0.9453125], "8": ["vertical_and_slash", 100, 800, 0.99609375], "9": ["vertical_and_slash", 100, 800, 0.99609375], "10": ["vertical_and_slash", 3500, 100, 0.9927955269813538], "11": ["vertical_and_slash", 3500, 100, 0.9970442056655884], "12": ["vertical_and_slash", 3500, 100, 0.986923098564148], "13": ["vertical_and_slash", 3500, 100, 0.7216698527336121], "14": ["vertical_and_slash", 3500, 100, 0.8158459663391113], "15": ["vertical_and_slash", 100, 800, 0.8515625], "16": ["vertical_and_slash", 500, 700, 0.997530996799469], "17": ["vertical_and_slash", 100, 800, 0.99609375], "18": ["vertical_and_slash", 100, 800, 0.98046875], "19": ["vertical_and_slash", 500, 700, 0.9970376491546631], "20": ["vertical_and_slash", 3500, 100, 0.9760066270828247], "21": ["vertical_and_slash", 3500, 100, 0.9728747010231018], "22": ["vertical_and_slash", 3500, 100, 0.9766891598701477], "23": ["vertical_and_slash", 3500, 100, 0.9976004362106323], "24": ["vertical_and_slash", 100, 800, 0.96484375], "25": ["vertical_and_slash", 100, 800, 0.96484375], "26": ["vertical_and_slash", 3500, 100, 0.9983320236206055], "27": ["vertical_and_slash", 3500, 100, 0.9873730540275574], "28": ["vertical_and_slash", 500, 700, 0.9973207116127014], "29": ["vertical_and_slash", 500, 700, 0.9969824552536011], "30": ["vertical_and_slash", 500, 700, 0.9982982277870178], "31": ["vertical_and_slash", 500, 700, 0.9983280301094055]}, {"0": ["vertical_and_slash", 100, 800, 0.9921875], "1": ["vertical_and_slash", 3500, 100, 0.9938210844993591], "2": ["vertical_and_slash", 100, 800, 0.98828125], "3": ["vertical_and_slash", 3500, 100, 0.9968554973602295], "4": ["vertical_and_slash", 1000, 6096, 0.9928692579269409], "5": ["vertical_and_slash", 3500, 100, 0.9889399409294128], "6": ["vertical_and_slash", 3500, 100, 0.9934308528900146], "7": ["vertical_and_slash", 3500, 100, 0.9926333427429199], "8": ["vertical_and_slash", 500, 700, 0.9881335496902466], "9": ["vertical_and_slash", 30, 800, 0.9847679734230042], "10": ["vertical_and_slash", 30, 800, 0.9852355122566223], "11": ["vertical_and_slash", 500, 700, 0.9858385920524597], "12": ["vertical_and_slash", 100, 800, 0.984375], "13": ["vertical_and_slash", 100, 800, 0.98828125], "14": ["vertical_and_slash", 30, 800, 0.876174807548523], "15": ["vertical_and_slash", 500, 700, 0.9925132393836975], "16": ["vertical_and_slash", 3500, 100, 0.980747640132904], "17": ["vertical_and_slash", 3500, 100, 0.9824157953262329], "18": ["vertical_and_slash", 3500, 100, 0.9821085333824158], "19": ["vertical_and_slash", 3500, 100, 0.9562081694602966], "20": ["vertical_and_slash", 1000, 6096, 0.9850624799728394], "21": ["vertical_and_slash", 3500, 100, 0.9780853390693665], "22": ["vertical_and_slash", 3500, 100, 0.9799768924713135], "23": ["vertical_and_slash", 3500, 100, 0.9772357940673828], "24": ["vertical_and_slash", 3500, 100, 0.9548801183700562], "25": ["vertical_and_slash", 3500, 100, 0.9809179306030273], "26": ["vertical_and_slash", 100, 800, 0.96484375], "27": ["vertical_and_slash", 100, 800, 0.96875], "28": ["vertical_and_slash", 3500, 100, 0.9743281602859497], "29": ["vertical_and_slash", 500, 700, 0.9775781631469727], "30": ["vertical_and_slash", 500, 700, 0.9927409291267395], "31": ["vertical_and_slash", 3500, 100, 0.992667555809021]}, {"0": ["vertical_and_slash", 3500, 100, 0.9837089776992798], "1": ["vertical_and_slash", 100, 800, 0.96484375], "2": ["vertical_and_slash", 3500, 100, 0.990432858467102], "3": ["vertical_and_slash", 3500, 100, 0.9577094912528992], "4": ["vertical_and_slash", 3500, 100, 0.9878129363059998], "5": ["vertical_and_slash", 1000, 6096, 0.9842414855957031], "6": ["vertical_and_slash", 3500, 100, 0.9822033047676086], "7": ["vertical_and_slash", 3500, 100, 0.9838908910751343], "8": ["vertical_and_slash", 100, 800, 0.9921875], "9": ["vertical_and_slash", 500, 700, 0.9924842715263367], "10": ["vertical_and_slash", 100, 800, 0.98828125], "11": ["vertical_and_slash", 100, 800, 0.99609375], "12": ["vertical_and_slash", 3500, 100, 0.983744740486145], "13": ["vertical_and_slash", 1000, 6096, 0.9854903817176819], "14": ["vertical_and_slash", 1000, 6096, 0.9890750050544739], "15": ["vertical_and_slash", 1000, 6096, 0.9871489405632019], "16": ["vertical_and_slash", 100, 800, 0.98828125], "17": ["vertical_and_slash", 100, 800, 0.99609375], "18": ["vertical_and_slash", 3500, 100, 0.9934486150741577], "19": ["vertical_and_slash", 1000, 6096, 0.9904783368110657], "20": ["vertical_and_slash", 30, 800, 0.9985806345939636], "21": ["vertical_and_slash", 30, 800, 0.9482468962669373], "22": ["vertical_and_slash", 30, 800, 0.9973258972167969], "23": ["vertical_and_slash", 30, 800, 0.9980663657188416], "24": ["vertical_and_slash", 500, 700, 0.9974254369735718], "25": ["vertical_and_slash", 100, 800, 0.99609375], "26": ["vertical_and_slash", 500, 700, 0.988605260848999], "27": ["vertical_and_slash", 100, 800, 0.98046875], "28": ["vertical_and_slash", 500, 700, 0.9811033010482788], "29": ["vertical_and_slash", 500, 700, 0.9804574847221375], "30": ["vertical_and_slash", 500, 700, 0.9865885376930237], "31": ["vertical_and_slash", 500, 700, 0.9892045259475708]}, {"0": ["vertical_and_slash", 100, 800, 0.69140625], "1": ["vertical_and_slash", 100, 800, 0.97265625], "2": ["vertical_and_slash", 100, 800, 0.62890625], "3": ["vertical_and_slash", 100, 800, 0.8828125], "4": ["vertical_and_slash", 100, 800, 0.95703125], "5": ["vertical_and_slash", 100, 800, 0.96875], "6": ["vertical_and_slash", 3500, 100, 0.9835638403892517], "7": ["vertical_and_slash", 3500, 100, 0.9768636226654053], "8": ["vertical_and_slash", 100, 800, 0.875], "9": ["vertical_and_slash", 500, 700, 0.9736713171005249], "10": ["vertical_and_slash", 3500, 100, 0.9867293238639832], "11": ["vertical_and_slash", 100, 800, 0.953125], "12": ["vertical_and_slash", 1000, 6096, 0.9799107909202576], "13": ["vertical_and_slash", 1000, 6096, 0.956547737121582], "14": ["vertical_and_slash", 1000, 6096, 0.9830597639083862], "15": ["vertical_and_slash", 1000, 6096, 0.9786513447761536], "16": ["vertical_and_slash", 1000, 6096, 0.9893789887428284], "17": ["vertical_and_slash", 100, 800, 0.984375], "18": ["vertical_and_slash", 1000, 6096, 0.9834024906158447], "19": ["vertical_and_slash", 3500, 100, 0.9586296081542969], "20": ["vertical_and_slash", 30, 800, 0.9566952586174011], "21": ["vertical_and_slash", 100, 800, 0.96484375], "22": ["vertical_and_slash", 30, 800, 0.9579682350158691], "23": ["vertical_and_slash", 100, 800, 0.97265625], "24": ["vertical_and_slash", 100, 800, 1.0], "25": ["vertical_and_slash", 500, 700, 0.9973883032798767], "26": ["vertical_and_slash", 100, 800, 1.0], "27": ["vertical_and_slash", 3500, 100, 0.9958353638648987], "28": ["vertical_and_slash", 100, 800, 0.92578125], "29": ["vertical_and_slash", 3500, 100, 0.9647402763366699], "30": ["vertical_and_slash", 100, 800, 0.9921875], "31": ["vertical_and_slash", 3500, 100, 0.9853026866912842]}, {"0": ["vertical_and_slash", 500, 700, 0.9965609312057495], "1": ["vertical_and_slash", 100, 800, 0.99609375], "2": ["vertical_and_slash", 100, 800, 0.99609375], "3": ["vertical_and_slash", 500, 700, 0.993408739566803], "4": ["vertical_and_slash", 1000, 6096, 0.9861562848091125], "5": ["vertical_and_slash", 1000, 6096, 0.9630166292190552], "6": ["vertical_and_slash", 1000, 6096, 0.9752892851829529], "7": ["vertical_and_slash", 1000, 6096, 0.9612733125686646], "8": ["vertical_and_slash", 100, 800, 0.98828125], "9": ["vertical_and_slash", 500, 700, 0.996844470500946], "10": ["vertical_and_slash", 100, 800, 0.99609375], "11": ["vertical_and_slash", 30, 800, 0.98490971326828], "12": ["vertical_and_slash", 3500, 100, 0.9845942854881287], "13": ["vertical_and_slash", 3500, 100, 0.9553945064544678], "14": ["vertical_and_slash", 500, 700, 0.9797781109809875], "15": ["vertical_and_slash", 500, 700, 0.9618297815322876], "16": ["vertical_and_slash", 3500, 100, 0.9837275147438049], "17": ["vertical_and_slash", 500, 700, 0.9883895516395569], "18": ["vertical_and_slash", 100, 800, 0.99609375], "19": ["vertical_and_slash", 3500, 100, 0.9968678951263428], "20": ["vertical_and_slash", 1000, 6096, 0.971379280090332], "21": ["vertical_and_slash", 1000, 6096, 0.9435017704963684], "22": ["vertical_and_slash", 1000, 6096, 0.9786460995674133], "23": ["vertical_and_slash", 1000, 6096, 0.9788541793823242], "24": ["vertical_and_slash", 500, 700, 0.9358993768692017], "25": ["vertical_and_slash", 30, 800, 0.9740927815437317], "26": ["vertical_and_slash", 500, 700, 0.9939063787460327], "27": ["vertical_and_slash", 500, 700, 0.9889722466468811], "28": ["vertical_and_slash", 100, 800, 0.6171875], "29": ["vertical_and_slash", 30, 800, 0.9943872094154358], "30": ["vertical_and_slash", 100, 800, 0.88671875], "31": ["vertical_and_slash", 100, 800, 0.9140625]}, {"0": ["vertical_and_slash", 100, 800, 0.96875], "1": ["vertical_and_slash", 100, 800, 0.99609375], "2": ["vertical_and_slash", 100, 800, 0.9765625], "3": ["vertical_and_slash", 100, 800, 0.9453125], "4": ["vertical_and_slash", 30, 800, 0.9959031939506531], "5": ["vertical_and_slash", 30, 800, 0.9976523518562317], "6": ["vertical_and_slash", 30, 800, 0.981220543384552], "7": ["vertical_and_slash", 30, 800, 0.996821939945221], "8": ["vertical_and_slash", 100, 800, 0.94140625], "9": ["vertical_and_slash", 100, 800, 0.99609375], "10": ["vertical_and_slash", 100, 800, 0.953125], "11": ["vertical_and_slash", 30, 800, 0.9451431632041931], "12": ["vertical_and_slash", 1000, 6096, 0.9946032166481018], "13": ["vertical_and_slash", 3500, 100, 0.9859157204627991], "14": ["vertical_and_slash", 1000, 6096, 0.986793577671051], "15": ["vertical_and_slash", 100, 800, 0.98046875], "16": ["vertical_and_slash", 3500, 100, 0.9806498885154724], "17": ["vertical_and_slash", 100, 800, 0.84765625], "18": ["vertical_and_slash", 100, 800, 0.8984375], "19": ["vertical_and_slash", 100, 800, 0.96484375], "20": ["vertical_and_slash", 3500, 100, 0.9754143357276917], "21": ["vertical_and_slash", 500, 700, 0.9632224440574646], "22": ["vertical_and_slash", 500, 700, 0.9907698631286621], "23": ["vertical_and_slash", 3500, 100, 0.9646450877189636], "24": ["vertical_and_slash", 3500, 100, 0.9653900265693665], "25": ["vertical_and_slash", 3500, 100, 0.9837985038757324], "26": ["vertical_and_slash", 30, 800, 0.9979236125946045], "27": ["vertical_and_slash", 3500, 100, 0.9754469394683838], "28": ["vertical_and_slash", 100, 800, 0.94921875], "29": ["vertical_and_slash", 100, 800, 0.79296875], "30": ["vertical_and_slash", 100, 800, 0.9453125], "31": ["vertical_and_slash", 500, 700, 0.9928015470504761]}, {"0": ["vertical_and_slash", 100, 800, 0.99609375], "1": ["vertical_and_slash", 100, 800, 0.9921875], "2": ["vertical_and_slash", 100, 800, 0.98828125], "3": ["vertical_and_slash", 500, 700, 0.9944132566452026], "4": ["vertical_and_slash", 100, 800, 0.99609375], "5": ["vertical_and_slash", 1000, 6096, 0.9978566765785217], "6": ["vertical_and_slash", 500, 700, 0.9965388774871826], "7": ["vertical_and_slash", 100, 800, 1.0], "8": ["vertical_and_slash", 100, 800, 0.98828125], "9": ["vertical_and_slash", 100, 800, 0.875], "10": ["vertical_and_slash", 30, 800, 0.9321915507316589], "11": ["vertical_and_slash", 100, 800, 0.9453125], "12": ["vertical_and_slash", 100, 800, 0.99609375], "13": ["vertical_and_slash", 500, 700, 0.9935281276702881], "14": ["vertical_and_slash", 500, 700, 0.9961588382720947], "15": ["vertical_and_slash", 500, 700, 0.9942258596420288], "16": ["vertical_and_slash", 100, 800, 0.921875], "17": ["vertical_and_slash", 100, 800, 0.98828125], "18": ["vertical_and_slash", 100, 800, 0.97265625], "19": ["vertical_and_slash", 100, 800, 0.97265625], "20": ["vertical_and_slash", 1000, 6096, 0.8195043206214905], "21": ["vertical_and_slash", 3500, 100, 0.69544917345047], "22": ["vertical_and_slash", 3500, 100, 0.808684229850769], "23": ["vertical_and_slash", 100, 800, 0.828125], "24": ["vertical_and_slash", 30, 800, 0.9985952973365784], "25": ["vertical_and_slash", 30, 800, 0.9965323805809021], "26": ["vertical_and_slash", 30, 800, 0.9991631507873535], "27": ["vertical_and_slash", 30, 800, 0.999081015586853], "28": ["vertical_and_slash", 3500, 100, 0.9952759146690369], "29": ["vertical_and_slash", 500, 700, 0.9537212252616882], "30": ["vertical_and_slash", 3500, 100, 0.9890041947364807], "31": ["vertical_and_slash", 3500, 100, 0.9856141805648804]}, {"0": ["vertical_and_slash", 500, 700, 0.9620710015296936], "1": ["vertical_and_slash", 3500, 100, 0.9257127046585083], "2": ["vertical_and_slash", 3500, 100, 0.960764467716217], "3": ["vertical_and_slash", 3500, 100, 0.9661008715629578], "4": ["vertical_and_slash", 1000, 6096, 0.9754106998443604], "5": ["vertical_and_slash", 100, 800, 0.93359375], "6": ["vertical_and_slash", 100, 800, 0.90625], "7": ["vertical_and_slash", 30, 800, 0.9456615447998047], "8": ["vertical_and_slash", 100, 800, 0.7578125], "9": ["vertical_and_slash", 3500, 100, 0.9354745745658875], "10": ["vertical_and_slash", 500, 700, 0.9463661313056946], "11": ["vertical_and_slash", 3500, 100, 0.684634268283844], "12": ["vertical_and_slash", 3500, 100, 0.9281976819038391], "13": ["vertical_and_slash", 100, 800, 0.8671875], "14": ["vertical_and_slash", 3500, 100, 0.9579365253448486], "15": ["vertical_and_slash", 3500, 100, 0.9362466931343079], "16": ["vertical_and_slash", 1000, 6096, 0.6537134647369385], "17": ["vertical_and_slash", 100, 800, 0.82421875], "18": ["vertical_and_slash", 100, 800, 0.65625], "19": ["vertical_and_slash", 100, 800, 0.9453125], "20": ["vertical_and_slash", 3500, 100, 0.9693450927734375], "21": ["vertical_and_slash", 3500, 100, 0.9710285663604736], "22": ["vertical_and_slash", 30, 800, 0.9574082493782043], "23": ["vertical_and_slash", 500, 700, 0.9826518297195435], "24": ["vertical_and_slash", 100, 800, 0.87109375], "25": ["vertical_and_slash", 100, 800, 0.81640625], "26": ["vertical_and_slash", 100, 800, 0.89453125], "27": ["vertical_and_slash", 3500, 100, 0.8967287540435791], "28": ["vertical_and_slash", 100, 800, 0.94140625], "29": ["vertical_and_slash", 3500, 100, 0.9489632844924927], "30": ["vertical_and_slash", 3500, 100, 0.9336714148521423], "31": ["vertical_and_slash", 100, 800, 0.921875]}, {"0": ["vertical_and_slash", 30, 800, 0.9627715349197388], "1": ["vertical_and_slash", 30, 800, 0.9686042070388794], "2": ["vertical_and_slash", 30, 800, 0.9637676477432251], "3": ["vertical_and_slash", 500, 700, 0.9802013635635376], "4": ["vertical_and_slash", 30, 800, 0.9397980570793152], "5": ["vertical_and_slash", 30, 800, 0.9883284568786621], "6": ["vertical_and_slash", 500, 700, 0.9728643894195557], "7": ["vertical_and_slash", 100, 800, 0.96484375], "8": ["vertical_and_slash", 30, 800, 0.9368970394134521], "9": ["vertical_and_slash", 30, 800, 0.9834686517715454], "10": ["vertical_and_slash", 30, 800, 0.9720800518989563], "11": ["vertical_and_slash", 30, 800, 0.7633146643638611], "12": ["vertical_and_slash", 30, 800, 0.9846735596656799], "13": ["vertical_and_slash", 30, 800, 0.8777114748954773], "14": ["vertical_and_slash", 1000, 6096, 0.9979324340820312], "15": ["vertical_and_slash", 30, 800, 0.9801661372184753], "16": ["vertical_and_slash", 100, 800, 0.875], "17": ["vertical_and_slash", 3500, 100, 0.9097177982330322], "18": ["vertical_and_slash", 30, 800, 0.9596639275550842], "19": ["vertical_and_slash", 100, 800, 0.78125], "20": ["vertical_and_slash", 3500, 100, 0.9271265268325806], "21": ["vertical_and_slash", 3500, 100, 0.9515058398246765], "22": ["vertical_and_slash", 100, 800, 0.90625], "23": ["vertical_and_slash", 30, 800, 0.992164134979248], "24": ["vertical_and_slash", 30, 800, 0.9773561358451843], "25": ["vertical_and_slash", 30, 800, 0.9886955618858337], "26": ["vertical_and_slash", 30, 800, 0.8982697129249573], "27": ["vertical_and_slash", 30, 800, 0.9301047325134277], "28": ["vertical_and_slash", 100, 800, 0.97265625], "29": ["vertical_and_slash", 100, 800, 0.90234375], "30": ["vertical_and_slash", 100, 800, 0.921875], "31": ["vertical_and_slash", 100, 800, 0.87109375]}]
+[{"0": ["vertical_and_slash", 1000, 6096, 0.9623493552207947], "1": ["vertical_and_slash", 1000, 6096, 0.9543083310127258], "2": ["vertical_and_slash", 1000, 6096, 0.9859212636947632], "3": ["vertical_and_slash", 1000, 6096, 0.884873628616333], "4": ["vertical_and_slash", 1000, 6096, 0.8294410109519958], "5": ["vertical_and_slash", 1000, 6096, 0.8265098333358765], "6": ["vertical_and_slash", 1000, 6096, 0.8764142394065857], "7": ["vertical_and_slash", 1000, 6096, 0.834898829460144], "8": ["vertical_and_slash", 1000, 6096, 0.8034278154373169], "9": ["vertical_and_slash", 1000, 6096, 0.8659768104553223], "10": ["vertical_and_slash", 1000, 6096, 0.9463561773300171], "11": ["vertical_and_slash", 1000, 6096, 0.8807589411735535], "12": ["vertical_and_slash", 1000, 6096, 0.5954520106315613], "13": ["vertical_and_slash", 1000, 6096, 0.5334455370903015], "14": ["vertical_and_slash", 1000, 6096, 0.6530376076698303], "15": ["vertical_and_slash", 1000, 6096, 0.6219685673713684], "16": ["vertical_and_slash", 30, 800, 0.5783088803291321], "17": ["vertical_and_slash", 30, 800, 0.6486243605613708], "18": ["vertical_and_slash", 30, 800, 0.5563837289810181], "19": ["vertical_and_slash", 30, 800, 0.5167779922485352], "20": ["vertical_and_slash", 1000, 6096, 0.5651902556419373], "21": ["vertical_and_slash", 1000, 6096, 0.502755343914032], "22": ["vertical_and_slash", 1000, 6096, 0.4645864963531494], "23": ["vertical_and_slash", 1000, 6096, 0.6853646039962769], "24": ["vertical_and_slash", 1000, 6096, 0.9108596444129944], "25": ["vertical_and_slash", 1000, 6096, 0.8522756695747375], "26": ["vertical_and_slash", 1000, 6096, 0.8810073137283325], "27": ["vertical_and_slash", 1000, 6096, 0.8361371159553528], "28": ["vertical_and_slash", 1000, 6096, 0.4399656355381012], "29": ["vertical_and_slash", 3500, 100, 0.9979082345962524], "30": ["vertical_and_slash", 3500, 100, 0.8938949704170227], "31": ["vertical_and_slash", 3500, 100, 1.0000008344650269]}, {"0": ["vertical_and_slash", 30, 800, 0.9742488861083984], "1": ["vertical_and_slash", 30, 800, 0.9788762927055359], "2": ["vertical_and_slash", 1000, 6096, 0.9051949977874756], "3": ["vertical_and_slash", 30, 800, 0.9755954146385193], "4": ["vertical_and_slash", 30, 800, 0.9866801500320435], "5": ["vertical_and_slash", 30, 800, 0.9911981821060181], "6": ["vertical_and_slash", 30, 800, 0.9378494024276733], "7": ["vertical_and_slash", 30, 800, 0.9651912450790405], "8": ["vertical_and_slash", 30, 800, 0.952520489692688], "9": ["vertical_and_slash", 1000, 6096, 0.6603082418441772], "10": ["vertical_and_slash", 30, 800, 0.9665190577507019], "11": ["vertical_and_slash", 30, 800, 0.9581809043884277], "12": ["vertical_and_slash", 1000, 6096, 0.9518635272979736], "13": ["vertical_and_slash", 30, 800, 0.9779806733131409], "14": ["vertical_and_slash", 30, 800, 0.89271080493927], "15": ["vertical_and_slash", 30, 800, 0.96027010679245], "16": ["vertical_and_slash", 30, 800, 0.9920278191566467], "17": ["vertical_and_slash", 30, 800, 0.9877189993858337], "18": ["vertical_and_slash", 30, 800, 0.995576024055481], "19": ["vertical_and_slash", 30, 800, 0.9701985120773315], "20": ["vertical_and_slash", 30, 800, 0.9970166087150574], "21": ["vertical_and_slash", 30, 800, 0.9962336421012878], "22": ["vertical_and_slash", 30, 800, 0.8178947567939758], "23": ["vertical_and_slash", 30, 800, 0.7966029644012451], "24": ["vertical_and_slash", 30, 800, 0.9736526608467102], "25": ["vertical_and_slash", 30, 800, 0.9269852638244629], "26": ["vertical_and_slash", 30, 800, 0.9770432114601135], "27": ["vertical_and_slash", 30, 800, 0.9720726609230042], "28": ["vertical_and_slash", 1000, 6096, 0.9068695902824402], "29": ["vertical_and_slash", 1000, 6096, 0.9494656920433044], "30": ["vertical_and_slash", 30, 800, 0.9789727926254272], "31": ["vertical_and_slash", 30, 800, 0.9188363552093506]}, {"0": ["vertical_and_slash", 30, 800, 0.9916821718215942], "1": ["vertical_and_slash", 1000, 6096, 0.9861699938774109], "2": ["vertical_and_slash", 30, 800, 0.9908547401428223], "3": ["vertical_and_slash", 1000, 6096, 0.9805020689964294], "4": ["vertical_and_slash", 30, 800, 0.9800416827201843], "5": ["vertical_and_slash", 30, 800, 0.9456108808517456], "6": ["vertical_and_slash", 100, 800, 0.86328125], "7": ["vertical_and_slash", 30, 800, 0.9905268549919128], "8": ["vertical_and_slash", 30, 800, 0.9948951005935669], "9": ["vertical_and_slash", 30, 800, 0.9933184385299683], "10": ["vertical_and_slash", 30, 800, 0.9832462072372437], "11": ["vertical_and_slash", 30, 800, 0.9870514273643494], "12": ["vertical_and_slash", 1000, 6096, 0.9968010783195496], "13": ["vertical_and_slash", 30, 800, 0.9833763837814331], "14": ["vertical_and_slash", 100, 800, 0.65234375], "15": ["vertical_and_slash", 1000, 6096, 0.8466139435768127], "16": ["vertical_and_slash", 1000, 6096, 0.9948402047157288], "17": ["vertical_and_slash", 1000, 6096, 0.9832385778427124], "18": ["vertical_and_slash", 30, 800, 0.9945462346076965], "19": ["vertical_and_slash", 1000, 6096, 0.9481031894683838], "20": ["vertical_and_slash", 1000, 6096, 0.9115096926689148], "21": ["vertical_and_slash", 1000, 6096, 0.9331101775169373], "22": ["vertical_and_slash", 1000, 6096, 0.9710216522216797], "23": ["vertical_and_slash", 1000, 6096, 0.9979152083396912], "24": ["vertical_and_slash", 30, 800, 0.9810062050819397], "25": ["vertical_and_slash", 30, 800, 0.9922820329666138], "26": ["vertical_and_slash", 30, 800, 0.994399905204773], "27": ["vertical_and_slash", 30, 800, 0.9931467175483704], "28": ["vertical_and_slash", 30, 800, 0.9915726184844971], "29": ["vertical_and_slash", 30, 800, 0.9912887215614319], "30": ["vertical_and_slash", 30, 800, 0.9941495656967163], "31": ["vertical_and_slash", 30, 800, 0.9939005970954895]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9837832450866699], "1": ["vertical_and_slash", 1000, 6096, 0.9791439175605774], "2": ["vertical_and_slash", 100, 800, 1.0], "3": ["vertical_and_slash", 1000, 6096, 0.9971569180488586], "4": ["vertical_and_slash", 30, 800, 0.9973106980323792], "5": ["vertical_and_slash", 30, 800, 0.9962120056152344], "6": ["vertical_and_slash", 30, 800, 0.9828794598579407], "7": ["vertical_and_slash", 30, 800, 0.9905376434326172], "8": ["vertical_and_slash", 30, 800, 0.9939618706703186], "9": ["vertical_and_slash", 30, 800, 0.9820200204849243], "10": ["vertical_and_slash", 30, 800, 0.989706814289093], "11": ["vertical_and_slash", 30, 800, 0.9937054514884949], "12": ["vertical_and_slash", 1000, 6096, 0.9380397796630859], "13": ["vertical_and_slash", 30, 800, 0.9371072053909302], "14": ["vertical_and_slash", 500, 700, 0.9933274984359741], "15": ["vertical_and_slash", 30, 800, 0.9924632906913757], "16": ["vertical_and_slash", 30, 800, 0.9889415502548218], "17": ["vertical_and_slash", 30, 800, 0.9969512820243835], "18": ["vertical_and_slash", 30, 800, 0.9957903027534485], "19": ["vertical_and_slash", 30, 800, 0.9710395932197571], "20": ["vertical_and_slash", 100, 800, 0.96484375], "21": ["vertical_and_slash", 100, 800, 0.95703125], "22": ["vertical_and_slash", 1000, 6096, 0.9383615255355835], "23": ["vertical_and_slash", 30, 800, 0.9005464911460876], "24": ["vertical_and_slash", 1000, 6096, 0.9980953335762024], "25": ["vertical_and_slash", 1000, 6096, 0.9897440075874329], "26": ["vertical_and_slash", 30, 800, 0.9923828840255737], "27": ["vertical_and_slash", 100, 800, 0.73828125], "28": ["vertical_and_slash", 30, 800, 0.9830290079116821], "29": ["vertical_and_slash", 30, 800, 0.9703660607337952], "30": ["vertical_and_slash", 30, 800, 0.9784981608390808], "31": ["vertical_and_slash", 30, 800, 0.9887380599975586]}, {"0": ["vertical_and_slash", 100, 800, 0.91015625], "1": ["vertical_and_slash", 100, 800, 0.953125], "2": ["vertical_and_slash", 100, 800, 0.9765625], "3": ["vertical_and_slash", 30, 800, 0.9728029370307922], "4": ["vertical_and_slash", 30, 800, 0.990367591381073], "5": ["vertical_and_slash", 30, 800, 0.8950048089027405], "6": ["vertical_and_slash", 1000, 6096, 0.8518299460411072], "7": ["vertical_and_slash", 30, 800, 0.9903310537338257], "8": ["vertical_and_slash", 100, 800, 0.8984375], "9": ["vertical_and_slash", 30, 800, 0.9666171073913574], "10": ["vertical_and_slash", 30, 800, 0.8871880769729614], "11": ["vertical_and_slash", 30, 800, 0.9558337926864624], "12": ["vertical_and_slash", 30, 800, 0.99130779504776], "13": ["vertical_and_slash", 30, 800, 0.9857202172279358], "14": ["vertical_and_slash", 30, 800, 0.992880642414093], "15": ["vertical_and_slash", 100, 800, 0.9375], "16": ["vertical_and_slash", 1000, 6096, 0.6494593620300293], "17": ["vertical_and_slash", 1000, 6096, 0.994994580745697], "18": ["vertical_and_slash", 1000, 6096, 0.9976939558982849], "19": ["vertical_and_slash", 1000, 6096, 0.5856859087944031], "20": ["vertical_and_slash", 1000, 6096, 0.9758481383323669], "21": ["vertical_and_slash", 1000, 6096, 0.9787202477455139], "22": ["vertical_and_slash", 30, 800, 0.9818038940429688], "23": ["vertical_and_slash", 1000, 6096, 0.9906389117240906], "24": ["vertical_and_slash", 30, 800, 0.9870673418045044], "25": ["vertical_and_slash", 30, 800, 0.9819732308387756], "26": ["vertical_and_slash", 30, 800, 0.894801914691925], "27": ["vertical_and_slash", 30, 800, 0.9789435267448425], "28": ["vertical_and_slash", 1000, 6096, 0.9558958411216736], "29": ["vertical_and_slash", 1000, 6096, 0.7887665033340454], "30": ["vertical_and_slash", 1000, 6096, 0.9841030240058899], "31": ["vertical_and_slash", 1000, 6096, 0.9959767460823059]}, {"0": ["vertical_and_slash", 1000, 6096, 0.985858678817749], "1": ["vertical_and_slash", 1000, 6096, 0.9785674214363098], "2": ["vertical_and_slash", 1000, 6096, 0.9867167472839355], "3": ["vertical_and_slash", 1000, 6096, 0.9883900880813599], "4": ["vertical_and_slash", 30, 800, 0.987496554851532], "5": ["vertical_and_slash", 1000, 6096, 0.8416361808776855], "6": ["vertical_and_slash", 30, 800, 0.9440363645553589], "7": ["vertical_and_slash", 100, 800, 0.8125], "8": ["vertical_and_slash", 1000, 6096, 0.9002290368080139], "9": ["vertical_and_slash", 1000, 6096, 0.9311371445655823], "10": ["vertical_and_slash", 1000, 6096, 0.8879005908966064], "11": ["vertical_and_slash", 1000, 6096, 0.9562467932701111], "12": ["vertical_and_slash", 30, 800, 0.9765503406524658], "13": ["vertical_and_slash", 30, 800, 0.9893233776092529], "14": ["vertical_and_slash", 30, 800, 0.9915202260017395], "15": ["vertical_and_slash", 30, 800, 0.9850893616676331], "16": ["vertical_and_slash", 100, 800, 0.98828125], "17": ["vertical_and_slash", 1000, 6096, 0.9221939444541931], "18": ["vertical_and_slash", 1000, 6096, 0.9556252956390381], "19": ["vertical_and_slash", 30, 800, 0.9365096688270569], "20": ["vertical_and_slash", 100, 800, 0.9765625], "21": ["vertical_and_slash", 1000, 6096, 0.9185587167739868], "22": ["vertical_and_slash", 100, 800, 0.86328125], "23": ["vertical_and_slash", 100, 800, 0.94921875], "24": ["vertical_and_slash", 1000, 6096, 0.9803503155708313], "25": ["vertical_and_slash", 30, 800, 0.9929267764091492], "26": ["vertical_and_slash", 30, 800, 0.9961451292037964], "27": ["vertical_and_slash", 1000, 6096, 0.9952177405357361], "28": ["vertical_and_slash", 30, 800, 0.9747095704078674], "29": ["vertical_and_slash", 30, 800, 0.9782059788703918], "30": ["vertical_and_slash", 30, 800, 0.9795500040054321], "31": ["vertical_and_slash", 30, 800, 0.9679626226425171]}, {"0": ["vertical_and_slash", 30, 800, 0.9898263216018677], "1": ["vertical_and_slash", 30, 800, 0.9882830381393433], "2": ["vertical_and_slash", 100, 800, 0.859375], "3": ["vertical_and_slash", 100, 800, 0.83984375], "4": ["vertical_and_slash", 30, 800, 0.9902689456939697], "5": ["vertical_and_slash", 30, 800, 0.9513372182846069], "6": ["vertical_and_slash", 100, 800, 0.90625], "7": ["vertical_and_slash", 30, 800, 0.9930790066719055], "8": ["vertical_and_slash", 30, 800, 0.9950608015060425], "9": ["vertical_and_slash", 30, 800, 0.9540755748748779], "10": ["vertical_and_slash", 30, 800, 0.9924449324607849], "11": ["vertical_and_slash", 30, 800, 0.9905205368995667], "12": ["vertical_and_slash", 30, 800, 0.9887551069259644], "13": ["vertical_and_slash", 30, 800, 0.9894125461578369], "14": ["vertical_and_slash", 30, 800, 0.9369159936904907], "15": ["vertical_and_slash", 30, 800, 0.9925025701522827], "16": ["vertical_and_slash", 30, 800, 0.9917540550231934], "17": ["vertical_and_slash", 30, 800, 0.9703994989395142], "18": ["vertical_and_slash", 30, 800, 0.9568591117858887], "19": ["vertical_and_slash", 30, 800, 0.9875468015670776], "20": ["vertical_and_slash", 100, 800, 0.921875], "21": ["vertical_and_slash", 1000, 6096, 0.7049875855445862], "22": ["vertical_and_slash", 100, 800, 0.91796875], "23": ["vertical_and_slash", 100, 800, 0.96875], "24": ["vertical_and_slash", 100, 800, 0.80078125], "25": ["vertical_and_slash", 1000, 6096, 0.8965499401092529], "26": ["vertical_and_slash", 1000, 6096, 0.8224286437034607], "27": ["vertical_and_slash", 30, 800, 0.9571169018745422], "28": ["vertical_and_slash", 100, 800, 0.71875], "29": ["vertical_and_slash", 100, 800, 0.98046875], "30": ["vertical_and_slash", 100, 800, 1.0], "31": ["vertical_and_slash", 100, 800, 0.90234375]}, {"0": ["vertical_and_slash", 30, 800, 0.9603511095046997], "1": ["vertical_and_slash", 30, 800, 0.9893207550048828], "2": ["vertical_and_slash", 30, 800, 0.9923656582832336], "3": ["vertical_and_slash", 100, 800, 0.458984375], "4": ["vertical_and_slash", 30, 800, 0.9753598570823669], "5": ["vertical_and_slash", 30, 800, 0.9915642142295837], "6": ["vertical_and_slash", 1000, 6096, 0.8411368727684021], "7": ["vertical_and_slash", 1000, 6096, 0.8587262034416199], "8": ["vertical_and_slash", 100, 800, 0.93359375], "9": ["vertical_and_slash", 100, 800, 0.96875], "10": ["vertical_and_slash", 30, 800, 0.8169850707054138], "11": ["vertical_and_slash", 30, 800, 0.9763987064361572], "12": ["vertical_and_slash", 30, 800, 0.9713965654373169], "13": ["vertical_and_slash", 30, 800, 0.8622501492500305], "14": ["vertical_and_slash", 30, 800, 0.9738231301307678], "15": ["vertical_and_slash", 100, 800, 0.9296875], "16": ["vertical_and_slash", 100, 800, 0.9296875], "17": ["vertical_and_slash", 30, 800, 0.9684662222862244], "18": ["vertical_and_slash", 100, 800, 0.9375], "19": ["vertical_and_slash", 100, 800, 0.94921875], "20": ["vertical_and_slash", 100, 800, 0.9765625], "21": ["vertical_and_slash", 3500, 100, 0.7724359035491943], "22": ["vertical_and_slash", 1000, 6096, 0.7844681143760681], "23": ["vertical_and_slash", 100, 800, 0.6796875], "24": ["vertical_and_slash", 100, 800, 0.9453125], "25": ["vertical_and_slash", 1000, 6096, 0.6538506150245667], "26": ["vertical_and_slash", 100, 800, 0.85546875], "27": ["vertical_and_slash", 100, 800, 0.94921875], "28": ["vertical_and_slash", 30, 800, 0.9509181380271912], "29": ["vertical_and_slash", 30, 800, 0.9657983183860779], "30": ["vertical_and_slash", 30, 800, 0.9171797633171082], "31": ["vertical_and_slash", 30, 800, 0.9357418417930603]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9873993992805481], "1": ["vertical_and_slash", 1000, 6096, 0.7471938729286194], "2": ["vertical_and_slash", 100, 800, 0.89453125], "3": ["vertical_and_slash", 100, 800, 0.6953125], "4": ["vertical_and_slash", 1000, 6096, 0.9859827160835266], "5": ["vertical_and_slash", 1000, 6096, 0.9493432641029358], "6": ["vertical_and_slash", 1000, 6096, 0.9839309453964233], "7": ["vertical_and_slash", 100, 800, 0.94921875], "8": ["vertical_and_slash", 1000, 6096, 0.8909018635749817], "9": ["vertical_and_slash", 1000, 6096, 0.5839027762413025], "10": ["vertical_and_slash", 1000, 6096, 0.8953289985656738], "11": ["vertical_and_slash", 1000, 6096, 0.5685684680938721], "12": ["vertical_and_slash", 1000, 6096, 0.9685022234916687], "13": ["vertical_and_slash", 30, 800, 0.9499353766441345], "14": ["vertical_and_slash", 1000, 6096, 0.9260799288749695], "15": ["vertical_and_slash", 1000, 6096, 0.9790918827056885], "16": ["vertical_and_slash", 1000, 6096, 0.8991709351539612], "17": ["vertical_and_slash", 1000, 6096, 0.9513922929763794], "18": ["vertical_and_slash", 1000, 6096, 0.9208007454872131], "19": ["vertical_and_slash", 1000, 6096, 0.9758914113044739], "20": ["vertical_and_slash", 100, 800, 0.98046875], "21": ["vertical_and_slash", 100, 800, 0.97265625], "22": ["vertical_and_slash", 500, 700, 0.9839867949485779], "23": ["vertical_and_slash", 30, 800, 0.9237134456634521], "24": ["vertical_and_slash", 1000, 6096, 0.9671244025230408], "25": ["vertical_and_slash", 1000, 6096, 0.982479989528656], "26": ["vertical_and_slash", 1000, 6096, 0.9730757474899292], "27": ["vertical_and_slash", 1000, 6096, 0.9641519784927368], "28": ["vertical_and_slash", 1000, 6096, 0.9639339447021484], "29": ["vertical_and_slash", 1000, 6096, 0.9427148699760437], "30": ["vertical_and_slash", 1000, 6096, 0.9345259666442871], "31": ["vertical_and_slash", 1000, 6096, 0.8792511820793152]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8634915947914124], "1": ["vertical_and_slash", 1000, 6096, 0.8989808559417725], "2": ["vertical_and_slash", 1000, 6096, 0.8503499031066895], "3": ["vertical_and_slash", 1000, 6096, 0.827835202217102], "4": ["vertical_and_slash", 1000, 6096, 0.9314107894897461], "5": ["vertical_and_slash", 1000, 6096, 0.9064517617225647], "6": ["vertical_and_slash", 1000, 6096, 0.9566625356674194], "7": ["vertical_and_slash", 1000, 6096, 0.9783104062080383], "8": ["vertical_and_slash", 500, 700, 0.9695003628730774], "9": ["vertical_and_slash", 500, 700, 0.9732148051261902], "10": ["vertical_and_slash", 1000, 6096, 0.9714348316192627], "11": ["vertical_and_slash", 500, 700, 0.9940823912620544], "12": ["vertical_and_slash", 1000, 6096, 0.706820547580719], "13": ["vertical_and_slash", 1000, 6096, 0.7126483917236328], "14": ["vertical_and_slash", 100, 800, 0.79296875], "15": ["vertical_and_slash", 1000, 6096, 0.8690503835678101], "16": ["vertical_and_slash", 1000, 6096, 0.9109070301055908], "17": ["vertical_and_slash", 1000, 6096, 0.8707846403121948], "18": ["vertical_and_slash", 500, 700, 0.8517163395881653], "19": ["vertical_and_slash", 1000, 6096, 0.834902286529541], "20": ["vertical_and_slash", 1000, 6096, 0.9286177158355713], "21": ["vertical_and_slash", 1000, 6096, 0.9553185105323792], "22": ["vertical_and_slash", 1000, 6096, 0.9636306166648865], "23": ["vertical_and_slash", 1000, 6096, 0.9541269540786743], "24": ["vertical_and_slash", 1000, 6096, 0.9912933707237244], "25": ["vertical_and_slash", 1000, 6096, 0.7418502569198608], "26": ["vertical_and_slash", 100, 800, 0.9921875], "27": ["vertical_and_slash", 1000, 6096, 0.8253664374351501], "28": ["vertical_and_slash", 1000, 6096, 0.6728078126907349], "29": ["vertical_and_slash", 1000, 6096, 0.8966277241706848], "30": ["vertical_and_slash", 100, 800, 0.94921875], "31": ["vertical_and_slash", 1000, 6096, 0.7612401247024536]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9580379128456116], "1": ["vertical_and_slash", 1000, 6096, 0.8889423608779907], "2": ["vertical_and_slash", 1000, 6096, 0.7774703502655029], "3": ["vertical_and_slash", 1000, 6096, 0.5454753637313843], "4": ["vertical_and_slash", 500, 700, 0.9631187319755554], "5": ["vertical_and_slash", 1000, 6096, 0.8950480818748474], "6": ["vertical_and_slash", 1000, 6096, 0.7367609143257141], "7": ["vertical_and_slash", 1000, 6096, 0.7220443487167358], "8": ["vertical_and_slash", 100, 800, 0.89453125], "9": ["vertical_and_slash", 500, 700, 0.969845712184906], "10": ["vertical_and_slash", 30, 800, 0.8675119876861572], "11": ["vertical_and_slash", 500, 700, 0.9803798794746399], "12": ["vertical_and_slash", 1000, 6096, 0.6585094332695007], "13": ["vertical_and_slash", 1000, 6096, 0.7352333664894104], "14": ["vertical_and_slash", 1000, 6096, 0.8553265333175659], "15": ["vertical_and_slash", 500, 700, 0.9600890874862671], "16": ["vertical_and_slash", 1000, 6096, 0.9677231311798096], "17": ["vertical_and_slash", 1000, 6096, 0.8380194306373596], "18": ["vertical_and_slash", 1000, 6096, 0.817642867565155], "19": ["vertical_and_slash", 1000, 6096, 0.7535743713378906], "20": ["vertical_and_slash", 3500, 100, 0.9120592474937439], "21": ["vertical_and_slash", 100, 800, 0.87109375], "22": ["vertical_and_slash", 500, 700, 0.8932908177375793], "23": ["vertical_and_slash", 100, 800, 0.91015625], "24": ["vertical_and_slash", 1000, 6096, 0.9268215298652649], "25": ["vertical_and_slash", 1000, 6096, 0.9567509293556213], "26": ["vertical_and_slash", 1000, 6096, 0.8897656798362732], "27": ["vertical_and_slash", 1000, 6096, 0.9563292264938354], "28": ["vertical_and_slash", 1000, 6096, 0.7572957277297974], "29": ["vertical_and_slash", 100, 800, 0.85546875], "30": ["vertical_and_slash", 500, 700, 0.9302328824996948], "31": ["vertical_and_slash", 1000, 6096, 0.7470130920410156]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9706114530563354], "1": ["vertical_and_slash", 1000, 6096, 0.9341138601303101], "2": ["vertical_and_slash", 1000, 6096, 0.88250732421875], "3": ["vertical_and_slash", 1000, 6096, 0.9176220893859863], "4": ["vertical_and_slash", 1000, 6096, 0.9890152215957642], "5": ["vertical_and_slash", 100, 800, 0.6015625], "6": ["vertical_and_slash", 100, 800, 0.53515625], "7": ["vertical_and_slash", 100, 800, 0.65625], "8": ["vertical_and_slash", 100, 800, 0.9140625], "9": ["vertical_and_slash", 100, 800, 0.91796875], "10": ["vertical_and_slash", 500, 700, 0.974812924861908], "11": ["vertical_and_slash", 100, 800, 0.93359375], "12": ["vertical_and_slash", 1000, 6096, 0.8015804886817932], "13": ["vertical_and_slash", 1000, 6096, 0.8895022869110107], "14": ["vertical_and_slash", 1000, 6096, 0.8971598744392395], "15": ["vertical_and_slash", 1000, 6096, 0.9316778779029846], "16": ["vertical_and_slash", 500, 700, 0.9727176427841187], "17": ["vertical_and_slash", 500, 700, 0.9684209227561951], "18": ["vertical_and_slash", 500, 700, 0.7916423082351685], "19": ["vertical_and_slash", 500, 700, 0.956729531288147], "20": ["vertical_and_slash", 100, 800, 0.7421875], "21": ["vertical_and_slash", 100, 800, 0.92578125], "22": ["vertical_and_slash", 1000, 6096, 0.6496378183364868], "23": ["vertical_and_slash", 100, 800, 0.75390625], "24": ["vertical_and_slash", 500, 700, 0.863329291343689], "25": ["vertical_and_slash", 500, 700, 0.9180703163146973], "26": ["vertical_and_slash", 100, 800, 0.8671875], "27": ["vertical_and_slash", 500, 700, 0.9076446890830994], "28": ["vertical_and_slash", 3500, 100, 0.7788168787956238], "29": ["vertical_and_slash", 100, 800, 0.734375], "30": ["vertical_and_slash", 100, 800, 0.66796875], "31": ["vertical_and_slash", 1000, 6096, 0.9902347922325134]}, {"0": ["vertical_and_slash", 100, 800, 0.95703125], "1": ["vertical_and_slash", 1000, 6096, 0.8122807145118713], "2": ["vertical_and_slash", 100, 800, 0.96484375], "3": ["vertical_and_slash", 100, 800, 0.96484375], "4": ["vertical_and_slash", 100, 800, 0.94921875], "5": ["vertical_and_slash", 100, 800, 0.9375], "6": ["vertical_and_slash", 100, 800, 0.90234375], "7": ["vertical_and_slash", 100, 800, 0.9453125], "8": ["vertical_and_slash", 500, 700, 0.8789168000221252], "9": ["vertical_and_slash", 500, 700, 0.9342090487480164], "10": ["vertical_and_slash", 100, 800, 0.78515625], "11": ["vertical_and_slash", 100, 800, 0.9140625], "12": ["vertical_and_slash", 500, 700, 0.9864805340766907], "13": ["vertical_and_slash", 500, 700, 0.9746032357215881], "14": ["vertical_and_slash", 100, 800, 0.6875], "15": ["vertical_and_slash", 100, 800, 0.953125], "16": ["vertical_and_slash", 500, 700, 0.9699565768241882], "17": ["vertical_and_slash", 3500, 100, 0.9764721393585205], "18": ["vertical_and_slash", 500, 700, 0.9664324522018433], "19": ["vertical_and_slash", 500, 700, 0.9909504652023315], "20": ["vertical_and_slash", 30, 800, 0.8712055683135986], "21": ["vertical_and_slash", 100, 800, 0.97265625], "22": ["vertical_and_slash", 100, 800, 0.98046875], "23": ["vertical_and_slash", 1000, 6096, 0.9733871221542358], "24": ["vertical_and_slash", 1000, 6096, 0.9706288576126099], "25": ["vertical_and_slash", 500, 700, 0.9624436497688293], "26": ["vertical_and_slash", 1000, 6096, 0.9703077077865601], "27": ["vertical_and_slash", 1000, 6096, 0.9371525049209595], "28": ["vertical_and_slash", 500, 700, 0.9811959266662598], "29": ["vertical_and_slash", 1000, 6096, 0.9622439742088318], "30": ["vertical_and_slash", 1000, 6096, 0.9234631061553955], "31": ["vertical_and_slash", 1000, 6096, 0.9721811413764954]}, {"0": ["vertical_and_slash", 3500, 100, 0.6208434104919434], "1": ["vertical_and_slash", 100, 800, 0.78515625], "2": ["vertical_and_slash", 1000, 6096, 0.682672917842865], "3": ["vertical_and_slash", 1000, 6096, 0.6564425230026245], "4": ["vertical_and_slash", 1000, 6096, 0.9247110486030579], "5": ["vertical_and_slash", 1000, 6096, 0.8886473774909973], "6": ["vertical_and_slash", 1000, 6096, 0.8195661902427673], "7": ["vertical_and_slash", 100, 800, 0.9609375], "8": ["vertical_and_slash", 1000, 6096, 0.6291986703872681], "9": ["vertical_and_slash", 3500, 100, 0.8490926623344421], "10": ["vertical_and_slash", 100, 800, 0.66796875], "11": ["vertical_and_slash", 3500, 100, 0.8477756977081299], "12": ["vertical_and_slash", 100, 800, 0.8828125], "13": ["vertical_and_slash", 100, 800, 0.87890625], "14": ["vertical_and_slash", 100, 800, 0.88671875], "15": ["vertical_and_slash", 100, 800, 0.90625], "16": ["vertical_and_slash", 1000, 6096, 0.6082240343093872], "17": ["vertical_and_slash", 3500, 100, 0.6033845543861389], "18": ["vertical_and_slash", 3500, 100, 0.7406787872314453], "19": ["vertical_and_slash", 100, 800, 0.5625], "20": ["vertical_and_slash", 3500, 100, 0.858406662940979], "21": ["vertical_and_slash", 3500, 100, 0.9015128016471863], "22": ["vertical_and_slash", 100, 800, 0.9453125], "23": ["vertical_and_slash", 3500, 100, 0.8777713179588318], "24": ["vertical_and_slash", 100, 800, 0.95703125], "25": ["vertical_and_slash", 100, 800, 0.8984375], "26": ["vertical_and_slash", 100, 800, 0.68359375], "27": ["vertical_and_slash", 100, 800, 0.5234375], "28": ["vertical_and_slash", 30, 800, 0.8055139183998108], "29": ["vertical_and_slash", 500, 700, 0.9727892875671387], "30": ["vertical_and_slash", 1000, 6096, 0.772057831287384], "31": ["vertical_and_slash", 1000, 6096, 0.6695537567138672]}, {"0": ["vertical_and_slash", 100, 800, 0.9453125], "1": ["vertical_and_slash", 100, 800, 0.95703125], "2": ["vertical_and_slash", 100, 800, 0.96484375], "3": ["vertical_and_slash", 3500, 100, 0.9689056277275085], "4": ["vertical_and_slash", 100, 800, 0.91015625], "5": ["vertical_and_slash", 3500, 100, 0.9235577583312988], "6": ["vertical_and_slash", 100, 800, 0.953125], "7": ["vertical_and_slash", 100, 800, 0.9140625], "8": ["vertical_and_slash", 500, 700, 0.9862722158432007], "9": ["vertical_and_slash", 500, 700, 0.977347731590271], "10": ["vertical_and_slash", 100, 800, 0.96875], "11": ["vertical_and_slash", 500, 700, 0.9818734526634216], "12": ["vertical_and_slash", 100, 800, 0.7890625], "13": ["vertical_and_slash", 1000, 6096, 0.7767282128334045], "14": ["vertical_and_slash", 100, 800, 0.796875], "15": ["vertical_and_slash", 100, 800, 0.9375], "16": ["vertical_and_slash", 100, 800, 0.69921875], "17": ["vertical_and_slash", 100, 800, 0.9140625], "18": ["vertical_and_slash", 100, 800, 0.89453125], "19": ["vertical_and_slash", 1000, 6096, 0.8810321092605591], "20": ["vertical_and_slash", 1000, 6096, 0.8856872320175171], "21": ["vertical_and_slash", 100, 800, 0.9453125], "22": ["vertical_and_slash", 100, 800, 0.87109375], "23": ["vertical_and_slash", 100, 800, 0.9453125], "24": ["vertical_and_slash", 1000, 6096, 0.9649856686592102], "25": ["vertical_and_slash", 1000, 6096, 0.9768476486206055], "26": ["vertical_and_slash", 30, 800, 0.9993871450424194], "27": ["vertical_and_slash", 100, 800, 0.921875], "28": ["vertical_and_slash", 1000, 6096, 0.8807946443557739], "29": ["vertical_and_slash", 1000, 6096, 0.8378621339797974], "30": ["vertical_and_slash", 1000, 6096, 0.9146285653114319], "31": ["vertical_and_slash", 1000, 6096, 0.9363900423049927]}, {"0": ["vertical_and_slash", 500, 700, 0.9848667979240417], "1": ["vertical_and_slash", 1000, 6096, 0.6662361025810242], "2": ["vertical_and_slash", 1000, 6096, 0.6793611645698547], "3": ["vertical_and_slash", 100, 800, 0.93359375], "4": ["vertical_and_slash", 100, 800, 0.93359375], "5": ["vertical_and_slash", 1000, 6096, 0.7997210621833801], "6": ["vertical_and_slash", 500, 700, 0.921200156211853], "7": ["vertical_and_slash", 500, 700, 0.9733778834342957], "8": ["vertical_and_slash", 100, 800, 0.9375], "9": ["vertical_and_slash", 100, 800, 0.88671875], "10": ["vertical_and_slash", 100, 800, 0.796875], "11": ["vertical_and_slash", 100, 800, 0.82421875], "12": ["vertical_and_slash", 500, 700, 0.981203019618988], "13": ["vertical_and_slash", 100, 800, 0.94921875], "14": ["vertical_and_slash", 3500, 100, 0.9745455980300903], "15": ["vertical_and_slash", 3500, 100, 0.9339883327484131], "16": ["vertical_and_slash", 100, 800, 0.79296875], "17": ["vertical_and_slash", 1000, 6096, 0.4775260090827942], "18": ["vertical_and_slash", 1000, 6096, 0.5015617609024048], "19": ["vertical_and_slash", 100, 800, 0.91796875], "20": ["vertical_and_slash", 100, 800, 0.91015625], "21": ["vertical_and_slash", 100, 800, 0.75390625], "22": ["vertical_and_slash", 100, 800, 0.92578125], "23": ["vertical_and_slash", 100, 800, 0.89453125], "24": ["vertical_and_slash", 100, 800, 0.84765625], "25": ["vertical_and_slash", 100, 800, 0.72265625], "26": ["vertical_and_slash", 100, 800, 0.87890625], "27": ["vertical_and_slash", 100, 800, 0.8984375], "28": ["vertical_and_slash", 1000, 6096, 0.676841139793396], "29": ["vertical_and_slash", 1000, 6096, 0.6767839193344116], "30": ["vertical_and_slash", 1000, 6096, 0.8959569334983826], "31": ["vertical_and_slash", 30, 800, 0.9896997213363647]}, {"0": ["vertical_and_slash", 3500, 100, 0.8137837052345276], "1": ["vertical_and_slash", 3500, 100, 0.8565242886543274], "2": ["vertical_and_slash", 100, 800, 0.84375], "3": ["vertical_and_slash", 1000, 6096, 0.9823415279388428], "4": ["vertical_and_slash", 3500, 100, 0.9782371520996094], "5": ["vertical_and_slash", 1000, 6096, 0.9068223237991333], "6": ["vertical_and_slash", 3500, 100, 0.9929006695747375], "7": ["vertical_and_slash", 1000, 6096, 0.9689693450927734], "8": ["vertical_and_slash", 3500, 100, 0.8670288920402527], "9": ["vertical_and_slash", 1000, 6096, 0.9820406436920166], "10": ["vertical_and_slash", 3500, 100, 0.9791944026947021], "11": ["vertical_and_slash", 100, 800, 0.97265625], "12": ["vertical_and_slash", 500, 700, 0.9788346886634827], "13": ["vertical_and_slash", 500, 700, 0.9739745259284973], "14": ["vertical_and_slash", 100, 800, 0.98046875], "15": ["vertical_and_slash", 500, 700, 0.9562920331954956], "16": ["vertical_and_slash", 100, 800, 0.84375], "17": ["vertical_and_slash", 100, 800, 0.9609375], "18": ["vertical_and_slash", 3500, 100, 0.9375513792037964], "19": ["vertical_and_slash", 100, 800, 0.9296875], "20": ["vertical_and_slash", 1000, 6096, 0.9592316746711731], "21": ["vertical_and_slash", 100, 800, 0.91796875], "22": ["vertical_and_slash", 100, 800, 0.9921875], "23": ["vertical_and_slash", 1000, 6096, 0.9259733557701111], "24": ["vertical_and_slash", 100, 800, 0.97265625], "25": ["vertical_and_slash", 100, 800, 0.84375], "26": ["vertical_and_slash", 100, 800, 0.96484375], "27": ["vertical_and_slash", 3500, 100, 0.9818596243858337], "28": ["vertical_and_slash", 500, 700, 0.9490596652030945], "29": ["vertical_and_slash", 500, 700, 0.9914187788963318], "30": ["vertical_and_slash", 30, 800, 0.9638843536376953], "31": ["vertical_and_slash", 500, 700, 0.9648941159248352]}, {"0": ["vertical_and_slash", 100, 800, 0.95703125], "1": ["vertical_and_slash", 100, 800, 0.9453125], "2": ["vertical_and_slash", 1000, 6096, 0.9201825857162476], "3": ["vertical_and_slash", 100, 800, 0.9765625], "4": ["vertical_and_slash", 100, 800, 0.96484375], "5": ["vertical_and_slash", 500, 700, 0.9729551076889038], "6": ["vertical_and_slash", 100, 800, 0.84765625], "7": ["vertical_and_slash", 500, 700, 0.9674957394599915], "8": ["vertical_and_slash", 1000, 6096, 0.5704637169837952], "9": ["vertical_and_slash", 100, 800, 0.8515625], "10": ["vertical_and_slash", 3500, 100, 0.8001803159713745], "11": ["vertical_and_slash", 500, 700, 0.9770910739898682], "12": ["vertical_and_slash", 500, 700, 0.9445182681083679], "13": ["vertical_and_slash", 3500, 100, 0.9648588299751282], "14": ["vertical_and_slash", 100, 800, 0.921875], "15": ["vertical_and_slash", 1000, 6096, 0.9490084648132324], "16": ["vertical_and_slash", 1000, 6096, 0.9791340231895447], "17": ["vertical_and_slash", 1000, 6096, 0.9951378107070923], "18": ["vertical_and_slash", 1000, 6096, 0.9684316515922546], "19": ["vertical_and_slash", 1000, 6096, 0.9796795845031738], "20": ["vertical_and_slash", 500, 700, 0.9908055067062378], "21": ["vertical_and_slash", 3500, 100, 0.9142110347747803], "22": ["vertical_and_slash", 100, 800, 0.9375], "23": ["vertical_and_slash", 100, 800, 0.9296875], "24": ["vertical_and_slash", 100, 800, 0.88671875], "25": ["vertical_and_slash", 100, 800, 0.8671875], "26": ["vertical_and_slash", 3500, 100, 0.9462155699729919], "27": ["vertical_and_slash", 100, 800, 0.8515625], "28": ["vertical_and_slash", 1000, 6096, 0.9939091801643372], "29": ["vertical_and_slash", 3500, 100, 0.9740011692047119], "30": ["vertical_and_slash", 3500, 100, 0.993536114692688], "31": ["vertical_and_slash", 3500, 100, 0.9895251393318176]}, {"0": ["vertical_and_slash", 100, 800, 0.9921875], "1": ["vertical_and_slash", 100, 800, 0.96484375], "2": ["vertical_and_slash", 100, 800, 0.96875], "3": ["vertical_and_slash", 500, 700, 0.9583335518836975], "4": ["vertical_and_slash", 3500, 100, 0.9164949059486389], "5": ["vertical_and_slash", 1000, 6096, 0.9469748139381409], "6": ["vertical_and_slash", 1000, 6096, 0.9885687232017517], "7": ["vertical_and_slash", 1000, 6096, 0.99079829454422], "8": ["vertical_and_slash", 3500, 100, 0.9748219847679138], "9": ["vertical_and_slash", 3500, 100, 0.9859137535095215], "10": ["vertical_and_slash", 100, 800, 0.97265625], "11": ["vertical_and_slash", 3500, 100, 0.9817909002304077], "12": ["vertical_and_slash", 100, 800, 0.9921875], "13": ["vertical_and_slash", 100, 800, 0.98046875], "14": ["vertical_and_slash", 100, 800, 0.9296875], "15": ["vertical_and_slash", 500, 700, 0.9796241521835327], "16": ["vertical_and_slash", 100, 800, 0.9453125], "17": ["vertical_and_slash", 100, 800, 0.953125], "18": ["vertical_and_slash", 100, 800, 0.953125], "19": ["vertical_and_slash", 1000, 6096, 0.9794105887413025], "20": ["vertical_and_slash", 100, 800, 0.8046875], "21": ["vertical_and_slash", 100, 800, 0.97265625], "22": ["vertical_and_slash", 100, 800, 0.87109375], "23": ["vertical_and_slash", 1000, 6096, 0.941199779510498], "24": ["vertical_and_slash", 1000, 6096, 0.9881477355957031], "25": ["vertical_and_slash", 1000, 6096, 0.992800235748291], "26": ["vertical_and_slash", 500, 700, 0.9911536574363708], "27": ["vertical_and_slash", 100, 800, 0.98828125], "28": ["vertical_and_slash", 100, 800, 0.890625], "29": ["vertical_and_slash", 100, 800, 0.91015625], "30": ["vertical_and_slash", 100, 800, 0.875], "31": ["vertical_and_slash", 100, 800, 0.90234375]}, {"0": ["vertical_and_slash", 100, 800, 0.5703125], "1": ["vertical_and_slash", 100, 800, 0.8984375], "2": ["vertical_and_slash", 1000, 6096, 0.8873317837715149], "3": ["vertical_and_slash", 1000, 6096, 0.9360411763191223], "4": ["vertical_and_slash", 1000, 6096, 0.9617438316345215], "5": ["vertical_and_slash", 1000, 6096, 0.9799512028694153], "6": ["vertical_and_slash", 1000, 6096, 0.9727489948272705], "7": ["vertical_and_slash", 1000, 6096, 0.9961766600608826], "8": ["vertical_and_slash", 500, 700, 0.9883258938789368], "9": ["vertical_and_slash", 100, 800, 0.8671875], "10": ["vertical_and_slash", 100, 800, 0.984375], "11": ["vertical_and_slash", 30, 800, 0.9185207486152649], "12": ["vertical_and_slash", 100, 800, 0.796875], "13": ["vertical_and_slash", 100, 800, 0.7265625], "14": ["vertical_and_slash", 1000, 6096, 0.9042388796806335], "15": ["vertical_and_slash", 3500, 100, 0.8820527195930481], "16": ["vertical_and_slash", 500, 700, 0.9611795544624329], "17": ["vertical_and_slash", 500, 700, 0.9778792262077332], "18": ["vertical_and_slash", 500, 700, 0.9646311402320862], "19": ["vertical_and_slash", 100, 800, 0.9296875], "20": ["vertical_and_slash", 100, 800, 0.89453125], "21": ["vertical_and_slash", 100, 800, 0.98046875], "22": ["vertical_and_slash", 100, 800, 0.94921875], "23": ["vertical_and_slash", 100, 800, 0.890625], "24": ["vertical_and_slash", 500, 700, 0.9629708528518677], "25": ["vertical_and_slash", 1000, 6096, 0.9756439328193665], "26": ["vertical_and_slash", 1000, 6096, 0.9903070330619812], "27": ["vertical_and_slash", 100, 800, 0.98046875], "28": ["vertical_and_slash", 500, 700, 0.9740398526191711], "29": ["vertical_and_slash", 100, 800, 0.9609375], "30": ["vertical_and_slash", 500, 700, 0.9582708477973938], "31": ["vertical_and_slash", 100, 800, 0.97265625]}, {"0": ["vertical_and_slash", 100, 800, 0.8359375], "1": ["vertical_and_slash", 100, 800, 0.83203125], "2": ["vertical_and_slash", 100, 800, 0.6875], "3": ["vertical_and_slash", 100, 800, 0.953125], "4": ["vertical_and_slash", 1000, 6096, 0.9807296991348267], "5": ["vertical_and_slash", 1000, 6096, 0.9786800742149353], "6": ["vertical_and_slash", 1000, 6096, 0.9897298216819763], "7": ["vertical_and_slash", 1000, 6096, 0.9931173324584961], "8": ["vertical_and_slash", 500, 700, 0.9886856079101562], "9": ["vertical_and_slash", 100, 800, 0.96484375], "10": ["vertical_and_slash", 100, 800, 0.98046875], "11": ["vertical_and_slash", 100, 800, 0.85546875], "12": ["vertical_and_slash", 100, 800, 0.859375], "13": ["vertical_and_slash", 3500, 100, 0.931843638420105], "14": ["vertical_and_slash", 100, 800, 0.8984375], "15": ["vertical_and_slash", 100, 800, 0.9140625], "16": ["vertical_and_slash", 1000, 6096, 0.9895663261413574], "17": ["vertical_and_slash", 1000, 6096, 0.9883778691291809], "18": ["vertical_and_slash", 1000, 6096, 0.9893786311149597], "19": ["vertical_and_slash", 1000, 6096, 0.9857285618782043], "20": ["vertical_and_slash", 1000, 6096, 0.9703198075294495], "21": ["vertical_and_slash", 1000, 6096, 0.9922442436218262], "22": ["vertical_and_slash", 500, 700, 0.9701283574104309], "23": ["vertical_and_slash", 3500, 100, 0.8291676044464111], "24": ["vertical_and_slash", 1000, 6096, 0.9685841798782349], "25": ["vertical_and_slash", 100, 800, 0.88671875], "26": ["vertical_and_slash", 1000, 6096, 0.983255922794342], "27": ["vertical_and_slash", 100, 800, 0.94140625], "28": ["vertical_and_slash", 500, 700, 0.9961317181587219], "29": ["vertical_and_slash", 100, 800, 0.99609375], "30": ["vertical_and_slash", 100, 800, 0.9921875], "31": ["vertical_and_slash", 500, 700, 0.9888145923614502]}, {"0": ["vertical_and_slash", 100, 800, 0.99609375], "1": ["vertical_and_slash", 100, 800, 0.9375], "2": ["vertical_and_slash", 100, 800, 0.8359375], "3": ["vertical_and_slash", 100, 800, 0.7421875], "4": ["vertical_and_slash", 500, 700, 0.9954570531845093], "5": ["vertical_and_slash", 500, 700, 0.9949510097503662], "6": ["vertical_and_slash", 500, 700, 0.9883001446723938], "7": ["vertical_and_slash", 500, 700, 0.9916946291923523], "8": ["vertical_and_slash", 100, 800, 0.93359375], "9": ["vertical_and_slash", 1000, 6096, 0.9987804293632507], "10": ["vertical_and_slash", 1000, 6096, 0.998569130897522], "11": ["vertical_and_slash", 1000, 6096, 0.9967666268348694], "12": ["vertical_and_slash", 3500, 100, 0.9984410405158997], "13": ["vertical_and_slash", 3500, 100, 0.9972114562988281], "14": ["vertical_and_slash", 3500, 100, 0.9971626996994019], "15": ["vertical_and_slash", 100, 800, 1.0], "16": ["vertical_and_slash", 100, 800, 0.99609375], "17": ["vertical_and_slash", 500, 700, 0.9962896704673767], "18": ["vertical_and_slash", 500, 700, 0.9771263599395752], "19": ["vertical_and_slash", 500, 700, 0.9975184202194214], "20": ["vertical_and_slash", 1000, 6096, 0.9886354207992554], "21": ["vertical_and_slash", 100, 800, 0.93359375], "22": ["vertical_and_slash", 3500, 100, 0.9530031681060791], "23": ["vertical_and_slash", 1000, 6096, 0.9577071666717529], "24": ["vertical_and_slash", 3500, 100, 0.9946836233139038], "25": ["vertical_and_slash", 100, 800, 0.98828125], "26": ["vertical_and_slash", 3500, 100, 0.9938666820526123], "27": ["vertical_and_slash", 3500, 100, 0.9809824824333191], "28": ["vertical_and_slash", 1000, 6096, 0.9874097108840942], "29": ["vertical_and_slash", 1000, 6096, 0.9902986884117126], "30": ["vertical_and_slash", 3500, 100, 0.9763473272323608], "31": ["vertical_and_slash", 3500, 100, 0.9897515773773193]}, {"0": ["vertical_and_slash", 3500, 100, 0.9748806953430176], "1": ["vertical_and_slash", 3500, 100, 0.9462656378746033], "2": ["vertical_and_slash", 3500, 100, 0.9676709175109863], "3": ["vertical_and_slash", 3500, 100, 0.9766073226928711], "4": ["vertical_and_slash", 100, 800, 0.94921875], "5": ["vertical_and_slash", 100, 800, 0.98046875], "6": ["vertical_and_slash", 3500, 100, 0.9683337211608887], "7": ["vertical_and_slash", 3500, 100, 0.9826856851577759], "8": ["vertical_and_slash", 100, 800, 0.9921875], "9": ["vertical_and_slash", 100, 800, 0.98046875], "10": ["vertical_and_slash", 100, 800, 0.99609375], "11": ["vertical_and_slash", 100, 800, 0.984375], "12": ["vertical_and_slash", 100, 800, 0.8828125], "13": ["vertical_and_slash", 3500, 100, 0.8072444200515747], "14": ["vertical_and_slash", 100, 800, 0.8515625], "15": ["vertical_and_slash", 100, 800, 0.84765625], "16": ["vertical_and_slash", 500, 700, 0.9871454834938049], "17": ["vertical_and_slash", 500, 700, 0.9812307357788086], "18": ["vertical_and_slash", 30, 800, 0.9256724715232849], "19": ["vertical_and_slash", 500, 700, 0.9851175546646118], "20": ["vertical_and_slash", 3500, 100, 0.9745984673500061], "21": ["vertical_and_slash", 500, 700, 0.9816620945930481], "22": ["vertical_and_slash", 500, 700, 0.9899131655693054], "23": ["vertical_and_slash", 100, 800, 0.99609375], "24": ["vertical_and_slash", 100, 800, 0.97265625], "25": ["vertical_and_slash", 100, 800, 0.96875], "26": ["vertical_and_slash", 100, 800, 0.953125], "27": ["vertical_and_slash", 3500, 100, 0.9898616671562195], "28": ["vertical_and_slash", 3500, 100, 0.9370988607406616], "29": ["vertical_and_slash", 3500, 100, 0.9257959723472595], "30": ["vertical_and_slash", 100, 800, 0.90234375], "31": ["vertical_and_slash", 3500, 100, 0.9679746627807617]}, {"0": ["vertical_and_slash", 100, 800, 0.96875], "1": ["vertical_and_slash", 100, 800, 0.98828125], "2": ["vertical_and_slash", 100, 800, 0.98046875], "3": ["vertical_and_slash", 100, 800, 0.98046875], "4": ["vertical_and_slash", 3500, 100, 0.9508832097053528], "5": ["vertical_and_slash", 3500, 100, 0.9683154225349426], "6": ["vertical_and_slash", 3500, 100, 0.9689172506332397], "7": ["vertical_and_slash", 100, 800, 0.9453125], "8": ["vertical_and_slash", 3500, 100, 0.9961307644844055], "9": ["vertical_and_slash", 100, 800, 0.99609375], "10": ["vertical_and_slash", 3500, 100, 0.993029773235321], "11": ["vertical_and_slash", 3500, 100, 0.9973375201225281], "12": ["vertical_and_slash", 3500, 100, 0.9874537587165833], "13": ["vertical_and_slash", 3500, 100, 0.7299361824989319], "14": ["vertical_and_slash", 3500, 100, 0.8232833743095398], "15": ["vertical_and_slash", 100, 800, 0.85546875], "16": ["vertical_and_slash", 500, 700, 0.9973604083061218], "17": ["vertical_and_slash", 500, 700, 0.9963013529777527], "18": ["vertical_and_slash", 500, 700, 0.9808886647224426], "19": ["vertical_and_slash", 500, 700, 0.9972676038742065], "20": ["vertical_and_slash", 3500, 100, 0.977578341960907], "21": ["vertical_and_slash", 3500, 100, 0.9749562740325928], "22": ["vertical_and_slash", 3500, 100, 0.9779685735702515], "23": ["vertical_and_slash", 3500, 100, 0.9976317882537842], "24": ["vertical_and_slash", 3500, 100, 0.9648603796958923], "25": ["vertical_and_slash", 3500, 100, 0.9690384268760681], "26": ["vertical_and_slash", 1000, 6096, 0.999101459980011], "27": ["vertical_and_slash", 3500, 100, 0.9886281490325928], "28": ["vertical_and_slash", 500, 700, 0.9974322319030762], "29": ["vertical_and_slash", 500, 700, 0.9969301819801331], "30": ["vertical_and_slash", 500, 700, 0.9982666373252869], "31": ["vertical_and_slash", 500, 700, 0.9982587695121765]}, {"0": ["vertical_and_slash", 100, 800, 0.9921875], "1": ["vertical_and_slash", 3500, 100, 0.9939185976982117], "2": ["vertical_and_slash", 100, 800, 0.98828125], "3": ["vertical_and_slash", 3500, 100, 0.9969795942306519], "4": ["vertical_and_slash", 1000, 6096, 0.9931398034095764], "5": ["vertical_and_slash", 3500, 100, 0.9898437261581421], "6": ["vertical_and_slash", 100, 800, 0.99609375], "7": ["vertical_and_slash", 3500, 100, 0.9928154945373535], "8": ["vertical_and_slash", 500, 700, 0.9874842166900635], "9": ["vertical_and_slash", 30, 800, 0.9839801788330078], "10": ["vertical_and_slash", 500, 700, 0.9848899841308594], "11": ["vertical_and_slash", 500, 700, 0.9852920174598694], "12": ["vertical_and_slash", 100, 800, 0.984375], "13": ["vertical_and_slash", 100, 800, 0.98828125], "14": ["vertical_and_slash", 30, 800, 0.8755365610122681], "15": ["vertical_and_slash", 100, 800, 0.9921875], "16": ["vertical_and_slash", 3500, 100, 0.9819779396057129], "17": ["vertical_and_slash", 3500, 100, 0.9832032322883606], "18": ["vertical_and_slash", 3500, 100, 0.983447253704071], "19": ["vertical_and_slash", 3500, 100, 0.9584761261940002], "20": ["vertical_and_slash", 3500, 100, 0.9834704399108887], "21": ["vertical_and_slash", 3500, 100, 0.9791264533996582], "22": ["vertical_and_slash", 3500, 100, 0.9806039929389954], "23": ["vertical_and_slash", 3500, 100, 0.9781394600868225], "24": ["vertical_and_slash", 3500, 100, 0.9544171094894409], "25": ["vertical_and_slash", 1000, 6096, 0.9811713695526123], "26": ["vertical_and_slash", 100, 800, 0.96484375], "27": ["vertical_and_slash", 100, 800, 0.96875], "28": ["vertical_and_slash", 3500, 100, 0.974831759929657], "29": ["vertical_and_slash", 500, 700, 0.9761193990707397], "30": ["vertical_and_slash", 500, 700, 0.9920510649681091], "31": ["vertical_and_slash", 3500, 100, 0.9924405813217163]}, {"0": ["vertical_and_slash", 3500, 100, 0.984471321105957], "1": ["vertical_and_slash", 3500, 100, 0.9659731388092041], "2": ["vertical_and_slash", 3500, 100, 0.99066561460495], "3": ["vertical_and_slash", 3500, 100, 0.9594177007675171], "4": ["vertical_and_slash", 1000, 6096, 0.9888981580734253], "5": ["vertical_and_slash", 3500, 100, 0.980528712272644], "6": ["vertical_and_slash", 3500, 100, 0.9838561415672302], "7": ["vertical_and_slash", 1000, 6096, 0.9883071184158325], "8": ["vertical_and_slash", 500, 700, 0.9888157248497009], "9": ["vertical_and_slash", 500, 700, 0.9923218488693237], "10": ["vertical_and_slash", 100, 800, 0.98828125], "11": ["vertical_and_slash", 100, 800, 0.99609375], "12": ["vertical_and_slash", 3500, 100, 0.9845908880233765], "13": ["vertical_and_slash", 1000, 6096, 0.985883891582489], "14": ["vertical_and_slash", 1000, 6096, 0.9861627817153931], "15": ["vertical_and_slash", 1000, 6096, 0.9870601296424866], "16": ["vertical_and_slash", 100, 800, 0.98828125], "17": ["vertical_and_slash", 3500, 100, 0.9962552785873413], "18": ["vertical_and_slash", 3500, 100, 0.9941535592079163], "19": ["vertical_and_slash", 1000, 6096, 0.9947088360786438], "20": ["vertical_and_slash", 30, 800, 0.9986183643341064], "21": ["vertical_and_slash", 30, 800, 0.949575662612915], "22": ["vertical_and_slash", 30, 800, 0.9973949193954468], "23": ["vertical_and_slash", 30, 800, 0.9980455636978149], "24": ["vertical_and_slash", 500, 700, 0.9974744319915771], "25": ["vertical_and_slash", 100, 800, 0.99609375], "26": ["vertical_and_slash", 500, 700, 0.9894559383392334], "27": ["vertical_and_slash", 100, 800, 0.98046875], "28": ["vertical_and_slash", 500, 700, 0.980691134929657], "29": ["vertical_and_slash", 500, 700, 0.9799355268478394], "30": ["vertical_and_slash", 500, 700, 0.985927402973175], "31": ["vertical_and_slash", 500, 700, 0.9888966083526611]}, {"0": ["vertical_and_slash", 100, 800, 0.6953125], "1": ["vertical_and_slash", 100, 800, 0.97265625], "2": ["vertical_and_slash", 100, 800, 0.625], "3": ["vertical_and_slash", 100, 800, 0.875], "4": ["vertical_and_slash", 100, 800, 0.9609375], "5": ["vertical_and_slash", 100, 800, 0.96875], "6": ["vertical_and_slash", 1000, 6096, 0.9836552143096924], "7": ["vertical_and_slash", 1000, 6096, 0.9845009446144104], "8": ["vertical_and_slash", 100, 800, 0.8828125], "9": ["vertical_and_slash", 500, 700, 0.971139132976532], "10": ["vertical_and_slash", 3500, 100, 0.9866796731948853], "11": ["vertical_and_slash", 100, 800, 0.95703125], "12": ["vertical_and_slash", 1000, 6096, 0.9799742102622986], "13": ["vertical_and_slash", 1000, 6096, 0.9558401703834534], "14": ["vertical_and_slash", 1000, 6096, 0.983681321144104], "15": ["vertical_and_slash", 1000, 6096, 0.9747731685638428], "16": ["vertical_and_slash", 1000, 6096, 0.9905271530151367], "17": ["vertical_and_slash", 1000, 6096, 0.994288444519043], "18": ["vertical_and_slash", 1000, 6096, 0.9693189263343811], "19": ["vertical_and_slash", 3500, 100, 0.9621111154556274], "20": ["vertical_and_slash", 30, 800, 0.9551467299461365], "21": ["vertical_and_slash", 100, 800, 0.96484375], "22": ["vertical_and_slash", 30, 800, 0.9537814259529114], "23": ["vertical_and_slash", 100, 800, 0.97265625], "24": ["vertical_and_slash", 100, 800, 1.0], "25": ["vertical_and_slash", 3500, 100, 0.9974595904350281], "26": ["vertical_and_slash", 3500, 100, 0.9985501766204834], "27": ["vertical_and_slash", 3500, 100, 0.9958456754684448], "28": ["vertical_and_slash", 3500, 100, 0.9298160076141357], "29": ["vertical_and_slash", 3500, 100, 0.9677495360374451], "30": ["vertical_and_slash", 3500, 100, 0.9930184483528137], "31": ["vertical_and_slash", 1000, 6096, 0.9892486333847046]}, {"0": ["vertical_and_slash", 500, 700, 0.9966373443603516], "1": ["vertical_and_slash", 100, 800, 0.99609375], "2": ["vertical_and_slash", 100, 800, 0.99609375], "3": ["vertical_and_slash", 500, 700, 0.9936943650245667], "4": ["vertical_and_slash", 1000, 6096, 0.9874167442321777], "5": ["vertical_and_slash", 3500, 100, 0.9508872628211975], "6": ["vertical_and_slash", 1000, 6096, 0.9765644073486328], "7": ["vertical_and_slash", 3500, 100, 0.9465764164924622], "8": ["vertical_and_slash", 100, 800, 0.98828125], "9": ["vertical_and_slash", 500, 700, 0.9964706897735596], "10": ["vertical_and_slash", 500, 700, 0.9941833019256592], "11": ["vertical_and_slash", 500, 700, 0.9840189218521118], "12": ["vertical_and_slash", 3500, 100, 0.9860485792160034], "13": ["vertical_and_slash", 3500, 100, 0.9577446579933167], "14": ["vertical_and_slash", 500, 700, 0.9811022877693176], "15": ["vertical_and_slash", 100, 800, 0.96484375], "16": ["vertical_and_slash", 3500, 100, 0.9848062992095947], "17": ["vertical_and_slash", 3500, 100, 0.9883707761764526], "18": ["vertical_and_slash", 100, 800, 0.99609375], "19": ["vertical_and_slash", 3500, 100, 0.9970006942749023], "20": ["vertical_and_slash", 1000, 6096, 0.9734271764755249], "21": ["vertical_and_slash", 1000, 6096, 0.9353823661804199], "22": ["vertical_and_slash", 1000, 6096, 0.98195481300354], "23": ["vertical_and_slash", 1000, 6096, 0.9744454026222229], "24": ["vertical_and_slash", 500, 700, 0.9329551458358765], "25": ["vertical_and_slash", 500, 700, 0.9738813638687134], "26": ["vertical_and_slash", 500, 700, 0.9938949942588806], "27": ["vertical_and_slash", 500, 700, 0.9889490604400635], "28": ["vertical_and_slash", 100, 800, 0.62109375], "29": ["vertical_and_slash", 30, 800, 0.9941080808639526], "30": ["vertical_and_slash", 100, 800, 0.890625], "31": ["vertical_and_slash", 100, 800, 0.91796875]}, {"0": ["vertical_and_slash", 100, 800, 0.96875], "1": ["vertical_and_slash", 100, 800, 0.99609375], "2": ["vertical_and_slash", 100, 800, 0.98046875], "3": ["vertical_and_slash", 100, 800, 0.9453125], "4": ["vertical_and_slash", 30, 800, 0.995571494102478], "5": ["vertical_and_slash", 30, 800, 0.9974302053451538], "6": ["vertical_and_slash", 30, 800, 0.980790913105011], "7": ["vertical_and_slash", 30, 800, 0.9964948892593384], "8": ["vertical_and_slash", 100, 800, 0.9453125], "9": ["vertical_and_slash", 100, 800, 0.99609375], "10": ["vertical_and_slash", 100, 800, 0.953125], "11": ["vertical_and_slash", 30, 800, 0.9461519718170166], "12": ["vertical_and_slash", 1000, 6096, 0.9942329525947571], "13": ["vertical_and_slash", 3500, 100, 0.986452043056488], "14": ["vertical_and_slash", 1000, 6096, 0.9859598278999329], "15": ["vertical_and_slash", 3500, 100, 0.9809826612472534], "16": ["vertical_and_slash", 3500, 100, 0.9811246991157532], "17": ["vertical_and_slash", 100, 800, 0.84375], "18": ["vertical_and_slash", 3500, 100, 0.8945657014846802], "19": ["vertical_and_slash", 3500, 100, 0.9630876779556274], "20": ["vertical_and_slash", 3500, 100, 0.977688193321228], "21": ["vertical_and_slash", 100, 800, 0.96484375], "22": ["vertical_and_slash", 500, 700, 0.9898525476455688], "23": ["vertical_and_slash", 3500, 100, 0.9673124551773071], "24": ["vertical_and_slash", 100, 800, 0.96484375], "25": ["vertical_and_slash", 3500, 100, 0.9844169020652771], "26": ["vertical_and_slash", 30, 800, 0.9976933598518372], "27": ["vertical_and_slash", 3500, 100, 0.9781811237335205], "28": ["vertical_and_slash", 100, 800, 0.94921875], "29": ["vertical_and_slash", 100, 800, 0.796875], "30": ["vertical_and_slash", 3500, 100, 0.9512074589729309], "31": ["vertical_and_slash", 500, 700, 0.9931371808052063]}, {"0": ["vertical_and_slash", 100, 800, 0.99609375], "1": ["vertical_and_slash", 100, 800, 0.9921875], "2": ["vertical_and_slash", 100, 800, 0.98828125], "3": ["vertical_and_slash", 500, 700, 0.9943420886993408], "4": ["vertical_and_slash", 100, 800, 0.99609375], "5": ["vertical_and_slash", 1000, 6096, 0.9979257583618164], "6": ["vertical_and_slash", 500, 700, 0.996625542640686], "7": ["vertical_and_slash", 100, 800, 1.0], "8": ["vertical_and_slash", 100, 800, 0.98828125], "9": ["vertical_and_slash", 3500, 100, 0.8759766221046448], "10": ["vertical_and_slash", 100, 800, 0.93359375], "11": ["vertical_and_slash", 100, 800, 0.9453125], "12": ["vertical_and_slash", 100, 800, 0.99609375], "13": ["vertical_and_slash", 500, 700, 0.9941465854644775], "14": ["vertical_and_slash", 500, 700, 0.996710479259491], "15": ["vertical_and_slash", 500, 700, 0.9947440028190613], "16": ["vertical_and_slash", 100, 800, 0.9296875], "17": ["vertical_and_slash", 500, 700, 0.9890033602714539], "18": ["vertical_and_slash", 100, 800, 0.97265625], "19": ["vertical_and_slash", 100, 800, 0.97265625], "20": ["vertical_and_slash", 3500, 100, 0.8076231479644775], "21": ["vertical_and_slash", 3500, 100, 0.7205219864845276], "22": ["vertical_and_slash", 3500, 100, 0.8295556306838989], "23": ["vertical_and_slash", 100, 800, 0.828125], "24": ["vertical_and_slash", 100, 750, 0.9983446598052979], "25": ["vertical_and_slash", 100, 750, 0.9962927103042603], "26": ["vertical_and_slash", 100, 750, 0.999026358127594], "27": ["vertical_and_slash", 100, 750, 0.9989404082298279], "28": ["vertical_and_slash", 3500, 100, 0.9959333539009094], "29": ["vertical_and_slash", 500, 700, 0.9554031491279602], "30": ["vertical_and_slash", 3500, 100, 0.9903731942176819], "31": ["vertical_and_slash", 3500, 100, 0.9875370860099792]}, {"0": ["vertical_and_slash", 500, 700, 0.9613699316978455], "1": ["vertical_and_slash", 3500, 100, 0.9259256720542908], "2": ["vertical_and_slash", 3500, 100, 0.9598730802536011], "3": ["vertical_and_slash", 3500, 100, 0.9660230875015259], "4": ["vertical_and_slash", 1000, 6096, 0.984009325504303], "5": ["vertical_and_slash", 1000, 6096, 0.9557270407676697], "6": ["vertical_and_slash", 1000, 6096, 0.946307897567749], "7": ["vertical_and_slash", 1000, 6096, 0.9521388411521912], "8": ["vertical_and_slash", 100, 800, 0.76171875], "9": ["vertical_and_slash", 3500, 100, 0.9380871653556824], "10": ["vertical_and_slash", 30, 800, 0.9453238248825073], "11": ["vertical_and_slash", 3500, 100, 0.7018069624900818], "12": ["vertical_and_slash", 3500, 100, 0.9353232979774475], "13": ["vertical_and_slash", 100, 800, 0.8671875], "14": ["vertical_and_slash", 3500, 100, 0.9630011916160583], "15": ["vertical_and_slash", 3500, 100, 0.9430934190750122], "16": ["vertical_and_slash", 1000, 6096, 0.6385319828987122], "17": ["vertical_and_slash", 100, 800, 0.84375], "18": ["vertical_and_slash", 100, 800, 0.671875], "19": ["vertical_and_slash", 30, 800, 0.945841908454895], "20": ["vertical_and_slash", 3500, 100, 0.9715811610221863], "21": ["vertical_and_slash", 3500, 100, 0.9713640213012695], "22": ["vertical_and_slash", 30, 800, 0.9576144218444824], "23": ["vertical_and_slash", 500, 700, 0.981230616569519], "24": ["vertical_and_slash", 100, 800, 0.87109375], "25": ["vertical_and_slash", 100, 800, 0.8203125], "26": ["vertical_and_slash", 100, 800, 0.89453125], "27": ["vertical_and_slash", 3500, 100, 0.8966026306152344], "28": ["vertical_and_slash", 100, 800, 0.9375], "29": ["vertical_and_slash", 3500, 100, 0.9490994215011597], "30": ["vertical_and_slash", 3500, 100, 0.9371445775032043], "31": ["vertical_and_slash", 100, 800, 0.921875]}, {"0": ["vertical_and_slash", 500, 700, 0.9600187540054321], "1": ["vertical_and_slash", 30, 800, 0.966858983039856], "2": ["vertical_and_slash", 30, 800, 0.9597631692886353], "3": ["vertical_and_slash", 500, 700, 0.9798767566680908], "4": ["vertical_and_slash", 30, 800, 0.9369503855705261], "5": ["vertical_and_slash", 30, 800, 0.9862673282623291], "6": ["vertical_and_slash", 500, 700, 0.9718878865242004], "7": ["vertical_and_slash", 100, 800, 0.96484375], "8": ["vertical_and_slash", 30, 800, 0.9311044216156006], "9": ["vertical_and_slash", 30, 800, 0.9824089407920837], "10": ["vertical_and_slash", 30, 800, 0.9688247442245483], "11": ["vertical_and_slash", 30, 800, 0.7707270383834839], "12": ["vertical_and_slash", 30, 800, 0.9828140735626221], "13": ["vertical_and_slash", 30, 800, 0.8726334571838379], "14": ["vertical_and_slash", 1000, 6096, 0.99779212474823], "15": ["vertical_and_slash", 30, 800, 0.9783028960227966], "16": ["vertical_and_slash", 100, 800, 0.87890625], "17": ["vertical_and_slash", 100, 800, 0.91015625], "18": ["vertical_and_slash", 30, 800, 0.9548130035400391], "19": ["vertical_and_slash", 100, 800, 0.79296875], "20": ["vertical_and_slash", 3500, 100, 0.9271127581596375], "21": ["vertical_and_slash", 3500, 100, 0.9487717151641846], "22": ["vertical_and_slash", 100, 800, 0.8984375], "23": ["vertical_and_slash", 1000, 6096, 0.9921852350234985], "24": ["vertical_and_slash", 30, 800, 0.9751260876655579], "25": ["vertical_and_slash", 30, 800, 0.9867735505104065], "26": ["vertical_and_slash", 30, 800, 0.8953668475151062], "27": ["vertical_and_slash", 30, 800, 0.925642728805542], "28": ["vertical_and_slash", 100, 800, 0.96875], "29": ["vertical_and_slash", 100, 800, 0.89453125], "30": ["vertical_and_slash", 100, 800, 0.921875], "31": ["vertical_and_slash", 100, 800, 0.87109375]}]
diff --git a/minference/configs/Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json b/minference/configs/Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json
new file mode 100644
index 00000000..c408ab11
--- /dev/null
+++ b/minference/configs/Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json
@@ -0,0 +1 @@
+[{"0": ["vertical_and_slash", 512, 2048, 0.9623493552207947], "1": ["vertical_and_slash", 512, 2048, 0.9543083310127258], "2": ["vertical_and_slash", 512, 2048, 0.9859212636947632], "3": ["vertical_and_slash", 512, 2048, 0.884873628616333], "4": ["vertical_and_slash", 512, 2048, 0.8294410109519958], "5": ["vertical_and_slash", 512, 2048, 0.8265098333358765], "6": ["vertical_and_slash", 512, 2048, 0.8764142394065857], "7": ["vertical_and_slash", 512, 2048, 0.834898829460144], "8": ["vertical_and_slash", 512, 2048, 0.8034278154373169], "9": ["vertical_and_slash", 512, 2048, 0.8659768104553223], "10": ["vertical_and_slash", 512, 2048, 0.9463561773300171], "11": ["vertical_and_slash", 512, 2048, 0.8807589411735535], "12": ["vertical_and_slash", 512, 2048, 0.5954520106315613], "13": ["vertical_and_slash", 512, 2048, 0.5334455370903015], "14": ["vertical_and_slash", 512, 2048, 0.6530376076698303], "15": ["vertical_and_slash", 512, 2048, 0.6219685673713684], "16": ["vertical_and_slash", 512, 2048, 0.5783088803291321], "17": ["vertical_and_slash", 512, 2048, 0.6486243605613708], "18": ["vertical_and_slash", 512, 2048, 0.5563837289810181], "19": ["vertical_and_slash", 512, 2048, 0.5167779922485352], "20": ["vertical_and_slash", 512, 2048, 0.5651902556419373], "21": ["vertical_and_slash", 512, 2048, 0.502755343914032], "22": ["vertical_and_slash", 512, 2048, 0.4645864963531494], "23": ["vertical_and_slash", 512, 2048, 0.6853646039962769], "24": ["vertical_and_slash", 512, 2048, 0.9108596444129944], "25": ["vertical_and_slash", 512, 2048, 0.8522756695747375], "26": ["vertical_and_slash", 512, 2048, 0.8810073137283325], "27": ["vertical_and_slash", 512, 2048, 0.8361371159553528], "28": ["vertical_and_slash", 512, 2048, 0.4399656355381012], "29": ["vertical_and_slash", 512, 2048, 0.9979082345962524], "30": ["vertical_and_slash", 512, 2048, 0.8938949704170227], "31": ["vertical_and_slash", 512, 2048, 1.0000008344650269]}, {"0": ["vertical_and_slash", 512, 2048, 0.9742488861083984], "1": ["vertical_and_slash", 512, 2048, 0.9788762927055359], "2": ["vertical_and_slash", 512, 2048, 0.9051949977874756], "3": ["vertical_and_slash", 512, 2048, 0.9755954146385193], "4": ["vertical_and_slash", 512, 2048, 0.9866801500320435], "5": ["vertical_and_slash", 512, 2048, 0.9911981821060181], "6": ["vertical_and_slash", 512, 2048, 0.9378494024276733], "7": ["vertical_and_slash", 512, 2048, 0.9651912450790405], "8": ["vertical_and_slash", 512, 2048, 0.952520489692688], "9": ["vertical_and_slash", 512, 2048, 0.6603082418441772], "10": ["vertical_and_slash", 512, 2048, 0.9665190577507019], "11": ["vertical_and_slash", 512, 2048, 0.9581809043884277], "12": ["vertical_and_slash", 512, 2048, 0.9518635272979736], "13": ["vertical_and_slash", 512, 2048, 0.9779806733131409], "14": ["vertical_and_slash", 512, 2048, 0.89271080493927], "15": ["vertical_and_slash", 512, 2048, 0.96027010679245], "16": ["vertical_and_slash", 512, 2048, 0.9920278191566467], "17": ["vertical_and_slash", 512, 2048, 0.9877189993858337], "18": ["vertical_and_slash", 512, 2048, 0.995576024055481], "19": ["vertical_and_slash", 512, 2048, 0.9701985120773315], "20": ["vertical_and_slash", 512, 2048, 0.9970166087150574], "21": ["vertical_and_slash", 512, 2048, 0.9962336421012878], "22": ["vertical_and_slash", 512, 2048, 0.8178947567939758], "23": ["vertical_and_slash", 512, 2048, 0.7966029644012451], "24": ["vertical_and_slash", 512, 2048, 0.9736526608467102], "25": ["vertical_and_slash", 512, 2048, 0.9269852638244629], "26": ["vertical_and_slash", 512, 2048, 0.9770432114601135], "27": ["vertical_and_slash", 512, 2048, 0.9720726609230042], "28": ["vertical_and_slash", 512, 2048, 0.9068695902824402], "29": ["vertical_and_slash", 512, 2048, 0.9494656920433044], "30": ["vertical_and_slash", 512, 2048, 0.9789727926254272], "31": ["vertical_and_slash", 512, 2048, 0.9188363552093506]}, {"0": ["vertical_and_slash", 512, 2048, 0.9916821718215942], "1": ["vertical_and_slash", 512, 2048, 0.9861699938774109], "2": ["vertical_and_slash", 512, 2048, 0.9908547401428223], "3": ["vertical_and_slash", 512, 2048, 0.9805020689964294], "4": ["vertical_and_slash", 512, 2048, 0.9800416827201843], "5": ["vertical_and_slash", 512, 2048, 0.9456108808517456], "6": ["vertical_and_slash", 512, 2048, 0.86328125], "7": ["vertical_and_slash", 512, 2048, 0.9905268549919128], "8": ["vertical_and_slash", 512, 2048, 0.9948951005935669], "9": ["vertical_and_slash", 512, 2048, 0.9933184385299683], "10": ["vertical_and_slash", 512, 2048, 0.9832462072372437], "11": ["vertical_and_slash", 512, 2048, 0.9870514273643494], "12": ["vertical_and_slash", 512, 2048, 0.9968010783195496], "13": ["vertical_and_slash", 512, 2048, 0.9833763837814331], "14": ["vertical_and_slash", 512, 2048, 0.65234375], "15": ["vertical_and_slash", 512, 2048, 0.8466139435768127], "16": ["vertical_and_slash", 512, 2048, 0.9948402047157288], "17": ["vertical_and_slash", 512, 2048, 0.9832385778427124], "18": ["vertical_and_slash", 512, 2048, 0.9945462346076965], "19": ["vertical_and_slash", 512, 2048, 0.9481031894683838], "20": ["vertical_and_slash", 512, 2048, 0.9115096926689148], "21": ["vertical_and_slash", 512, 2048, 0.9331101775169373], "22": ["vertical_and_slash", 512, 2048, 0.9710216522216797], "23": ["vertical_and_slash", 512, 2048, 0.9979152083396912], "24": ["vertical_and_slash", 512, 2048, 0.9810062050819397], "25": ["vertical_and_slash", 512, 2048, 0.9922820329666138], "26": ["vertical_and_slash", 512, 2048, 0.994399905204773], "27": ["vertical_and_slash", 512, 2048, 0.9931467175483704], "28": ["vertical_and_slash", 512, 2048, 0.9915726184844971], "29": ["vertical_and_slash", 512, 2048, 0.9912887215614319], "30": ["vertical_and_slash", 512, 2048, 0.9941495656967163], "31": ["vertical_and_slash", 512, 2048, 0.9939005970954895]}, {"0": ["vertical_and_slash", 512, 2048, 0.9837832450866699], "1": ["vertical_and_slash", 512, 2048, 0.9791439175605774], "2": ["vertical_and_slash", 512, 2048, 1.0], "3": ["vertical_and_slash", 512, 2048, 0.9971569180488586], "4": ["vertical_and_slash", 512, 2048, 0.9973106980323792], "5": ["vertical_and_slash", 512, 2048, 0.9962120056152344], "6": ["vertical_and_slash", 512, 2048, 0.9828794598579407], "7": ["vertical_and_slash", 512, 2048, 0.9905376434326172], "8": ["vertical_and_slash", 512, 2048, 0.9939618706703186], "9": ["vertical_and_slash", 512, 2048, 0.9820200204849243], "10": ["vertical_and_slash", 512, 2048, 0.989706814289093], "11": ["vertical_and_slash", 512, 2048, 0.9937054514884949], "12": ["vertical_and_slash", 512, 2048, 0.9380397796630859], "13": ["vertical_and_slash", 512, 2048, 0.9371072053909302], "14": ["vertical_and_slash", 512, 2048, 0.9933274984359741], "15": ["vertical_and_slash", 512, 2048, 0.9924632906913757], "16": ["vertical_and_slash", 512, 2048, 0.9889415502548218], "17": ["vertical_and_slash", 512, 2048, 0.9969512820243835], "18": ["vertical_and_slash", 512, 2048, 0.9957903027534485], "19": ["vertical_and_slash", 512, 2048, 0.9710395932197571], "20": ["vertical_and_slash", 512, 2048, 0.96484375], "21": ["vertical_and_slash", 512, 2048, 0.95703125], "22": ["vertical_and_slash", 512, 2048, 0.9383615255355835], "23": ["vertical_and_slash", 512, 2048, 0.9005464911460876], "24": ["vertical_and_slash", 512, 2048, 0.9980953335762024], "25": ["vertical_and_slash", 512, 2048, 0.9897440075874329], "26": ["vertical_and_slash", 512, 2048, 0.9923828840255737], "27": ["vertical_and_slash", 512, 2048, 0.73828125], "28": ["vertical_and_slash", 512, 2048, 0.9830290079116821], "29": ["vertical_and_slash", 512, 2048, 0.9703660607337952], "30": ["vertical_and_slash", 512, 2048, 0.9784981608390808], "31": ["vertical_and_slash", 512, 2048, 0.9887380599975586]}, {"0": ["vertical_and_slash", 512, 2048, 0.91015625], "1": ["vertical_and_slash", 512, 2048, 0.953125], "2": ["vertical_and_slash", 512, 2048, 0.9765625], "3": ["vertical_and_slash", 512, 2048, 0.9728029370307922], "4": ["vertical_and_slash", 512, 2048, 0.990367591381073], "5": ["vertical_and_slash", 512, 2048, 0.8950048089027405], "6": ["vertical_and_slash", 512, 2048, 0.8518299460411072], "7": ["vertical_and_slash", 512, 2048, 0.9903310537338257], "8": ["vertical_and_slash", 512, 2048, 0.8984375], "9": ["vertical_and_slash", 512, 2048, 0.9666171073913574], "10": ["vertical_and_slash", 512, 2048, 0.8871880769729614], "11": ["vertical_and_slash", 512, 2048, 0.9558337926864624], "12": ["vertical_and_slash", 512, 2048, 0.99130779504776], "13": ["vertical_and_slash", 512, 2048, 0.9857202172279358], "14": ["vertical_and_slash", 512, 2048, 0.992880642414093], "15": ["vertical_and_slash", 512, 2048, 0.9375], "16": ["vertical_and_slash", 512, 2048, 0.6494593620300293], "17": ["vertical_and_slash", 512, 2048, 0.994994580745697], "18": ["vertical_and_slash", 512, 2048, 0.9976939558982849], "19": ["vertical_and_slash", 512, 2048, 0.5856859087944031], "20": ["vertical_and_slash", 512, 2048, 0.9758481383323669], "21": ["vertical_and_slash", 512, 2048, 0.9787202477455139], "22": ["vertical_and_slash", 512, 2048, 0.9818038940429688], "23": ["vertical_and_slash", 512, 2048, 0.9906389117240906], "24": ["vertical_and_slash", 512, 2048, 0.9870673418045044], "25": ["vertical_and_slash", 512, 2048, 0.9819732308387756], "26": ["vertical_and_slash", 512, 2048, 0.894801914691925], "27": ["vertical_and_slash", 512, 2048, 0.9789435267448425], "28": ["vertical_and_slash", 512, 2048, 0.9558958411216736], "29": ["vertical_and_slash", 512, 2048, 0.7887665033340454], "30": ["vertical_and_slash", 512, 2048, 0.9841030240058899], "31": ["vertical_and_slash", 512, 2048, 0.9959767460823059]}, {"0": ["vertical_and_slash", 512, 2048, 0.985858678817749], "1": ["vertical_and_slash", 512, 2048, 0.9785674214363098], "2": ["vertical_and_slash", 512, 2048, 0.9867167472839355], "3": ["vertical_and_slash", 512, 2048, 0.9883900880813599], "4": ["vertical_and_slash", 512, 2048, 0.987496554851532], "5": ["vertical_and_slash", 512, 2048, 0.8416361808776855], "6": ["vertical_and_slash", 512, 2048, 0.9440363645553589], "7": ["vertical_and_slash", 512, 2048, 0.8125], "8": ["vertical_and_slash", 512, 2048, 0.9002290368080139], "9": ["vertical_and_slash", 512, 2048, 0.9311371445655823], "10": ["vertical_and_slash", 512, 2048, 0.8879005908966064], "11": ["vertical_and_slash", 512, 2048, 0.9562467932701111], "12": ["vertical_and_slash", 512, 2048, 0.9765503406524658], "13": ["vertical_and_slash", 512, 2048, 0.9893233776092529], "14": ["vertical_and_slash", 512, 2048, 0.9915202260017395], "15": ["vertical_and_slash", 512, 2048, 0.9850893616676331], "16": ["vertical_and_slash", 512, 2048, 0.98828125], "17": ["vertical_and_slash", 512, 2048, 0.9221939444541931], "18": ["vertical_and_slash", 512, 2048, 0.9556252956390381], "19": ["vertical_and_slash", 512, 2048, 0.9365096688270569], "20": ["vertical_and_slash", 512, 2048, 0.9765625], "21": ["vertical_and_slash", 512, 2048, 0.9185587167739868], "22": ["vertical_and_slash", 512, 2048, 0.86328125], "23": ["vertical_and_slash", 512, 2048, 0.94921875], "24": ["vertical_and_slash", 512, 2048, 0.9803503155708313], "25": ["vertical_and_slash", 512, 2048, 0.9929267764091492], "26": ["vertical_and_slash", 512, 2048, 0.9961451292037964], "27": ["vertical_and_slash", 512, 2048, 0.9952177405357361], "28": ["vertical_and_slash", 512, 2048, 0.9747095704078674], "29": ["vertical_and_slash", 512, 2048, 0.9782059788703918], "30": ["vertical_and_slash", 512, 2048, 0.9795500040054321], "31": ["vertical_and_slash", 512, 2048, 0.9679626226425171]}, {"0": ["vertical_and_slash", 512, 2048, 0.9898263216018677], "1": ["vertical_and_slash", 512, 2048, 0.9882830381393433], "2": ["vertical_and_slash", 512, 2048, 0.859375], "3": ["vertical_and_slash", 512, 2048, 0.83984375], "4": ["vertical_and_slash", 512, 2048, 0.9902689456939697], "5": ["vertical_and_slash", 512, 2048, 0.9513372182846069], "6": ["vertical_and_slash", 512, 2048, 0.90625], "7": ["vertical_and_slash", 512, 2048, 0.9930790066719055], "8": ["vertical_and_slash", 512, 2048, 0.9950608015060425], "9": ["vertical_and_slash", 512, 2048, 0.9540755748748779], "10": ["vertical_and_slash", 512, 2048, 0.9924449324607849], "11": ["vertical_and_slash", 512, 2048, 0.9905205368995667], "12": ["vertical_and_slash", 512, 2048, 0.9887551069259644], "13": ["vertical_and_slash", 512, 2048, 0.9894125461578369], "14": ["vertical_and_slash", 512, 2048, 0.9369159936904907], "15": ["vertical_and_slash", 512, 2048, 0.9925025701522827], "16": ["vertical_and_slash", 512, 2048, 0.9917540550231934], "17": ["vertical_and_slash", 512, 2048, 0.9703994989395142], "18": ["vertical_and_slash", 512, 2048, 0.9568591117858887], "19": ["vertical_and_slash", 512, 2048, 0.9875468015670776], "20": ["vertical_and_slash", 512, 2048, 0.921875], "21": ["vertical_and_slash", 512, 2048, 0.7049875855445862], "22": ["vertical_and_slash", 512, 2048, 0.91796875], "23": ["vertical_and_slash", 512, 2048, 0.96875], "24": ["vertical_and_slash", 512, 2048, 0.80078125], "25": ["vertical_and_slash", 512, 2048, 0.8965499401092529], "26": ["vertical_and_slash", 512, 2048, 0.8224286437034607], "27": ["vertical_and_slash", 512, 2048, 0.9571169018745422], "28": ["vertical_and_slash", 512, 2048, 0.71875], "29": ["vertical_and_slash", 512, 2048, 0.98046875], "30": ["vertical_and_slash", 512, 2048, 1.0], "31": ["vertical_and_slash", 512, 2048, 0.90234375]}, {"0": ["vertical_and_slash", 512, 2048, 0.9603511095046997], "1": ["vertical_and_slash", 512, 2048, 0.9893207550048828], "2": ["vertical_and_slash", 512, 2048, 0.9923656582832336], "3": ["vertical_and_slash", 512, 2048, 0.458984375], "4": ["vertical_and_slash", 512, 2048, 0.9753598570823669], "5": ["vertical_and_slash", 512, 2048, 0.9915642142295837], "6": ["vertical_and_slash", 512, 2048, 0.8411368727684021], "7": ["vertical_and_slash", 512, 2048, 0.8587262034416199], "8": ["vertical_and_slash", 512, 2048, 0.93359375], "9": ["vertical_and_slash", 512, 2048, 0.96875], "10": ["vertical_and_slash", 512, 2048, 0.8169850707054138], "11": ["vertical_and_slash", 512, 2048, 0.9763987064361572], "12": ["vertical_and_slash", 512, 2048, 0.9713965654373169], "13": ["vertical_and_slash", 512, 2048, 0.8622501492500305], "14": ["vertical_and_slash", 512, 2048, 0.9738231301307678], "15": ["vertical_and_slash", 512, 2048, 0.9296875], "16": ["vertical_and_slash", 512, 2048, 0.9296875], "17": ["vertical_and_slash", 512, 2048, 0.9684662222862244], "18": ["vertical_and_slash", 512, 2048, 0.9375], "19": ["vertical_and_slash", 512, 2048, 0.94921875], "20": ["vertical_and_slash", 512, 2048, 0.9765625], "21": ["vertical_and_slash", 512, 2048, 0.7724359035491943], "22": ["vertical_and_slash", 512, 2048, 0.7844681143760681], "23": ["vertical_and_slash", 512, 2048, 0.6796875], "24": ["vertical_and_slash", 512, 2048, 0.9453125], "25": ["vertical_and_slash", 512, 2048, 0.6538506150245667], "26": ["vertical_and_slash", 512, 2048, 0.85546875], "27": ["vertical_and_slash", 512, 2048, 0.94921875], "28": ["vertical_and_slash", 512, 2048, 0.9509181380271912], "29": ["vertical_and_slash", 512, 2048, 0.9657983183860779], "30": ["vertical_and_slash", 512, 2048, 0.9171797633171082], "31": ["vertical_and_slash", 512, 2048, 0.9357418417930603]}, {"0": ["vertical_and_slash", 512, 2048, 0.9873993992805481], "1": ["vertical_and_slash", 512, 2048, 0.7471938729286194], "2": ["vertical_and_slash", 512, 2048, 0.89453125], "3": ["vertical_and_slash", 512, 2048, 0.6953125], "4": ["vertical_and_slash", 512, 2048, 0.9859827160835266], "5": ["vertical_and_slash", 512, 2048, 0.9493432641029358], "6": ["vertical_and_slash", 512, 2048, 0.9839309453964233], "7": ["vertical_and_slash", 512, 2048, 0.94921875], "8": ["vertical_and_slash", 512, 2048, 0.8909018635749817], "9": ["vertical_and_slash", 512, 2048, 0.5839027762413025], "10": ["vertical_and_slash", 512, 2048, 0.8953289985656738], "11": ["vertical_and_slash", 512, 2048, 0.5685684680938721], "12": ["vertical_and_slash", 512, 2048, 0.9685022234916687], "13": ["vertical_and_slash", 512, 2048, 0.9499353766441345], "14": ["vertical_and_slash", 512, 2048, 0.9260799288749695], "15": ["vertical_and_slash", 512, 2048, 0.9790918827056885], "16": ["vertical_and_slash", 512, 2048, 0.8991709351539612], "17": ["vertical_and_slash", 512, 2048, 0.9513922929763794], "18": ["vertical_and_slash", 512, 2048, 0.9208007454872131], "19": ["vertical_and_slash", 512, 2048, 0.9758914113044739], "20": ["vertical_and_slash", 512, 2048, 0.98046875], "21": ["vertical_and_slash", 512, 2048, 0.97265625], "22": ["vertical_and_slash", 512, 2048, 0.9839867949485779], "23": ["vertical_and_slash", 512, 2048, 0.9237134456634521], "24": ["vertical_and_slash", 512, 2048, 0.9671244025230408], "25": ["vertical_and_slash", 512, 2048, 0.982479989528656], "26": ["vertical_and_slash", 512, 2048, 0.9730757474899292], "27": ["vertical_and_slash", 512, 2048, 0.9641519784927368], "28": ["vertical_and_slash", 512, 2048, 0.9639339447021484], "29": ["vertical_and_slash", 512, 2048, 0.9427148699760437], "30": ["vertical_and_slash", 512, 2048, 0.9345259666442871], "31": ["vertical_and_slash", 512, 2048, 0.8792511820793152]}, {"0": ["vertical_and_slash", 512, 2048, 0.8634915947914124], "1": ["vertical_and_slash", 512, 2048, 0.8989808559417725], "2": ["vertical_and_slash", 512, 2048, 0.8503499031066895], "3": ["vertical_and_slash", 512, 2048, 0.827835202217102], "4": ["vertical_and_slash", 512, 2048, 0.9314107894897461], "5": ["vertical_and_slash", 512, 2048, 0.9064517617225647], "6": ["vertical_and_slash", 512, 2048, 0.9566625356674194], "7": ["vertical_and_slash", 512, 2048, 0.9783104062080383], "8": ["vertical_and_slash", 512, 2048, 0.9695003628730774], "9": ["vertical_and_slash", 512, 2048, 0.9732148051261902], "10": ["vertical_and_slash", 512, 2048, 0.9714348316192627], "11": ["vertical_and_slash", 512, 2048, 0.9940823912620544], "12": ["vertical_and_slash", 512, 2048, 0.706820547580719], "13": ["vertical_and_slash", 512, 2048, 0.7126483917236328], "14": ["vertical_and_slash", 512, 2048, 0.79296875], "15": ["vertical_and_slash", 512, 2048, 0.8690503835678101], "16": ["vertical_and_slash", 512, 2048, 0.9109070301055908], "17": ["vertical_and_slash", 512, 2048, 0.8707846403121948], "18": ["vertical_and_slash", 512, 2048, 0.8517163395881653], "19": ["vertical_and_slash", 512, 2048, 0.834902286529541], "20": ["vertical_and_slash", 512, 2048, 0.9286177158355713], "21": ["vertical_and_slash", 512, 2048, 0.9553185105323792], "22": ["vertical_and_slash", 512, 2048, 0.9636306166648865], "23": ["vertical_and_slash", 512, 2048, 0.9541269540786743], "24": ["vertical_and_slash", 512, 2048, 0.9912933707237244], "25": ["vertical_and_slash", 512, 2048, 0.7418502569198608], "26": ["vertical_and_slash", 512, 2048, 0.9921875], "27": ["vertical_and_slash", 512, 2048, 0.8253664374351501], "28": ["vertical_and_slash", 512, 2048, 0.6728078126907349], "29": ["vertical_and_slash", 512, 2048, 0.8966277241706848], "30": ["vertical_and_slash", 512, 2048, 0.94921875], "31": ["vertical_and_slash", 512, 2048, 0.7612401247024536]}, {"0": ["vertical_and_slash", 512, 2048, 0.9580379128456116], "1": ["vertical_and_slash", 512, 2048, 0.8889423608779907], "2": ["vertical_and_slash", 512, 2048, 0.7774703502655029], "3": ["vertical_and_slash", 512, 2048, 0.5454753637313843], "4": ["vertical_and_slash", 512, 2048, 0.9631187319755554], "5": ["vertical_and_slash", 512, 2048, 0.8950480818748474], "6": ["vertical_and_slash", 512, 2048, 0.7367609143257141], "7": ["vertical_and_slash", 512, 2048, 0.7220443487167358], "8": ["vertical_and_slash", 512, 2048, 0.89453125], "9": ["vertical_and_slash", 512, 2048, 0.969845712184906], "10": ["vertical_and_slash", 512, 2048, 0.8675119876861572], "11": ["vertical_and_slash", 512, 2048, 0.9803798794746399], "12": ["vertical_and_slash", 512, 2048, 0.6585094332695007], "13": ["vertical_and_slash", 512, 2048, 0.7352333664894104], "14": ["vertical_and_slash", 512, 2048, 0.8553265333175659], "15": ["vertical_and_slash", 512, 2048, 0.9600890874862671], "16": ["vertical_and_slash", 512, 2048, 0.9677231311798096], "17": ["vertical_and_slash", 512, 2048, 0.8380194306373596], "18": ["vertical_and_slash", 512, 2048, 0.817642867565155], "19": ["vertical_and_slash", 512, 2048, 0.7535743713378906], "20": ["vertical_and_slash", 512, 2048, 0.9120592474937439], "21": ["vertical_and_slash", 512, 2048, 0.87109375], "22": ["vertical_and_slash", 512, 2048, 0.8932908177375793], "23": ["vertical_and_slash", 512, 2048, 0.91015625], "24": ["vertical_and_slash", 512, 2048, 0.9268215298652649], "25": ["vertical_and_slash", 512, 2048, 0.9567509293556213], "26": ["vertical_and_slash", 512, 2048, 0.8897656798362732], "27": ["vertical_and_slash", 512, 2048, 0.9563292264938354], "28": ["vertical_and_slash", 512, 2048, 0.7572957277297974], "29": ["vertical_and_slash", 512, 2048, 0.85546875], "30": ["vertical_and_slash", 512, 2048, 0.9302328824996948], "31": ["vertical_and_slash", 512, 2048, 0.7470130920410156]}, {"0": ["vertical_and_slash", 512, 2048, 0.9706114530563354], "1": ["vertical_and_slash", 512, 2048, 0.9341138601303101], "2": ["vertical_and_slash", 512, 2048, 0.88250732421875], "3": ["vertical_and_slash", 512, 2048, 0.9176220893859863], "4": ["vertical_and_slash", 512, 2048, 0.9890152215957642], "5": ["vertical_and_slash", 512, 2048, 0.6015625], "6": ["vertical_and_slash", 512, 2048, 0.53515625], "7": ["vertical_and_slash", 512, 2048, 0.65625], "8": ["vertical_and_slash", 512, 2048, 0.9140625], "9": ["vertical_and_slash", 512, 2048, 0.91796875], "10": ["vertical_and_slash", 512, 2048, 0.974812924861908], "11": ["vertical_and_slash", 512, 2048, 0.93359375], "12": ["vertical_and_slash", 512, 2048, 0.8015804886817932], "13": ["vertical_and_slash", 512, 2048, 0.8895022869110107], "14": ["vertical_and_slash", 512, 2048, 0.8971598744392395], "15": ["vertical_and_slash", 512, 2048, 0.9316778779029846], "16": ["vertical_and_slash", 512, 2048, 0.9727176427841187], "17": ["vertical_and_slash", 512, 2048, 0.9684209227561951], "18": ["vertical_and_slash", 512, 2048, 0.7916423082351685], "19": ["vertical_and_slash", 512, 2048, 0.956729531288147], "20": ["vertical_and_slash", 512, 2048, 0.7421875], "21": ["vertical_and_slash", 512, 2048, 0.92578125], "22": ["vertical_and_slash", 512, 2048, 0.6496378183364868], "23": ["vertical_and_slash", 512, 2048, 0.75390625], "24": ["vertical_and_slash", 512, 2048, 0.863329291343689], "25": ["vertical_and_slash", 512, 2048, 0.9180703163146973], "26": ["vertical_and_slash", 512, 2048, 0.8671875], "27": ["vertical_and_slash", 512, 2048, 0.9076446890830994], "28": ["vertical_and_slash", 512, 2048, 0.7788168787956238], "29": ["vertical_and_slash", 512, 2048, 0.734375], "30": ["vertical_and_slash", 512, 2048, 0.66796875], "31": ["vertical_and_slash", 512, 2048, 0.9902347922325134]}, {"0": ["vertical_and_slash", 512, 2048, 0.95703125], "1": ["vertical_and_slash", 512, 2048, 0.8122807145118713], "2": ["vertical_and_slash", 512, 2048, 0.96484375], "3": ["vertical_and_slash", 512, 2048, 0.96484375], "4": ["vertical_and_slash", 512, 2048, 0.94921875], "5": ["vertical_and_slash", 512, 2048, 0.9375], "6": ["vertical_and_slash", 512, 2048, 0.90234375], "7": ["vertical_and_slash", 512, 2048, 0.9453125], "8": ["vertical_and_slash", 512, 2048, 0.8789168000221252], "9": ["vertical_and_slash", 512, 2048, 0.9342090487480164], "10": ["vertical_and_slash", 512, 2048, 0.78515625], "11": ["vertical_and_slash", 512, 2048, 0.9140625], "12": ["vertical_and_slash", 512, 2048, 0.9864805340766907], "13": ["vertical_and_slash", 512, 2048, 0.9746032357215881], "14": ["vertical_and_slash", 512, 2048, 0.6875], "15": ["vertical_and_slash", 512, 2048, 0.953125], "16": ["vertical_and_slash", 512, 2048, 0.9699565768241882], "17": ["vertical_and_slash", 512, 2048, 0.9764721393585205], "18": ["vertical_and_slash", 512, 2048, 0.9664324522018433], "19": ["vertical_and_slash", 512, 2048, 0.9909504652023315], "20": ["vertical_and_slash", 512, 2048, 0.8712055683135986], "21": ["vertical_and_slash", 512, 2048, 0.97265625], "22": ["vertical_and_slash", 512, 2048, 0.98046875], "23": ["vertical_and_slash", 512, 2048, 0.9733871221542358], "24": ["vertical_and_slash", 512, 2048, 0.9706288576126099], "25": ["vertical_and_slash", 512, 2048, 0.9624436497688293], "26": ["vertical_and_slash", 512, 2048, 0.9703077077865601], "27": ["vertical_and_slash", 512, 2048, 0.9371525049209595], "28": ["vertical_and_slash", 512, 2048, 0.9811959266662598], "29": ["vertical_and_slash", 512, 2048, 0.9622439742088318], "30": ["vertical_and_slash", 512, 2048, 0.9234631061553955], "31": ["vertical_and_slash", 512, 2048, 0.9721811413764954]}, {"0": ["vertical_and_slash", 512, 2048, 0.6208434104919434], "1": ["vertical_and_slash", 512, 2048, 0.78515625], "2": ["vertical_and_slash", 512, 2048, 0.682672917842865], "3": ["vertical_and_slash", 512, 2048, 0.6564425230026245], "4": ["vertical_and_slash", 512, 2048, 0.9247110486030579], "5": ["vertical_and_slash", 512, 2048, 0.8886473774909973], "6": ["vertical_and_slash", 512, 2048, 0.8195661902427673], "7": ["vertical_and_slash", 512, 2048, 0.9609375], "8": ["vertical_and_slash", 512, 2048, 0.6291986703872681], "9": ["vertical_and_slash", 512, 2048, 0.8490926623344421], "10": ["vertical_and_slash", 512, 2048, 0.66796875], "11": ["vertical_and_slash", 512, 2048, 0.8477756977081299], "12": ["vertical_and_slash", 512, 2048, 0.8828125], "13": ["vertical_and_slash", 512, 2048, 0.87890625], "14": ["vertical_and_slash", 512, 2048, 0.88671875], "15": ["vertical_and_slash", 512, 2048, 0.90625], "16": ["vertical_and_slash", 512, 2048, 0.6082240343093872], "17": ["vertical_and_slash", 512, 2048, 0.6033845543861389], "18": ["vertical_and_slash", 512, 2048, 0.7406787872314453], "19": ["vertical_and_slash", 512, 2048, 0.5625], "20": ["vertical_and_slash", 512, 2048, 0.858406662940979], "21": ["vertical_and_slash", 512, 2048, 0.9015128016471863], "22": ["vertical_and_slash", 512, 2048, 0.9453125], "23": ["vertical_and_slash", 512, 2048, 0.8777713179588318], "24": ["vertical_and_slash", 512, 2048, 0.95703125], "25": ["vertical_and_slash", 512, 2048, 0.8984375], "26": ["vertical_and_slash", 512, 2048, 0.68359375], "27": ["vertical_and_slash", 512, 2048, 0.5234375], "28": ["vertical_and_slash", 512, 2048, 0.8055139183998108], "29": ["vertical_and_slash", 512, 2048, 0.9727892875671387], "30": ["vertical_and_slash", 512, 2048, 0.772057831287384], "31": ["vertical_and_slash", 512, 2048, 0.6695537567138672]}, {"0": ["vertical_and_slash", 512, 2048, 0.9453125], "1": ["vertical_and_slash", 512, 2048, 0.95703125], "2": ["vertical_and_slash", 512, 2048, 0.96484375], "3": ["vertical_and_slash", 512, 2048, 0.9689056277275085], "4": ["vertical_and_slash", 512, 2048, 0.91015625], "5": ["vertical_and_slash", 512, 2048, 0.9235577583312988], "6": ["vertical_and_slash", 512, 2048, 0.953125], "7": ["vertical_and_slash", 512, 2048, 0.9140625], "8": ["vertical_and_slash", 512, 2048, 0.9862722158432007], "9": ["vertical_and_slash", 512, 2048, 0.977347731590271], "10": ["vertical_and_slash", 512, 2048, 0.96875], "11": ["vertical_and_slash", 512, 2048, 0.9818734526634216], "12": ["vertical_and_slash", 512, 2048, 0.7890625], "13": ["vertical_and_slash", 512, 2048, 0.7767282128334045], "14": ["vertical_and_slash", 512, 2048, 0.796875], "15": ["vertical_and_slash", 512, 2048, 0.9375], "16": ["vertical_and_slash", 512, 2048, 0.69921875], "17": ["vertical_and_slash", 512, 2048, 0.9140625], "18": ["vertical_and_slash", 512, 2048, 0.89453125], "19": ["vertical_and_slash", 512, 2048, 0.8810321092605591], "20": ["vertical_and_slash", 512, 2048, 0.8856872320175171], "21": ["vertical_and_slash", 512, 2048, 0.9453125], "22": ["vertical_and_slash", 512, 2048, 0.87109375], "23": ["vertical_and_slash", 512, 2048, 0.9453125], "24": ["vertical_and_slash", 512, 2048, 0.9649856686592102], "25": ["vertical_and_slash", 512, 2048, 0.9768476486206055], "26": ["vertical_and_slash", 512, 2048, 0.9993871450424194], "27": ["vertical_and_slash", 512, 2048, 0.921875], "28": ["vertical_and_slash", 512, 2048, 0.8807946443557739], "29": ["vertical_and_slash", 512, 2048, 0.8378621339797974], "30": ["vertical_and_slash", 512, 2048, 0.9146285653114319], "31": ["vertical_and_slash", 512, 2048, 0.9363900423049927]}, {"0": ["vertical_and_slash", 512, 2048, 0.9848667979240417], "1": ["vertical_and_slash", 512, 2048, 0.6662361025810242], "2": ["vertical_and_slash", 512, 2048, 0.6793611645698547], "3": ["vertical_and_slash", 512, 2048, 0.93359375], "4": ["vertical_and_slash", 512, 2048, 0.93359375], "5": ["vertical_and_slash", 512, 2048, 0.7997210621833801], "6": ["vertical_and_slash", 512, 2048, 0.921200156211853], "7": ["vertical_and_slash", 512, 2048, 0.9733778834342957], "8": ["vertical_and_slash", 512, 2048, 0.9375], "9": ["vertical_and_slash", 512, 2048, 0.88671875], "10": ["vertical_and_slash", 512, 2048, 0.796875], "11": ["vertical_and_slash", 512, 2048, 0.82421875], "12": ["vertical_and_slash", 512, 2048, 0.981203019618988], "13": ["vertical_and_slash", 512, 2048, 0.94921875], "14": ["vertical_and_slash", 512, 2048, 0.9745455980300903], "15": ["vertical_and_slash", 512, 2048, 0.9339883327484131], "16": ["vertical_and_slash", 512, 2048, 0.79296875], "17": ["vertical_and_slash", 512, 2048, 0.4775260090827942], "18": ["vertical_and_slash", 512, 2048, 0.5015617609024048], "19": ["vertical_and_slash", 512, 2048, 0.91796875], "20": ["vertical_and_slash", 512, 2048, 0.91015625], "21": ["vertical_and_slash", 512, 2048, 0.75390625], "22": ["vertical_and_slash", 512, 2048, 0.92578125], "23": ["vertical_and_slash", 512, 2048, 0.89453125], "24": ["vertical_and_slash", 512, 2048, 0.84765625], "25": ["vertical_and_slash", 512, 2048, 0.72265625], "26": ["vertical_and_slash", 512, 2048, 0.87890625], "27": ["vertical_and_slash", 512, 2048, 0.8984375], "28": ["vertical_and_slash", 512, 2048, 0.676841139793396], "29": ["vertical_and_slash", 512, 2048, 0.6767839193344116], "30": ["vertical_and_slash", 512, 2048, 0.8959569334983826], "31": ["vertical_and_slash", 512, 2048, 0.9896997213363647]}, {"0": ["vertical_and_slash", 512, 2048, 0.8137837052345276], "1": ["vertical_and_slash", 512, 2048, 0.8565242886543274], "2": ["vertical_and_slash", 512, 2048, 0.84375], "3": ["vertical_and_slash", 512, 2048, 0.9823415279388428], "4": ["vertical_and_slash", 512, 2048, 0.9782371520996094], "5": ["vertical_and_slash", 512, 2048, 0.9068223237991333], "6": ["vertical_and_slash", 512, 2048, 0.9929006695747375], "7": ["vertical_and_slash", 512, 2048, 0.9689693450927734], "8": ["vertical_and_slash", 512, 2048, 0.8670288920402527], "9": ["vertical_and_slash", 512, 2048, 0.9820406436920166], "10": ["vertical_and_slash", 512, 2048, 0.9791944026947021], "11": ["vertical_and_slash", 512, 2048, 0.97265625], "12": ["vertical_and_slash", 512, 2048, 0.9788346886634827], "13": ["vertical_and_slash", 512, 2048, 0.9739745259284973], "14": ["vertical_and_slash", 512, 2048, 0.98046875], "15": ["vertical_and_slash", 512, 2048, 0.9562920331954956], "16": ["vertical_and_slash", 512, 2048, 0.84375], "17": ["vertical_and_slash", 512, 2048, 0.9609375], "18": ["vertical_and_slash", 512, 2048, 0.9375513792037964], "19": ["vertical_and_slash", 512, 2048, 0.9296875], "20": ["vertical_and_slash", 512, 2048, 0.9592316746711731], "21": ["vertical_and_slash", 512, 2048, 0.91796875], "22": ["vertical_and_slash", 512, 2048, 0.9921875], "23": ["vertical_and_slash", 512, 2048, 0.9259733557701111], "24": ["vertical_and_slash", 512, 2048, 0.97265625], "25": ["vertical_and_slash", 512, 2048, 0.84375], "26": ["vertical_and_slash", 512, 2048, 0.96484375], "27": ["vertical_and_slash", 512, 2048, 0.9818596243858337], "28": ["vertical_and_slash", 512, 2048, 0.9490596652030945], "29": ["vertical_and_slash", 512, 2048, 0.9914187788963318], "30": ["vertical_and_slash", 512, 2048, 0.9638843536376953], "31": ["vertical_and_slash", 512, 2048, 0.9648941159248352]}, {"0": ["vertical_and_slash", 512, 2048, 0.95703125], "1": ["vertical_and_slash", 512, 2048, 0.9453125], "2": ["vertical_and_slash", 512, 2048, 0.9201825857162476], "3": ["vertical_and_slash", 512, 2048, 0.9765625], "4": ["vertical_and_slash", 512, 2048, 0.96484375], "5": ["vertical_and_slash", 512, 2048, 0.9729551076889038], "6": ["vertical_and_slash", 512, 2048, 0.84765625], "7": ["vertical_and_slash", 512, 2048, 0.9674957394599915], "8": ["vertical_and_slash", 512, 2048, 0.5704637169837952], "9": ["vertical_and_slash", 512, 2048, 0.8515625], "10": ["vertical_and_slash", 512, 2048, 0.8001803159713745], "11": ["vertical_and_slash", 512, 2048, 0.9770910739898682], "12": ["vertical_and_slash", 512, 2048, 0.9445182681083679], "13": ["vertical_and_slash", 512, 2048, 0.9648588299751282], "14": ["vertical_and_slash", 512, 2048, 0.921875], "15": ["vertical_and_slash", 512, 2048, 0.9490084648132324], "16": ["vertical_and_slash", 512, 2048, 0.9791340231895447], "17": ["vertical_and_slash", 512, 2048, 0.9951378107070923], "18": ["vertical_and_slash", 512, 2048, 0.9684316515922546], "19": ["vertical_and_slash", 512, 2048, 0.9796795845031738], "20": ["vertical_and_slash", 512, 2048, 0.9908055067062378], "21": ["vertical_and_slash", 512, 2048, 0.9142110347747803], "22": ["vertical_and_slash", 512, 2048, 0.9375], "23": ["vertical_and_slash", 512, 2048, 0.9296875], "24": ["vertical_and_slash", 512, 2048, 0.88671875], "25": ["vertical_and_slash", 512, 2048, 0.8671875], "26": ["vertical_and_slash", 512, 2048, 0.9462155699729919], "27": ["vertical_and_slash", 512, 2048, 0.8515625], "28": ["vertical_and_slash", 512, 2048, 0.9939091801643372], "29": ["vertical_and_slash", 512, 2048, 0.9740011692047119], "30": ["vertical_and_slash", 512, 2048, 0.993536114692688], "31": ["vertical_and_slash", 512, 2048, 0.9895251393318176]}, {"0": ["vertical_and_slash", 512, 2048, 0.9921875], "1": ["vertical_and_slash", 512, 2048, 0.96484375], "2": ["vertical_and_slash", 512, 2048, 0.96875], "3": ["vertical_and_slash", 512, 2048, 0.9583335518836975], "4": ["vertical_and_slash", 512, 2048, 0.9164949059486389], "5": ["vertical_and_slash", 512, 2048, 0.9469748139381409], "6": ["vertical_and_slash", 512, 2048, 0.9885687232017517], "7": ["vertical_and_slash", 512, 2048, 0.99079829454422], "8": ["vertical_and_slash", 512, 2048, 0.9748219847679138], "9": ["vertical_and_slash", 512, 2048, 0.9859137535095215], "10": ["vertical_and_slash", 512, 2048, 0.97265625], "11": ["vertical_and_slash", 512, 2048, 0.9817909002304077], "12": ["vertical_and_slash", 512, 2048, 0.9921875], "13": ["vertical_and_slash", 512, 2048, 0.98046875], "14": ["vertical_and_slash", 512, 2048, 0.9296875], "15": ["vertical_and_slash", 512, 2048, 0.9796241521835327], "16": ["vertical_and_slash", 512, 2048, 0.9453125], "17": ["vertical_and_slash", 512, 2048, 0.953125], "18": ["vertical_and_slash", 512, 2048, 0.953125], "19": ["vertical_and_slash", 512, 2048, 0.9794105887413025], "20": ["vertical_and_slash", 512, 2048, 0.8046875], "21": ["vertical_and_slash", 512, 2048, 0.97265625], "22": ["vertical_and_slash", 512, 2048, 0.87109375], "23": ["vertical_and_slash", 512, 2048, 0.941199779510498], "24": ["vertical_and_slash", 512, 2048, 0.9881477355957031], "25": ["vertical_and_slash", 512, 2048, 0.992800235748291], "26": ["vertical_and_slash", 512, 2048, 0.9911536574363708], "27": ["vertical_and_slash", 512, 2048, 0.98828125], "28": ["vertical_and_slash", 512, 2048, 0.890625], "29": ["vertical_and_slash", 512, 2048, 0.91015625], "30": ["vertical_and_slash", 512, 2048, 0.875], "31": ["vertical_and_slash", 512, 2048, 0.90234375]}, {"0": ["vertical_and_slash", 512, 2048, 0.5703125], "1": ["vertical_and_slash", 512, 2048, 0.8984375], "2": ["vertical_and_slash", 512, 2048, 0.8873317837715149], "3": ["vertical_and_slash", 512, 2048, 0.9360411763191223], "4": ["vertical_and_slash", 512, 2048, 0.9617438316345215], "5": ["vertical_and_slash", 512, 2048, 0.9799512028694153], "6": ["vertical_and_slash", 512, 2048, 0.9727489948272705], "7": ["vertical_and_slash", 512, 2048, 0.9961766600608826], "8": ["vertical_and_slash", 512, 2048, 0.9883258938789368], "9": ["vertical_and_slash", 512, 2048, 0.8671875], "10": ["vertical_and_slash", 512, 2048, 0.984375], "11": ["vertical_and_slash", 512, 2048, 0.9185207486152649], "12": ["vertical_and_slash", 512, 2048, 0.796875], "13": ["vertical_and_slash", 512, 2048, 0.7265625], "14": ["vertical_and_slash", 512, 2048, 0.9042388796806335], "15": ["vertical_and_slash", 512, 2048, 0.8820527195930481], "16": ["vertical_and_slash", 512, 2048, 0.9611795544624329], "17": ["vertical_and_slash", 512, 2048, 0.9778792262077332], "18": ["vertical_and_slash", 512, 2048, 0.9646311402320862], "19": ["vertical_and_slash", 512, 2048, 0.9296875], "20": ["vertical_and_slash", 512, 2048, 0.89453125], "21": ["vertical_and_slash", 512, 2048, 0.98046875], "22": ["vertical_and_slash", 512, 2048, 0.94921875], "23": ["vertical_and_slash", 512, 2048, 0.890625], "24": ["vertical_and_slash", 512, 2048, 0.9629708528518677], "25": ["vertical_and_slash", 512, 2048, 0.9756439328193665], "26": ["vertical_and_slash", 512, 2048, 0.9903070330619812], "27": ["vertical_and_slash", 512, 2048, 0.98046875], "28": ["vertical_and_slash", 512, 2048, 0.9740398526191711], "29": ["vertical_and_slash", 512, 2048, 0.9609375], "30": ["vertical_and_slash", 512, 2048, 0.9582708477973938], "31": ["vertical_and_slash", 512, 2048, 0.97265625]}, {"0": ["vertical_and_slash", 512, 2048, 0.8359375], "1": ["vertical_and_slash", 512, 2048, 0.83203125], "2": ["vertical_and_slash", 512, 2048, 0.6875], "3": ["vertical_and_slash", 512, 2048, 0.953125], "4": ["vertical_and_slash", 512, 2048, 0.9807296991348267], "5": ["vertical_and_slash", 512, 2048, 0.9786800742149353], "6": ["vertical_and_slash", 512, 2048, 0.9897298216819763], "7": ["vertical_and_slash", 512, 2048, 0.9931173324584961], "8": ["vertical_and_slash", 512, 2048, 0.9886856079101562], "9": ["vertical_and_slash", 512, 2048, 0.96484375], "10": ["vertical_and_slash", 512, 2048, 0.98046875], "11": ["vertical_and_slash", 512, 2048, 0.85546875], "12": ["vertical_and_slash", 512, 2048, 0.859375], "13": ["vertical_and_slash", 512, 2048, 0.931843638420105], "14": ["vertical_and_slash", 512, 2048, 0.8984375], "15": ["vertical_and_slash", 512, 2048, 0.9140625], "16": ["vertical_and_slash", 512, 2048, 0.9895663261413574], "17": ["vertical_and_slash", 512, 2048, 0.9883778691291809], "18": ["vertical_and_slash", 512, 2048, 0.9893786311149597], "19": ["vertical_and_slash", 512, 2048, 0.9857285618782043], "20": ["vertical_and_slash", 512, 2048, 0.9703198075294495], "21": ["vertical_and_slash", 512, 2048, 0.9922442436218262], "22": ["vertical_and_slash", 512, 2048, 0.9701283574104309], "23": ["vertical_and_slash", 512, 2048, 0.8291676044464111], "24": ["vertical_and_slash", 512, 2048, 0.9685841798782349], "25": ["vertical_and_slash", 512, 2048, 0.88671875], "26": ["vertical_and_slash", 512, 2048, 0.983255922794342], "27": ["vertical_and_slash", 512, 2048, 0.94140625], "28": ["vertical_and_slash", 512, 2048, 0.9961317181587219], "29": ["vertical_and_slash", 512, 2048, 0.99609375], "30": ["vertical_and_slash", 512, 2048, 0.9921875], "31": ["vertical_and_slash", 512, 2048, 0.9888145923614502]}, {"0": ["vertical_and_slash", 512, 2048, 0.99609375], "1": ["vertical_and_slash", 512, 2048, 0.9375], "2": ["vertical_and_slash", 512, 2048, 0.8359375], "3": ["vertical_and_slash", 512, 2048, 0.7421875], "4": ["vertical_and_slash", 512, 2048, 0.9954570531845093], "5": ["vertical_and_slash", 512, 2048, 0.9949510097503662], "6": ["vertical_and_slash", 512, 2048, 0.9883001446723938], "7": ["vertical_and_slash", 512, 2048, 0.9916946291923523], "8": ["vertical_and_slash", 512, 2048, 0.93359375], "9": ["vertical_and_slash", 512, 2048, 0.9987804293632507], "10": ["vertical_and_slash", 512, 2048, 0.998569130897522], "11": ["vertical_and_slash", 512, 2048, 0.9967666268348694], "12": ["vertical_and_slash", 512, 2048, 0.9984410405158997], "13": ["vertical_and_slash", 512, 2048, 0.9972114562988281], "14": ["vertical_and_slash", 512, 2048, 0.9971626996994019], "15": ["vertical_and_slash", 512, 2048, 1.0], "16": ["vertical_and_slash", 512, 2048, 0.99609375], "17": ["vertical_and_slash", 512, 2048, 0.9962896704673767], "18": ["vertical_and_slash", 512, 2048, 0.9771263599395752], "19": ["vertical_and_slash", 512, 2048, 0.9975184202194214], "20": ["vertical_and_slash", 512, 2048, 0.9886354207992554], "21": ["vertical_and_slash", 512, 2048, 0.93359375], "22": ["vertical_and_slash", 512, 2048, 0.9530031681060791], "23": ["vertical_and_slash", 512, 2048, 0.9577071666717529], "24": ["vertical_and_slash", 512, 2048, 0.9946836233139038], "25": ["vertical_and_slash", 512, 2048, 0.98828125], "26": ["vertical_and_slash", 512, 2048, 0.9938666820526123], "27": ["vertical_and_slash", 512, 2048, 0.9809824824333191], "28": ["vertical_and_slash", 512, 2048, 0.9874097108840942], "29": ["vertical_and_slash", 512, 2048, 0.9902986884117126], "30": ["vertical_and_slash", 512, 2048, 0.9763473272323608], "31": ["vertical_and_slash", 512, 2048, 0.9897515773773193]}, {"0": ["vertical_and_slash", 512, 2048, 0.9748806953430176], "1": ["vertical_and_slash", 512, 2048, 0.9462656378746033], "2": ["vertical_and_slash", 512, 2048, 0.9676709175109863], "3": ["vertical_and_slash", 512, 2048, 0.9766073226928711], "4": ["vertical_and_slash", 512, 2048, 0.94921875], "5": ["vertical_and_slash", 512, 2048, 0.98046875], "6": ["vertical_and_slash", 512, 2048, 0.9683337211608887], "7": ["vertical_and_slash", 512, 2048, 0.9826856851577759], "8": ["vertical_and_slash", 512, 2048, 0.9921875], "9": ["vertical_and_slash", 512, 2048, 0.98046875], "10": ["vertical_and_slash", 512, 2048, 0.99609375], "11": ["vertical_and_slash", 512, 2048, 0.984375], "12": ["vertical_and_slash", 512, 2048, 0.8828125], "13": ["vertical_and_slash", 512, 2048, 0.8072444200515747], "14": ["vertical_and_slash", 512, 2048, 0.8515625], "15": ["vertical_and_slash", 512, 2048, 0.84765625], "16": ["vertical_and_slash", 512, 2048, 0.9871454834938049], "17": ["vertical_and_slash", 512, 2048, 0.9812307357788086], "18": ["vertical_and_slash", 512, 2048, 0.9256724715232849], "19": ["vertical_and_slash", 512, 2048, 0.9851175546646118], "20": ["vertical_and_slash", 512, 2048, 0.9745984673500061], "21": ["vertical_and_slash", 512, 2048, 0.9816620945930481], "22": ["vertical_and_slash", 512, 2048, 0.9899131655693054], "23": ["vertical_and_slash", 512, 2048, 0.99609375], "24": ["vertical_and_slash", 512, 2048, 0.97265625], "25": ["vertical_and_slash", 512, 2048, 0.96875], "26": ["vertical_and_slash", 512, 2048, 0.953125], "27": ["vertical_and_slash", 512, 2048, 0.9898616671562195], "28": ["vertical_and_slash", 512, 2048, 0.9370988607406616], "29": ["vertical_and_slash", 512, 2048, 0.9257959723472595], "30": ["vertical_and_slash", 512, 2048, 0.90234375], "31": ["vertical_and_slash", 512, 2048, 0.9679746627807617]}, {"0": ["vertical_and_slash", 512, 2048, 0.96875], "1": ["vertical_and_slash", 512, 2048, 0.98828125], "2": ["vertical_and_slash", 512, 2048, 0.98046875], "3": ["vertical_and_slash", 512, 2048, 0.98046875], "4": ["vertical_and_slash", 512, 2048, 0.9508832097053528], "5": ["vertical_and_slash", 512, 2048, 0.9683154225349426], "6": ["vertical_and_slash", 512, 2048, 0.9689172506332397], "7": ["vertical_and_slash", 512, 2048, 0.9453125], "8": ["vertical_and_slash", 512, 2048, 0.9961307644844055], "9": ["vertical_and_slash", 512, 2048, 0.99609375], "10": ["vertical_and_slash", 512, 2048, 0.993029773235321], "11": ["vertical_and_slash", 512, 2048, 0.9973375201225281], "12": ["vertical_and_slash", 512, 2048, 0.9874537587165833], "13": ["vertical_and_slash", 512, 2048, 0.7299361824989319], "14": ["vertical_and_slash", 512, 2048, 0.8232833743095398], "15": ["vertical_and_slash", 512, 2048, 0.85546875], "16": ["vertical_and_slash", 512, 2048, 0.9973604083061218], "17": ["vertical_and_slash", 512, 2048, 0.9963013529777527], "18": ["vertical_and_slash", 512, 2048, 0.9808886647224426], "19": ["vertical_and_slash", 512, 2048, 0.9972676038742065], "20": ["vertical_and_slash", 512, 2048, 0.977578341960907], "21": ["vertical_and_slash", 512, 2048, 0.9749562740325928], "22": ["vertical_and_slash", 512, 2048, 0.9779685735702515], "23": ["vertical_and_slash", 512, 2048, 0.9976317882537842], "24": ["vertical_and_slash", 512, 2048, 0.9648603796958923], "25": ["vertical_and_slash", 512, 2048, 0.9690384268760681], "26": ["vertical_and_slash", 512, 2048, 0.999101459980011], "27": ["vertical_and_slash", 512, 2048, 0.9886281490325928], "28": ["vertical_and_slash", 512, 2048, 0.9974322319030762], "29": ["vertical_and_slash", 512, 2048, 0.9969301819801331], "30": ["vertical_and_slash", 512, 2048, 0.9982666373252869], "31": ["vertical_and_slash", 512, 2048, 0.9982587695121765]}, {"0": ["vertical_and_slash", 512, 2048, 0.9921875], "1": ["vertical_and_slash", 512, 2048, 0.9939185976982117], "2": ["vertical_and_slash", 512, 2048, 0.98828125], "3": ["vertical_and_slash", 512, 2048, 0.9969795942306519], "4": ["vertical_and_slash", 512, 2048, 0.9931398034095764], "5": ["vertical_and_slash", 512, 2048, 0.9898437261581421], "6": ["vertical_and_slash", 512, 2048, 0.99609375], "7": ["vertical_and_slash", 512, 2048, 0.9928154945373535], "8": ["vertical_and_slash", 512, 2048, 0.9874842166900635], "9": ["vertical_and_slash", 512, 2048, 0.9839801788330078], "10": ["vertical_and_slash", 512, 2048, 0.9848899841308594], "11": ["vertical_and_slash", 512, 2048, 0.9852920174598694], "12": ["vertical_and_slash", 512, 2048, 0.984375], "13": ["vertical_and_slash", 512, 2048, 0.98828125], "14": ["vertical_and_slash", 512, 2048, 0.8755365610122681], "15": ["vertical_and_slash", 512, 2048, 0.9921875], "16": ["vertical_and_slash", 512, 2048, 0.9819779396057129], "17": ["vertical_and_slash", 512, 2048, 0.9832032322883606], "18": ["vertical_and_slash", 512, 2048, 0.983447253704071], "19": ["vertical_and_slash", 512, 2048, 0.9584761261940002], "20": ["vertical_and_slash", 512, 2048, 0.9834704399108887], "21": ["vertical_and_slash", 512, 2048, 0.9791264533996582], "22": ["vertical_and_slash", 512, 2048, 0.9806039929389954], "23": ["vertical_and_slash", 512, 2048, 0.9781394600868225], "24": ["vertical_and_slash", 512, 2048, 0.9544171094894409], "25": ["vertical_and_slash", 512, 2048, 0.9811713695526123], "26": ["vertical_and_slash", 512, 2048, 0.96484375], "27": ["vertical_and_slash", 512, 2048, 0.96875], "28": ["vertical_and_slash", 512, 2048, 0.974831759929657], "29": ["vertical_and_slash", 512, 2048, 0.9761193990707397], "30": ["vertical_and_slash", 512, 2048, 0.9920510649681091], "31": ["vertical_and_slash", 512, 2048, 0.9924405813217163]}, {"0": ["vertical_and_slash", 512, 2048, 0.984471321105957], "1": ["vertical_and_slash", 512, 2048, 0.9659731388092041], "2": ["vertical_and_slash", 512, 2048, 0.99066561460495], "3": ["vertical_and_slash", 512, 2048, 0.9594177007675171], "4": ["vertical_and_slash", 512, 2048, 0.9888981580734253], "5": ["vertical_and_slash", 512, 2048, 0.980528712272644], "6": ["vertical_and_slash", 512, 2048, 0.9838561415672302], "7": ["vertical_and_slash", 512, 2048, 0.9883071184158325], "8": ["vertical_and_slash", 512, 2048, 0.9888157248497009], "9": ["vertical_and_slash", 512, 2048, 0.9923218488693237], "10": ["vertical_and_slash", 512, 2048, 0.98828125], "11": ["vertical_and_slash", 512, 2048, 0.99609375], "12": ["vertical_and_slash", 512, 2048, 0.9845908880233765], "13": ["vertical_and_slash", 512, 2048, 0.985883891582489], "14": ["vertical_and_slash", 512, 2048, 0.9861627817153931], "15": ["vertical_and_slash", 512, 2048, 0.9870601296424866], "16": ["vertical_and_slash", 512, 2048, 0.98828125], "17": ["vertical_and_slash", 512, 2048, 0.9962552785873413], "18": ["vertical_and_slash", 512, 2048, 0.9941535592079163], "19": ["vertical_and_slash", 512, 2048, 0.9947088360786438], "20": ["vertical_and_slash", 512, 2048, 0.9986183643341064], "21": ["vertical_and_slash", 512, 2048, 0.949575662612915], "22": ["vertical_and_slash", 512, 2048, 0.9973949193954468], "23": ["vertical_and_slash", 512, 2048, 0.9980455636978149], "24": ["vertical_and_slash", 512, 2048, 0.9974744319915771], "25": ["vertical_and_slash", 512, 2048, 0.99609375], "26": ["vertical_and_slash", 512, 2048, 0.9894559383392334], "27": ["vertical_and_slash", 512, 2048, 0.98046875], "28": ["vertical_and_slash", 512, 2048, 0.980691134929657], "29": ["vertical_and_slash", 512, 2048, 0.9799355268478394], "30": ["vertical_and_slash", 512, 2048, 0.985927402973175], "31": ["vertical_and_slash", 512, 2048, 0.9888966083526611]}, {"0": ["vertical_and_slash", 512, 2048, 0.6953125], "1": ["vertical_and_slash", 512, 2048, 0.97265625], "2": ["vertical_and_slash", 512, 2048, 0.625], "3": ["vertical_and_slash", 512, 2048, 0.875], "4": ["vertical_and_slash", 512, 2048, 0.9609375], "5": ["vertical_and_slash", 512, 2048, 0.96875], "6": ["vertical_and_slash", 512, 2048, 0.9836552143096924], "7": ["vertical_and_slash", 512, 2048, 0.9845009446144104], "8": ["vertical_and_slash", 512, 2048, 0.8828125], "9": ["vertical_and_slash", 512, 2048, 0.971139132976532], "10": ["vertical_and_slash", 512, 2048, 0.9866796731948853], "11": ["vertical_and_slash", 512, 2048, 0.95703125], "12": ["vertical_and_slash", 512, 2048, 0.9799742102622986], "13": ["vertical_and_slash", 512, 2048, 0.9558401703834534], "14": ["vertical_and_slash", 512, 2048, 0.983681321144104], "15": ["vertical_and_slash", 512, 2048, 0.9747731685638428], "16": ["vertical_and_slash", 512, 2048, 0.9905271530151367], "17": ["vertical_and_slash", 512, 2048, 0.994288444519043], "18": ["vertical_and_slash", 512, 2048, 0.9693189263343811], "19": ["vertical_and_slash", 512, 2048, 0.9621111154556274], "20": ["vertical_and_slash", 512, 2048, 0.9551467299461365], "21": ["vertical_and_slash", 512, 2048, 0.96484375], "22": ["vertical_and_slash", 512, 2048, 0.9537814259529114], "23": ["vertical_and_slash", 512, 2048, 0.97265625], "24": ["vertical_and_slash", 512, 2048, 1.0], "25": ["vertical_and_slash", 512, 2048, 0.9974595904350281], "26": ["vertical_and_slash", 512, 2048, 0.9985501766204834], "27": ["vertical_and_slash", 512, 2048, 0.9958456754684448], "28": ["vertical_and_slash", 512, 2048, 0.9298160076141357], "29": ["vertical_and_slash", 512, 2048, 0.9677495360374451], "30": ["vertical_and_slash", 512, 2048, 0.9930184483528137], "31": ["vertical_and_slash", 512, 2048, 0.9892486333847046]}, {"0": ["vertical_and_slash", 512, 2048, 0.9966373443603516], "1": ["vertical_and_slash", 512, 2048, 0.99609375], "2": ["vertical_and_slash", 512, 2048, 0.99609375], "3": ["vertical_and_slash", 512, 2048, 0.9936943650245667], "4": ["vertical_and_slash", 512, 2048, 0.9874167442321777], "5": ["vertical_and_slash", 512, 2048, 0.9508872628211975], "6": ["vertical_and_slash", 512, 2048, 0.9765644073486328], "7": ["vertical_and_slash", 512, 2048, 0.9465764164924622], "8": ["vertical_and_slash", 512, 2048, 0.98828125], "9": ["vertical_and_slash", 512, 2048, 0.9964706897735596], "10": ["vertical_and_slash", 512, 2048, 0.9941833019256592], "11": ["vertical_and_slash", 512, 2048, 0.9840189218521118], "12": ["vertical_and_slash", 512, 2048, 0.9860485792160034], "13": ["vertical_and_slash", 512, 2048, 0.9577446579933167], "14": ["vertical_and_slash", 512, 2048, 0.9811022877693176], "15": ["vertical_and_slash", 512, 2048, 0.96484375], "16": ["vertical_and_slash", 512, 2048, 0.9848062992095947], "17": ["vertical_and_slash", 512, 2048, 0.9883707761764526], "18": ["vertical_and_slash", 512, 2048, 0.99609375], "19": ["vertical_and_slash", 512, 2048, 0.9970006942749023], "20": ["vertical_and_slash", 512, 2048, 0.9734271764755249], "21": ["vertical_and_slash", 512, 2048, 0.9353823661804199], "22": ["vertical_and_slash", 512, 2048, 0.98195481300354], "23": ["vertical_and_slash", 512, 2048, 0.9744454026222229], "24": ["vertical_and_slash", 512, 2048, 0.9329551458358765], "25": ["vertical_and_slash", 512, 2048, 0.9738813638687134], "26": ["vertical_and_slash", 512, 2048, 0.9938949942588806], "27": ["vertical_and_slash", 512, 2048, 0.9889490604400635], "28": ["vertical_and_slash", 512, 2048, 0.62109375], "29": ["vertical_and_slash", 512, 2048, 0.9941080808639526], "30": ["vertical_and_slash", 512, 2048, 0.890625], "31": ["vertical_and_slash", 512, 2048, 0.91796875]}, {"0": ["vertical_and_slash", 512, 2048, 0.96875], "1": ["vertical_and_slash", 512, 2048, 0.99609375], "2": ["vertical_and_slash", 512, 2048, 0.98046875], "3": ["vertical_and_slash", 512, 2048, 0.9453125], "4": ["vertical_and_slash", 512, 2048, 0.995571494102478], "5": ["vertical_and_slash", 512, 2048, 0.9974302053451538], "6": ["vertical_and_slash", 512, 2048, 0.980790913105011], "7": ["vertical_and_slash", 512, 2048, 0.9964948892593384], "8": ["vertical_and_slash", 512, 2048, 0.9453125], "9": ["vertical_and_slash", 512, 2048, 0.99609375], "10": ["vertical_and_slash", 512, 2048, 0.953125], "11": ["vertical_and_slash", 512, 2048, 0.9461519718170166], "12": ["vertical_and_slash", 512, 2048, 0.9942329525947571], "13": ["vertical_and_slash", 512, 2048, 0.986452043056488], "14": ["vertical_and_slash", 512, 2048, 0.9859598278999329], "15": ["vertical_and_slash", 512, 2048, 0.9809826612472534], "16": ["vertical_and_slash", 512, 2048, 0.9811246991157532], "17": ["vertical_and_slash", 512, 2048, 0.84375], "18": ["vertical_and_slash", 512, 2048, 0.8945657014846802], "19": ["vertical_and_slash", 512, 2048, 0.9630876779556274], "20": ["vertical_and_slash", 512, 2048, 0.977688193321228], "21": ["vertical_and_slash", 512, 2048, 0.96484375], "22": ["vertical_and_slash", 512, 2048, 0.9898525476455688], "23": ["vertical_and_slash", 512, 2048, 0.9673124551773071], "24": ["vertical_and_slash", 512, 2048, 0.96484375], "25": ["vertical_and_slash", 512, 2048, 0.9844169020652771], "26": ["vertical_and_slash", 512, 2048, 0.9976933598518372], "27": ["vertical_and_slash", 512, 2048, 0.9781811237335205], "28": ["vertical_and_slash", 512, 2048, 0.94921875], "29": ["vertical_and_slash", 512, 2048, 0.796875], "30": ["vertical_and_slash", 512, 2048, 0.9512074589729309], "31": ["vertical_and_slash", 512, 2048, 0.9931371808052063]}, {"0": ["vertical_and_slash", 512, 2048, 0.99609375], "1": ["vertical_and_slash", 512, 2048, 0.9921875], "2": ["vertical_and_slash", 512, 2048, 0.98828125], "3": ["vertical_and_slash", 512, 2048, 0.9943420886993408], "4": ["vertical_and_slash", 512, 2048, 0.99609375], "5": ["vertical_and_slash", 512, 2048, 0.9979257583618164], "6": ["vertical_and_slash", 512, 2048, 0.996625542640686], "7": ["vertical_and_slash", 512, 2048, 1.0], "8": ["vertical_and_slash", 512, 2048, 0.98828125], "9": ["vertical_and_slash", 512, 2048, 0.8759766221046448], "10": ["vertical_and_slash", 512, 2048, 0.93359375], "11": ["vertical_and_slash", 512, 2048, 0.9453125], "12": ["vertical_and_slash", 512, 2048, 0.99609375], "13": ["vertical_and_slash", 512, 2048, 0.9941465854644775], "14": ["vertical_and_slash", 512, 2048, 0.996710479259491], "15": ["vertical_and_slash", 512, 2048, 0.9947440028190613], "16": ["vertical_and_slash", 512, 2048, 0.9296875], "17": ["vertical_and_slash", 512, 2048, 0.9890033602714539], "18": ["vertical_and_slash", 512, 2048, 0.97265625], "19": ["vertical_and_slash", 512, 2048, 0.97265625], "20": ["vertical_and_slash", 512, 2048, 0.8076231479644775], "21": ["vertical_and_slash", 512, 2048, 0.7205219864845276], "22": ["vertical_and_slash", 512, 2048, 0.8295556306838989], "23": ["vertical_and_slash", 512, 2048, 0.828125], "24": ["vertical_and_slash", 512, 2048, 0.9983446598052979], "25": ["vertical_and_slash", 512, 2048, 0.9962927103042603], "26": ["vertical_and_slash", 512, 2048, 0.999026358127594], "27": ["vertical_and_slash", 512, 2048, 0.9989404082298279], "28": ["vertical_and_slash", 512, 2048, 0.9959333539009094], "29": ["vertical_and_slash", 512, 2048, 0.9554031491279602], "30": ["vertical_and_slash", 512, 2048, 0.9903731942176819], "31": ["vertical_and_slash", 512, 2048, 0.9875370860099792]}, {"0": ["vertical_and_slash", 512, 2048, 0.9613699316978455], "1": ["vertical_and_slash", 512, 2048, 0.9259256720542908], "2": ["vertical_and_slash", 512, 2048, 0.9598730802536011], "3": ["vertical_and_slash", 512, 2048, 0.9660230875015259], "4": ["vertical_and_slash", 512, 2048, 0.984009325504303], "5": ["vertical_and_slash", 512, 2048, 0.9557270407676697], "6": ["vertical_and_slash", 512, 2048, 0.946307897567749], "7": ["vertical_and_slash", 512, 2048, 0.9521388411521912], "8": ["vertical_and_slash", 512, 2048, 0.76171875], "9": ["vertical_and_slash", 512, 2048, 0.9380871653556824], "10": ["vertical_and_slash", 512, 2048, 0.9453238248825073], "11": ["vertical_and_slash", 512, 2048, 0.7018069624900818], "12": ["vertical_and_slash", 512, 2048, 0.9353232979774475], "13": ["vertical_and_slash", 512, 2048, 0.8671875], "14": ["vertical_and_slash", 512, 2048, 0.9630011916160583], "15": ["vertical_and_slash", 512, 2048, 0.9430934190750122], "16": ["vertical_and_slash", 512, 2048, 0.6385319828987122], "17": ["vertical_and_slash", 512, 2048, 0.84375], "18": ["vertical_and_slash", 512, 2048, 0.671875], "19": ["vertical_and_slash", 512, 2048, 0.945841908454895], "20": ["vertical_and_slash", 512, 2048, 0.9715811610221863], "21": ["vertical_and_slash", 512, 2048, 0.9713640213012695], "22": ["vertical_and_slash", 512, 2048, 0.9576144218444824], "23": ["vertical_and_slash", 512, 2048, 0.981230616569519], "24": ["vertical_and_slash", 512, 2048, 0.87109375], "25": ["vertical_and_slash", 512, 2048, 0.8203125], "26": ["vertical_and_slash", 512, 2048, 0.89453125], "27": ["vertical_and_slash", 512, 2048, 0.8966026306152344], "28": ["vertical_and_slash", 512, 2048, 0.9375], "29": ["vertical_and_slash", 512, 2048, 0.9490994215011597], "30": ["vertical_and_slash", 512, 2048, 0.9371445775032043], "31": ["vertical_and_slash", 512, 2048, 0.921875]}, {"0": ["vertical_and_slash", 512, 2048, 0.9600187540054321], "1": ["vertical_and_slash", 512, 2048, 0.966858983039856], "2": ["vertical_and_slash", 512, 2048, 0.9597631692886353], "3": ["vertical_and_slash", 512, 2048, 0.9798767566680908], "4": ["vertical_and_slash", 512, 2048, 0.9369503855705261], "5": ["vertical_and_slash", 512, 2048, 0.9862673282623291], "6": ["vertical_and_slash", 512, 2048, 0.9718878865242004], "7": ["vertical_and_slash", 512, 2048, 0.96484375], "8": ["vertical_and_slash", 512, 2048, 0.9311044216156006], "9": ["vertical_and_slash", 512, 2048, 0.9824089407920837], "10": ["vertical_and_slash", 512, 2048, 0.9688247442245483], "11": ["vertical_and_slash", 512, 2048, 0.7707270383834839], "12": ["vertical_and_slash", 512, 2048, 0.9828140735626221], "13": ["vertical_and_slash", 512, 2048, 0.8726334571838379], "14": ["vertical_and_slash", 512, 2048, 0.99779212474823], "15": ["vertical_and_slash", 512, 2048, 0.9783028960227966], "16": ["vertical_and_slash", 512, 2048, 0.87890625], "17": ["vertical_and_slash", 512, 2048, 0.91015625], "18": ["vertical_and_slash", 512, 2048, 0.9548130035400391], "19": ["vertical_and_slash", 512, 2048, 0.79296875], "20": ["vertical_and_slash", 512, 2048, 0.9271127581596375], "21": ["vertical_and_slash", 512, 2048, 0.9487717151641846], "22": ["vertical_and_slash", 512, 2048, 0.8984375], "23": ["vertical_and_slash", 512, 2048, 0.9921852350234985], "24": ["vertical_and_slash", 512, 2048, 0.9751260876655579], "25": ["vertical_and_slash", 512, 2048, 0.9867735505104065], "26": ["vertical_and_slash", 512, 2048, 0.8953668475151062], "27": ["vertical_and_slash", 512, 2048, 0.925642728805542], "28": ["vertical_and_slash", 512, 2048, 0.96875], "29": ["vertical_and_slash", 512, 2048, 0.89453125], "30": ["vertical_and_slash", 512, 2048, 0.921875], "31": ["vertical_and_slash", 512, 2048, 0.87109375]}]
diff --git a/minference/configs/Phi-3-mini-4k-instruct-LongRoPE-128k.json b/minference/configs/Phi-3-mini-4k-instruct-LongRoPE-128k.json
new file mode 100644
index 00000000..836d9efd
--- /dev/null
+++ b/minference/configs/Phi-3-mini-4k-instruct-LongRoPE-128k.json
@@ -0,0 +1,6210 @@
+[
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.3332791030406952
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4416683614253998
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4923180937767029
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.3880477547645569
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.38015398383140564
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.34974828362464905
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4002125859260559
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8959001898765564
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.44538143277168274
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.364785760641098
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4276016652584076
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.3688332438468933
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.46328139305114746
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.3348214626312256
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.392171710729599
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.34772568941116333
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4066217541694641
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.360747754573822
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.34830111265182495
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.47614070773124695
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.47739607095718384
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.376796156167984
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5052061080932617
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7984429001808167
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4089720547199249
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.512876033782959
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.45735129714012146
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8220791220664978
+ ],
+ "28": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9416881203651428
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4253023564815521
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6658170819282532
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5853910446166992
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5987299084663391
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5018280148506165
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4191092550754547
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5811007618904114
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5408463478088379
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.47298333048820496
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.786112368106842
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6012060642242432
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5791159272193909
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9051483869552612
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.58222496509552
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.48059725761413574
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7023431062698364
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.43349939584732056
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5206228494644165
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6523349285125732
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.605652391910553
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4064832925796509
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5884730219841003
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8767980337142944
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9045116305351257
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7437348365783691
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.48772233724594116
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.46409618854522705
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.49005016684532166
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6720733046531677
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9864497184753418
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8722768425941467
+ ],
+ "28": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4877939820289612
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.575534462928772
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.42707663774490356
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6121442317962646
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7372177839279175
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7894041538238525
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5279066562652588
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.4761664569377899
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4725611209869385
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8257285952568054
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5990859866142273
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6145595908164978
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7928207516670227
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.707308292388916
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7120367884635925
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.687991201877594
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.673988401889801
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8554307222366333
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.755358874797821
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7254285216331482
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7025570273399353
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9539948105812073
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6157228946685791
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.733711838722229
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5779848694801331
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6323117017745972
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5607789754867554
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6604608297348022
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8311918377876282
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5358595848083496
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8005751967430115
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5391202569007874
+ ],
+ "28": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7308750152587891
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6740477085113525
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5843774676322937
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9640750885009766
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9977744221687317
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "5": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.997894287109375
+ ],
+ "6": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9970235824584961
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9453125
+ ],
+ "9": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9917697310447693
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9911014437675476
+ ],
+ "11": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9628551602363586
+ ],
+ "12": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9096153378486633
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9951955080032349
+ ],
+ "14": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9851846098899841
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.996168315410614
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.96484375
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9974663853645325
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.995905339717865
+ ],
+ "19": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.998296320438385
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9968391060829163
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9973934888839722
+ ],
+ "22": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.998639702796936
+ ],
+ "23": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9961214661598206
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.97265625
+ ],
+ "25": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9963303804397583
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.96484375
+ ],
+ "27": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "28": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9944915175437927
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9453125
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7727152705192566
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9912976622581482
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9765625
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9993456602096558
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 1.0000362396240234
+ ],
+ "3": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9966914057731628
+ ],
+ "4": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.8796641826629639
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "6": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9982466697692871
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.96875
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9937134385108948
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9967017769813538
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9959616661071777
+ ],
+ "14": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9792157411575317
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9846665859222412
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9296243190765381
+ ],
+ "18": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9900624752044678
+ ],
+ "19": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "20": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9983267188072205
+ ],
+ "21": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9900224804878235
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.997448205947876
+ ],
+ "24": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9974275827407837
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9921515583992004
+ ],
+ "26": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9961837530136108
+ ],
+ "27": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.99609375
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9986920952796936
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9839296936988831
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9926077127456665
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.99609375
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9920685291290283
+ ],
+ "2": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9854402542114258
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9949761033058167
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8777214288711548
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9941083192825317
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9981333613395691
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9964705109596252
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9998161196708679
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9989191293716431
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9850229620933533
+ ],
+ "11": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9929664731025696
+ ],
+ "12": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9970735311508179
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9854613542556763
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.998539924621582
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9988572597503662
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9937215447425842
+ ],
+ "20": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9997386336326599
+ ],
+ "21": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9956527352333069
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "23": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9980509877204895
+ ],
+ "24": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9594733119010925
+ ],
+ "25": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9973872900009155
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8722200989723206
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9793213605880737
+ ],
+ "29": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9947999119758606
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.999260425567627
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.991969883441925
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9714643955230713
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9889455437660217
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9993591904640198
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "4": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9979987740516663
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9925215244293213
+ ],
+ "6": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9974944591522217
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.733367919921875
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9970583319664001
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.997306227684021
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.99609375
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9741608500480652
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9867183566093445
+ ],
+ "15": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9974902868270874
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8923004865646362
+ ],
+ "17": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9976084232330322
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9930179119110107
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9756211638450623
+ ],
+ "20": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9988991022109985
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9600447416305542
+ ],
+ "22": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9966569542884827
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9705502986907959
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.996631383895874
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7431671023368835
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9833155274391174
+ ],
+ "27": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.995357096195221
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9988372921943665
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6221311688423157
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.962378740310669
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8013869524002075
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9569835662841797
+ ],
+ "2": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9765567779541016
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9659812450408936
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.893607497215271
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9978155493736267
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9035964608192444
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9964177012443542
+ ],
+ "8": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9885393381118774
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9989112019538879
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9980699419975281
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9978247284889221
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9864377379417419
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9934346675872803
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8716491460800171
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9965083003044128
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9953898191452026
+ ],
+ "17": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.906114399433136
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9365297555923462
+ ],
+ "19": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9918379783630371
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.998047411441803
+ ],
+ "21": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9964088797569275
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9905833601951599
+ ],
+ "23": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9906750917434692
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "25": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9951276183128357
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.99609375
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9964181780815125
+ ],
+ "28": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9454283118247986
+ ],
+ "29": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.98893141746521
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9951130747795105
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.997049868106842
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.706658124923706
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9980204701423645
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6844134330749512
+ ],
+ "3": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9990592002868652
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8842496871948242
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9998780488967896
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9981557130813599
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9960853457450867
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9998636841773987
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9711626172065735
+ ],
+ "10": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9983590245246887
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9998638033866882
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8798794150352478
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9990912079811096
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9776530265808105
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9992070198059082
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.999956488609314
+ ],
+ "17": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9974350929260254
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9929025173187256
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9999449253082275
+ ],
+ "20": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9973562359809875
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9999447464942932
+ ],
+ "22": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9998283982276917
+ ],
+ "23": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9996301531791687
+ ],
+ "24": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.99957674741745
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8067362904548645
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9952501654624939
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.987522304058075
+ ],
+ "28": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9994516968727112
+ ],
+ "29": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9991733431816101
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9994316697120667
+ ],
+ "31": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9992942214012146
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9960434436798096
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9954621195793152
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9846372604370117
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.866407036781311
+ ],
+ "4": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9988003373146057
+ ],
+ "5": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9943948984146118
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9850127696990967
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9895413517951965
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8989573121070862
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9728281497955322
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9660746455192566
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9935301542282104
+ ],
+ "12": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9088181257247925
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9997004270553589
+ ],
+ "14": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9992449879646301
+ ],
+ "15": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9945877194404602
+ ],
+ "16": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9919179677963257
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9887120127677917
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8854116797447205
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9977339506149292
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9949434995651245
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9963594675064087
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8057686686515808
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7062434554100037
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.714179515838623
+ ],
+ "25": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9638258814811707
+ ],
+ "26": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9974403381347656
+ ],
+ "27": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9426991939544678
+ ],
+ "28": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9981192946434021
+ ],
+ "29": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.997894287109375
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9979783892631531
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9956291317939758
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9942532777786255
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9382870197296143
+ ],
+ "2": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9994586706161499
+ ],
+ "3": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9935740232467651
+ ],
+ "4": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9681561589241028
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7282125949859619
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9943098425865173
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6868635416030884
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9987292885780334
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8973380923271179
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.995192289352417
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6858950257301331
+ ],
+ "12": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9975180625915527
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9925879836082458
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9210449457168579
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9997319579124451
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9917876720428467
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6827043890953064
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9853598475456238
+ ],
+ "19": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9982296824455261
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9968504905700684
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9502132534980774
+ ],
+ "22": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9775718450546265
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7796767354011536
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.76953125
+ ],
+ "25": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9922406673431396
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9383352994918823
+ ],
+ "27": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8227579593658447
+ ],
+ "28": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9977031946182251
+ ],
+ "29": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9920535087585449
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9953324198722839
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6449344754219055
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.652643620967865
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9490343928337097
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.6628352403640747
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9904823899269104
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7459101676940918
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8223817348480225
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8061914443969727
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9979249238967896
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.956091582775116
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9958360195159912
+ ],
+ "10": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.6650176644325256
+ ],
+ "11": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.946822464466095
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8161380887031555
+ ],
+ "13": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9891030192375183
+ ],
+ "14": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9935292601585388
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7558044195175171
+ ],
+ "16": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9654762744903564
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.57756507396698
+ ],
+ "18": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9946773648262024
+ ],
+ "19": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.6871338486671448
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9582348465919495
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8474758267402649
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9841222763061523
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9883868098258972
+ ],
+ "24": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8085432052612305
+ ],
+ "25": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8961470127105713
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7980116009712219
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9630904197692871
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9312359094619751
+ ],
+ "29": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.7192952036857605
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9928464889526367
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6763450503349304
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9938258528709412
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9992395043373108
+ ],
+ "2": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9985423684120178
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5909091234207153
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9045535326004028
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5327474474906921
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7697188258171082
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9967195987701416
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7490115761756897
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9701148271560669
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9769464135169983
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9512021541595459
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.945134162902832
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9831423759460449
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5774247050285339
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.992691695690155
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9788296222686768
+ ],
+ "17": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9921766519546509
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9580382108688354
+ ],
+ "19": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8484612703323364
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9827266931533813
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6860563158988953
+ ],
+ "22": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9984534978866577
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7839770913124084
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7702468633651733
+ ],
+ "25": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9819672107696533
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7506661415100098
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8615745902061462
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9768449068069458
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9400221109390259
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.99903404712677
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.814863383769989
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9924616813659668
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7962270975112915
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8446550965309143
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.986892819404602
+ ],
+ "4": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9911617636680603
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9796133637428284
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9944987893104553
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.989579975605011
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9553632736206055
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9968169331550598
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9912428855895996
+ ],
+ "11": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9946438074111938
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.960080087184906
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9448127150535583
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.992832362651825
+ ],
+ "15": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9532816410064697
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9048861265182495
+ ],
+ "17": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.994652509689331
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9620020389556885
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6745208501815796
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5609733462333679
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8874548673629761
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6557503342628479
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.6937400102615356
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9920935034751892
+ ],
+ "25": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9974891543388367
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.771544873714447
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9795010685920715
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9977417588233948
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9876147508621216
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7052534222602844
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9702804684638977
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8113299608230591
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.75982666015625
+ ],
+ "2": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9722777009010315
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6454917788505554
+ ],
+ "4": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9591151475906372
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9245743751525879
+ ],
+ "6": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.672450602054596
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8690704703330994
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.77734375
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9322983622550964
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6070910692214966
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8282676935195923
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7981728315353394
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9669613838195801
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7259189486503601
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7247616052627563
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.927749514579773
+ ],
+ "17": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9843335747718811
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.939363956451416
+ ],
+ "19": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9057216048240662
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8851215839385986
+ ],
+ "21": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9476007223129272
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8492496013641357
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9673789143562317
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9385042190551758
+ ],
+ "25": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9418122172355652
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.890839159488678
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9094856977462769
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9846156239509583
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.926313042640686
+ ],
+ "30": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8919520974159241
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7341398596763611
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9748796820640564
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.918652355670929
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5971909165382385
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7076131105422974
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.84599369764328
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.548804521560669
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6747919917106628
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9607553482055664
+ ],
+ "8": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9529062509536743
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9745543003082275
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5900142788887024
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6382952332496643
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8348908424377441
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8871970772743225
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6412845849990845
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9772831201553345
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7201339602470398
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9578301310539246
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.97627192735672
+ ],
+ "19": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.802989661693573
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9934332370758057
+ ],
+ "21": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.988477885723114
+ ],
+ "22": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.977401614189148
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8183449506759644
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8428875803947449
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9019399285316467
+ ],
+ "26": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.850577175617218
+ ],
+ "27": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8797270655632019
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9850448369979858
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7459467649459839
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6797484159469604
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7080639004707336
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.684343695640564
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6700385808944702
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6516374945640564
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9504920840263367
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9940043687820435
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8019426465034485
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.67058926820755
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9486842751502991
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9951285719871521
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7573692202568054
+ ],
+ "10": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9824524521827698
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6431483030319214
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7401513457298279
+ ],
+ "13": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9602683186531067
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5100213289260864
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.620374321937561
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9879629015922546
+ ],
+ "17": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9683711528778076
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9918504953384399
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9663371443748474
+ ],
+ "20": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9958233833312988
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.731646716594696
+ ],
+ "22": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9967772364616394
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7126162052154541
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.73592609167099
+ ],
+ "25": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9757681488990784
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.821488082408905
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.940662145614624
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7413780689239502
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.855651319026947
+ ],
+ "30": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.957796573638916
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9116591215133667
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9933403730392456
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9730038642883301
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9607142210006714
+ ],
+ "3": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9766934514045715
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8421128988265991
+ ],
+ "5": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9946009516716003
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6585829257965088
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8131377100944519
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9548000693321228
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9604054093360901
+ ],
+ "10": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9921189546585083
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9823326468467712
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9664893746376038
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.991500735282898
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9875308275222778
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9863171577453613
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9744452238082886
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8115602731704712
+ ],
+ "18": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8509863018989563
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7890191674232483
+ ],
+ "20": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9685430526733398
+ ],
+ "21": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9879798293113708
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8662706017494202
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9373542666435242
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9943336248397827
+ ],
+ "25": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9302772283554077
+ ],
+ "26": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.977992832660675
+ ],
+ "27": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.932984471321106
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9221320152282715
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6031616926193237
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9101414680480957
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9133468270301819
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9161027669906616
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8264802694320679
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9612565636634827
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8154476881027222
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.870027482509613
+ ],
+ "5": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8858229517936707
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.732231616973877
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8734871745109558
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7596957683563232
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8638753890991211
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8574302792549133
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7212074398994446
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8646799921989441
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6482542753219604
+ ],
+ "14": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.83883136510849
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8585593104362488
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9678143858909607
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7909027934074402
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8701796531677246
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7488895654678345
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8781315088272095
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7022296190261841
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7967407703399658
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7607766389846802
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.5860172510147095
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8311766982078552
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9960868954658508
+ ],
+ "27": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9676917195320129
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9716269373893738
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7922632098197937
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9029964804649353
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7617987990379333
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7531226873397827
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9829939007759094
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9867648482322693
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7495420575141907
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8411062359809875
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9238793253898621
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8017065525054932
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.949862539768219
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7137445211410522
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8886378407478333
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6042924523353577
+ ],
+ "11": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9610161781311035
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6790781617164612
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8115764856338501
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8262994885444641
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5916704535484314
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.817267894744873
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8543776869773865
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.875899612903595
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.804548442363739
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7843059301376343
+ ],
+ "21": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9910483360290527
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7706062197685242
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8355559706687927
+ ],
+ "24": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8440632820129395
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8883750438690186
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5762335062026978
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.6402088403701782
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9510595202445984
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6832257509231567
+ ],
+ "30": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8940309882164001
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6904930472373962
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9712978005409241
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.715650200843811
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9092068076133728
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5512199401855469
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9947230815887451
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.942888617515564
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8079208135604858
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9934179186820984
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9797422885894775
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9305815100669861
+ ],
+ "10": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.988544762134552
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9259185791015625
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9055561423301697
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9950724244117737
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9936328530311584
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7676860094070435
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9986890554428101
+ ],
+ "17": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9774577021598816
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9445663094520569
+ ],
+ "19": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9051742553710938
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9963957071304321
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6996874809265137
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9648709297180176
+ ],
+ "23": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8904076814651489
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6945720314979553
+ ],
+ "25": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9790938496589661
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9644208550453186
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9994762539863586
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8030149936676025
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9580996632575989
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9973891973495483
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9492622017860413
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9596408009529114
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9903782606124878
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.985963761806488
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8823582530021667
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.78907310962677
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7436686754226685
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9900132417678833
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9172103404998779
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9941288232803345
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9992177486419678
+ ],
+ "10": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9879436492919922
+ ],
+ "11": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9889773726463318
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9961833953857422
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.992790699005127
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.879410445690155
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9506082534790039
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9580295085906982
+ ],
+ "17": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9992238879203796
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8985172510147095
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9653815031051636
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8116862177848816
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8597212433815002
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8310312032699585
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9763169288635254
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9825117588043213
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9893282651901245
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9984802007675171
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9936201572418213
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.966774582862854
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9496166110038757
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9060468077659607
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9787821769714355
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.98621666431427
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9871621131896973
+ ],
+ "2": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.998849630355835
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9951006174087524
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9684059619903564
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9988571405410767
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9973605275154114
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9547178745269775
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9439932107925415
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.761074960231781
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9970728158950806
+ ],
+ "11": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9856483340263367
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9948339462280273
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9889204502105713
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.991661548614502
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9836190938949585
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9960607290267944
+ ],
+ "17": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9184228181838989
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9532352685928345
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9791851043701172
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9993872046470642
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8049466013908386
+ ],
+ "22": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9487335681915283
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9942803382873535
+ ],
+ "24": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9795239567756653
+ ],
+ "25": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9796655774116516
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9073926210403442
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8279032707214355
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.928718090057373
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9749822616577148
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8956699371337891
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7362069487571716
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.948788583278656
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9491262435913086
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8548357486724854
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9972718954086304
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.995877742767334
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9962888360023499
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.950820803642273
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.940021276473999
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.97650146484375
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9919331669807434
+ ],
+ "10": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7328993082046509
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.992123007774353
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9887773394584656
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9808080792427063
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9803031086921692
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7681567072868347
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9663677215576172
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9978005886077881
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9956686496734619
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9876529574394226
+ ],
+ "20": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9083545207977295
+ ],
+ "21": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9825258851051331
+ ],
+ "22": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9929331541061401
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9937247037887573
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9849582314491272
+ ],
+ "25": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9303218722343445
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9582874774932861
+ ],
+ "27": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9650914669036865
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9943416714668274
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9150344133377075
+ ],
+ "30": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8997454047203064
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9907521605491638
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9935974478721619
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9944353103637695
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9993149638175964
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9968475103378296
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7440210580825806
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.99021315574646
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9938840270042419
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.93567955493927
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9877127408981323
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9873458743095398
+ ],
+ "10": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9998465776443481
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9908269047737122
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9990859031677246
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8770437836647034
+ ],
+ "14": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9989800453186035
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9958862662315369
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9739276766777039
+ ],
+ "17": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9918177723884583
+ ],
+ "18": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9817363023757935
+ ],
+ "19": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9980490207672119
+ ],
+ "20": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9854499101638794
+ ],
+ "21": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9956621527671814
+ ],
+ "22": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9646536111831665
+ ],
+ "23": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8399244546890259
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9599056243896484
+ ],
+ "25": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9969561100006104
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8741656541824341
+ ],
+ "27": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9881818890571594
+ ],
+ "28": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9986366629600525
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8203835487365723
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.916657567024231
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9909099340438843
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9891338348388672
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9982934594154358
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7269482016563416
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.97837895154953
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9985787868499756
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9378053545951843
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8923203945159912
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9933837056159973
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9831008911132812
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9890069961547852
+ ],
+ "10": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9977155923843384
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9636794328689575
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9993752837181091
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9918390512466431
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9965466856956482
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.979774534702301
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9794058799743652
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.998450517654419
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7310967445373535
+ ],
+ "19": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9953096508979797
+ ],
+ "20": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9857947826385498
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.987230658531189
+ ],
+ "22": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9985311031341553
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9923253655433655
+ ],
+ "24": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9921882152557373
+ ],
+ "25": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9417669773101807
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9951248168945312
+ ],
+ "27": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9957342743873596
+ ],
+ "28": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8214721083641052
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9924106001853943
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9996931552886963
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9912320375442505
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9739575982093811
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9948337078094482
+ ],
+ "2": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9912586808204651
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9957679510116577
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9885770678520203
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9814969301223755
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9366181492805481
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9948270320892334
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9994694590568542
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9889045357704163
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9929386973381042
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9881108999252319
+ ],
+ "12": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9796789288520813
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9889234304428101
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9873424768447876
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9901317358016968
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9893797039985657
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9819779992103577
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.989522397518158
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9819537997245789
+ ],
+ "20": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9925962686538696
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9989944696426392
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9997721314430237
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9876223802566528
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9952347874641418
+ ],
+ "25": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9843642711639404
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9960111975669861
+ ],
+ "27": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.6954624652862549
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.970451295375824
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.991379976272583
+ ],
+ "30": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8738142848014832
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9786747694015503
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.997829258441925
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.990919291973114
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9968075156211853
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9982627630233765
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9957785606384277
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.974946141242981
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.995257556438446
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8554062247276306
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9880555272102356
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9956945776939392
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9789673089981079
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.991595447063446
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9686179161071777
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9943218231201172
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9565165042877197
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9816807508468628
+ ],
+ "16": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9969928860664368
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9579240679740906
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7692553400993347
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9934751987457275
+ ],
+ "20": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.996086597442627
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9795346260070801
+ ],
+ "22": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9099371433258057
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9606084823608398
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9944002032279968
+ ],
+ "25": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9969326257705688
+ ],
+ "26": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.943459689617157
+ ],
+ "27": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9907713532447815
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9855557084083557
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.911635160446167
+ ],
+ "30": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9951326847076416
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9821126461029053
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9654005765914917
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.999093770980835
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.99101722240448
+ ],
+ "3": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9934068918228149
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9982517957687378
+ ],
+ "5": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9996331334114075
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9952266216278076
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.880059003829956
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9872965216636658
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9972522854804993
+ ],
+ "10": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9663414359092712
+ ],
+ "11": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9989503622055054
+ ],
+ "12": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9980217218399048
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9978732466697693
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.990183413028717
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9975069761276245
+ ],
+ "16": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9249386787414551
+ ],
+ "17": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9966362118721008
+ ],
+ "18": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9951748847961426
+ ],
+ "19": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9986919164657593
+ ],
+ "20": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9912869930267334
+ ],
+ "21": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9970594644546509
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.998475193977356
+ ],
+ "23": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9993215799331665
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9980448484420776
+ ],
+ "25": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9916543364524841
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.980556845664978
+ ],
+ "27": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9921435117721558
+ ],
+ "28": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9989830255508423
+ ],
+ "29": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9973907470703125
+ ],
+ "30": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9833565354347229
+ ],
+ "31": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9759599566459656
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9991001486778259
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9909220933914185
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9492173194885254
+ ],
+ "3": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9961900115013123
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8713532090187073
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9998117089271545
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.998404324054718
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9998683333396912
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9000679850578308
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9757489562034607
+ ],
+ "10": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9937180876731873
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9938128590583801
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9991152882575989
+ ],
+ "13": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9996658563613892
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9966751933097839
+ ],
+ "15": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.999762773513794
+ ],
+ "16": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9985853433609009
+ ],
+ "17": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7297303080558777
+ ],
+ "18": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9985169768333435
+ ],
+ "19": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9998067617416382
+ ],
+ "20": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8747161030769348
+ ],
+ "21": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9923343658447266
+ ],
+ "22": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9940261840820312
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9998047351837158
+ ],
+ "24": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9748029112815857
+ ],
+ "25": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9991946816444397
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8475115299224854
+ ],
+ "27": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9997408390045166
+ ],
+ "28": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9990043044090271
+ ],
+ "29": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8996012806892395
+ ],
+ "30": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9358092546463013
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9944736361503601
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8248796463012695
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7729880213737488
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7208629250526428
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7399256825447083
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7422590851783752
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.949510395526886
+ ],
+ "6": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8432893753051758
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9128398299217224
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8389910459518433
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8408676981925964
+ ],
+ "10": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8649052381515503
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9523816704750061
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7945832014083862
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8890054225921631
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.755133330821991
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9407838582992554
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.978395938873291
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.841075599193573
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7965966463088989
+ ],
+ "19": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.7598339319229126
+ ],
+ "20": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.7436662316322327
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8721699714660645
+ ],
+ "22": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.872313916683197
+ ],
+ "23": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9902216792106628
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7798812985420227
+ ],
+ "25": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9631245732307434
+ ],
+ "26": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.845567524433136
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8043644428253174
+ ],
+ "28": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9540744423866272
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9055390357971191
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9507457613945007
+ ],
+ "31": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.827296793460846
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9915655255317688
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9915629029273987
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9245554804801941
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9129937291145325
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9958124160766602
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.806128203868866
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8036627769470215
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9017021656036377
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9740359783172607
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.968455970287323
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9159662127494812
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8428217172622681
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9581736326217651
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.970429539680481
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9745091199874878
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.894728422164917
+ ],
+ "16": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9496183395385742
+ ],
+ "17": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9325363039970398
+ ],
+ "18": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9315022826194763
+ ],
+ "19": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8017199635505676
+ ],
+ "20": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9268004298210144
+ ],
+ "21": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8929623365402222
+ ],
+ "22": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8346715569496155
+ ],
+ "23": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8660512566566467
+ ],
+ "24": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9183820486068726
+ ],
+ "25": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8379555344581604
+ ],
+ "26": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.911184549331665
+ ],
+ "27": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8829504251480103
+ ],
+ "28": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9138942956924438
+ ],
+ "29": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.872784435749054
+ ],
+ "30": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9097738862037659
+ ],
+ "31": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9275451898574829
+ ]
+ }
+]
diff --git a/minference/configs/Qwen2.5_0.5B_flex_0.90.json b/minference/configs/Qwen2.5_0.5B_flex_0.90.json
new file mode 100644
index 00000000..4c80b399
--- /dev/null
+++ b/minference/configs/Qwen2.5_0.5B_flex_0.90.json
@@ -0,0 +1,364 @@
+[
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]
+ },
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]
+ }
+]
diff --git a/minference/configs/Qwen2.5_14B_Instruct_1M.json b/minference/configs/Qwen2.5_14B_Instruct_1M.json
new file mode 100644
index 00000000..0fb7a318
--- /dev/null
+++ b/minference/configs/Qwen2.5_14B_Instruct_1M.json
@@ -0,0 +1 @@
+[{"0": ["vertical_and_slash", 1000, 6096, 0.9018375873565674], "1": ["vertical_and_slash", 5096, 10192, 0.7878596186637878], "2": ["vertical_and_slash", 1000, 6096, 0.9263450503349304], "3": ["vertical_and_slash", 1000, 6096, 0.9307640194892883], "4": ["vertical_and_slash", 9192, 14288, 0.6277983784675598], "5": ["vertical_and_slash", 3048, 8144, 0.8169577121734619], "6": ["vertical_and_slash", 9192, 14288, 0.6064115166664124], "7": ["vertical_and_slash", 3048, 8144, 0.8215791583061218], "8": ["vertical_and_slash", 13244, 18340, 0.46584364771842957], "9": ["vertical_and_slash", 3048, 8144, 0.8256490230560303], "10": ["vertical_and_slash", 13244, 18340, 0.5764999389648438], "11": ["vertical_and_slash", 1000, 6096, 0.9868255853652954], "12": ["vertical_and_slash", 13244, 18340, 0.5919196009635925], "13": ["vertical_and_slash", 5096, 10192, 0.7234704494476318], "14": ["vertical_and_slash", 1000, 6096, 0.9526177644729614], "15": ["vertical_and_slash", 1000, 6096, 0.9157508015632629], "16": ["vertical_and_slash", 5096, 10192, 0.7700856328010559], "17": ["vertical_and_slash", 5096, 10192, 0.7806406617164612], "18": ["vertical_and_slash", 1000, 6096, 0.9238468408584595], "19": ["vertical_and_slash", 3048, 8144, 0.8358588218688965], "20": ["vertical_and_slash", 3048, 8144, 0.8766817450523376], "21": ["vertical_and_slash", 9192, 14288, 0.6166951060295105], "22": ["vertical_and_slash", 3048, 8144, 0.8657681941986084], "23": ["vertical_and_slash", 3048, 8144, 0.8927581310272217], "24": ["vertical_and_slash", 3048, 8144, 0.8810591697692871], "25": ["vertical_and_slash", 5096, 10192, 0.7451760768890381], "26": ["vertical_and_slash", 1000, 6096, 0.906029999256134], "27": ["vertical_and_slash", 1000, 6096, 0.9528238773345947], "28": ["vertical_and_slash", 13244, 18340, 0.599810004234314], "29": ["vertical_and_slash", 13244, 18340, 0.5487516522407532], "30": ["vertical_and_slash", 5096, 10192, 0.7330573797225952], "31": ["vertical_and_slash", 1000, 6096, 0.9265012741088867], "32": ["vertical_and_slash", 5096, 10192, 0.7356006503105164], "33": ["vertical_and_slash", 9192, 14288, 0.6968193650245667], "34": ["vertical_and_slash", 5096, 10192, 0.7402594089508057], "35": ["vertical_and_slash", 1000, 6096, 0.9530928134918213], "36": ["vertical_and_slash", 5096, 10192, 0.776784360408783], "37": ["vertical_and_slash", 5096, 10192, 0.7916855812072754], "38": ["vertical_and_slash", 1000, 6096, 0.9673947691917419], "39": ["vertical_and_slash", 9192, 14288, 0.6088058352470398]}, {"0": ["vertical_and_slash", 13244, 18340, 0.5150268077850342], "1": ["vertical_and_slash", 13244, 18340, 0.5097425580024719], "2": ["vertical_and_slash", 13244, 18340, 0.5086504817008972], "3": ["vertical_and_slash", 13244, 18340, 0.4610571265220642], "4": ["vertical_and_slash", 1000, 6096, 0.9450822472572327], "5": ["vertical_and_slash", 7596, 4196, 0.7855445742607117], "6": ["vertical_and_slash", 1000, 6096, 0.9682983160018921], "7": ["vertical_and_slash", 30, 800, 0.9995935559272766], "8": ["vertical_and_slash", 1000, 6096, 0.9883986115455627], "9": ["vertical_and_slash", 1000, 6096, 0.9990907907485962], "10": ["vertical_and_slash", 9192, 14288, 0.6363117694854736], "11": ["vertical_and_slash", 3048, 8144, 0.8724498152732849], "12": ["vertical_and_slash", 13244, 18340, 0.5760000348091125], "13": ["vertical_and_slash", 3048, 8144, 0.8185386657714844], "14": ["vertical_and_slash", 3048, 8144, 0.8035999536514282], "15": ["vertical_and_slash", 5548, 2148, 0.8558602333068848], "16": ["vertical_and_slash", 1000, 6096, 0.9993630647659302], "17": ["vertical_and_slash", 1000, 6096, 0.9980729222297668], "18": ["vertical_and_slash", 1000, 6096, 0.9123708009719849], "19": ["vertical_and_slash", 9192, 14288, 0.6062105298042297], "20": ["vertical_and_slash", 9192, 14288, 0.6334558725357056], "21": ["vertical_and_slash", 13244, 18340, 0.5699681639671326], "22": ["vertical_and_slash", 9192, 14288, 0.6320412158966064], "23": ["vertical_and_slash", 5096, 10192, 0.7509382963180542], "24": ["vertical_and_slash", 5096, 10192, 0.7321545481681824], "25": ["vertical_and_slash", 13244, 18340, 0.49753791093826294], "26": ["vertical_and_slash", 1000, 6096, 0.9609797596931458], "27": ["vertical_and_slash", 1000, 6096, 0.9980624914169312], "28": ["vertical_and_slash", 30, 800, 0.9995718002319336], "29": ["vertical_and_slash", 1000, 6096, 0.9888953566551208], "30": ["vertical_and_slash", 13244, 18340, 0.5494968891143799], "31": ["vertical_and_slash", 13244, 18340, 0.5925032496452332], "32": ["vertical_and_slash", 9192, 14288, 0.6027927398681641], "33": ["vertical_and_slash", 13244, 18340, 0.454701691865921], "34": ["vertical_and_slash", 13244, 18340, 0.4541953504085541], "35": ["vertical_and_slash", 13244, 18340, 0.518548846244812], "36": ["vertical_and_slash", 13244, 18340, 0.5190585851669312], "37": ["vertical_and_slash", 13244, 18340, 0.5445966124534607], "38": ["vertical_and_slash", 13244, 18340, 0.4791974127292633], "39": ["vertical_and_slash", 13244, 18340, 0.5071344375610352]}, {"0": ["vertical_and_slash", 1000, 6096, 0.990366518497467], "1": ["vertical_and_slash", 9192, 14288, 0.6608825922012329], "2": ["vertical_and_slash", 13244, 18340, 0.537653386592865], "3": ["vertical_and_slash", 3048, 8144, 0.8360852003097534], "4": ["vertical_and_slash", 1000, 6096, 0.9898924827575684], "5": ["vertical_and_slash", 9192, 14288, 0.621884822845459], "6": ["vertical_and_slash", 13244, 18340, 0.4904407858848572], "7": ["vertical_and_slash", 13244, 18340, 0.5056431293487549], "8": ["vertical_and_slash", 13244, 18340, 0.5329692959785461], "9": ["vertical_and_slash", 13244, 18340, 0.5387783646583557], "10": ["vertical_and_slash", 30, 800, 0.960912823677063], "11": ["vertical_and_slash", 3048, 8144, 0.8302602171897888], "12": ["vertical_and_slash", 1000, 6096, 0.9147046804428101], "13": ["vertical_and_slash", 30, 800, 0.9643442630767822], "14": ["vertical_and_slash", 1000, 6096, 0.9143072962760925], "15": ["vertical_and_slash", 1000, 6096, 0.9777354598045349], "16": ["vertical_and_slash", 30, 800, 0.9628884196281433], "17": ["vertical_and_slash", 5096, 10192, 0.7169287800788879], "18": ["vertical_and_slash", 1000, 6096, 0.9596256017684937], "19": ["vertical_and_slash", 30, 800, 0.9327686429023743], "20": ["vertical_and_slash", 1000, 6096, 0.9946181774139404], "21": ["vertical_and_slash", 5096, 10192, 0.7257689833641052], "22": ["vertical_and_slash", 3048, 8144, 0.8649487495422363], "23": ["vertical_and_slash", 5096, 10192, 0.751032292842865], "24": ["vertical_and_slash", 5096, 10192, 0.7728216052055359], "25": ["vertical_and_slash", 3048, 8144, 0.8253000378608704], "26": ["vertical_and_slash", 7596, 4196, 0.7152419686317444], "27": ["vertical_and_slash", 5096, 10192, 0.7344471216201782], "28": ["vertical_and_slash", 3048, 8144, 0.8496546149253845], "29": ["vertical_and_slash", 1000, 6096, 0.9245868921279907], "30": ["vertical_and_slash", 15744, 12344, 0.5711127519607544], "31": ["vertical_and_slash", 13244, 18340, 0.5184959769248962], "32": ["vertical_and_slash", 3048, 8144, 0.8797417283058167], "33": ["vertical_and_slash", 11692, 8292, 0.6475806832313538], "34": ["vertical_and_slash", 15744, 12344, 0.4383830726146698], "35": ["vertical_and_slash", 3048, 8144, 0.8047581911087036], "36": ["vertical_and_slash", 1000, 6096, 0.9922201037406921], "37": ["vertical_and_slash", 1000, 6096, 0.92972332239151], "38": ["vertical_and_slash", 1000, 6096, 0.9839180707931519], "39": ["vertical_and_slash", 1000, 6096, 0.9086654782295227]}, {"0": ["vertical_and_slash", 3048, 8144, 0.8991352319717407], "1": ["vertical_and_slash", 9192, 14288, 0.6092080473899841], "2": ["vertical_and_slash", 9192, 14288, 0.6249264478683472], "3": ["vertical_and_slash", 5096, 10192, 0.7502530217170715], "4": ["vertical_and_slash", 5096, 10192, 0.7992927432060242], "5": ["vertical_and_slash", 3048, 8144, 0.8347184658050537], "6": ["vertical_and_slash", 5096, 10192, 0.7923355102539062], "7": ["vertical_and_slash", 5548, 2148, 0.8533896803855896], "8": ["vertical_and_slash", 5096, 10192, 0.7782379984855652], "9": ["vertical_and_slash", 3048, 8144, 0.8715920448303223], "10": ["vertical_and_slash", 1000, 6096, 0.9884403347969055], "11": ["vertical_and_slash", 3048, 8144, 0.8903265595436096], "12": ["vertical_and_slash", 1000, 6096, 0.9965521693229675], "13": ["vertical_and_slash", 1000, 6096, 0.998411238193512], "14": ["vertical_and_slash", 1000, 6096, 0.9817443490028381], "15": ["vertical_and_slash", 3048, 8144, 0.8739714026451111], "16": ["vertical_and_slash", 9192, 14288, 0.6736558079719543], "17": ["vertical_and_slash", 13244, 18340, 0.5943540334701538], "18": ["vertical_and_slash", 11692, 8292, 0.646263062953949], "19": ["vertical_and_slash", 5096, 10192, 0.7653915882110596], "20": ["vertical_and_slash", 3048, 8144, 0.8192989230155945], "21": ["vertical_and_slash", 9192, 14288, 0.6920157074928284], "22": ["vertical_and_slash", 5096, 10192, 0.7000462412834167], "23": ["vertical_and_slash", 5096, 10192, 0.7828812003135681], "24": ["vertical_and_slash", 5096, 10192, 0.7342385053634644], "25": ["vertical_and_slash", 3048, 8144, 0.8018438220024109], "26": ["vertical_and_slash", 5096, 10192, 0.7552486062049866], "27": ["vertical_and_slash", 5096, 10192, 0.7548491358757019], "28": ["vertical_and_slash", 5096, 10192, 0.7865061163902283], "29": ["vertical_and_slash", 7596, 4196, 0.7603948712348938], "30": ["vertical_and_slash", 9192, 14288, 0.6803387999534607], "31": ["vertical_and_slash", 5096, 10192, 0.7792625427246094], "32": ["vertical_and_slash", 5096, 10192, 0.7432223558425903], "33": ["vertical_and_slash", 9192, 14288, 0.6876270174980164], "34": ["vertical_and_slash", 9192, 14288, 0.6975982785224915], "35": ["vertical_and_slash", 7596, 4196, 0.7447095513343811], "36": ["vertical_and_slash", 5548, 2148, 0.8158477544784546], "37": ["vertical_and_slash", 5548, 2148, 0.8786969184875488], "38": ["vertical_and_slash", 5548, 2148, 0.8939892053604126], "39": ["vertical_and_slash", 5096, 10192, 0.7587383389472961]}, {"0": ["vertical_and_slash", 1000, 6096, 0.981295108795166], "1": ["vertical_and_slash", 3048, 8144, 0.8451018929481506], "2": ["vertical_and_slash", 1000, 6096, 0.9367514252662659], "3": ["vertical_and_slash", 3048, 8144, 0.8582007884979248], "4": ["vertical_and_slash", 1000, 6096, 0.9324629902839661], "5": ["vertical_and_slash", 3048, 8144, 0.8004482984542847], "6": ["vertical_and_slash", 1000, 6096, 0.9305800199508667], "7": ["vertical_and_slash", 3048, 8144, 0.8491737842559814], "8": ["vertical_and_slash", 9192, 14288, 0.6909979581832886], "9": ["vertical_and_slash", 3048, 8144, 0.8827236294746399], "10": ["vertical_and_slash", 3048, 8144, 0.8520296216011047], "11": ["vertical_and_slash", 5096, 10192, 0.7804857492446899], "12": ["vertical_and_slash", 9192, 14288, 0.6057842969894409], "13": ["vertical_and_slash", 3048, 8144, 0.8864642977714539], "14": ["vertical_and_slash", 5096, 10192, 0.7542927265167236], "15": ["vertical_and_slash", 5096, 10192, 0.7964098453521729], "16": ["vertical_and_slash", 3048, 8144, 0.8656834959983826], "17": ["vertical_and_slash", 1000, 6096, 0.9584798812866211], "18": ["vertical_and_slash", 5096, 10192, 0.7804146409034729], "19": ["vertical_and_slash", 5096, 10192, 0.7777124643325806], "20": ["vertical_and_slash", 1000, 6096, 0.9467268586158752], "21": ["vertical_and_slash", 9192, 14288, 0.6990000009536743], "22": ["vertical_and_slash", 3048, 8144, 0.8394556641578674], "23": ["vertical_and_slash", 1000, 6096, 0.9044105410575867], "24": ["vertical_and_slash", 3048, 8144, 0.8972065448760986], "25": ["vertical_and_slash", 3048, 8144, 0.8993157744407654], "26": ["vertical_and_slash", 1000, 6096, 0.9031649827957153], "27": ["vertical_and_slash", 3048, 8144, 0.8498262166976929], "28": ["vertical_and_slash", 1000, 6096, 0.9266154170036316], "29": ["vertical_and_slash", 1000, 6096, 0.9532366991043091], "30": ["vertical_and_slash", 3048, 8144, 0.817142903804779], "31": ["vertical_and_slash", 3048, 8144, 0.8082360029220581], "32": ["vertical_and_slash", 3048, 8144, 0.8663895726203918], "33": ["vertical_and_slash", 1000, 6096, 0.9508757591247559], "34": ["vertical_and_slash", 3048, 8144, 0.8607431054115295], "35": ["vertical_and_slash", 3048, 8144, 0.8339462280273438], "36": ["vertical_and_slash", 3048, 8144, 0.8136036992073059], "37": ["vertical_and_slash", 3048, 8144, 0.8692225813865662], "38": ["vertical_and_slash", 3048, 8144, 0.8974177241325378], "39": ["vertical_and_slash", 3048, 8144, 0.8474618792533875]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9231444597244263], "1": ["vertical_and_slash", 1000, 6096, 0.9963938593864441], "2": ["vertical_and_slash", 1000, 6096, 0.9928590059280396], "3": ["vertical_and_slash", 3048, 8144, 0.8113066554069519], "4": ["vertical_and_slash", 1000, 6096, 0.9855546951293945], "5": ["vertical_and_slash", 1000, 6096, 0.9740644693374634], "6": ["vertical_and_slash", 1000, 6096, 0.9935879111289978], "7": ["vertical_and_slash", 1000, 6096, 0.9974786639213562], "8": ["vertical_and_slash", 1000, 6096, 0.9800047278404236], "9": ["vertical_and_slash", 1000, 6096, 0.9961841106414795], "10": ["vertical_and_slash", 30, 800, 0.9749283194541931], "11": ["vertical_and_slash", 30, 800, 0.9870263934135437], "12": ["vertical_and_slash", 1000, 6096, 0.9915913343429565], "13": ["vertical_and_slash", 1000, 6096, 0.9904038310050964], "14": ["vertical_and_slash", 30, 800, 0.9765282869338989], "15": ["vertical_and_slash", 1000, 6096, 0.9880887866020203], "16": ["vertical_and_slash", 1000, 6096, 0.992650032043457], "17": ["vertical_and_slash", 3500, 100, 0.9828685522079468], "18": ["vertical_and_slash", 1000, 6096, 0.978222131729126], "19": ["vertical_and_slash", 1000, 6096, 0.9733692407608032], "20": ["vertical_and_slash", 3500, 100, 0.9588627815246582], "21": ["vertical_and_slash", 1000, 6096, 0.9916520118713379], "22": ["vertical_and_slash", 30, 800, 0.983111560344696], "23": ["vertical_and_slash", 30, 800, 0.9838811755180359], "24": ["vertical_and_slash", 500, 700, 0.9932025671005249], "25": ["vertical_and_slash", 3048, 8144, 0.8006608486175537], "26": ["vertical_and_slash", 1000, 6096, 0.9890327453613281], "27": ["vertical_and_slash", 1000, 6096, 0.9970406889915466], "28": ["vertical_and_slash", 1000, 6096, 0.9906882643699646], "29": ["vertical_and_slash", 1000, 6096, 0.9338343739509583], "30": ["vertical_and_slash", 1000, 6096, 0.9383223652839661], "31": ["vertical_and_slash", 1000, 6096, 0.9205225110054016], "32": ["vertical_and_slash", 1000, 6096, 0.9476842880249023], "33": ["vertical_and_slash", 1000, 6096, 0.9877256155014038], "34": ["vertical_and_slash", 1000, 6096, 0.9942063689231873], "35": ["vertical_and_slash", 30, 800, 0.9843602180480957], "36": ["vertical_and_slash", 1000, 6096, 0.9770064353942871], "37": ["vertical_and_slash", 1000, 6096, 0.9875185489654541], "38": ["vertical_and_slash", 5096, 10192, 0.7903820872306824], "39": ["vertical_and_slash", 1000, 6096, 0.9458699226379395]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9596244692802429], "1": ["vertical_and_slash", 1000, 6096, 0.9850636124610901], "2": ["vertical_and_slash", 1000, 6096, 0.9702422022819519], "3": ["vertical_and_slash", 1000, 6096, 0.9212018847465515], "4": ["vertical_and_slash", 1000, 6096, 0.9595897197723389], "5": ["vertical_and_slash", 3048, 8144, 0.8073392510414124], "6": ["vertical_and_slash", 1000, 6096, 0.9670282602310181], "7": ["vertical_and_slash", 5096, 10192, 0.7541435360908508], "8": ["vertical_and_slash", 9192, 14288, 0.6708145141601562], "9": ["vertical_and_slash", 3048, 8144, 0.8442608714103699], "10": ["vertical_and_slash", 1000, 6096, 0.979655921459198], "11": ["vertical_and_slash", 1000, 6096, 0.9757721424102783], "12": ["vertical_and_slash", 1000, 6096, 0.9864847660064697], "13": ["vertical_and_slash", 1000, 6096, 0.9509450197219849], "14": ["vertical_and_slash", 1000, 6096, 0.9703799486160278], "15": ["vertical_and_slash", 1000, 6096, 0.991069495677948], "16": ["vertical_and_slash", 100, 800, 1.0], "17": ["vertical_and_slash", 3500, 100, 0.925412118434906], "18": ["vertical_and_slash", 1000, 6096, 0.9921606183052063], "19": ["vertical_and_slash", 100, 800, 1.0], "20": ["vertical_and_slash", 1000, 6096, 0.9603196978569031], "21": ["vertical_and_slash", 1000, 6096, 0.9967119693756104], "22": ["vertical_and_slash", 5096, 10192, 0.7992810606956482], "23": ["vertical_and_slash", 1000, 6096, 0.9834567904472351], "24": ["vertical_and_slash", 1000, 6096, 0.9716013669967651], "25": ["vertical_and_slash", 1000, 6096, 0.9792769551277161], "26": ["vertical_and_slash", 1000, 6096, 0.9958730340003967], "27": ["vertical_and_slash", 3048, 8144, 0.8667415380477905], "28": ["vertical_and_slash", 1000, 6096, 0.9593064188957214], "29": ["vertical_and_slash", 1000, 6096, 0.9945090413093567], "30": ["vertical_and_slash", 1000, 6096, 0.9878947138786316], "31": ["vertical_and_slash", 1000, 6096, 0.9829334616661072], "32": ["vertical_and_slash", 1000, 6096, 0.99587482213974], "33": ["vertical_and_slash", 1000, 6096, 0.9934815764427185], "34": ["vertical_and_slash", 1000, 6096, 0.9562831521034241], "35": ["vertical_and_slash", 1000, 6096, 0.9430533647537231], "36": ["vertical_and_slash", 1000, 6096, 0.9976186752319336], "37": ["vertical_and_slash", 1000, 6096, 0.9983856678009033], "38": ["vertical_and_slash", 3048, 8144, 0.8759037852287292], "39": ["vertical_and_slash", 1000, 6096, 0.9944774508476257]}, {"0": ["vertical_and_slash", 1000, 6096, 0.991961658000946], "1": ["vertical_and_slash", 1000, 6096, 0.9753410816192627], "2": ["vertical_and_slash", 1000, 6096, 0.9834523797035217], "3": ["vertical_and_slash", 1000, 6096, 0.9954426884651184], "4": ["vertical_and_slash", 1000, 6096, 0.9974134564399719], "5": ["vertical_and_slash", 1000, 6096, 0.9672316312789917], "6": ["vertical_and_slash", 5096, 10192, 0.7588592767715454], "7": ["vertical_and_slash", 1000, 6096, 0.9632255434989929], "8": ["vertical_and_slash", 1000, 6096, 0.930942952632904], "9": ["vertical_and_slash", 1000, 6096, 0.9956786632537842], "10": ["vertical_and_slash", 1000, 6096, 0.9898648858070374], "11": ["vertical_and_slash", 1000, 6096, 0.9539744853973389], "12": ["vertical_and_slash", 1000, 6096, 0.9462095499038696], "13": ["vertical_and_slash", 1000, 6096, 0.991100549697876], "14": ["vertical_and_slash", 3048, 8144, 0.89292973279953], "15": ["vertical_and_slash", 1000, 6096, 0.9984208345413208], "16": ["vertical_and_slash", 1000, 6096, 0.9965934157371521], "17": ["vertical_and_slash", 1000, 6096, 0.9565901160240173], "18": ["vertical_and_slash", 1000, 6096, 0.9797224402427673], "19": ["vertical_and_slash", 1000, 6096, 0.989121675491333], "20": ["vertical_and_slash", 1000, 6096, 0.9915088415145874], "21": ["vertical_and_slash", 1000, 6096, 0.9978752732276917], "22": ["vertical_and_slash", 3500, 100, 0.9967882037162781], "23": ["vertical_and_slash", 1000, 6096, 0.9928992390632629], "24": ["vertical_and_slash", 1000, 6096, 0.9969044327735901], "25": ["vertical_and_slash", 1000, 6096, 0.9995604753494263], "26": ["vertical_and_slash", 1000, 6096, 0.998979926109314], "27": ["vertical_and_slash", 1000, 6096, 0.9990330338478088], "28": ["vertical_and_slash", 100, 800, 1.0], "29": ["vertical_and_slash", 1000, 6096, 0.9944055676460266], "30": ["vertical_and_slash", 1000, 6096, 0.9940977692604065], "31": ["vertical_and_slash", 1000, 6096, 0.9875434041023254], "32": ["vertical_and_slash", 1000, 6096, 0.9825462102890015], "33": ["vertical_and_slash", 1000, 6096, 0.9688763618469238], "34": ["vertical_and_slash", 1000, 6096, 0.9669585227966309], "35": ["vertical_and_slash", 1000, 6096, 0.9861962199211121], "36": ["vertical_and_slash", 1000, 6096, 0.9463567137718201], "37": ["vertical_and_slash", 1000, 6096, 0.9912019371986389], "38": ["vertical_and_slash", 1000, 6096, 0.9946266412734985], "39": ["vertical_and_slash", 1000, 6096, 0.9696652293205261]}, {"0": ["vertical_and_slash", 30, 800, 0.9741811156272888], "1": ["vertical_and_slash", 30, 800, 0.9625585675239563], "2": ["vertical_and_slash", 30, 800, 0.979397714138031], "3": ["vertical_and_slash", 1000, 6096, 0.9287881851196289], "4": ["vertical_and_slash", 500, 700, 0.973983645439148], "5": ["vertical_and_slash", 1000, 6096, 0.9063910841941833], "6": ["vertical_and_slash", 1000, 6096, 0.9896420836448669], "7": ["vertical_and_slash", 3048, 8144, 0.8757166266441345], "8": ["vertical_and_slash", 1000, 6096, 0.9786465167999268], "9": ["vertical_and_slash", 1000, 6096, 0.9754707217216492], "10": ["vertical_and_slash", 1000, 6096, 0.9498945474624634], "11": ["vertical_and_slash", 3500, 100, 0.9845443367958069], "12": ["vertical_and_slash", 1000, 6096, 0.9394834041595459], "13": ["vertical_and_slash", 5096, 10192, 0.713840901851654], "14": ["vertical_and_slash", 1000, 6096, 0.9268712997436523], "15": ["vertical_and_slash", 1000, 6096, 0.9895875453948975], "16": ["vertical_and_slash", 1000, 6096, 0.9960319995880127], "17": ["vertical_and_slash", 5548, 2148, 0.8952298164367676], "18": ["vertical_and_slash", 1000, 6096, 0.996100127696991], "19": ["vertical_and_slash", 1000, 6096, 0.9952865839004517], "20": ["vertical_and_slash", 30, 800, 0.9824029207229614], "21": ["vertical_and_slash", 30, 800, 0.9882795810699463], "22": ["vertical_and_slash", 30, 800, 0.9683751463890076], "23": ["vertical_and_slash", 30, 800, 0.9864210486412048], "24": ["vertical_and_slash", 30, 800, 0.9651835560798645], "25": ["vertical_and_slash", 30, 800, 0.9887890815734863], "26": ["vertical_and_slash", 3048, 8144, 0.8331947922706604], "27": ["vertical_and_slash", 100, 800, 0.984375], "28": ["vertical_and_slash", 1000, 6096, 0.9391244053840637], "29": ["vertical_and_slash", 100, 800, 0.97265625], "30": ["vertical_and_slash", 3500, 100, 0.9845428466796875], "31": ["vertical_and_slash", 3500, 100, 0.9979265928268433], "32": ["vertical_and_slash", 3048, 8144, 0.838711142539978], "33": ["vertical_and_slash", 3048, 8144, 0.8753407597541809], "34": ["vertical_and_slash", 1000, 6096, 0.9702343940734863], "35": ["vertical_and_slash", 1000, 6096, 0.9688568711280823], "36": ["vertical_and_slash", 1000, 6096, 0.9924439787864685], "37": ["vertical_and_slash", 1000, 6096, 0.9959765076637268], "38": ["vertical_and_slash", 3048, 8144, 0.8976634740829468], "39": ["vertical_and_slash", 1000, 6096, 0.9936397075653076]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9411018490791321], "1": ["vertical_and_slash", 1000, 6096, 0.9236558079719543], "2": ["vertical_and_slash", 1000, 6096, 0.9963605403900146], "3": ["vertical_and_slash", 1000, 6096, 0.9447300434112549], "4": ["vertical_and_slash", 1000, 6096, 0.9898527264595032], "5": ["vertical_and_slash", 1000, 6096, 0.9866098761558533], "6": ["vertical_and_slash", 1000, 6096, 0.9683012962341309], "7": ["vertical_and_slash", 1000, 6096, 0.9433973431587219], "8": ["vertical_and_slash", 1000, 6096, 0.990349531173706], "9": ["vertical_and_slash", 1000, 6096, 0.9941514730453491], "10": ["vertical_and_slash", 3048, 8144, 0.838062584400177], "11": ["vertical_and_slash", 3048, 8144, 0.8057646155357361], "12": ["vertical_and_slash", 3048, 8144, 0.8987709283828735], "13": ["vertical_and_slash", 1000, 6096, 0.9666357040405273], "14": ["vertical_and_slash", 1000, 6096, 0.9036163091659546], "15": ["vertical_and_slash", 30, 800, 0.9766568541526794], "16": ["vertical_and_slash", 30, 800, 0.9884012937545776], "17": ["vertical_and_slash", 30, 800, 0.9689221978187561], "18": ["vertical_and_slash", 1000, 6096, 0.9241542816162109], "19": ["vertical_and_slash", 1000, 6096, 0.9793452620506287], "20": ["vertical_and_slash", 1000, 6096, 0.9986910820007324], "21": ["vertical_and_slash", 1000, 6096, 0.998926043510437], "22": ["vertical_and_slash", 1000, 6096, 0.9229971766471863], "23": ["vertical_and_slash", 1000, 6096, 0.9844131469726562], "24": ["vertical_and_slash", 1000, 6096, 0.9980815052986145], "25": ["vertical_and_slash", 3048, 8144, 0.8069590330123901], "26": ["vertical_and_slash", 1000, 6096, 0.9851098656654358], "27": ["vertical_and_slash", 11692, 8292, 0.622993528842926], "28": ["vertical_and_slash", 1000, 6096, 0.9896459579467773], "29": ["vertical_and_slash", 1000, 6096, 0.9563962817192078], "30": ["vertical_and_slash", 3500, 100, 0.9862285256385803], "31": ["vertical_and_slash", 1000, 6096, 0.980183482170105], "32": ["vertical_and_slash", 1000, 6096, 0.9923521876335144], "33": ["vertical_and_slash", 1000, 6096, 0.9417382478713989], "34": ["vertical_and_slash", 1000, 6096, 0.9960012435913086], "35": ["vertical_and_slash", 1000, 6096, 0.9829400181770325], "36": ["vertical_and_slash", 1000, 6096, 0.9293447732925415], "37": ["vertical_and_slash", 1000, 6096, 0.9797482490539551], "38": ["vertical_and_slash", 1000, 6096, 0.9816072583198547], "39": ["vertical_and_slash", 1000, 6096, 0.9459104537963867]}, {"0": ["vertical_and_slash", 30, 800, 0.9848767518997192], "1": ["vertical_and_slash", 30, 800, 0.979809045791626], "2": ["vertical_and_slash", 30, 800, 0.996708869934082], "3": ["vertical_and_slash", 30, 800, 0.9847413301467896], "4": ["vertical_and_slash", 30, 800, 0.9832382798194885], "5": ["vertical_and_slash", 1000, 6096, 0.966720461845398], "6": ["vertical_and_slash", 1000, 6096, 0.9751442670822144], "7": ["vertical_and_slash", 1000, 6096, 0.9881802797317505], "8": ["vertical_and_slash", 3048, 8144, 0.8721657395362854], "9": ["vertical_and_slash", 1000, 6096, 0.962170422077179], "10": ["vertical_and_slash", 1000, 6096, 0.9940729737281799], "11": ["vertical_and_slash", 1000, 6096, 0.9260163307189941], "12": ["vertical_and_slash", 1000, 6096, 0.9328430891036987], "13": ["vertical_and_slash", 1000, 6096, 0.9902431964874268], "14": ["vertical_and_slash", 1000, 6096, 0.9434567093849182], "15": ["vertical_and_slash", 5096, 10192, 0.7504247426986694], "16": ["vertical_and_slash", 30, 800, 0.9653999209403992], "17": ["vertical_and_slash", 1000, 6096, 0.9662322998046875], "18": ["vertical_and_slash", 1000, 6096, 0.9893771409988403], "19": ["vertical_and_slash", 30, 800, 0.9717207551002502], "20": ["vertical_and_slash", 1000, 6096, 0.910992443561554], "21": ["vertical_and_slash", 1000, 6096, 0.9672078490257263], "22": ["vertical_and_slash", 1000, 6096, 0.996392011642456], "23": ["vertical_and_slash", 1000, 6096, 0.9477424025535583], "24": ["vertical_and_slash", 1000, 6096, 0.992936372756958], "25": ["vertical_and_slash", 1000, 6096, 0.9753215312957764], "26": ["vertical_and_slash", 1000, 6096, 0.9847853779792786], "27": ["vertical_and_slash", 1000, 6096, 0.98066645860672], "28": ["vertical_and_slash", 3048, 8144, 0.86658775806427], "29": ["vertical_and_slash", 1000, 6096, 0.9826641082763672], "30": ["vertical_and_slash", 30, 800, 0.9801075458526611], "31": ["vertical_and_slash", 1000, 6096, 0.9420411586761475], "32": ["vertical_and_slash", 1000, 6096, 0.92023104429245], "33": ["vertical_and_slash", 30, 800, 0.981113076210022], "34": ["vertical_and_slash", 1000, 6096, 0.9839187860488892], "35": ["vertical_and_slash", 1000, 6096, 0.912300169467926], "36": ["vertical_and_slash", 1000, 6096, 0.9928078651428223], "37": ["vertical_and_slash", 3048, 8144, 0.8704795837402344], "38": ["vertical_and_slash", 1000, 6096, 0.9941791892051697], "39": ["vertical_and_slash", 1000, 6096, 0.9894456267356873]}, {"0": ["vertical_and_slash", 30, 800, 0.989588737487793], "1": ["vertical_and_slash", 30, 800, 0.9737234711647034], "2": ["vertical_and_slash", 30, 800, 0.9846724271774292], "3": ["vertical_and_slash", 30, 800, 0.9825773239135742], "4": ["vertical_and_slash", 30, 800, 0.9626457691192627], "5": ["vertical_and_slash", 30, 800, 0.9817877411842346], "6": ["vertical_and_slash", 30, 800, 0.9829357862472534], "7": ["vertical_and_slash", 3048, 8144, 0.8770850896835327], "8": ["vertical_and_slash", 30, 800, 0.9948512315750122], "9": ["vertical_and_slash", 3048, 8144, 0.8725793957710266], "10": ["vertical_and_slash", 30, 800, 0.9732138514518738], "11": ["vertical_and_slash", 1000, 6096, 0.9820443391799927], "12": ["vertical_and_slash", 30, 800, 0.9644232392311096], "13": ["vertical_and_slash", 1000, 6096, 0.9330013394355774], "14": ["vertical_and_slash", 9192, 14288, 0.6753383874893188], "15": ["vertical_and_slash", 30, 800, 0.9514920711517334], "16": ["vertical_and_slash", 30, 800, 0.9647454619407654], "17": ["vertical_and_slash", 30, 800, 0.9813026189804077], "18": ["vertical_and_slash", 1000, 6096, 0.971619725227356], "19": ["vertical_and_slash", 30, 800, 0.9617007970809937], "20": ["vertical_and_slash", 1000, 6096, 0.9188520908355713], "21": ["vertical_and_slash", 1000, 6096, 0.9383499622344971], "22": ["vertical_and_slash", 1000, 6096, 0.9722764492034912], "23": ["vertical_and_slash", 1000, 6096, 0.9632120728492737], "24": ["vertical_and_slash", 1000, 6096, 0.9911702871322632], "25": ["vertical_and_slash", 1000, 6096, 0.9276264905929565], "26": ["vertical_and_slash", 1000, 6096, 0.9347774386405945], "27": ["vertical_and_slash", 5096, 10192, 0.7526459693908691], "28": ["vertical_and_slash", 1000, 6096, 0.9952964186668396], "29": ["vertical_and_slash", 1000, 6096, 0.9829565286636353], "30": ["vertical_and_slash", 30, 800, 0.9941303730010986], "31": ["vertical_and_slash", 30, 800, 0.9915730357170105], "32": ["vertical_and_slash", 30, 800, 0.993859589099884], "33": ["vertical_and_slash", 30, 800, 0.9755209684371948], "34": ["vertical_and_slash", 30, 800, 0.9820301532745361], "35": ["vertical_and_slash", 1000, 6096, 0.9902023673057556], "36": ["vertical_and_slash", 1000, 6096, 0.9382041692733765], "37": ["vertical_and_slash", 3500, 100, 0.9736177325248718], "38": ["vertical_and_slash", 1000, 6096, 0.9118228554725647], "39": ["vertical_and_slash", 1000, 6096, 0.9687713384628296]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9326781630516052], "1": ["vertical_and_slash", 3048, 8144, 0.807319700717926], "2": ["vertical_and_slash", 1000, 6096, 0.9966117143630981], "3": ["vertical_and_slash", 3048, 8144, 0.8531867265701294], "4": ["vertical_and_slash", 1000, 6096, 0.9775977730751038], "5": ["vertical_and_slash", 1000, 6096, 0.990051805973053], "6": ["vertical_and_slash", 1000, 6096, 0.9987436532974243], "7": ["vertical_and_slash", 1000, 6096, 0.9958208203315735], "8": ["vertical_and_slash", 3500, 100, 0.953998327255249], "9": ["vertical_and_slash", 5096, 10192, 0.7210197448730469], "10": ["vertical_and_slash", 1000, 6096, 0.9385311007499695], "11": ["vertical_and_slash", 1000, 6096, 0.9789326190948486], "12": ["vertical_and_slash", 3048, 8144, 0.8863975405693054], "13": ["vertical_and_slash", 1000, 6096, 0.945769190788269], "14": ["vertical_and_slash", 30, 800, 0.9861490726470947], "15": ["vertical_and_slash", 30, 800, 0.9370501041412354], "16": ["vertical_and_slash", 30, 800, 0.9717501401901245], "17": ["vertical_and_slash", 30, 800, 0.9774892330169678], "18": ["vertical_and_slash", 500, 700, 0.9710538983345032], "19": ["vertical_and_slash", 500, 700, 0.9681988954544067], "20": ["vertical_and_slash", 1000, 6096, 0.9875897169113159], "21": ["vertical_and_slash", 1000, 6096, 0.9766654372215271], "22": ["vertical_and_slash", 1000, 6096, 0.9790798425674438], "23": ["vertical_and_slash", 3048, 8144, 0.8904788494110107], "24": ["vertical_and_slash", 1000, 6096, 0.972406268119812], "25": ["vertical_and_slash", 30, 800, 0.962500274181366], "26": ["vertical_and_slash", 30, 800, 0.9730808734893799], "27": ["vertical_and_slash", 3048, 8144, 0.8528904914855957], "28": ["vertical_and_slash", 30, 800, 0.9800687432289124], "29": ["vertical_and_slash", 30, 800, 0.972234308719635], "30": ["vertical_and_slash", 1000, 6096, 0.9893993139266968], "31": ["vertical_and_slash", 1000, 6096, 0.9948370456695557], "32": ["vertical_and_slash", 1000, 6096, 0.9880678057670593], "33": ["vertical_and_slash", 1000, 6096, 0.9443345069885254], "34": ["vertical_and_slash", 3048, 8144, 0.8866196870803833], "35": ["vertical_and_slash", 3048, 8144, 0.8620410561561584], "36": ["vertical_and_slash", 1000, 6096, 0.9847792983055115], "37": ["vertical_and_slash", 30, 800, 0.9723518490791321], "38": ["vertical_and_slash", 5096, 10192, 0.7818230390548706], "39": ["vertical_and_slash", 30, 800, 0.9561707377433777]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9142560362815857], "1": ["vertical_and_slash", 30, 800, 0.9706120491027832], "2": ["vertical_and_slash", 30, 800, 0.9784256219863892], "3": ["vertical_and_slash", 30, 800, 0.9734562635421753], "4": ["vertical_and_slash", 3048, 8144, 0.8942901492118835], "5": ["vertical_and_slash", 30, 800, 0.9617255926132202], "6": ["vertical_and_slash", 1000, 6096, 0.9507123231887817], "7": ["vertical_and_slash", 3048, 8144, 0.863666832447052], "8": ["vertical_and_slash", 30, 800, 0.9771642684936523], "9": ["vertical_and_slash", 30, 800, 0.9802729487419128], "10": ["vertical_and_slash", 500, 700, 0.9184444546699524], "11": ["vertical_and_slash", 30, 800, 0.9767218828201294], "12": ["vertical_and_slash", 30, 800, 0.9822949767112732], "13": ["vertical_and_slash", 30, 800, 0.976490318775177], "14": ["vertical_and_slash", 30, 800, 0.9724388718605042], "15": ["vertical_and_slash", 1000, 6096, 0.9714520573616028], "16": ["vertical_and_slash", 9192, 14288, 0.6361825466156006], "17": ["vertical_and_slash", 1000, 6096, 0.9817716479301453], "18": ["vertical_and_slash", 1000, 6096, 0.9768180847167969], "19": ["vertical_and_slash", 1000, 6096, 0.9573886394500732], "20": ["vertical_and_slash", 1000, 6096, 0.9939083456993103], "21": ["vertical_and_slash", 1000, 6096, 0.996833324432373], "22": ["vertical_and_slash", 1000, 6096, 0.9977316856384277], "23": ["vertical_and_slash", 1000, 6096, 0.9709166884422302], "24": ["vertical_and_slash", 1000, 6096, 0.990997850894928], "25": ["vertical_and_slash", 13244, 18340, 0.5217705965042114], "26": ["vertical_and_slash", 1000, 6096, 0.9873850345611572], "27": ["vertical_and_slash", 100, 800, 1.0], "28": ["vertical_and_slash", 9192, 14288, 0.6857542991638184], "29": ["vertical_and_slash", 15744, 12344, 0.5942282676696777], "30": ["vertical_and_slash", 1000, 6096, 0.9264698624610901], "31": ["vertical_and_slash", 1000, 6096, 0.9631531834602356], "32": ["vertical_and_slash", 1000, 6096, 0.9457237720489502], "33": ["vertical_and_slash", 1000, 6096, 0.9533750414848328], "34": ["vertical_and_slash", 1000, 6096, 0.953086256980896], "35": ["vertical_and_slash", 9192, 14288, 0.6891908645629883], "36": ["vertical_and_slash", 13244, 18340, 0.47972041368484497], "37": ["vertical_and_slash", 9192, 14288, 0.6461042761802673], "38": ["vertical_and_slash", 3048, 8144, 0.8099544048309326], "39": ["vertical_and_slash", 3048, 8144, 0.8640173673629761]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9735789895057678], "1": ["vertical_and_slash", 500, 700, 0.9710066318511963], "2": ["vertical_and_slash", 30, 800, 0.959937572479248], "3": ["vertical_and_slash", 30, 800, 0.9533385634422302], "4": ["vertical_and_slash", 30, 800, 0.9814746379852295], "5": ["vertical_and_slash", 3500, 100, 0.9852659106254578], "6": ["vertical_and_slash", 30, 800, 0.9844872951507568], "7": ["vertical_and_slash", 13244, 18340, 0.38921576738357544], "8": ["vertical_and_slash", 3048, 8144, 0.8466800451278687], "9": ["vertical_and_slash", 1000, 6096, 0.986076831817627], "10": ["vertical_and_slash", 1000, 6096, 0.9812390208244324], "11": ["vertical_and_slash", 13244, 18340, 0.5635663270950317], "12": ["vertical_and_slash", 1000, 6096, 0.994143009185791], "13": ["vertical_and_slash", 1000, 6096, 0.9191070795059204], "14": ["vertical_and_slash", 1000, 6096, 0.968726396560669], "15": ["vertical_and_slash", 30, 800, 0.9566224217414856], "16": ["vertical_and_slash", 30, 800, 0.9671502709388733], "17": ["vertical_and_slash", 1000, 6096, 0.9669968485832214], "18": ["vertical_and_slash", 30, 800, 0.9731153249740601], "19": ["vertical_and_slash", 1000, 6096, 0.9675655364990234], "20": ["vertical_and_slash", 1000, 6096, 0.9758727550506592], "21": ["vertical_and_slash", 3500, 100, 0.949756383895874], "22": ["vertical_and_slash", 1000, 6096, 0.9233805537223816], "23": ["vertical_and_slash", 3048, 8144, 0.8024644255638123], "24": ["vertical_and_slash", 1000, 6096, 0.9889033436775208], "25": ["vertical_and_slash", 500, 700, 0.9749848246574402], "26": ["vertical_and_slash", 30, 800, 0.9713113903999329], "27": ["vertical_and_slash", 3048, 8144, 0.850206196308136], "28": ["vertical_and_slash", 30, 800, 0.9772977232933044], "29": ["vertical_and_slash", 1000, 6096, 0.9685791730880737], "30": ["vertical_and_slash", 500, 700, 0.9800508618354797], "31": ["vertical_and_slash", 15744, 12344, 0.48130443692207336], "32": ["vertical_and_slash", 30, 800, 0.9261408448219299], "33": ["vertical_and_slash", 100, 800, 0.921875], "34": ["vertical_and_slash", 100, 800, 0.90234375], "35": ["vertical_and_slash", 1000, 6096, 0.9947608709335327], "36": ["vertical_and_slash", 1000, 6096, 0.9631083607673645], "37": ["vertical_and_slash", 3048, 8144, 0.8504776954650879], "38": ["vertical_and_slash", 3048, 8144, 0.8187342882156372], "39": ["vertical_and_slash", 1000, 6096, 0.9965180158615112]}, {"0": ["vertical_and_slash", 3048, 8144, 0.8062371611595154], "1": ["vertical_and_slash", 1000, 6096, 0.9491556286811829], "2": ["vertical_and_slash", 5096, 10192, 0.7901508212089539], "3": ["vertical_and_slash", 3048, 8144, 0.8464810848236084], "4": ["vertical_and_slash", 3048, 8144, 0.8667616248130798], "5": ["vertical_and_slash", 3500, 100, 0.9400447607040405], "6": ["vertical_and_slash", 5548, 2148, 0.8646203279495239], "7": ["vertical_and_slash", 1000, 6096, 0.9377368688583374], "8": ["vertical_and_slash", 3048, 8144, 0.8815305233001709], "9": ["vertical_and_slash", 1000, 6096, 0.9138360619544983], "10": ["vertical_and_slash", 30, 800, 0.9511128664016724], "11": ["vertical_and_slash", 30, 800, 0.9488115310668945], "12": ["vertical_and_slash", 1000, 6096, 0.9296771287918091], "13": ["vertical_and_slash", 1000, 6096, 0.9540526866912842], "14": ["vertical_and_slash", 500, 700, 0.9471203088760376], "15": ["vertical_and_slash", 1000, 6096, 0.9824201464653015], "16": ["vertical_and_slash", 1000, 6096, 0.978899359703064], "17": ["vertical_and_slash", 5548, 2148, 0.8538793921470642], "18": ["vertical_and_slash", 1000, 6096, 0.9451289772987366], "19": ["vertical_and_slash", 3048, 8144, 0.8827020525932312], "20": ["vertical_and_slash", 1000, 6096, 0.9397249221801758], "21": ["vertical_and_slash", 30, 800, 0.9468895196914673], "22": ["vertical_and_slash", 4126, 4896, 0.70733243227005], "23": ["vertical_and_slash", 30, 800, 0.9705603718757629], "24": ["vertical_and_slash", 1000, 6096, 0.9283720254898071], "25": ["vertical_and_slash", 1000, 6096, 0.9888479113578796], "26": ["vertical_and_slash", 1000, 6096, 0.9025041460990906], "27": ["vertical_and_slash", 1000, 6096, 0.9352433085441589], "28": ["vertical_and_slash", 5548, 2148, 0.821130096912384], "29": ["vertical_and_slash", 1000, 6096, 0.9921875596046448], "30": ["vertical_and_slash", 3500, 100, 0.9412122368812561], "31": ["vertical_and_slash", 1000, 6096, 0.9970604181289673], "32": ["vertical_and_slash", 3048, 8144, 0.8965034484863281], "33": ["vertical_and_slash", 13244, 18340, 0.5008286833763123], "34": ["vertical_and_slash", 1000, 6096, 0.9211915731430054], "35": ["vertical_and_slash", 1000, 6096, 0.9577488303184509], "36": ["vertical_and_slash", 3048, 8144, 0.893515408039093], "37": ["vertical_and_slash", 1000, 6096, 0.9742791056632996], "38": ["vertical_and_slash", 1000, 6096, 0.9890986084938049], "39": ["vertical_and_slash", 1000, 6096, 0.9847546815872192]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9861096739768982], "1": ["vertical_and_slash", 1000, 6096, 0.9749796390533447], "2": ["vertical_and_slash", 500, 700, 0.9929495453834534], "3": ["vertical_and_slash", 500, 700, 0.9943677186965942], "4": ["vertical_and_slash", 500, 700, 0.9941945672035217], "5": ["vertical_and_slash", 5548, 2148, 0.8918614983558655], "6": ["vertical_and_slash", 1000, 6096, 0.9808422923088074], "7": ["vertical_and_slash", 1000, 6096, 0.9785822033882141], "8": ["vertical_and_slash", 1000, 6096, 0.9401575922966003], "9": ["vertical_and_slash", 3048, 8144, 0.8788498044013977], "10": ["vertical_and_slash", 1000, 6096, 0.9909613728523254], "11": ["vertical_and_slash", 1000, 6096, 0.972564160823822], "12": ["vertical_and_slash", 9192, 14288, 0.6153343319892883], "13": ["vertical_and_slash", 1000, 6096, 0.9754350185394287], "14": ["vertical_and_slash", 1000, 6096, 0.9866433143615723], "15": ["vertical_and_slash", 3048, 8144, 0.8399285674095154], "16": ["vertical_and_slash", 1000, 6096, 0.9291568398475647], "17": ["vertical_and_slash", 30, 800, 0.967564582824707], "18": ["vertical_and_slash", 1000, 6096, 0.9781951308250427], "19": ["vertical_and_slash", 1000, 6096, 0.9488160014152527], "20": ["vertical_and_slash", 11692, 8292, 0.6895484328269958], "21": ["vertical_and_slash", 1000, 6096, 0.9623690247535706], "22": ["vertical_and_slash", 1000, 6096, 0.9049995541572571], "23": ["vertical_and_slash", 7596, 4196, 0.785365641117096], "24": ["vertical_and_slash", 1000, 6096, 0.9926794171333313], "25": ["vertical_and_slash", 1000, 6096, 0.9892585277557373], "26": ["vertical_and_slash", 1000, 6096, 0.9660714864730835], "27": ["vertical_and_slash", 1000, 6096, 0.9942393898963928], "28": ["vertical_and_slash", 30, 800, 0.9780371785163879], "29": ["vertical_and_slash", 30, 800, 0.973516583442688], "30": ["vertical_and_slash", 500, 700, 0.9823615550994873], "31": ["vertical_and_slash", 500, 700, 0.9649654030799866], "32": ["vertical_and_slash", 1000, 6096, 0.9399592876434326], "33": ["vertical_and_slash", 500, 700, 0.9971593022346497], "34": ["vertical_and_slash", 1000, 6096, 0.9068447947502136], "35": ["vertical_and_slash", 5096, 10192, 0.720664381980896], "36": ["vertical_and_slash", 1000, 6096, 0.9785795211791992], "37": ["vertical_and_slash", 1000, 6096, 0.9887612462043762], "38": ["vertical_and_slash", 1000, 6096, 0.921016275882721], "39": ["vertical_and_slash", 1000, 6096, 0.9707241058349609]}, {"0": ["vertical_and_slash", 9192, 14288, 0.6682038307189941], "1": ["vertical_and_slash", 1000, 6096, 0.9773355722427368], "2": ["vertical_and_slash", 1000, 6096, 0.9742084741592407], "3": ["vertical_and_slash", 1000, 6096, 0.9491426348686218], "4": ["vertical_and_slash", 5096, 10192, 0.7786505222320557], "5": ["vertical_and_slash", 1000, 6096, 0.9935159683227539], "6": ["vertical_and_slash", 1000, 6096, 0.9450880885124207], "7": ["vertical_and_slash", 1000, 6096, 0.9799599051475525], "8": ["vertical_and_slash", 1000, 6096, 0.9950014352798462], "9": ["vertical_and_slash", 1000, 6096, 0.9893255233764648], "10": ["vertical_and_slash", 1000, 6096, 0.9711648225784302], "11": ["vertical_and_slash", 9192, 14288, 0.6750795841217041], "12": ["vertical_and_slash", 1000, 6096, 0.9636266231536865], "13": ["vertical_and_slash", 1000, 6096, 0.9106791615486145], "14": ["vertical_and_slash", 1000, 6096, 0.960976243019104], "15": ["vertical_and_slash", 13244, 18340, 0.550165057182312], "16": ["vertical_and_slash", 1000, 6096, 0.9891451597213745], "17": ["vertical_and_slash", 1000, 6096, 0.9922628998756409], "18": ["vertical_and_slash", 1000, 6096, 0.9899148344993591], "19": ["vertical_and_slash", 1000, 6096, 0.9901177883148193], "20": ["vertical_and_slash", 1000, 6096, 0.9854280352592468], "21": ["vertical_and_slash", 3048, 8144, 0.8529897928237915], "22": ["vertical_and_slash", 1000, 6096, 0.920750081539154], "23": ["vertical_and_slash", 3048, 8144, 0.8537185192108154], "24": ["vertical_and_slash", 500, 700, 0.9805704355239868], "25": ["vertical_and_slash", 1000, 6096, 0.9620708227157593], "26": ["vertical_and_slash", 1000, 6096, 0.9443960785865784], "27": ["vertical_and_slash", 1000, 6096, 0.9915533661842346], "28": ["vertical_and_slash", 1000, 6096, 0.9478697776794434], "29": ["vertical_and_slash", 1000, 6096, 0.9636085629463196], "30": ["vertical_and_slash", 1000, 6096, 0.988511323928833], "31": ["vertical_and_slash", 1000, 6096, 0.993701159954071], "32": ["vertical_and_slash", 1000, 6096, 0.9057166576385498], "33": ["vertical_and_slash", 1000, 6096, 0.9937421083450317], "34": ["vertical_and_slash", 1000, 6096, 0.9582727551460266], "35": ["vertical_and_slash", 5096, 10192, 0.7277109026908875], "36": ["vertical_and_slash", 4196, 4896, 0.76953125], "37": ["vertical_and_slash", 1000, 6096, 0.9394615888595581], "38": ["vertical_and_slash", 100, 800, 0.97265625], "39": ["vertical_and_slash", 15744, 12344, 0.5534924268722534]}, {"0": ["vertical_and_slash", 30, 800, 0.9666914343833923], "1": ["vertical_and_slash", 30, 800, 0.9835317134857178], "2": ["vertical_and_slash", 30, 800, 0.996649444103241], "3": ["vertical_and_slash", 30, 800, 0.988285481929779], "4": ["vertical_and_slash", 1000, 6096, 0.9287636280059814], "5": ["vertical_and_slash", 1000, 6096, 0.9825299978256226], "6": ["vertical_and_slash", 11692, 8292, 0.627365231513977], "7": ["vertical_and_slash", 8292, 8992, 0.6640625], "8": ["vertical_and_slash", 1000, 6096, 0.9578100442886353], "9": ["vertical_and_slash", 3500, 100, 0.9666172862052917], "10": ["vertical_and_slash", 5096, 10192, 0.7665071487426758], "11": ["vertical_and_slash", 1000, 6096, 0.9648350477218628], "12": ["vertical_and_slash", 1000, 6096, 0.9793229103088379], "13": ["vertical_and_slash", 1000, 6096, 0.9990435838699341], "14": ["vertical_and_slash", 1000, 6096, 0.9761689901351929], "15": ["vertical_and_slash", 1000, 6096, 0.9790050983428955], "16": ["vertical_and_slash", 1000, 6096, 0.9287222623825073], "17": ["vertical_and_slash", 30, 800, 0.9846031069755554], "18": ["vertical_and_slash", 100, 800, 0.953125], "19": ["vertical_and_slash", 1000, 6096, 0.954155683517456], "20": ["vertical_and_slash", 1000, 6096, 0.9913740158081055], "21": ["vertical_and_slash", 1000, 6096, 0.902725100517273], "22": ["vertical_and_slash", 3048, 8144, 0.8796353936195374], "23": ["vertical_and_slash", 1000, 6096, 0.9359667301177979], "24": ["vertical_and_slash", 1000, 6096, 0.9666111469268799], "25": ["vertical_and_slash", 1000, 6096, 0.9388531446456909], "26": ["vertical_and_slash", 1000, 6096, 0.941346287727356], "27": ["vertical_and_slash", 3048, 8144, 0.8844871520996094], "28": ["vertical_and_slash", 3048, 8144, 0.8360744118690491], "29": ["vertical_and_slash", 3048, 8144, 0.8600526452064514], "30": ["vertical_and_slash", 1000, 6096, 0.9594017863273621], "31": ["vertical_and_slash", 1000, 6096, 0.904701292514801], "32": ["vertical_and_slash", 1000, 6096, 0.9712159037590027], "33": ["vertical_and_slash", 1000, 6096, 0.9163137674331665], "34": ["vertical_and_slash", 1000, 6096, 0.9831938743591309], "35": ["vertical_and_slash", 3048, 8144, 0.881057620048523], "36": ["vertical_and_slash", 1000, 6096, 0.972842812538147], "37": ["vertical_and_slash", 5096, 10192, 0.7613016366958618], "38": ["vertical_and_slash", 1000, 6096, 0.9242376685142517], "39": ["vertical_and_slash", 5096, 10192, 0.7733613848686218]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9372579455375671], "1": ["vertical_and_slash", 3500, 100, 0.963448703289032], "2": ["vertical_and_slash", 1000, 6096, 0.915566623210907], "3": ["vertical_and_slash", 1000, 6096, 0.9903600215911865], "4": ["vertical_and_slash", 1000, 6096, 0.9775757193565369], "5": ["vertical_and_slash", 1000, 6096, 0.9188745021820068], "6": ["vertical_and_slash", 7596, 4196, 0.7525462508201599], "7": ["vertical_and_slash", 1000, 6096, 0.9361878037452698], "8": ["vertical_and_slash", 1000, 6096, 0.9336079955101013], "9": ["vertical_and_slash", 1000, 6096, 0.9642080664634705], "10": ["vertical_and_slash", 3048, 8144, 0.818486750125885], "11": ["vertical_and_slash", 1000, 6096, 0.9533711075782776], "12": ["vertical_and_slash", 5548, 2148, 0.8555048108100891], "13": ["vertical_and_slash", 1000, 6096, 0.9076502919197083], "14": ["vertical_and_slash", 3048, 8144, 0.8762632012367249], "15": ["vertical_and_slash", 30, 800, 0.9289241433143616], "16": ["vertical_and_slash", 30, 800, 0.9480916857719421], "17": ["vertical_and_slash", 30, 800, 0.9559626579284668], "18": ["vertical_and_slash", 30, 800, 0.9267023205757141], "19": ["vertical_and_slash", 30, 800, 0.9629817008972168], "20": ["vertical_and_slash", 1000, 6096, 0.9591138362884521], "21": ["vertical_and_slash", 1000, 6096, 0.9063478708267212], "22": ["vertical_and_slash", 3500, 100, 1.0000309944152832], "23": ["vertical_and_slash", 3048, 8144, 0.8029321432113647], "24": ["vertical_and_slash", 3048, 8144, 0.8376001715660095], "25": ["vertical_and_slash", 1000, 6096, 0.9292928576469421], "26": ["vertical_and_slash", 100, 800, 0.97265625], "27": ["vertical_and_slash", 9192, 14288, 0.6681580543518066], "28": ["vertical_and_slash", 9192, 14288, 0.659645676612854], "29": ["vertical_and_slash", 9192, 14288, 0.6086839437484741], "30": ["vertical_and_slash", 1000, 6096, 0.9845035076141357], "31": ["vertical_and_slash", 1000, 6096, 0.9445158243179321], "32": ["vertical_and_slash", 1000, 6096, 0.9398161172866821], "33": ["vertical_and_slash", 9192, 14288, 0.6169046759605408], "34": ["vertical_and_slash", 5096, 10192, 0.7220165133476257], "35": ["vertical_and_slash", 1000, 6096, 0.9794515371322632], "36": ["vertical_and_slash", 1000, 6096, 0.9599097967147827], "37": ["vertical_and_slash", 1000, 6096, 0.9847788214683533], "38": ["vertical_and_slash", 9192, 14288, 0.6966208815574646], "39": ["vertical_and_slash", 1000, 6096, 0.9615061283111572]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9309116005897522], "1": ["vertical_and_slash", 30, 800, 0.9808964133262634], "2": ["vertical_and_slash", 30, 800, 0.9813137650489807], "3": ["vertical_and_slash", 1000, 6096, 0.9108445048332214], "4": ["vertical_and_slash", 30, 800, 0.9647480845451355], "5": ["vertical_and_slash", 1000, 6096, 0.9700267910957336], "6": ["vertical_and_slash", 1000, 6096, 0.9898602366447449], "7": ["vertical_and_slash", 5548, 2148, 0.8581121563911438], "8": ["vertical_and_slash", 1000, 6096, 0.9889950156211853], "9": ["vertical_and_slash", 3048, 8144, 0.885164201259613], "10": ["vertical_and_slash", 30, 800, 0.9671134352684021], "11": ["vertical_and_slash", 3048, 8144, 0.8993249535560608], "12": ["vertical_and_slash", 30, 800, 0.9827681183815002], "13": ["vertical_and_slash", 500, 700, 0.9576374292373657], "14": ["vertical_and_slash", 100, 800, 0.9765625], "15": ["vertical_and_slash", 5096, 10192, 0.7794672250747681], "16": ["vertical_and_slash", 3048, 8144, 0.8598983883857727], "17": ["vertical_and_slash", 13244, 18340, 0.5485108494758606], "18": ["vertical_and_slash", 3048, 8144, 0.8285969495773315], "19": ["vertical_and_slash", 3048, 8144, 0.8010903000831604], "20": ["vertical_and_slash", 1000, 6096, 0.9821910262107849], "21": ["vertical_and_slash", 5096, 10192, 0.7920611500740051], "22": ["vertical_and_slash", 13244, 18340, 0.587577760219574], "23": ["vertical_and_slash", 13244, 18340, 0.4525212049484253], "24": ["vertical_and_slash", 5096, 10192, 0.7511809468269348], "25": ["vertical_and_slash", 3048, 8144, 0.8870682120323181], "26": ["vertical_and_slash", 1000, 6096, 0.9818049669265747], "27": ["vertical_and_slash", 3048, 8144, 0.8193631172180176], "28": ["vertical_and_slash", 1000, 6096, 0.9139252305030823], "29": ["vertical_and_slash", 3048, 8144, 0.8937361836433411], "30": ["vertical_and_slash", 1000, 6096, 0.9952555894851685], "31": ["vertical_and_slash", 9192, 14288, 0.6008565425872803], "32": ["vertical_and_slash", 1000, 6096, 0.9891071915626526], "33": ["vertical_and_slash", 13244, 18340, 0.5951302647590637], "34": ["vertical_and_slash", 1000, 6096, 0.9958813786506653], "35": ["vertical_and_slash", 3048, 8144, 0.867950975894928], "36": ["vertical_and_slash", 7596, 4196, 0.737917959690094], "37": ["vertical_and_slash", 1000, 6096, 0.9649951457977295], "38": ["vertical_and_slash", 1000, 6096, 0.9848694205284119], "39": ["vertical_and_slash", 9192, 14288, 0.6458787322044373]}, {"0": ["vertical_and_slash", 500, 700, 0.9618247747421265], "1": ["vertical_and_slash", 500, 700, 0.9880454540252686], "2": ["vertical_and_slash", 500, 700, 0.9561574459075928], "3": ["vertical_and_slash", 500, 700, 0.9810187220573425], "4": ["vertical_and_slash", 1000, 6096, 0.9555935263633728], "5": ["vertical_and_slash", 5548, 2148, 0.8264198303222656], "6": ["vertical_and_slash", 7596, 4196, 0.7397205829620361], "7": ["vertical_and_slash", 1000, 6096, 0.9418384432792664], "8": ["vertical_and_slash", 1000, 6096, 0.9737698435783386], "9": ["vertical_and_slash", 1000, 6096, 0.9327544569969177], "10": ["vertical_and_slash", 1000, 6096, 0.9872485399246216], "11": ["vertical_and_slash", 1000, 6096, 0.9863738417625427], "12": ["vertical_and_slash", 1000, 6096, 0.9722111225128174], "13": ["vertical_and_slash", 1000, 6096, 0.9638990759849548], "14": ["vertical_and_slash", 1000, 6096, 0.9632765650749207], "15": ["vertical_and_slash", 3048, 8144, 0.8685882687568665], "16": ["vertical_and_slash", 9192, 14288, 0.6808444261550903], "17": ["vertical_and_slash", 3048, 8144, 0.8680838942527771], "18": ["vertical_and_slash", 1000, 6096, 0.9291468262672424], "19": ["vertical_and_slash", 3048, 8144, 0.8905760645866394], "20": ["vertical_and_slash", 13244, 18340, 0.5592277646064758], "21": ["vertical_and_slash", 1000, 6096, 0.9831752181053162], "22": ["vertical_and_slash", 3048, 8144, 0.8375348448753357], "23": ["vertical_and_slash", 1000, 6096, 0.9761329293251038], "24": ["vertical_and_slash", 3048, 8144, 0.8210043907165527], "25": ["vertical_and_slash", 1000, 6096, 0.9455711841583252], "26": ["vertical_and_slash", 3048, 8144, 0.8151703476905823], "27": ["vertical_and_slash", 3048, 8144, 0.8478951454162598], "28": ["vertical_and_slash", 3048, 8144, 0.859541118144989], "29": ["vertical_and_slash", 5096, 10192, 0.7953957319259644], "30": ["vertical_and_slash", 5096, 10192, 0.7969784736633301], "31": ["vertical_and_slash", 1000, 6096, 0.9309409260749817], "32": ["vertical_and_slash", 3048, 8144, 0.883380115032196], "33": ["vertical_and_slash", 30, 800, 0.9114272594451904], "34": ["vertical_and_slash", 30, 800, 0.9253450036048889], "35": ["vertical_and_slash", 1000, 6096, 0.9698923826217651], "36": ["vertical_and_slash", 1000, 6096, 0.9810246229171753], "37": ["vertical_and_slash", 1000, 6096, 0.9694713354110718], "38": ["vertical_and_slash", 3048, 8144, 0.8544138669967651], "39": ["vertical_and_slash", 1000, 6096, 0.9567173719406128]}, {"0": ["vertical_and_slash", 3500, 100, 0.9237840175628662], "1": ["vertical_and_slash", 5548, 2148, 0.8335959315299988], "2": ["vertical_and_slash", 3500, 100, 0.9204753041267395], "3": ["vertical_and_slash", 3048, 8144, 0.8086016178131104], "4": ["vertical_and_slash", 1000, 6096, 0.9749594330787659], "5": ["vertical_and_slash", 1000, 6096, 0.9776788949966431], "6": ["vertical_and_slash", 3048, 8144, 0.8158401846885681], "7": ["vertical_and_slash", 1000, 6096, 0.9496205449104309], "8": ["vertical_and_slash", 5096, 10192, 0.7622160911560059], "9": ["vertical_and_slash", 3048, 8144, 0.870159924030304], "10": ["vertical_and_slash", 100, 800, 0.90234375], "11": ["vertical_and_slash", 3048, 8144, 0.8432259559631348], "12": ["vertical_and_slash", 13244, 18340, 0.5246782302856445], "13": ["vertical_and_slash", 1000, 6096, 0.9186107516288757], "14": ["vertical_and_slash", 3048, 8144, 0.8611202239990234], "15": ["vertical_and_slash", 5096, 10192, 0.7256661057472229], "16": ["vertical_and_slash", 1000, 6096, 0.9242069125175476], "17": ["vertical_and_slash", 3500, 100, 0.970562219619751], "18": ["vertical_and_slash", 9192, 14288, 0.6779265403747559], "19": ["vertical_and_slash", 7596, 4196, 0.7945611476898193], "20": ["vertical_and_slash", 3048, 8144, 0.8773882985115051], "21": ["vertical_and_slash", 3048, 8144, 0.8867231607437134], "22": ["vertical_and_slash", 1000, 6096, 0.9776970744132996], "23": ["vertical_and_slash", 7596, 4196, 0.7997610569000244], "24": ["vertical_and_slash", 1000, 6096, 0.9595460295677185], "25": ["vertical_and_slash", 3048, 8144, 0.8236842751502991], "26": ["vertical_and_slash", 500, 700, 0.9317872524261475], "27": ["vertical_and_slash", 8292, 8992, 0.640625], "28": ["vertical_and_slash", 500, 700, 0.9379465579986572], "29": ["vertical_and_slash", 500, 700, 0.9078335762023926], "30": ["vertical_and_slash", 1000, 6096, 0.9230840802192688], "31": ["vertical_and_slash", 3048, 8144, 0.8407047986984253], "32": ["vertical_and_slash", 3048, 8144, 0.8672984838485718], "33": ["vertical_and_slash", 1000, 6096, 0.9426825642585754], "34": ["vertical_and_slash", 1000, 6096, 0.9346023797988892], "35": ["vertical_and_slash", 13244, 18340, 0.5531452894210815], "36": ["vertical_and_slash", 1000, 6096, 0.9420048594474792], "37": ["vertical_and_slash", 9192, 14288, 0.6537720561027527], "38": ["vertical_and_slash", 1000, 6096, 0.9412205815315247], "39": ["vertical_and_slash", 3048, 8144, 0.8141274452209473]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9160153269767761], "1": ["vertical_and_slash", 1000, 6096, 0.9297031164169312], "2": ["vertical_and_slash", 3048, 8144, 0.868784487247467], "3": ["vertical_and_slash", 1000, 6096, 0.9413763284683228], "4": ["vertical_and_slash", 1000, 6096, 0.9268408417701721], "5": ["vertical_and_slash", 5096, 10192, 0.7852370738983154], "6": ["vertical_and_slash", 5548, 2148, 0.8340354561805725], "7": ["vertical_and_slash", 3048, 8144, 0.8852138519287109], "8": ["vertical_and_slash", 1000, 6096, 0.9051670432090759], "9": ["vertical_and_slash", 5096, 10192, 0.7790438532829285], "10": ["vertical_and_slash", 1000, 6096, 0.9090266227722168], "11": ["vertical_and_slash", 1000, 6096, 0.9056149125099182], "12": ["vertical_and_slash", 1000, 6096, 0.927521288394928], "13": ["vertical_and_slash", 1000, 6096, 0.9502477645874023], "14": ["vertical_and_slash", 1000, 6096, 0.9323114156723022], "15": ["vertical_and_slash", 3048, 8144, 0.8727365136146545], "16": ["vertical_and_slash", 1000, 6096, 0.9240326285362244], "17": ["vertical_and_slash", 3048, 8144, 0.8289337158203125], "18": ["vertical_and_slash", 9192, 14288, 0.6566157937049866], "19": ["vertical_and_slash", 9192, 14288, 0.6850197315216064], "20": ["vertical_and_slash", 3048, 8144, 0.8966965675354004], "21": ["vertical_and_slash", 1000, 6096, 0.904083251953125], "22": ["vertical_and_slash", 500, 700, 0.9651522636413574], "23": ["vertical_and_slash", 1000, 6096, 0.9536517858505249], "24": ["vertical_and_slash", 3048, 8144, 0.8635534644126892], "25": ["vertical_and_slash", 9192, 14288, 0.6930084228515625], "26": ["vertical_and_slash", 5096, 10192, 0.7243042588233948], "27": ["vertical_and_slash", 5096, 10192, 0.7654940485954285], "28": ["vertical_and_slash", 1000, 6096, 0.9601783156394958], "29": ["vertical_and_slash", 5096, 10192, 0.7544548511505127], "30": ["vertical_and_slash", 5096, 10192, 0.7687339186668396], "31": ["vertical_and_slash", 1000, 6096, 0.9261904954910278], "32": ["vertical_and_slash", 1000, 6096, 0.901333212852478], "33": ["vertical_and_slash", 1000, 6096, 0.9389258623123169], "34": ["vertical_and_slash", 1000, 6096, 0.9238626956939697], "35": ["vertical_and_slash", 1000, 6096, 0.9880523681640625], "36": ["vertical_and_slash", 1000, 6096, 0.9750658869743347], "37": ["vertical_and_slash", 3048, 8144, 0.863137423992157], "38": ["vertical_and_slash", 3500, 100, 0.9969508647918701], "39": ["vertical_and_slash", 5096, 10192, 0.7445880174636841]}, {"0": ["vertical_and_slash", 1000, 6096, 0.930405855178833], "1": ["vertical_and_slash", 3048, 8144, 0.8925277590751648], "2": ["vertical_and_slash", 1000, 6096, 0.9411126375198364], "3": ["vertical_and_slash", 1000, 6096, 0.916792094707489], "4": ["vertical_and_slash", 3048, 8144, 0.8727935552597046], "5": ["vertical_and_slash", 3048, 8144, 0.8737069368362427], "6": ["vertical_and_slash", 1000, 6096, 0.9147027134895325], "7": ["vertical_and_slash", 5096, 10192, 0.7998125553131104], "8": ["vertical_and_slash", 3048, 8144, 0.8751596808433533], "9": ["vertical_and_slash", 5096, 10192, 0.7415482401847839], "10": ["vertical_and_slash", 1000, 6096, 0.978778600692749], "11": ["vertical_and_slash", 1000, 6096, 0.9731906056404114], "12": ["vertical_and_slash", 1000, 6096, 0.977104902267456], "13": ["vertical_and_slash", 3048, 8144, 0.87949538230896], "14": ["vertical_and_slash", 3500, 100, 0.9634283781051636], "15": ["vertical_and_slash", 1000, 6096, 0.9382895827293396], "16": ["vertical_and_slash", 9192, 14288, 0.6115815043449402], "17": ["vertical_and_slash", 1000, 6096, 0.9641004204750061], "18": ["vertical_and_slash", 1000, 6096, 0.9710224270820618], "19": ["vertical_and_slash", 30, 800, 0.9874494671821594], "20": ["vertical_and_slash", 3048, 8144, 0.8827521800994873], "21": ["vertical_and_slash", 3048, 8144, 0.8781187534332275], "22": ["vertical_and_slash", 1000, 6096, 0.9577769041061401], "23": ["vertical_and_slash", 1000, 6096, 0.9475281834602356], "24": ["vertical_and_slash", 1000, 6096, 0.9711649417877197], "25": ["vertical_and_slash", 3048, 8144, 0.8742477297782898], "26": ["vertical_and_slash", 1000, 6096, 0.9779566526412964], "27": ["vertical_and_slash", 1000, 6096, 0.9513294100761414], "28": ["vertical_and_slash", 1000, 6096, 0.9312472939491272], "29": ["vertical_and_slash", 5096, 10192, 0.788093626499176], "30": ["vertical_and_slash", 1000, 6096, 0.972289502620697], "31": ["vertical_and_slash", 1000, 6096, 0.9675362706184387], "32": ["vertical_and_slash", 1000, 6096, 0.9675027132034302], "33": ["vertical_and_slash", 3048, 8144, 0.8831809759140015], "34": ["vertical_and_slash", 1000, 6096, 0.9205896258354187], "35": ["vertical_and_slash", 1000, 6096, 0.9477973580360413], "36": ["vertical_and_slash", 1000, 6096, 0.979175329208374], "37": ["vertical_and_slash", 5096, 10192, 0.7828466296195984], "38": ["vertical_and_slash", 9192, 14288, 0.6794345378875732], "39": ["vertical_and_slash", 1000, 6096, 0.9483242034912109]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9520151615142822], "1": ["vertical_and_slash", 3048, 8144, 0.8832715153694153], "2": ["vertical_and_slash", 3048, 8144, 0.8702895641326904], "3": ["vertical_and_slash", 1000, 6096, 0.9546113610267639], "4": ["vertical_and_slash", 1000, 6096, 0.931942880153656], "5": ["vertical_and_slash", 1000, 6096, 0.9201483130455017], "6": ["vertical_and_slash", 3048, 8144, 0.8662661910057068], "7": ["vertical_and_slash", 1000, 6096, 0.9192161560058594], "8": ["vertical_and_slash", 3048, 8144, 0.8625533580780029], "9": ["vertical_and_slash", 1000, 6096, 0.9223607182502747], "10": ["vertical_and_slash", 1000, 6096, 0.9279952049255371], "11": ["vertical_and_slash", 3048, 8144, 0.8745982646942139], "12": ["vertical_and_slash", 1000, 6096, 0.9463031888008118], "13": ["vertical_and_slash", 1000, 6096, 0.9421766996383667], "14": ["vertical_and_slash", 3048, 8144, 0.8900583386421204], "15": ["vertical_and_slash", 3048, 8144, 0.8898603916168213], "16": ["vertical_and_slash", 1000, 6096, 0.9868079423904419], "17": ["vertical_and_slash", 3048, 8144, 0.8957553505897522], "18": ["vertical_and_slash", 3048, 8144, 0.8089627623558044], "19": ["vertical_and_slash", 1000, 6096, 0.9605066180229187], "20": ["vertical_and_slash", 3048, 8144, 0.8866044878959656], "21": ["vertical_and_slash", 3048, 8144, 0.877276599407196], "22": ["vertical_and_slash", 3048, 8144, 0.8349625468254089], "23": ["vertical_and_slash", 1000, 6096, 0.9153252840042114], "24": ["vertical_and_slash", 3048, 8144, 0.8394809365272522], "25": ["vertical_and_slash", 5096, 10192, 0.7842531800270081], "26": ["vertical_and_slash", 5096, 10192, 0.7737446427345276], "27": ["vertical_and_slash", 5096, 10192, 0.7114154696464539], "28": ["vertical_and_slash", 9192, 14288, 0.6664968729019165], "29": ["vertical_and_slash", 1000, 6096, 0.9532284736633301], "30": ["vertical_and_slash", 1000, 6096, 0.964769721031189], "31": ["vertical_and_slash", 1000, 6096, 0.9775221943855286], "32": ["vertical_and_slash", 1000, 6096, 0.9235398769378662], "33": ["vertical_and_slash", 1000, 6096, 0.9735510349273682], "34": ["vertical_and_slash", 1000, 6096, 0.9730265140533447], "35": ["vertical_and_slash", 5096, 10192, 0.7498651146888733], "36": ["vertical_and_slash", 3048, 8144, 0.8040428757667542], "37": ["vertical_and_slash", 5096, 10192, 0.7644263505935669], "38": ["vertical_and_slash", 1000, 6096, 0.9909747838973999], "39": ["vertical_and_slash", 9192, 14288, 0.6429763436317444]}, {"0": ["vertical_and_slash", 5096, 10192, 0.749825119972229], "1": ["vertical_and_slash", 1000, 6096, 0.9673517942428589], "2": ["vertical_and_slash", 3048, 8144, 0.8564919233322144], "3": ["vertical_and_slash", 5096, 10192, 0.7475799918174744], "4": ["vertical_and_slash", 1000, 6096, 0.9447060823440552], "5": ["vertical_and_slash", 1000, 6096, 0.9264246225357056], "6": ["vertical_and_slash", 1000, 6096, 0.9425864815711975], "7": ["vertical_and_slash", 3500, 100, 0.9214184284210205], "8": ["vertical_and_slash", 1000, 6096, 0.932872474193573], "9": ["vertical_and_slash", 1000, 6096, 0.9508489966392517], "10": ["vertical_and_slash", 3048, 8144, 0.8351834416389465], "11": ["vertical_and_slash", 1000, 6096, 0.9318627715110779], "12": ["vertical_and_slash", 3048, 8144, 0.8569568991661072], "13": ["vertical_and_slash", 5096, 10192, 0.729631781578064], "14": ["vertical_and_slash", 3048, 8144, 0.8594312071800232], "15": ["vertical_and_slash", 1000, 6096, 0.9777551293373108], "16": ["vertical_and_slash", 1000, 6096, 0.9523215293884277], "17": ["vertical_and_slash", 3048, 8144, 0.89132159948349], "18": ["vertical_and_slash", 1000, 6096, 0.9576021432876587], "19": ["vertical_and_slash", 1000, 6096, 0.9523268938064575], "20": ["vertical_and_slash", 1000, 6096, 0.9325323104858398], "21": ["vertical_and_slash", 3048, 8144, 0.8949593901634216], "22": ["vertical_and_slash", 5096, 10192, 0.7421680092811584], "23": ["vertical_and_slash", 5096, 10192, 0.7344619631767273], "24": ["vertical_and_slash", 9192, 14288, 0.6544512510299683], "25": ["vertical_and_slash", 3048, 8144, 0.8633208870887756], "26": ["vertical_and_slash", 100, 800, 0.9921875], "27": ["vertical_and_slash", 7596, 4196, 0.7696033120155334], "28": ["vertical_and_slash", 3500, 100, 0.9214956164360046], "29": ["vertical_and_slash", 3048, 8144, 0.8381994962692261], "30": ["vertical_and_slash", 1000, 6096, 0.9123284220695496], "31": ["vertical_and_slash", 1000, 6096, 0.9104674458503723], "32": ["vertical_and_slash", 3048, 8144, 0.8971980214118958], "33": ["vertical_and_slash", 1000, 6096, 0.9609608054161072], "34": ["vertical_and_slash", 1000, 6096, 0.9028342366218567], "35": ["vertical_and_slash", 1000, 6096, 0.9805289506912231], "36": ["vertical_and_slash", 1000, 6096, 0.9730293154716492], "37": ["vertical_and_slash", 1000, 6096, 0.9381327033042908], "38": ["vertical_and_slash", 1000, 6096, 0.9766618609428406], "39": ["vertical_and_slash", 1000, 6096, 0.9864060878753662]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9365179538726807], "1": ["vertical_and_slash", 1000, 6096, 0.9662060141563416], "2": ["vertical_and_slash", 1000, 6096, 0.9527161717414856], "3": ["vertical_and_slash", 1000, 6096, 0.959445595741272], "4": ["vertical_and_slash", 1000, 6096, 0.9544845819473267], "5": ["vertical_and_slash", 1000, 6096, 0.9366532564163208], "6": ["vertical_and_slash", 3500, 100, 0.9519115686416626], "7": ["vertical_and_slash", 1000, 6096, 0.9440216422080994], "8": ["vertical_and_slash", 1000, 6096, 0.9295591115951538], "9": ["vertical_and_slash", 1000, 6096, 0.9195898175239563], "10": ["vertical_and_slash", 3048, 8144, 0.875956118106842], "11": ["vertical_and_slash", 1000, 6096, 0.9364949464797974], "12": ["vertical_and_slash", 100, 800, 0.9140625], "13": ["vertical_and_slash", 3048, 8144, 0.8202493786811829], "14": ["vertical_and_slash", 5096, 10192, 0.7856994271278381], "15": ["vertical_and_slash", 30, 800, 0.9952673316001892], "16": ["vertical_and_slash", 30, 800, 0.9865661263465881], "17": ["vertical_and_slash", 30, 800, 0.9432961344718933], "18": ["vertical_and_slash", 30, 800, 0.990509033203125], "19": ["vertical_and_slash", 30, 800, 0.963677704334259], "20": ["vertical_and_slash", 1000, 6096, 0.9887117147445679], "21": ["vertical_and_slash", 5096, 10192, 0.797829806804657], "22": ["vertical_and_slash", 1000, 6096, 0.9806796908378601], "23": ["vertical_and_slash", 3048, 8144, 0.848339319229126], "24": ["vertical_and_slash", 1000, 6096, 0.9791911244392395], "25": ["vertical_and_slash", 1000, 6096, 0.9097675085067749], "26": ["vertical_and_slash", 5096, 10192, 0.775140106678009], "27": ["vertical_and_slash", 3048, 8144, 0.8685469031333923], "28": ["vertical_and_slash", 1000, 6096, 0.9645570516586304], "29": ["vertical_and_slash", 5548, 2148, 0.8029168248176575], "30": ["vertical_and_slash", 1000, 6096, 0.9222044348716736], "31": ["vertical_and_slash", 1000, 6096, 0.9458315968513489], "32": ["vertical_and_slash", 1000, 6096, 0.9468505382537842], "33": ["vertical_and_slash", 1000, 6096, 0.9270690679550171], "34": ["vertical_and_slash", 1000, 6096, 0.9465615153312683], "35": ["vertical_and_slash", 1000, 6096, 0.9811714291572571], "36": ["vertical_and_slash", 1000, 6096, 0.9898707866668701], "37": ["vertical_and_slash", 1000, 6096, 0.9552093148231506], "38": ["vertical_and_slash", 1000, 6096, 0.9451280832290649], "39": ["vertical_and_slash", 1000, 6096, 0.9339846968650818]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9524639248847961], "1": ["vertical_and_slash", 1000, 6096, 0.9777984023094177], "2": ["vertical_and_slash", 3048, 8144, 0.8134886026382446], "3": ["vertical_and_slash", 1000, 6096, 0.9204867482185364], "4": ["vertical_and_slash", 1000, 6096, 0.9173088669776917], "5": ["vertical_and_slash", 9192, 14288, 0.6577295064926147], "6": ["vertical_and_slash", 5096, 10192, 0.7291984558105469], "7": ["vertical_and_slash", 9192, 14288, 0.6912325024604797], "8": ["vertical_and_slash", 3048, 8144, 0.880233108997345], "9": ["vertical_and_slash", 1000, 6096, 0.9726386070251465], "10": ["vertical_and_slash", 1000, 6096, 0.9763084053993225], "11": ["vertical_and_slash", 1000, 6096, 0.9385591745376587], "12": ["vertical_and_slash", 1000, 6096, 0.9820747375488281], "13": ["vertical_and_slash", 1000, 6096, 0.9689509272575378], "14": ["vertical_and_slash", 1000, 6096, 0.9511028528213501], "15": ["vertical_and_slash", 1000, 6096, 0.9480229616165161], "16": ["vertical_and_slash", 3048, 8144, 0.8825543522834778], "17": ["vertical_and_slash", 1000, 6096, 0.9397943615913391], "18": ["vertical_and_slash", 1000, 6096, 0.9749455451965332], "19": ["vertical_and_slash", 1000, 6096, 0.948927104473114], "20": ["vertical_and_slash", 3048, 8144, 0.8113911151885986], "21": ["vertical_and_slash", 3048, 8144, 0.8356988430023193], "22": ["vertical_and_slash", 500, 700, 0.9661133885383606], "23": ["vertical_and_slash", 1000, 6096, 0.9770509004592896], "24": ["vertical_and_slash", 3048, 8144, 0.8591731190681458], "25": ["vertical_and_slash", 3500, 100, 0.9063153266906738], "26": ["vertical_and_slash", 2148, 2848, 0.8359375], "27": ["vertical_and_slash", 100, 800, 0.94921875], "28": ["vertical_and_slash", 9192, 14288, 0.6955689191818237], "29": ["vertical_and_slash", 100, 800, 0.9375], "30": ["vertical_and_slash", 3048, 8144, 0.8937103152275085], "31": ["vertical_and_slash", 1000, 6096, 0.9863076210021973], "32": ["vertical_and_slash", 3048, 8144, 0.874880313873291], "33": ["vertical_and_slash", 9192, 14288, 0.6529630422592163], "34": ["vertical_and_slash", 1000, 6096, 0.9175313711166382], "35": ["vertical_and_slash", 1000, 6096, 0.9157241582870483], "36": ["vertical_and_slash", 1000, 6096, 0.9548712968826294], "37": ["vertical_and_slash", 3048, 8144, 0.8382976651191711], "38": ["vertical_and_slash", 100, 800, 0.95703125], "39": ["vertical_and_slash", 1000, 6096, 0.9144138693809509]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9664146304130554], "1": ["vertical_and_slash", 3048, 8144, 0.8994136452674866], "2": ["vertical_and_slash", 1000, 6096, 0.9246600866317749], "3": ["vertical_and_slash", 1000, 6096, 0.9626349806785583], "4": ["vertical_and_slash", 1000, 6096, 0.9298959374427795], "5": ["vertical_and_slash", 5096, 10192, 0.7527567744255066], "6": ["vertical_and_slash", 5096, 10192, 0.7178046703338623], "7": ["vertical_and_slash", 5096, 10192, 0.7343249917030334], "8": ["vertical_and_slash", 1000, 6096, 0.9027687311172485], "9": ["vertical_and_slash", 5096, 10192, 0.7417930364608765], "10": ["vertical_and_slash", 3048, 8144, 0.8957345485687256], "11": ["vertical_and_slash", 1000, 6096, 0.9307156205177307], "12": ["vertical_and_slash", 1000, 6096, 0.9807167649269104], "13": ["vertical_and_slash", 3048, 8144, 0.8694334626197815], "14": ["vertical_and_slash", 3048, 8144, 0.8008334636688232], "15": ["vertical_and_slash", 3048, 8144, 0.894042432308197], "16": ["vertical_and_slash", 1000, 6096, 0.9402773976325989], "17": ["vertical_and_slash", 100, 800, 0.92578125], "18": ["vertical_and_slash", 1000, 6096, 0.9612575173377991], "19": ["vertical_and_slash", 3048, 8144, 0.8793848156929016], "20": ["vertical_and_slash", 5096, 10192, 0.7723790407180786], "21": ["vertical_and_slash", 1000, 6096, 0.96921306848526], "22": ["vertical_and_slash", 3048, 8144, 0.8878388404846191], "23": ["vertical_and_slash", 1000, 6096, 0.9056389927864075], "24": ["vertical_and_slash", 3048, 8144, 0.8951614499092102], "25": ["vertical_and_slash", 5096, 10192, 0.7445169687271118], "26": ["vertical_and_slash", 13244, 18340, 0.5608470439910889], "27": ["vertical_and_slash", 13244, 18340, 0.5830637812614441], "28": ["vertical_and_slash", 1000, 6096, 0.9086351990699768], "29": ["vertical_and_slash", 5096, 10192, 0.7210283279418945], "30": ["vertical_and_slash", 1000, 6096, 0.9157561659812927], "31": ["vertical_and_slash", 1000, 6096, 0.9615012407302856], "32": ["vertical_and_slash", 1000, 6096, 0.9358956813812256], "33": ["vertical_and_slash", 100, 800, 0.90625], "34": ["vertical_and_slash", 1000, 6096, 0.9394569396972656], "35": ["vertical_and_slash", 5096, 10192, 0.70567786693573], "36": ["vertical_and_slash", 1000, 6096, 0.9779843688011169], "37": ["vertical_and_slash", 9192, 14288, 0.6307677030563354], "38": ["vertical_and_slash", 11692, 8292, 0.6909995079040527], "39": ["vertical_and_slash", 100, 800, 0.96484375]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9465231895446777], "1": ["vertical_and_slash", 1000, 6096, 0.9138509035110474], "2": ["vertical_and_slash", 1000, 6096, 0.9675502181053162], "3": ["vertical_and_slash", 1000, 6096, 0.9728706479072571], "4": ["vertical_and_slash", 1000, 6096, 0.9462876915931702], "5": ["vertical_and_slash", 1000, 6096, 0.9061281681060791], "6": ["vertical_and_slash", 1000, 6096, 0.93622887134552], "7": ["vertical_and_slash", 1000, 6096, 0.9633945226669312], "8": ["vertical_and_slash", 1000, 6096, 0.9740681648254395], "9": ["vertical_and_slash", 3048, 8144, 0.8339638710021973], "10": ["vertical_and_slash", 3048, 8144, 0.852887749671936], "11": ["vertical_and_slash", 1000, 6096, 0.9873525500297546], "12": ["vertical_and_slash", 1000, 6096, 0.9143085479736328], "13": ["vertical_and_slash", 1000, 6096, 0.9968701601028442], "14": ["vertical_and_slash", 1000, 6096, 0.990261971950531], "15": ["vertical_and_slash", 500, 700, 0.9616547226905823], "16": ["vertical_and_slash", 3048, 8144, 0.8094536662101746], "17": ["vertical_and_slash", 1000, 6096, 0.9368354082107544], "18": ["vertical_and_slash", 3048, 8144, 0.8561015129089355], "19": ["vertical_and_slash", 30, 800, 0.9600914120674133], "20": ["vertical_and_slash", 3500, 100, 0.9555844068527222], "21": ["vertical_and_slash", 2148, 2848, 0.87109375], "22": ["vertical_and_slash", 500, 700, 0.9493725895881653], "23": ["vertical_and_slash", 3500, 100, 0.9703266024589539], "24": ["vertical_and_slash", 2148, 2848, 0.8984375], "25": ["vertical_and_slash", 1000, 6096, 0.9095062017440796], "26": ["vertical_and_slash", 1000, 6096, 0.9677131772041321], "27": ["vertical_and_slash", 1000, 6096, 0.9356470704078674], "28": ["vertical_and_slash", 3048, 8144, 0.880037784576416], "29": ["vertical_and_slash", 1000, 6096, 0.9710361361503601], "30": ["vertical_and_slash", 1000, 6096, 0.9353585839271545], "31": ["vertical_and_slash", 100, 800, 0.94921875], "32": ["vertical_and_slash", 500, 700, 0.938251256942749], "33": ["vertical_and_slash", 500, 700, 0.9629893898963928], "34": ["vertical_and_slash", 1000, 6096, 0.9683235287666321], "35": ["vertical_and_slash", 1000, 6096, 0.9374136328697205], "36": ["vertical_and_slash", 3048, 8144, 0.8996721506118774], "37": ["vertical_and_slash", 1000, 6096, 0.9801483154296875], "38": ["vertical_and_slash", 1000, 6096, 0.9616703391075134], "39": ["vertical_and_slash", 1000, 6096, 0.9212111234664917]}, {"0": ["vertical_and_slash", 3500, 100, 0.9586876630783081], "1": ["vertical_and_slash", 500, 700, 0.922041654586792], "2": ["vertical_and_slash", 3048, 8144, 0.8944562673568726], "3": ["vertical_and_slash", 100, 800, 0.96484375], "4": ["vertical_and_slash", 1000, 6096, 0.9378555417060852], "5": ["vertical_and_slash", 1000, 6096, 0.951851487159729], "6": ["vertical_and_slash", 1000, 6096, 0.9714210629463196], "7": ["vertical_and_slash", 1000, 6096, 0.978588342666626], "8": ["vertical_and_slash", 1000, 6096, 0.9517953991889954], "9": ["vertical_and_slash", 1000, 6096, 0.9405440092086792], "10": ["vertical_and_slash", 1000, 6096, 0.9865923523902893], "11": ["vertical_and_slash", 1000, 6096, 0.9901511669158936], "12": ["vertical_and_slash", 5096, 10192, 0.7888402342796326], "13": ["vertical_and_slash", 3048, 8144, 0.899949312210083], "14": ["vertical_and_slash", 1000, 6096, 0.9905305504798889], "15": ["vertical_and_slash", 3048, 8144, 0.893486738204956], "16": ["vertical_and_slash", 1000, 6096, 0.938328742980957], "17": ["vertical_and_slash", 1000, 6096, 0.9295006394386292], "18": ["vertical_and_slash", 1000, 6096, 0.9633724689483643], "19": ["vertical_and_slash", 1000, 6096, 0.9281322360038757], "20": ["vertical_and_slash", 1000, 6096, 0.9647982120513916], "21": ["vertical_and_slash", 1000, 6096, 0.9377158880233765], "22": ["vertical_and_slash", 1000, 6096, 0.926757276058197], "23": ["vertical_and_slash", 1000, 6096, 0.9625777006149292], "24": ["vertical_and_slash", 1000, 6096, 0.9773386120796204], "25": ["vertical_and_slash", 500, 700, 0.9598379731178284], "26": ["vertical_and_slash", 3048, 8144, 0.8856900930404663], "27": ["vertical_and_slash", 500, 700, 0.9113988280296326], "28": ["vertical_and_slash", 500, 700, 0.930976152420044], "29": ["vertical_and_slash", 500, 700, 0.9451494216918945], "30": ["vertical_and_slash", 3048, 8144, 0.858772337436676], "31": ["vertical_and_slash", 1000, 6096, 0.9873988032341003], "32": ["vertical_and_slash", 1000, 6096, 0.9259295463562012], "33": ["vertical_and_slash", 1000, 6096, 0.9667267799377441], "34": ["vertical_and_slash", 5096, 10192, 0.7674205899238586], "35": ["vertical_and_slash", 500, 700, 0.959222137928009], "36": ["vertical_and_slash", 500, 700, 0.946297287940979], "37": ["vertical_and_slash", 3048, 8144, 0.8870705366134644], "38": ["vertical_and_slash", 100, 800, 0.90625], "39": ["vertical_and_slash", 100, 800, 0.9765625]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9884225726127625], "1": ["vertical_and_slash", 1000, 6096, 0.9657701253890991], "2": ["vertical_and_slash", 1000, 6096, 0.9556513428688049], "3": ["vertical_and_slash", 3048, 8144, 0.8814572095870972], "4": ["vertical_and_slash", 1000, 6096, 0.9555386304855347], "5": ["vertical_and_slash", 1000, 6096, 0.9497918486595154], "6": ["vertical_and_slash", 1000, 6096, 0.989371657371521], "7": ["vertical_and_slash", 1000, 6096, 0.9649036526679993], "8": ["vertical_and_slash", 1000, 6096, 0.992281436920166], "9": ["vertical_and_slash", 1000, 6096, 0.9572242498397827], "10": ["vertical_and_slash", 3500, 100, 0.9864726662635803], "11": ["vertical_and_slash", 1000, 6096, 0.9427834153175354], "12": ["vertical_and_slash", 5096, 10192, 0.7666211128234863], "13": ["vertical_and_slash", 1000, 6096, 0.9958125352859497], "14": ["vertical_and_slash", 1000, 6096, 0.9761656522750854], "15": ["vertical_and_slash", 1000, 6096, 0.9713703989982605], "16": ["vertical_and_slash", 3048, 8144, 0.8100246787071228], "17": ["vertical_and_slash", 1000, 6096, 0.9495450258255005], "18": ["vertical_and_slash", 1000, 6096, 0.9780734777450562], "19": ["vertical_and_slash", 5096, 10192, 0.7560385465621948], "20": ["vertical_and_slash", 30, 800, 0.9713151454925537], "21": ["vertical_and_slash", 1000, 6096, 0.9195780754089355], "22": ["vertical_and_slash", 500, 700, 0.9770663976669312], "23": ["vertical_and_slash", 30, 800, 0.9754247665405273], "24": ["vertical_and_slash", 30, 800, 0.9565082788467407], "25": ["vertical_and_slash", 1000, 6096, 0.9658346176147461], "26": ["vertical_and_slash", 1000, 6096, 0.957155168056488], "27": ["vertical_and_slash", 1000, 6096, 0.9153123497962952], "28": ["vertical_and_slash", 1000, 6096, 0.9578511118888855], "29": ["vertical_and_slash", 1000, 6096, 0.9598592519760132], "30": ["vertical_and_slash", 3048, 8144, 0.8983713388442993], "31": ["vertical_and_slash", 1000, 6096, 0.9626600742340088], "32": ["vertical_and_slash", 1000, 6096, 0.9411455392837524], "33": ["vertical_and_slash", 1000, 6096, 0.9700142741203308], "34": ["vertical_and_slash", 1000, 6096, 0.9299137592315674], "35": ["vertical_and_slash", 1000, 6096, 0.9755858778953552], "36": ["vertical_and_slash", 1000, 6096, 0.9415363073348999], "37": ["vertical_and_slash", 1000, 6096, 0.9732873439788818], "38": ["vertical_and_slash", 1000, 6096, 0.9618871808052063], "39": ["vertical_and_slash", 1000, 6096, 0.9316824674606323]}, {"0": ["vertical_and_slash", 1000, 6096, 0.976745069026947], "1": ["vertical_and_slash", 1000, 6096, 0.9937078952789307], "2": ["vertical_and_slash", 1000, 6096, 0.9501501321792603], "3": ["vertical_and_slash", 1000, 6096, 0.9828822016716003], "4": ["vertical_and_slash", 1000, 6096, 0.9479304552078247], "5": ["vertical_and_slash", 3048, 8144, 0.8534132242202759], "6": ["vertical_and_slash", 1000, 6096, 0.9095656275749207], "7": ["vertical_and_slash", 100, 800, 0.96484375], "8": ["vertical_and_slash", 13244, 18340, 0.5630600452423096], "9": ["vertical_and_slash", 500, 700, 0.986583948135376], "10": ["vertical_and_slash", 3048, 8144, 0.8997471332550049], "11": ["vertical_and_slash", 1000, 6096, 0.9101494550704956], "12": ["vertical_and_slash", 100, 800, 0.96875], "13": ["vertical_and_slash", 1000, 6096, 0.9631344079971313], "14": ["vertical_and_slash", 1000, 6096, 0.9670076370239258], "15": ["vertical_and_slash", 1000, 6096, 0.9837666153907776], "16": ["vertical_and_slash", 1000, 6096, 0.9556997418403625], "17": ["vertical_and_slash", 1000, 6096, 0.9847139716148376], "18": ["vertical_and_slash", 1000, 6096, 0.9815922975540161], "19": ["vertical_and_slash", 1000, 6096, 0.9613214731216431], "20": ["vertical_and_slash", 500, 700, 0.9729388356208801], "21": ["vertical_and_slash", 30, 800, 0.9646701812744141], "22": ["vertical_and_slash", 500, 700, 0.9276658892631531], "23": ["vertical_and_slash", 3500, 100, 0.9294955134391785], "24": ["vertical_and_slash", 30, 800, 0.9609096050262451], "25": ["vertical_and_slash", 3048, 8144, 0.8426336050033569], "26": ["vertical_and_slash", 9192, 14288, 0.6223312020301819], "27": ["vertical_and_slash", 4196, 4896, 0.75390625], "28": ["vertical_and_slash", 1000, 6096, 0.9666014313697815], "29": ["vertical_and_slash", 1000, 6096, 0.9895376563072205], "30": ["vertical_and_slash", 1000, 6096, 0.9871048331260681], "31": ["vertical_and_slash", 1000, 6096, 0.9764318466186523], "32": ["vertical_and_slash", 1000, 6096, 0.97807377576828], "33": ["vertical_and_slash", 1000, 6096, 0.9896739721298218], "34": ["vertical_and_slash", 1000, 6096, 0.9874780774116516], "35": ["vertical_and_slash", 1000, 6096, 0.9911878705024719], "36": ["vertical_and_slash", 1000, 6096, 0.9856313467025757], "37": ["vertical_and_slash", 1000, 6096, 0.980508029460907], "38": ["vertical_and_slash", 1000, 6096, 0.9775713086128235], "39": ["vertical_and_slash", 1000, 6096, 0.9921905398368835]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9310158491134644], "1": ["vertical_and_slash", 13244, 18340, 0.5703044533729553], "2": ["vertical_and_slash", 1000, 6096, 0.988821268081665], "3": ["vertical_and_slash", 1000, 6096, 0.9716726541519165], "4": ["vertical_and_slash", 1000, 6096, 0.990047037601471], "5": ["vertical_and_slash", 1000, 6096, 0.9577871561050415], "6": ["vertical_and_slash", 1000, 6096, 0.926821768283844], "7": ["vertical_and_slash", 1000, 6096, 0.9884239435195923], "8": ["vertical_and_slash", 1000, 6096, 0.9619236588478088], "9": ["vertical_and_slash", 3048, 8144, 0.868797779083252], "10": ["vertical_and_slash", 3048, 8144, 0.8773615956306458], "11": ["vertical_and_slash", 1000, 6096, 0.9647166728973389], "12": ["vertical_and_slash", 1000, 6096, 0.9558796882629395], "13": ["vertical_and_slash", 3048, 8144, 0.8282517790794373], "14": ["vertical_and_slash", 1000, 6096, 0.9746207594871521], "15": ["vertical_and_slash", 3048, 8144, 0.8827066421508789], "16": ["vertical_and_slash", 1000, 6096, 0.9920842051506042], "17": ["vertical_and_slash", 1000, 6096, 0.9099866151809692], "18": ["vertical_and_slash", 1000, 6096, 0.9899295568466187], "19": ["vertical_and_slash", 1000, 6096, 0.9881210923194885], "20": ["vertical_and_slash", 3048, 8144, 0.8171201944351196], "21": ["vertical_and_slash", 3048, 8144, 0.8365668058395386], "22": ["vertical_and_slash", 3048, 8144, 0.8064008355140686], "23": ["vertical_and_slash", 9192, 14288, 0.6598557233810425], "24": ["vertical_and_slash", 13244, 18340, 0.5866872072219849], "25": ["vertical_and_slash", 1000, 6096, 0.9873585104942322], "26": ["vertical_and_slash", 1000, 6096, 0.9873473644256592], "27": ["vertical_and_slash", 1000, 6096, 0.9099739193916321], "28": ["vertical_and_slash", 1000, 6096, 0.9914581775665283], "29": ["vertical_and_slash", 1000, 6096, 0.9219077229499817], "30": ["vertical_and_slash", 1000, 6096, 0.9014546871185303], "31": ["vertical_and_slash", 5096, 10192, 0.7969730496406555], "32": ["vertical_and_slash", 1000, 6096, 0.9636660814285278], "33": ["vertical_and_slash", 9192, 14288, 0.6710490584373474], "34": ["vertical_and_slash", 1000, 6096, 0.9463331699371338], "35": ["vertical_and_slash", 1000, 6096, 0.9615098834037781], "36": ["vertical_and_slash", 1000, 6096, 0.9770929217338562], "37": ["vertical_and_slash", 3048, 8144, 0.8738768100738525], "38": ["vertical_and_slash", 1000, 6096, 0.9173521399497986], "39": ["vertical_and_slash", 1000, 6096, 0.962170422077179]}, {"0": ["vertical_and_slash", 30, 800, 0.9701426029205322], "1": ["vertical_and_slash", 30, 800, 0.9806621074676514], "2": ["vertical_and_slash", 30, 800, 0.9920138120651245], "3": ["vertical_and_slash", 30, 800, 0.9978371262550354], "4": ["vertical_and_slash", 30, 800, 0.9945651292800903], "5": ["vertical_and_slash", 1000, 6096, 0.96164870262146], "6": ["vertical_and_slash", 100, 800, 0.984375], "7": ["vertical_and_slash", 1000, 6096, 0.9062989354133606], "8": ["vertical_and_slash", 100, 800, 0.98046875], "9": ["vertical_and_slash", 1000, 6096, 0.979543149471283], "10": ["vertical_and_slash", 100, 800, 0.98046875], "11": ["vertical_and_slash", 1000, 6096, 0.9547487497329712], "12": ["vertical_and_slash", 1000, 6096, 0.9773856997489929], "13": ["vertical_and_slash", 1000, 6096, 0.9363219738006592], "14": ["vertical_and_slash", 1000, 6096, 0.9826802015304565], "15": ["vertical_and_slash", 1000, 6096, 0.973421573638916], "16": ["vertical_and_slash", 1000, 6096, 0.9769363403320312], "17": ["vertical_and_slash", 1000, 6096, 0.9858410954475403], "18": ["vertical_and_slash", 1000, 6096, 0.9773050546646118], "19": ["vertical_and_slash", 1000, 6096, 0.9432402849197388], "20": ["vertical_and_slash", 1000, 6096, 0.944821834564209], "21": ["vertical_and_slash", 5096, 10192, 0.7476618885993958], "22": ["vertical_and_slash", 1000, 6096, 0.9538031220436096], "23": ["vertical_and_slash", 1000, 6096, 0.9577361941337585], "24": ["vertical_and_slash", 1000, 6096, 0.9413042068481445], "25": ["vertical_and_slash", 1000, 6096, 0.9912337064743042], "26": ["vertical_and_slash", 1000, 6096, 0.9335371255874634], "27": ["vertical_and_slash", 1000, 6096, 0.9766976833343506], "28": ["vertical_and_slash", 3048, 8144, 0.8681369423866272], "29": ["vertical_and_slash", 1000, 6096, 0.9775100946426392], "30": ["vertical_and_slash", 500, 700, 0.9612457156181335], "31": ["vertical_and_slash", 3500, 100, 0.9780910015106201], "32": ["vertical_and_slash", 1000, 6096, 0.9744886159896851], "33": ["vertical_and_slash", 1000, 6096, 0.9103143215179443], "34": ["vertical_and_slash", 3048, 8144, 0.8747583627700806], "35": ["vertical_and_slash", 1000, 6096, 0.9491387605667114], "36": ["vertical_and_slash", 1000, 6096, 0.9191687703132629], "37": ["vertical_and_slash", 500, 700, 0.9597871899604797], "38": ["vertical_and_slash", 1000, 6096, 0.9040086269378662], "39": ["vertical_and_slash", 1000, 6096, 0.9794480204582214]}, {"0": ["vertical_and_slash", 3500, 100, 0.9929783940315247], "1": ["vertical_and_slash", 500, 700, 0.9721273183822632], "2": ["vertical_and_slash", 500, 700, 0.9861060976982117], "3": ["vertical_and_slash", 500, 700, 0.9804486632347107], "4": ["vertical_and_slash", 500, 700, 0.9751427173614502], "5": ["vertical_and_slash", 1000, 6096, 0.9008292555809021], "6": ["vertical_and_slash", 3500, 100, 0.9847376942634583], "7": ["vertical_and_slash", 1000, 6096, 0.9825294613838196], "8": ["vertical_and_slash", 1000, 6096, 0.9633070826530457], "9": ["vertical_and_slash", 3048, 8144, 0.8541744947433472], "10": ["vertical_and_slash", 3048, 8144, 0.8666449785232544], "11": ["vertical_and_slash", 1000, 6096, 0.9733067154884338], "12": ["vertical_and_slash", 3048, 8144, 0.853722870349884], "13": ["vertical_and_slash", 1000, 6096, 0.9262656569480896], "14": ["vertical_and_slash", 1000, 6096, 0.9423971772193909], "15": ["vertical_and_slash", 1000, 6096, 0.9616424441337585], "16": ["vertical_and_slash", 1000, 6096, 0.9820241928100586], "17": ["vertical_and_slash", 1000, 6096, 0.9749463200569153], "18": ["vertical_and_slash", 1000, 6096, 0.9941414594650269], "19": ["vertical_and_slash", 1000, 6096, 0.9765053391456604], "20": ["vertical_and_slash", 1000, 6096, 0.9748213291168213], "21": ["vertical_and_slash", 1000, 6096, 0.9756632447242737], "22": ["vertical_and_slash", 1000, 6096, 0.972671627998352], "23": ["vertical_and_slash", 1000, 6096, 0.9309594035148621], "24": ["vertical_and_slash", 3048, 8144, 0.8672316670417786], "25": ["vertical_and_slash", 1000, 6096, 0.9865481853485107], "26": ["vertical_and_slash", 1000, 6096, 0.985664427280426], "27": ["vertical_and_slash", 1000, 6096, 0.9955885410308838], "28": ["vertical_and_slash", 1000, 6096, 0.9880906343460083], "29": ["vertical_and_slash", 1000, 6096, 0.9593304991722107], "30": ["vertical_and_slash", 3048, 8144, 0.8896874785423279], "31": ["vertical_and_slash", 3048, 8144, 0.8928280472755432], "32": ["vertical_and_slash", 5096, 10192, 0.7950940728187561], "33": ["vertical_and_slash", 3048, 8144, 0.8981066346168518], "34": ["vertical_and_slash", 3500, 100, 0.9970122575759888], "35": ["vertical_and_slash", 1000, 6096, 0.9654149413108826], "36": ["vertical_and_slash", 1000, 6096, 0.9510290026664734], "37": ["vertical_and_slash", 1000, 6096, 0.9254100918769836], "38": ["vertical_and_slash", 1000, 6096, 0.9603153467178345], "39": ["vertical_and_slash", 1000, 6096, 0.9440348744392395]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9896321296691895], "1": ["vertical_and_slash", 1000, 6096, 0.9903612732887268], "2": ["vertical_and_slash", 1000, 6096, 0.9970978498458862], "3": ["vertical_and_slash", 1000, 6096, 0.9892421960830688], "4": ["vertical_and_slash", 1000, 6096, 0.9593987464904785], "5": ["vertical_and_slash", 500, 700, 0.9436636567115784], "6": ["vertical_and_slash", 5096, 10192, 0.7883052825927734], "7": ["vertical_and_slash", 1000, 6096, 0.9571496248245239], "8": ["vertical_and_slash", 500, 700, 0.9827621579170227], "9": ["vertical_and_slash", 500, 700, 0.941496729850769], "10": ["vertical_and_slash", 3500, 100, 0.9946107268333435], "11": ["vertical_and_slash", 1000, 6096, 0.9758301973342896], "12": ["vertical_and_slash", 3500, 100, 0.990632176399231], "13": ["vertical_and_slash", 3048, 8144, 0.8859867453575134], "14": ["vertical_and_slash", 100, 800, 0.98046875], "15": ["vertical_and_slash", 3048, 8144, 0.8729467988014221], "16": ["vertical_and_slash", 1000, 6096, 0.949256956577301], "17": ["vertical_and_slash", 3048, 8144, 0.8809787034988403], "18": ["vertical_and_slash", 3048, 8144, 0.8972460031509399], "19": ["vertical_and_slash", 1000, 6096, 0.9035227298736572], "20": ["vertical_and_slash", 3500, 100, 0.9799007177352905], "21": ["vertical_and_slash", 3500, 100, 0.983356237411499], "22": ["vertical_and_slash", 1000, 6096, 0.9903866052627563], "23": ["vertical_and_slash", 1000, 6096, 0.9652894139289856], "24": ["vertical_and_slash", 1000, 6096, 0.9593511819839478], "25": ["vertical_and_slash", 1000, 6096, 0.9899460673332214], "26": ["vertical_and_slash", 1000, 6096, 0.9908138513565063], "27": ["vertical_and_slash", 1000, 6096, 0.9944829940795898], "28": ["vertical_and_slash", 1000, 6096, 0.98954176902771], "29": ["vertical_and_slash", 1000, 6096, 0.9958754181861877], "30": ["vertical_and_slash", 3500, 100, 0.9906379580497742], "31": ["vertical_and_slash", 3048, 8144, 0.8398972749710083], "32": ["vertical_and_slash", 1000, 6096, 0.9777312278747559], "33": ["vertical_and_slash", 1000, 6096, 0.9940976500511169], "34": ["vertical_and_slash", 1000, 6096, 0.9836350679397583], "35": ["vertical_and_slash", 1000, 6096, 0.9388172030448914], "36": ["vertical_and_slash", 1000, 6096, 0.9898322224617004], "37": ["vertical_and_slash", 1000, 6096, 0.9698703289031982], "38": ["vertical_and_slash", 1000, 6096, 0.9420815706253052], "39": ["vertical_and_slash", 1000, 6096, 0.9932281374931335]}, {"0": ["vertical_and_slash", 3048, 8144, 0.8140926361083984], "1": ["vertical_and_slash", 1000, 6096, 0.9152762293815613], "2": ["vertical_and_slash", 3048, 8144, 0.8508564233779907], "3": ["vertical_and_slash", 1000, 6096, 0.903032124042511], "4": ["vertical_and_slash", 1000, 6096, 0.9492875933647156], "5": ["vertical_and_slash", 1000, 6096, 0.9953514933586121], "6": ["vertical_and_slash", 3500, 100, 0.992725670337677], "7": ["vertical_and_slash", 3500, 100, 0.9945627450942993], "8": ["vertical_and_slash", 3500, 100, 0.9948824644088745], "9": ["vertical_and_slash", 1000, 6096, 0.991593062877655], "10": ["vertical_and_slash", 500, 700, 0.9929046034812927], "11": ["vertical_and_slash", 500, 700, 0.9880997538566589], "12": ["vertical_and_slash", 1000, 6096, 0.9780631065368652], "13": ["vertical_and_slash", 1000, 6096, 0.9651744365692139], "14": ["vertical_and_slash", 500, 700, 0.9855435490608215], "15": ["vertical_and_slash", 3500, 100, 0.9960723519325256], "16": ["vertical_and_slash", 3500, 100, 0.9835113286972046], "17": ["vertical_and_slash", 3500, 100, 0.9834434390068054], "18": ["vertical_and_slash", 3500, 100, 0.9815698862075806], "19": ["vertical_and_slash", 500, 700, 0.9798949956893921], "20": ["vertical_and_slash", 1000, 6096, 0.9769665598869324], "21": ["vertical_and_slash", 1000, 6096, 0.9899415969848633], "22": ["vertical_and_slash", 1000, 6096, 0.9411264061927795], "23": ["vertical_and_slash", 1000, 6096, 0.9851979613304138], "24": ["vertical_and_slash", 1000, 6096, 0.9894612431526184], "25": ["vertical_and_slash", 1000, 6096, 0.939525842666626], "26": ["vertical_and_slash", 1000, 6096, 0.9679979085922241], "27": ["vertical_and_slash", 1000, 6096, 0.989242434501648], "28": ["vertical_and_slash", 1000, 6096, 0.9967673420906067], "29": ["vertical_and_slash", 1000, 6096, 0.9966699481010437], "30": ["vertical_and_slash", 1000, 6096, 0.968306839466095], "31": ["vertical_and_slash", 3048, 8144, 0.8543223142623901], "32": ["vertical_and_slash", 1000, 6096, 0.9696008563041687], "33": ["vertical_and_slash", 1000, 6096, 0.9603421688079834], "34": ["vertical_and_slash", 1000, 6096, 0.9791843295097351], "35": ["vertical_and_slash", 1000, 6096, 0.966170072555542], "36": ["vertical_and_slash", 100, 800, 0.98828125], "37": ["vertical_and_slash", 1000, 6096, 0.9940048456192017], "38": ["vertical_and_slash", 1000, 6096, 0.914739727973938], "39": ["vertical_and_slash", 1000, 6096, 0.944419801235199]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9879574775695801], "1": ["vertical_and_slash", 1000, 6096, 0.9740590453147888], "2": ["vertical_and_slash", 1000, 6096, 0.9738840460777283], "3": ["vertical_and_slash", 1000, 6096, 0.9723190069198608], "4": ["vertical_and_slash", 1000, 6096, 0.9642390608787537], "5": ["vertical_and_slash", 1000, 6096, 0.9808192253112793], "6": ["vertical_and_slash", 1000, 6096, 0.9966481328010559], "7": ["vertical_and_slash", 500, 700, 0.9783027768135071], "8": ["vertical_and_slash", 500, 700, 0.991054892539978], "9": ["vertical_and_slash", 500, 700, 0.995037853717804], "10": ["vertical_and_slash", 3048, 8144, 0.8974602818489075], "11": ["vertical_and_slash", 1000, 6096, 0.9298633933067322], "12": ["vertical_and_slash", 1000, 6096, 0.991539716720581], "13": ["vertical_and_slash", 3048, 8144, 0.8874135613441467], "14": ["vertical_and_slash", 3048, 8144, 0.882474958896637], "15": ["vertical_and_slash", 3500, 100, 0.993442952632904], "16": ["vertical_and_slash", 100, 800, 1.0], "17": ["vertical_and_slash", 3500, 100, 0.9973551034927368], "18": ["vertical_and_slash", 500, 700, 0.9943462610244751], "19": ["vertical_and_slash", 3500, 100, 0.996669352054596], "20": ["vertical_and_slash", 3048, 8144, 0.8694844245910645], "21": ["vertical_and_slash", 1000, 6096, 0.9902501702308655], "22": ["vertical_and_slash", 1000, 6096, 0.9728785157203674], "23": ["vertical_and_slash", 500, 700, 0.9846650958061218], "24": ["vertical_and_slash", 5096, 10192, 0.7963665127754211], "25": ["vertical_and_slash", 1000, 6096, 0.9894538521766663], "26": ["vertical_and_slash", 1000, 6096, 0.9933777451515198], "27": ["vertical_and_slash", 3500, 100, 0.991401731967926], "28": ["vertical_and_slash", 1000, 6096, 0.9732900857925415], "29": ["vertical_and_slash", 1000, 6096, 0.9927235245704651], "30": ["vertical_and_slash", 1000, 6096, 0.9877493977546692], "31": ["vertical_and_slash", 1000, 6096, 0.9927010536193848], "32": ["vertical_and_slash", 3500, 100, 0.9957630038261414], "33": ["vertical_and_slash", 1000, 6096, 0.9948281645774841], "34": ["vertical_and_slash", 3500, 100, 0.9951621890068054], "35": ["vertical_and_slash", 500, 700, 0.989303469657898], "36": ["vertical_and_slash", 1000, 6096, 0.9829508066177368], "37": ["vertical_and_slash", 500, 700, 0.9914119839668274], "38": ["vertical_and_slash", 500, 700, 0.9890760183334351], "39": ["vertical_and_slash", 500, 700, 0.9733577966690063]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9936951994895935], "1": ["vertical_and_slash", 1000, 6096, 0.9931851029396057], "2": ["vertical_and_slash", 1000, 6096, 0.983021080493927], "3": ["vertical_and_slash", 1000, 6096, 0.9794268608093262], "4": ["vertical_and_slash", 1000, 6096, 0.9924114942550659], "5": ["vertical_and_slash", 3500, 100, 0.9976242184638977], "6": ["vertical_and_slash", 3500, 100, 0.9992868900299072], "7": ["vertical_and_slash", 3500, 100, 0.9984087347984314], "8": ["vertical_and_slash", 3500, 100, 0.995461642742157], "9": ["vertical_and_slash", 3500, 100, 0.9956874847412109], "10": ["vertical_and_slash", 1000, 6096, 0.956352949142456], "11": ["vertical_and_slash", 1000, 6096, 0.9623122811317444], "12": ["vertical_and_slash", 1000, 6096, 0.9846339821815491], "13": ["vertical_and_slash", 1000, 6096, 0.9336572885513306], "14": ["vertical_and_slash", 1000, 6096, 0.956778347492218], "15": ["vertical_and_slash", 1000, 6096, 0.9144752621650696], "16": ["vertical_and_slash", 30, 800, 0.9652518630027771], "17": ["vertical_and_slash", 11692, 8292, 0.6437919735908508], "18": ["vertical_and_slash", 30, 800, 0.9200955033302307], "19": ["vertical_and_slash", 500, 700, 0.9749646782875061], "20": ["vertical_and_slash", 3500, 100, 0.994928240776062], "21": ["vertical_and_slash", 3500, 100, 0.9980435967445374], "22": ["vertical_and_slash", 3500, 100, 0.998473048210144], "23": ["vertical_and_slash", 3500, 100, 0.9963645339012146], "24": ["vertical_and_slash", 3500, 100, 0.9936484694480896], "25": ["vertical_and_slash", 1000, 6096, 0.9736685752868652], "26": ["vertical_and_slash", 1000, 6096, 0.9939982891082764], "27": ["vertical_and_slash", 1000, 6096, 0.9823991060256958], "28": ["vertical_and_slash", 1000, 6096, 0.9564589858055115], "29": ["vertical_and_slash", 1000, 6096, 0.9090656042098999], "30": ["vertical_and_slash", 3500, 100, 0.9598578214645386], "31": ["vertical_and_slash", 1000, 6096, 0.9804523587226868], "32": ["vertical_and_slash", 3500, 100, 0.9931389093399048], "33": ["vertical_and_slash", 3500, 100, 0.9430963397026062], "34": ["vertical_and_slash", 1000, 6096, 0.9467871785163879], "35": ["vertical_and_slash", 1000, 6096, 0.9741998314857483], "36": ["vertical_and_slash", 3500, 100, 0.9946966171264648], "37": ["vertical_and_slash", 1000, 6096, 0.9907556772232056], "38": ["vertical_and_slash", 3500, 100, 0.9634305238723755], "39": ["vertical_and_slash", 1000, 6096, 0.980162501335144]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9567875862121582], "1": ["vertical_and_slash", 3500, 100, 0.993293046951294], "2": ["vertical_and_slash", 1000, 6096, 0.9477084875106812], "3": ["vertical_and_slash", 3500, 100, 0.9939444065093994], "4": ["vertical_and_slash", 3500, 100, 0.9793328046798706], "5": ["vertical_and_slash", 500, 700, 0.9721090197563171], "6": ["vertical_and_slash", 500, 700, 0.992363691329956], "7": ["vertical_and_slash", 3500, 100, 0.9863381385803223], "8": ["vertical_and_slash", 3500, 100, 0.9969126582145691], "9": ["vertical_and_slash", 500, 700, 0.9905705451965332], "10": ["vertical_and_slash", 1000, 6096, 0.9820885062217712], "11": ["vertical_and_slash", 1000, 6096, 0.9890661239624023], "12": ["vertical_and_slash", 1000, 6096, 0.9916929602622986], "13": ["vertical_and_slash", 1000, 6096, 0.9843471050262451], "14": ["vertical_and_slash", 3500, 100, 0.9750230312347412], "15": ["vertical_and_slash", 1000, 6096, 0.9809290766716003], "16": ["vertical_and_slash", 1000, 6096, 0.9055142998695374], "17": ["vertical_and_slash", 1000, 6096, 0.9710241556167603], "18": ["vertical_and_slash", 3500, 100, 0.9973068833351135], "19": ["vertical_and_slash", 3048, 8144, 0.8116639256477356], "20": ["vertical_and_slash", 1000, 6096, 0.9891541600227356], "21": ["vertical_and_slash", 3500, 100, 0.9848664999008179], "22": ["vertical_and_slash", 1000, 6096, 0.9908791184425354], "23": ["vertical_and_slash", 1000, 6096, 0.96574866771698], "24": ["vertical_and_slash", 1000, 6096, 0.9901608228683472], "25": ["vertical_and_slash", 3500, 100, 0.9879163503646851], "26": ["vertical_and_slash", 3500, 100, 0.9780656695365906], "27": ["vertical_and_slash", 1000, 6096, 0.9821938872337341], "28": ["vertical_and_slash", 1000, 6096, 0.9579916596412659], "29": ["vertical_and_slash", 3500, 100, 0.9972220063209534], "30": ["vertical_and_slash", 3500, 100, 0.9899366497993469], "31": ["vertical_and_slash", 1000, 6096, 0.9665626287460327], "32": ["vertical_and_slash", 1000, 6096, 0.976994514465332], "33": ["vertical_and_slash", 3500, 100, 0.9796446561813354], "34": ["vertical_and_slash", 1000, 6096, 0.9836405515670776], "35": ["vertical_and_slash", 1000, 6096, 0.960895299911499], "36": ["vertical_and_slash", 1000, 6096, 0.9917722344398499], "37": ["vertical_and_slash", 3500, 100, 0.9963958263397217], "38": ["vertical_and_slash", 1000, 6096, 0.931512713432312], "39": ["vertical_and_slash", 1000, 6096, 0.9448882937431335]}, {"0": ["vertical_and_slash", 3500, 100, 0.9666692018508911], "1": ["vertical_and_slash", 3500, 100, 0.9812990427017212], "2": ["vertical_and_slash", 3500, 100, 0.9737275242805481], "3": ["vertical_and_slash", 1000, 6096, 0.9874204397201538], "4": ["vertical_and_slash", 3500, 100, 0.9860012531280518], "5": ["vertical_and_slash", 3500, 100, 0.9855799078941345], "6": ["vertical_and_slash", 1000, 6096, 0.9647853970527649], "7": ["vertical_and_slash", 1000, 6096, 0.9848135709762573], "8": ["vertical_and_slash", 1000, 6096, 0.9917643070220947], "9": ["vertical_and_slash", 1000, 6096, 0.985798716545105], "10": ["vertical_and_slash", 1000, 6096, 0.9924654364585876], "11": ["vertical_and_slash", 500, 700, 0.9660372734069824], "12": ["vertical_and_slash", 1000, 6096, 0.9932405352592468], "13": ["vertical_and_slash", 1000, 6096, 0.9921973347663879], "14": ["vertical_and_slash", 3500, 100, 0.9834864139556885], "15": ["vertical_and_slash", 1000, 6096, 0.9737875461578369], "16": ["vertical_and_slash", 3500, 100, 0.9799361228942871], "17": ["vertical_and_slash", 1000, 6096, 0.9283269047737122], "18": ["vertical_and_slash", 1000, 6096, 0.9899070858955383], "19": ["vertical_and_slash", 3500, 100, 0.9745963215827942], "20": ["vertical_and_slash", 1000, 6096, 0.9931060671806335], "21": ["vertical_and_slash", 1000, 6096, 0.9896429777145386], "22": ["vertical_and_slash", 1000, 6096, 0.9687962532043457], "23": ["vertical_and_slash", 3500, 100, 0.9916325807571411], "24": ["vertical_and_slash", 3500, 100, 0.9840542078018188], "25": ["vertical_and_slash", 1000, 6096, 0.9909347891807556], "26": ["vertical_and_slash", 3500, 100, 0.9802504181861877], "27": ["vertical_and_slash", 1000, 6096, 0.9848265647888184], "28": ["vertical_and_slash", 1000, 6096, 0.9548919796943665], "29": ["vertical_and_slash", 3500, 100, 0.9806832075119019], "30": ["vertical_and_slash", 5548, 2148, 0.8537518382072449], "31": ["vertical_and_slash", 1000, 6096, 0.9728904962539673], "32": ["vertical_and_slash", 1000, 6096, 0.9965689182281494], "33": ["vertical_and_slash", 1000, 6096, 0.9895464777946472], "34": ["vertical_and_slash", 1000, 6096, 0.9471479058265686], "35": ["vertical_and_slash", 1000, 6096, 0.9140689373016357], "36": ["vertical_and_slash", 1000, 6096, 0.906638503074646], "37": ["vertical_and_slash", 1000, 6096, 0.9913777709007263], "38": ["vertical_and_slash", 1000, 6096, 0.9803243279457092], "39": ["vertical_and_slash", 1000, 6096, 0.995938777923584]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9967458248138428], "1": ["vertical_and_slash", 1000, 6096, 0.9964662790298462], "2": ["vertical_and_slash", 500, 700, 0.9932101368904114], "3": ["vertical_and_slash", 1000, 6096, 0.9962498545646667], "4": ["vertical_and_slash", 1000, 6096, 0.9974588751792908], "5": ["vertical_and_slash", 3500, 100, 0.9931450486183167], "6": ["vertical_and_slash", 3500, 100, 0.9994134902954102], "7": ["vertical_and_slash", 3500, 100, 0.9933943152427673], "8": ["vertical_and_slash", 1000, 6096, 0.99267578125], "9": ["vertical_and_slash", 1000, 6096, 0.9902094006538391], "10": ["vertical_and_slash", 1000, 6096, 0.9750677943229675], "11": ["vertical_and_slash", 3500, 100, 0.9888282418251038], "12": ["vertical_and_slash", 1000, 6096, 0.9916014671325684], "13": ["vertical_and_slash", 1000, 6096, 0.9756740927696228], "14": ["vertical_and_slash", 3500, 100, 0.9856880903244019], "15": ["vertical_and_slash", 3500, 100, 0.9853808879852295], "16": ["vertical_and_slash", 1000, 6096, 0.9838837385177612], "17": ["vertical_and_slash", 3500, 100, 0.9948219656944275], "18": ["vertical_and_slash", 3500, 100, 0.9841027855873108], "19": ["vertical_and_slash", 1000, 6096, 0.9704200625419617], "20": ["vertical_and_slash", 3500, 100, 0.9740341901779175], "21": ["vertical_and_slash", 3500, 100, 0.9934924244880676], "22": ["vertical_and_slash", 3500, 100, 0.954302966594696], "23": ["vertical_and_slash", 1000, 6096, 0.9160425066947937], "24": ["vertical_and_slash", 1000, 6096, 0.9053351283073425], "25": ["vertical_and_slash", 3500, 100, 0.9968559741973877], "26": ["vertical_and_slash", 3500, 100, 0.9548030495643616], "27": ["vertical_and_slash", 1000, 6096, 0.925932765007019], "28": ["vertical_and_slash", 1000, 6096, 0.9829452037811279], "29": ["vertical_and_slash", 500, 700, 0.9597297310829163], "30": ["vertical_and_slash", 1000, 6096, 0.9579432606697083], "31": ["vertical_and_slash", 3500, 100, 0.9813975095748901], "32": ["vertical_and_slash", 1000, 6096, 0.968403697013855], "33": ["vertical_and_slash", 3500, 100, 0.98689866065979], "34": ["vertical_and_slash", 1000, 6096, 0.9683440923690796], "35": ["vertical_and_slash", 3500, 100, 0.9752838611602783], "36": ["vertical_and_slash", 1000, 6096, 0.9359539151191711], "37": ["vertical_and_slash", 3500, 100, 0.9808348417282104], "38": ["vertical_and_slash", 1000, 6096, 0.9855327606201172], "39": ["vertical_and_slash", 1000, 6096, 0.9680958390235901]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9662633538246155], "1": ["vertical_and_slash", 1000, 6096, 0.9462399482727051], "2": ["vertical_and_slash", 3048, 8144, 0.8000611662864685], "3": ["vertical_and_slash", 1000, 6096, 0.971875786781311], "4": ["vertical_and_slash", 3500, 100, 0.9851511120796204], "5": ["vertical_and_slash", 3500, 100, 0.9923555254936218], "6": ["vertical_and_slash", 3500, 100, 0.9821395874023438], "7": ["vertical_and_slash", 3500, 100, 0.9812511205673218], "8": ["vertical_and_slash", 1000, 6096, 0.9642708897590637], "9": ["vertical_and_slash", 3500, 100, 0.9546874761581421], "10": ["vertical_and_slash", 3048, 8144, 0.8525933623313904], "11": ["vertical_and_slash", 5096, 10192, 0.784069836139679], "12": ["vertical_and_slash", 3500, 100, 0.9128150343894958], "13": ["vertical_and_slash", 1000, 6096, 0.9687226414680481], "14": ["vertical_and_slash", 3048, 8144, 0.8573071956634521], "15": ["vertical_and_slash", 1000, 6096, 0.9302921295166016], "16": ["vertical_and_slash", 3500, 100, 0.9678751230239868], "17": ["vertical_and_slash", 1000, 6096, 0.9653796553611755], "18": ["vertical_and_slash", 1000, 6096, 0.909930408000946], "19": ["vertical_and_slash", 5096, 10192, 0.7216154336929321], "20": ["vertical_and_slash", 1000, 6096, 0.9944660663604736], "21": ["vertical_and_slash", 1000, 6096, 0.953434944152832], "22": ["vertical_and_slash", 3500, 100, 0.9922366142272949], "23": ["vertical_and_slash", 1000, 6096, 0.9776286482810974], "24": ["vertical_and_slash", 1000, 6096, 0.9923290610313416], "25": ["vertical_and_slash", 1000, 6096, 0.9234032034873962], "26": ["vertical_and_slash", 3500, 100, 0.9815073609352112], "27": ["vertical_and_slash", 1000, 6096, 0.9707662463188171], "28": ["vertical_and_slash", 1000, 6096, 0.9796174764633179], "29": ["vertical_and_slash", 1000, 6096, 0.9215888381004333], "30": ["vertical_and_slash", 3500, 100, 0.9955319166183472], "31": ["vertical_and_slash", 3500, 100, 0.9869089126586914], "32": ["vertical_and_slash", 3500, 100, 0.9438732266426086], "33": ["vertical_and_slash", 3500, 100, 0.9391714930534363], "34": ["vertical_and_slash", 1000, 6096, 0.937117874622345], "35": ["vertical_and_slash", 3500, 100, 0.9977035522460938], "36": ["vertical_and_slash", 1000, 6096, 0.9844733476638794], "37": ["vertical_and_slash", 3500, 100, 0.9935635924339294], "38": ["vertical_and_slash", 3500, 100, 0.992626428604126], "39": ["vertical_and_slash", 3500, 100, 0.9929220676422119]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9575890898704529], "1": ["vertical_and_slash", 3500, 100, 0.9677919149398804], "2": ["vertical_and_slash", 1000, 6096, 0.9869283437728882], "3": ["vertical_and_slash", 1000, 6096, 0.9735446572303772], "4": ["vertical_and_slash", 1000, 6096, 0.9788641929626465], "5": ["vertical_and_slash", 3500, 100, 0.9961890578269958], "6": ["vertical_and_slash", 1000, 6096, 0.9445140361785889], "7": ["vertical_and_slash", 1000, 6096, 0.9850566387176514], "8": ["vertical_and_slash", 1000, 6096, 0.9307213425636292], "9": ["vertical_and_slash", 3500, 100, 0.952678918838501], "10": ["vertical_and_slash", 3048, 8144, 0.878973126411438], "11": ["vertical_and_slash", 1000, 6096, 0.9581291675567627], "12": ["vertical_and_slash", 1000, 6096, 0.9932008385658264], "13": ["vertical_and_slash", 5548, 2148, 0.8739012479782104], "14": ["vertical_and_slash", 1000, 6096, 0.9912850260734558], "15": ["vertical_and_slash", 1000, 6096, 0.9906399250030518], "16": ["vertical_and_slash", 5548, 2148, 0.819602906703949], "17": ["vertical_and_slash", 9192, 14288, 0.6665210127830505], "18": ["vertical_and_slash", 1000, 6096, 0.9600022435188293], "19": ["vertical_and_slash", 1000, 6096, 0.9038325548171997], "20": ["vertical_and_slash", 3500, 100, 0.9577525854110718], "21": ["vertical_and_slash", 3500, 100, 0.9570968151092529], "22": ["vertical_and_slash", 3500, 100, 0.9430463910102844], "23": ["vertical_and_slash", 3500, 100, 0.9774989485740662], "24": ["vertical_and_slash", 3500, 100, 0.9738900661468506], "25": ["vertical_and_slash", 1000, 6096, 0.9335026144981384], "26": ["vertical_and_slash", 3500, 100, 0.9642995595932007], "27": ["vertical_and_slash", 3500, 100, 0.9700027108192444], "28": ["vertical_and_slash", 1000, 6096, 0.9605782628059387], "29": ["vertical_and_slash", 3500, 100, 0.9887697100639343], "30": ["vertical_and_slash", 1000, 6096, 0.9909459352493286], "31": ["vertical_and_slash", 1000, 6096, 0.9731295704841614], "32": ["vertical_and_slash", 3500, 100, 0.9750890731811523], "33": ["vertical_and_slash", 3500, 100, 0.9796292781829834], "34": ["vertical_and_slash", 1000, 6096, 0.9822960495948792], "35": ["vertical_and_slash", 3500, 100, 0.9732731580734253], "36": ["vertical_and_slash", 3500, 100, 0.972599983215332], "37": ["vertical_and_slash", 1000, 6096, 0.9310420751571655], "38": ["vertical_and_slash", 1000, 6096, 0.9628490209579468], "39": ["vertical_and_slash", 1000, 6096, 0.9681499004364014]}, {"0": ["vertical_and_slash", 3048, 8144, 0.8893836736679077], "1": ["vertical_and_slash", 3048, 8144, 0.8703296184539795], "2": ["vertical_and_slash", 1000, 6096, 0.961355984210968], "3": ["vertical_and_slash", 3048, 8144, 0.8530225157737732], "4": ["vertical_and_slash", 3048, 8144, 0.8924871683120728], "5": ["vertical_and_slash", 1000, 6096, 0.9278416633605957], "6": ["vertical_and_slash", 1000, 6096, 0.9618312120437622], "7": ["vertical_and_slash", 1000, 6096, 0.9560527801513672], "8": ["vertical_and_slash", 1000, 6096, 0.9683689475059509], "9": ["vertical_and_slash", 1000, 6096, 0.9673488736152649], "10": ["vertical_and_slash", 1000, 6096, 0.9783194065093994], "11": ["vertical_and_slash", 3048, 8144, 0.8913053274154663], "12": ["vertical_and_slash", 1000, 6096, 0.975897490978241], "13": ["vertical_and_slash", 3500, 100, 0.9924948811531067], "14": ["vertical_and_slash", 1000, 6096, 0.9531598687171936], "15": ["vertical_and_slash", 30, 800, 0.9965916872024536], "16": ["vertical_and_slash", 30, 800, 0.998519241809845], "17": ["vertical_and_slash", 500, 700, 0.9787888526916504], "18": ["vertical_and_slash", 500, 700, 0.9552795886993408], "19": ["vertical_and_slash", 30, 800, 0.9988910555839539], "20": ["vertical_and_slash", 3048, 8144, 0.82850581407547], "21": ["vertical_and_slash", 1000, 6096, 0.9505853652954102], "22": ["vertical_and_slash", 3048, 8144, 0.8548622131347656], "23": ["vertical_and_slash", 1000, 6096, 0.9273167252540588], "24": ["vertical_and_slash", 1000, 6096, 0.9734153747558594], "25": ["vertical_and_slash", 3048, 8144, 0.8255495429039001], "26": ["vertical_and_slash", 5096, 10192, 0.7682231664657593], "27": ["vertical_and_slash", 9192, 14288, 0.6783349514007568], "28": ["vertical_and_slash", 9192, 14288, 0.6578035950660706], "29": ["vertical_and_slash", 9192, 14288, 0.674615204334259], "30": ["vertical_and_slash", 3048, 8144, 0.8909530639648438], "31": ["vertical_and_slash", 1000, 6096, 0.9051004648208618], "32": ["vertical_and_slash", 5096, 10192, 0.7926168441772461], "33": ["vertical_and_slash", 5096, 10192, 0.7715818285942078], "34": ["vertical_and_slash", 1000, 6096, 0.9410092234611511], "35": ["vertical_and_slash", 500, 700, 0.9406668543815613], "36": ["vertical_and_slash", 1000, 6096, 0.9548261165618896], "37": ["vertical_and_slash", 1000, 6096, 0.9808189272880554], "38": ["vertical_and_slash", 3500, 100, 0.9917089343070984], "39": ["vertical_and_slash", 100, 800, 0.98828125]}, {"0": ["vertical_and_slash", 9192, 14288, 0.6226118206977844], "1": ["vertical_and_slash", 5096, 10192, 0.7815380692481995], "2": ["vertical_and_slash", 9192, 14288, 0.6278335452079773], "3": ["vertical_and_slash", 3048, 8144, 0.8173134326934814], "4": ["vertical_and_slash", 3048, 8144, 0.8831840753555298], "5": ["vertical_and_slash", 3048, 8144, 0.8794034719467163], "6": ["vertical_and_slash", 1000, 6096, 0.9384722709655762], "7": ["vertical_and_slash", 1000, 6096, 0.9250066876411438], "8": ["vertical_and_slash", 3048, 8144, 0.8958603739738464], "9": ["vertical_and_slash", 3048, 8144, 0.86203932762146], "10": ["vertical_and_slash", 1000, 6096, 0.9036557674407959], "11": ["vertical_and_slash", 1000, 6096, 0.925635576248169], "12": ["vertical_and_slash", 1000, 6096, 0.9378361701965332], "13": ["vertical_and_slash", 1000, 6096, 0.9263177514076233], "14": ["vertical_and_slash", 1000, 6096, 0.9518314599990845], "15": ["vertical_and_slash", 1000, 6096, 0.9230427145957947], "16": ["vertical_and_slash", 3048, 8144, 0.841392457485199], "17": ["vertical_and_slash", 3048, 8144, 0.8089483380317688], "18": ["vertical_and_slash", 1000, 6096, 0.9049822092056274], "19": ["vertical_and_slash", 1000, 6096, 0.9400890469551086], "20": ["vertical_and_slash", 1000, 6096, 0.9325522184371948], "21": ["vertical_and_slash", 1000, 6096, 0.944596529006958], "22": ["vertical_and_slash", 3048, 8144, 0.8669170141220093], "23": ["vertical_and_slash", 1000, 6096, 0.9166103601455688], "24": ["vertical_and_slash", 1000, 6096, 0.9630905389785767], "25": ["vertical_and_slash", 5096, 10192, 0.778262734413147], "26": ["vertical_and_slash", 5096, 10192, 0.7899712324142456], "27": ["vertical_and_slash", 9192, 14288, 0.6216180920600891], "28": ["vertical_and_slash", 9192, 14288, 0.6805283427238464], "29": ["vertical_and_slash", 5096, 10192, 0.7642425894737244], "30": ["vertical_and_slash", 3048, 8144, 0.8946554660797119], "31": ["vertical_and_slash", 1000, 6096, 0.9593181610107422], "32": ["vertical_and_slash", 1000, 6096, 0.9003524780273438], "33": ["vertical_and_slash", 1000, 6096, 0.9250353574752808], "34": ["vertical_and_slash", 5096, 10192, 0.7112997770309448], "35": ["vertical_and_slash", 1000, 6096, 0.9370287656784058], "36": ["vertical_and_slash", 3048, 8144, 0.8055636882781982], "37": ["vertical_and_slash", 1000, 6096, 0.9981359243392944], "38": ["vertical_and_slash", 5096, 10192, 0.7454675436019897], "39": ["vertical_and_slash", 1000, 6096, 0.9895527362823486]}]
diff --git a/minference/configs/Qwen2.5_32B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json b/minference/configs/Qwen2.5_32B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json
new file mode 100644
index 00000000..0c84ebb0
--- /dev/null
+++ b/minference/configs/Qwen2.5_32B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json
@@ -0,0 +1 @@
+[{"0": ["vertical_and_slash", 1000, 6096, 0.8595879673957825], "1": ["vertical_and_slash", 1000, 6096, 0.677900493144989], "2": ["vertical_and_slash", 1000, 6096, 0.937256395816803], "3": ["vertical_and_slash", 1000, 6096, 0.9350263476371765], "4": ["vertical_and_slash", 1000, 6096, 0.635826826095581], "5": ["vertical_and_slash", 1000, 6096, 0.9377691149711609], "6": ["vertical_and_slash", 1000, 6096, 0.5481041669845581], "7": ["vertical_and_slash", 1000, 6096, 0.9348520040512085], "8": ["vertical_and_slash", 1000, 6096, 0.4695983827114105], "9": ["vertical_and_slash", 1000, 6096, 0.9326639175415039], "10": ["vertical_and_slash", 1000, 6096, 0.5698804259300232], "11": ["vertical_and_slash", 1000, 6096, 0.993532121181488], "12": ["vertical_and_slash", 1000, 6096, 0.564853847026825], "13": ["vertical_and_slash", 1000, 6096, 0.7515659332275391], "14": ["vertical_and_slash", 1000, 6096, 0.9742660522460938], "15": ["vertical_and_slash", 1000, 6096, 0.9012133479118347], "16": ["vertical_and_slash", 1000, 6096, 0.6600093841552734], "17": ["vertical_and_slash", 1000, 6096, 0.6914014220237732], "18": ["vertical_and_slash", 1000, 6096, 0.906983494758606], "19": ["vertical_and_slash", 1000, 6096, 0.7274374961853027], "20": ["vertical_and_slash", 1000, 6096, 0.9316114187240601], "21": ["vertical_and_slash", 1000, 6096, 0.7053929567337036], "22": ["vertical_and_slash", 1000, 6096, 0.851769208908081], "23": ["vertical_and_slash", 1000, 6096, 0.9100949764251709], "24": ["vertical_and_slash", 1000, 6096, 0.9337905049324036], "25": ["vertical_and_slash", 1000, 6096, 0.6528579592704773], "26": ["vertical_and_slash", 1000, 6096, 0.7829988598823547], "27": ["vertical_and_slash", 1000, 6096, 0.9594557881355286], "28": ["vertical_and_slash", 3500, 100, 0.7299822568893433], "29": ["vertical_and_slash", 1000, 6096, 0.4569512903690338], "30": ["vertical_and_slash", 1000, 6096, 0.7135136723518372], "31": ["vertical_and_slash", 1000, 6096, 0.9315493106842041], "32": ["vertical_and_slash", 1000, 6096, 0.712002694606781], "33": ["vertical_and_slash", 1000, 6096, 0.6580793261528015], "34": ["vertical_and_slash", 1000, 6096, 0.7191514372825623], "35": ["vertical_and_slash", 1000, 6096, 0.9677614569664001], "36": ["vertical_and_slash", 1000, 6096, 0.7741791605949402], "37": ["vertical_and_slash", 1000, 6096, 0.8102108240127563], "38": ["vertical_and_slash", 1000, 6096, 0.9771817922592163], "39": ["vertical_and_slash", 1000, 6096, 0.6008121371269226]}, {"0": ["vertical_and_slash", 1000, 6096, 0.5780692100524902], "1": ["vertical_and_slash", 1000, 6096, 0.590826690196991], "2": ["vertical_and_slash", 1000, 6096, 0.5855319499969482], "3": ["vertical_and_slash", 1000, 6096, 0.557890772819519], "4": ["vertical_and_slash", 1000, 6096, 0.9574500322341919], "5": ["vertical_and_slash", 3500, 100, 0.8711386919021606], "6": ["vertical_and_slash", 1000, 6096, 0.9455059766769409], "7": ["vertical_and_slash", 30, 800, 0.999399721622467], "8": ["vertical_and_slash", 1000, 6096, 0.9881018996238708], "9": ["vertical_and_slash", 30, 800, 0.9904633164405823], "10": ["vertical_and_slash", 1000, 6096, 0.718948245048523], "11": ["vertical_and_slash", 1000, 6096, 0.9442924857139587], "12": ["vertical_and_slash", 1000, 6096, 0.6176681518554688], "13": ["vertical_and_slash", 1000, 6096, 0.8869352340698242], "14": ["vertical_and_slash", 1000, 6096, 0.8472557663917542], "15": ["vertical_and_slash", 3500, 100, 0.8700689077377319], "16": ["vertical_and_slash", 1000, 6096, 1.000004529953003], "17": ["vertical_and_slash", 1000, 6096, 0.9969146251678467], "18": ["vertical_and_slash", 1000, 6096, 0.9858112931251526], "19": ["vertical_and_slash", 1000, 6096, 0.7257927060127258], "20": ["vertical_and_slash", 1000, 6096, 0.6835055351257324], "21": ["vertical_and_slash", 1000, 6096, 0.5905671715736389], "22": ["vertical_and_slash", 1000, 6096, 0.6847525835037231], "23": ["vertical_and_slash", 1000, 6096, 0.8070325255393982], "24": ["vertical_and_slash", 1000, 6096, 0.7993292808532715], "25": ["vertical_and_slash", 1000, 6096, 0.4577499330043793], "26": ["vertical_and_slash", 1000, 6096, 0.9792490005493164], "27": ["vertical_and_slash", 1000, 6096, 0.9969779849052429], "28": ["vertical_and_slash", 30, 800, 0.9998692274093628], "29": ["vertical_and_slash", 1000, 6096, 0.9915031790733337], "30": ["vertical_and_slash", 1000, 6096, 0.6287465691566467], "31": ["vertical_and_slash", 1000, 6096, 0.5590917468070984], "32": ["vertical_and_slash", 1000, 6096, 0.6530432105064392], "33": ["vertical_and_slash", 1000, 6096, 0.5379100441932678], "34": ["vertical_and_slash", 1000, 6096, 0.47713610529899597], "35": ["vertical_and_slash", 1000, 6096, 0.579090416431427], "36": ["vertical_and_slash", 1000, 6096, 0.5331117510795593], "37": ["vertical_and_slash", 1000, 6096, 0.5972039699554443], "38": ["vertical_and_slash", 1000, 6096, 0.50728440284729], "39": ["vertical_and_slash", 1000, 6096, 0.5399038791656494]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9850784540176392], "1": ["vertical_and_slash", 1000, 6096, 0.638654351234436], "2": ["vertical_and_slash", 1000, 6096, 0.6308850049972534], "3": ["vertical_and_slash", 1000, 6096, 0.8572124242782593], "4": ["vertical_and_slash", 1000, 6096, 0.98296719789505], "5": ["vertical_and_slash", 1000, 6096, 0.7682300209999084], "6": ["vertical_and_slash", 1000, 6096, 0.5544219613075256], "7": ["vertical_and_slash", 1000, 6096, 0.5641868114471436], "8": ["vertical_and_slash", 1000, 6096, 0.5686959028244019], "9": ["vertical_and_slash", 1000, 6096, 0.5687942504882812], "10": ["vertical_and_slash", 30, 800, 0.9543052911758423], "11": ["vertical_and_slash", 1000, 6096, 0.8730300664901733], "12": ["vertical_and_slash", 1000, 6096, 0.8694428205490112], "13": ["vertical_and_slash", 30, 800, 0.9776438474655151], "14": ["vertical_and_slash", 30, 800, 0.876179575920105], "15": ["vertical_and_slash", 1000, 6096, 0.9740710258483887], "16": ["vertical_and_slash", 30, 800, 0.960694432258606], "17": ["vertical_and_slash", 1000, 6096, 0.7393561601638794], "18": ["vertical_and_slash", 1000, 6096, 0.9435052275657654], "19": ["vertical_and_slash", 30, 800, 0.9432980418205261], "20": ["vertical_and_slash", 1000, 6096, 0.9941500425338745], "21": ["vertical_and_slash", 1000, 6096, 0.7369858622550964], "22": ["vertical_and_slash", 1000, 6096, 0.9618136882781982], "23": ["vertical_and_slash", 1000, 6096, 0.6403058767318726], "24": ["vertical_and_slash", 1000, 6096, 0.7177169322967529], "25": ["vertical_and_slash", 1000, 6096, 0.7451578974723816], "26": ["vertical_and_slash", 1000, 6096, 0.6188691854476929], "27": ["vertical_and_slash", 1000, 6096, 0.6789723038673401], "28": ["vertical_and_slash", 1000, 6096, 0.7766066789627075], "29": ["vertical_and_slash", 1000, 6096, 0.9109611511230469], "30": ["vertical_and_slash", 3500, 100, 0.5188373923301697], "31": ["vertical_and_slash", 1000, 6096, 0.5005264282226562], "32": ["vertical_and_slash", 1000, 6096, 0.9254875183105469], "33": ["vertical_and_slash", 3500, 100, 0.650166928768158], "34": ["vertical_and_slash", 1000, 6096, 0.47144293785095215], "35": ["vertical_and_slash", 1000, 6096, 0.8767935633659363], "36": ["vertical_and_slash", 30, 800, 0.991529107093811], "37": ["vertical_and_slash", 1000, 6096, 0.9263222813606262], "38": ["vertical_and_slash", 1000, 6096, 0.9725203514099121], "39": ["vertical_and_slash", 1000, 6096, 0.9031548500061035]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9089404940605164], "1": ["vertical_and_slash", 3500, 100, 0.4982472360134125], "2": ["vertical_and_slash", 1000, 6096, 0.6751903295516968], "3": ["vertical_and_slash", 1000, 6096, 0.6492144465446472], "4": ["vertical_and_slash", 1000, 6096, 0.6620914340019226], "5": ["vertical_and_slash", 1000, 6096, 0.8109068870544434], "6": ["vertical_and_slash", 1000, 6096, 0.7313815355300903], "7": ["vertical_and_slash", 3500, 100, 0.7917609214782715], "8": ["vertical_and_slash", 1000, 6096, 0.7251050472259521], "9": ["vertical_and_slash", 1000, 6096, 0.7627575397491455], "10": ["vertical_and_slash", 1000, 6096, 0.9764474630355835], "11": ["vertical_and_slash", 1000, 6096, 0.8691239953041077], "12": ["vertical_and_slash", 1000, 6096, 0.9927623271942139], "13": ["vertical_and_slash", 1000, 6096, 0.9978007674217224], "14": ["vertical_and_slash", 1000, 6096, 0.9584883451461792], "15": ["vertical_and_slash", 1000, 6096, 0.8787619471549988], "16": ["vertical_and_slash", 1000, 6096, 0.5700175166130066], "17": ["vertical_and_slash", 1000, 6096, 0.5371003746986389], "18": ["vertical_and_slash", 3500, 100, 0.5999955534934998], "19": ["vertical_and_slash", 1000, 6096, 0.8238154649734497], "20": ["vertical_and_slash", 1000, 6096, 0.7391127347946167], "21": ["vertical_and_slash", 1000, 6096, 0.6254953742027283], "22": ["vertical_and_slash", 3500, 100, 0.5731092095375061], "23": ["vertical_and_slash", 1000, 6096, 0.6657670140266418], "24": ["vertical_and_slash", 1000, 6096, 0.6749957203865051], "25": ["vertical_and_slash", 1000, 6096, 0.7382640242576599], "26": ["vertical_and_slash", 3500, 100, 0.6491906046867371], "27": ["vertical_and_slash", 1000, 6096, 0.6951600909233093], "28": ["vertical_and_slash", 3500, 100, 0.7050278782844543], "29": ["vertical_and_slash", 3500, 100, 0.710655152797699], "30": ["vertical_and_slash", 1000, 6096, 0.6554528474807739], "31": ["vertical_and_slash", 1000, 6096, 0.852859616279602], "32": ["vertical_and_slash", 1000, 6096, 0.7995194792747498], "33": ["vertical_and_slash", 1000, 6096, 0.6767405867576599], "34": ["vertical_and_slash", 1000, 6096, 0.7768736481666565], "35": ["vertical_and_slash", 1000, 6096, 0.606532633304596], "36": ["vertical_and_slash", 3500, 100, 0.7186007499694824], "37": ["vertical_and_slash", 3500, 100, 0.7520785331726074], "38": ["vertical_and_slash", 3500, 100, 0.78960782289505], "39": ["vertical_and_slash", 1000, 6096, 0.6743583083152771]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9934982657432556], "1": ["vertical_and_slash", 1000, 6096, 0.873843789100647], "2": ["vertical_and_slash", 1000, 6096, 0.9684486985206604], "3": ["vertical_and_slash", 1000, 6096, 0.9060821533203125], "4": ["vertical_and_slash", 1000, 6096, 0.9540987014770508], "5": ["vertical_and_slash", 1000, 6096, 0.7840837240219116], "6": ["vertical_and_slash", 1000, 6096, 0.9283869862556458], "7": ["vertical_and_slash", 1000, 6096, 0.8708937168121338], "8": ["vertical_and_slash", 1000, 6096, 0.6653714776039124], "9": ["vertical_and_slash", 1000, 6096, 0.8596433997154236], "10": ["vertical_and_slash", 1000, 6096, 0.8991130590438843], "11": ["vertical_and_slash", 1000, 6096, 0.8370591402053833], "12": ["vertical_and_slash", 3500, 100, 0.5860728621482849], "13": ["vertical_and_slash", 1000, 6096, 0.8663804531097412], "14": ["vertical_and_slash", 1000, 6096, 0.7903067469596863], "15": ["vertical_and_slash", 1000, 6096, 0.7797677516937256], "16": ["vertical_and_slash", 1000, 6096, 0.8842828273773193], "17": ["vertical_and_slash", 1000, 6096, 0.9253506660461426], "18": ["vertical_and_slash", 1000, 6096, 0.7866469621658325], "19": ["vertical_and_slash", 1000, 6096, 0.7856847643852234], "20": ["vertical_and_slash", 1000, 6096, 0.9678407311439514], "21": ["vertical_and_slash", 1000, 6096, 0.7002184391021729], "22": ["vertical_and_slash", 1000, 6096, 0.8666926026344299], "23": ["vertical_and_slash", 1000, 6096, 0.9517683386802673], "24": ["vertical_and_slash", 1000, 6096, 0.9537820219993591], "25": ["vertical_and_slash", 1000, 6096, 0.9233936071395874], "26": ["vertical_and_slash", 1000, 6096, 0.9277289509773254], "27": ["vertical_and_slash", 1000, 6096, 0.8538787961006165], "28": ["vertical_and_slash", 1000, 6096, 0.9460838437080383], "29": ["vertical_and_slash", 1000, 6096, 0.9745223522186279], "30": ["vertical_and_slash", 1000, 6096, 0.8628096580505371], "31": ["vertical_and_slash", 1000, 6096, 0.8571760058403015], "32": ["vertical_and_slash", 1000, 6096, 0.8962575793266296], "33": ["vertical_and_slash", 1000, 6096, 0.9752947688102722], "34": ["vertical_and_slash", 1000, 6096, 0.8990426659584045], "35": ["vertical_and_slash", 1000, 6096, 0.8321795463562012], "36": ["vertical_and_slash", 1000, 6096, 0.8449130058288574], "37": ["vertical_and_slash", 1000, 6096, 0.8621129393577576], "38": ["vertical_and_slash", 1000, 6096, 0.9407624006271362], "39": ["vertical_and_slash", 1000, 6096, 0.8657121062278748]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8684644103050232], "1": ["vertical_and_slash", 1000, 6096, 0.9791953563690186], "2": ["vertical_and_slash", 1000, 6096, 0.9715188145637512], "3": ["vertical_and_slash", 1000, 6096, 0.7482463121414185], "4": ["vertical_and_slash", 1000, 6096, 0.9274041652679443], "5": ["vertical_and_slash", 1000, 6096, 0.8926094770431519], "6": ["vertical_and_slash", 1000, 6096, 0.9710202813148499], "7": ["vertical_and_slash", 500, 700, 0.9670369029045105], "8": ["vertical_and_slash", 1000, 6096, 0.8788617849349976], "9": ["vertical_and_slash", 1000, 6096, 0.9570327997207642], "10": ["vertical_and_slash", 30, 800, 0.9441091418266296], "11": ["vertical_and_slash", 30, 800, 0.9712821245193481], "12": ["vertical_and_slash", 1000, 6096, 0.960600733757019], "13": ["vertical_and_slash", 1000, 6096, 0.9548667073249817], "14": ["vertical_and_slash", 30, 800, 0.9545784592628479], "15": ["vertical_and_slash", 1000, 6096, 0.899615466594696], "16": ["vertical_and_slash", 1000, 6096, 0.9688994884490967], "17": ["vertical_and_slash", 1000, 6096, 0.9068484902381897], "18": ["vertical_and_slash", 1000, 6096, 0.8963817954063416], "19": ["vertical_and_slash", 1000, 6096, 0.8309125304222107], "20": ["vertical_and_slash", 3500, 100, 0.8527111411094666], "21": ["vertical_and_slash", 3500, 100, 0.8795992136001587], "22": ["vertical_and_slash", 500, 700, 0.9465757608413696], "23": ["vertical_and_slash", 500, 700, 0.948398768901825], "24": ["vertical_and_slash", 500, 700, 0.9661059379577637], "25": ["vertical_and_slash", 1000, 6096, 0.7353717684745789], "26": ["vertical_and_slash", 1000, 6096, 0.9116237163543701], "27": ["vertical_and_slash", 1000, 6096, 0.9839115142822266], "28": ["vertical_and_slash", 1000, 6096, 0.9642530083656311], "29": ["vertical_and_slash", 1000, 6096, 0.8103916645050049], "30": ["vertical_and_slash", 30, 800, 0.888765811920166], "31": ["vertical_and_slash", 30, 800, 0.8178173899650574], "32": ["vertical_and_slash", 1000, 6096, 0.9057892560958862], "33": ["vertical_and_slash", 30, 800, 0.9637014865875244], "34": ["vertical_and_slash", 30, 800, 0.958552360534668], "35": ["vertical_and_slash", 30, 800, 0.932081401348114], "36": ["vertical_and_slash", 30, 800, 0.9248875975608826], "37": ["vertical_and_slash", 30, 800, 0.9606475830078125], "38": ["vertical_and_slash", 100, 800, 0.6015625], "39": ["vertical_and_slash", 30, 800, 0.8779636025428772]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8917055726051331], "1": ["vertical_and_slash", 1000, 6096, 0.9580313563346863], "2": ["vertical_and_slash", 1000, 6096, 0.888550341129303], "3": ["vertical_and_slash", 1000, 6096, 0.8866209983825684], "4": ["vertical_and_slash", 1000, 6096, 0.8364959359169006], "5": ["vertical_and_slash", 1000, 6096, 0.9249329566955566], "6": ["vertical_and_slash", 30, 800, 0.9634563326835632], "7": ["vertical_and_slash", 1000, 6096, 0.8508889675140381], "8": ["vertical_and_slash", 1000, 6096, 0.6474734544754028], "9": ["vertical_and_slash", 1000, 6096, 0.8387067914009094], "10": ["vertical_and_slash", 1000, 6096, 0.91255122423172], "11": ["vertical_and_slash", 30, 800, 0.9507998824119568], "12": ["vertical_and_slash", 30, 800, 0.9479179382324219], "13": ["vertical_and_slash", 1000, 6096, 0.8843726515769958], "14": ["vertical_and_slash", 30, 800, 0.9132011532783508], "15": ["vertical_and_slash", 1000, 6096, 0.9811457395553589], "16": ["vertical_and_slash", 1000, 6096, 0.9988837242126465], "17": ["vertical_and_slash", 3500, 100, 0.877736508846283], "18": ["vertical_and_slash", 3500, 100, 0.9923533201217651], "19": ["vertical_and_slash", 1000, 6096, 0.9976898431777954], "20": ["vertical_and_slash", 1000, 6096, 0.9134724140167236], "21": ["vertical_and_slash", 30, 800, 0.9687848091125488], "22": ["vertical_and_slash", 1000, 6096, 0.6705645322799683], "23": ["vertical_and_slash", 1000, 6096, 0.9525309801101685], "24": ["vertical_and_slash", 1000, 6096, 0.8896396160125732], "25": ["vertical_and_slash", 30, 800, 0.9063323140144348], "26": ["vertical_and_slash", 1000, 6096, 0.9885446429252625], "27": ["vertical_and_slash", 1000, 6096, 0.8261077404022217], "28": ["vertical_and_slash", 1000, 6096, 0.9185212254524231], "29": ["vertical_and_slash", 1000, 6096, 0.9754278659820557], "30": ["vertical_and_slash", 30, 800, 0.9673750996589661], "31": ["vertical_and_slash", 1000, 6096, 0.9374644756317139], "32": ["vertical_and_slash", 100, 800, 0.97265625], "33": ["vertical_and_slash", 30, 800, 0.9876686930656433], "34": ["vertical_and_slash", 3500, 100, 0.8608510494232178], "35": ["vertical_and_slash", 1000, 6096, 0.8273220658302307], "36": ["vertical_and_slash", 1000, 6096, 0.9646387696266174], "37": ["vertical_and_slash", 30, 800, 0.9684654474258423], "38": ["vertical_and_slash", 1000, 6096, 0.8494899868965149], "39": ["vertical_and_slash", 1000, 6096, 0.9535810947418213]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9761648774147034], "1": ["vertical_and_slash", 1000, 6096, 0.9528639912605286], "2": ["vertical_and_slash", 30, 800, 0.9782282114028931], "3": ["vertical_and_slash", 1000, 6096, 0.9852755069732666], "4": ["vertical_and_slash", 1000, 6096, 0.9936963319778442], "5": ["vertical_and_slash", 1000, 6096, 0.9580302238464355], "6": ["vertical_and_slash", 3500, 100, 0.9062021970748901], "7": ["vertical_and_slash", 1000, 6096, 0.9610993266105652], "8": ["vertical_and_slash", 1000, 6096, 0.9191659688949585], "9": ["vertical_and_slash", 1000, 6096, 0.9897433519363403], "10": ["vertical_and_slash", 1000, 6096, 0.9940584301948547], "11": ["vertical_and_slash", 1000, 6096, 0.8756449222564697], "12": ["vertical_and_slash", 1000, 6096, 0.8686608076095581], "13": ["vertical_and_slash", 1000, 6096, 0.9901939630508423], "14": ["vertical_and_slash", 1000, 6096, 0.849760115146637], "15": ["vertical_and_slash", 1000, 6096, 0.9955243468284607], "16": ["vertical_and_slash", 1000, 6096, 0.9943691492080688], "17": ["vertical_and_slash", 1000, 6096, 0.9463406205177307], "18": ["vertical_and_slash", 1000, 6096, 0.9301040172576904], "19": ["vertical_and_slash", 1000, 6096, 0.9776157736778259], "20": ["vertical_and_slash", 3500, 100, 0.9696680307388306], "21": ["vertical_and_slash", 3500, 100, 0.9948687553405762], "22": ["vertical_and_slash", 3500, 100, 0.9964054226875305], "23": ["vertical_and_slash", 1000, 6096, 0.9652619957923889], "24": ["vertical_and_slash", 1000, 6096, 0.9966368079185486], "25": ["vertical_and_slash", 1000, 6096, 0.999036967754364], "26": ["vertical_and_slash", 1000, 6096, 0.995461106300354], "27": ["vertical_and_slash", 1000, 6096, 0.9977635145187378], "28": ["vertical_and_slash", 1000, 6096, 0.9989631175994873], "29": ["vertical_and_slash", 1000, 6096, 0.9897971749305725], "30": ["vertical_and_slash", 1000, 6096, 0.9911646246910095], "31": ["vertical_and_slash", 1000, 6096, 0.9687688946723938], "32": ["vertical_and_slash", 1000, 6096, 0.9792739152908325], "33": ["vertical_and_slash", 1000, 6096, 0.9688437581062317], "34": ["vertical_and_slash", 1000, 6096, 0.9536206126213074], "35": ["vertical_and_slash", 1000, 6096, 0.9807616472244263], "36": ["vertical_and_slash", 1000, 6096, 0.9463276863098145], "37": ["vertical_and_slash", 1000, 6096, 0.9949150085449219], "38": ["vertical_and_slash", 1000, 6096, 0.9885256886482239], "39": ["vertical_and_slash", 1000, 6096, 0.954365611076355]}, {"0": ["vertical_and_slash", 30, 800, 0.972132682800293], "1": ["vertical_and_slash", 30, 800, 0.9528712034225464], "2": ["vertical_and_slash", 500, 700, 0.9763285517692566], "3": ["vertical_and_slash", 1000, 6096, 0.9301146864891052], "4": ["vertical_and_slash", 30, 800, 0.9680641889572144], "5": ["vertical_and_slash", 1000, 6096, 0.8602001070976257], "6": ["vertical_and_slash", 1000, 6096, 0.9705460667610168], "7": ["vertical_and_slash", 1000, 6096, 0.9078283309936523], "8": ["vertical_and_slash", 1000, 6096, 0.9794800877571106], "9": ["vertical_and_slash", 1000, 6096, 0.9529551863670349], "10": ["vertical_and_slash", 1000, 6096, 0.8923068642616272], "11": ["vertical_and_slash", 1000, 6096, 0.9917446374893188], "12": ["vertical_and_slash", 1000, 6096, 0.928623616695404], "13": ["vertical_and_slash", 1000, 6096, 0.7550984621047974], "14": ["vertical_and_slash", 1000, 6096, 0.9626830816268921], "15": ["vertical_and_slash", 1000, 6096, 0.9842640161514282], "16": ["vertical_and_slash", 1000, 6096, 0.994007408618927], "17": ["vertical_and_slash", 1000, 6096, 0.9195037484169006], "18": ["vertical_and_slash", 1000, 6096, 0.9844812750816345], "19": ["vertical_and_slash", 1000, 6096, 0.981484055519104], "20": ["vertical_and_slash", 30, 800, 0.9922513961791992], "21": ["vertical_and_slash", 30, 800, 0.9860044717788696], "22": ["vertical_and_slash", 30, 800, 0.980429470539093], "23": ["vertical_and_slash", 30, 800, 0.9917383193969727], "24": ["vertical_and_slash", 30, 800, 0.9803014993667603], "25": ["vertical_and_slash", 30, 800, 0.985116183757782], "26": ["vertical_and_slash", 1000, 6096, 0.8794309496879578], "27": ["vertical_and_slash", 500, 700, 0.9841223955154419], "28": ["vertical_and_slash", 1000, 6096, 0.9378510117530823], "29": ["vertical_and_slash", 30, 800, 0.9695103764533997], "30": ["vertical_and_slash", 3500, 100, 0.9831942319869995], "31": ["vertical_and_slash", 1000, 6096, 0.9679700136184692], "32": ["vertical_and_slash", 1000, 6096, 0.9591716527938843], "33": ["vertical_and_slash", 1000, 6096, 0.8719543218612671], "34": ["vertical_and_slash", 1000, 6096, 0.9604390859603882], "35": ["vertical_and_slash", 1000, 6096, 0.977486789226532], "36": ["vertical_and_slash", 1000, 6096, 0.9894447326660156], "37": ["vertical_and_slash", 1000, 6096, 0.9948815703392029], "38": ["vertical_and_slash", 1000, 6096, 0.8925806879997253], "39": ["vertical_and_slash", 1000, 6096, 0.989867091178894]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9585041403770447], "1": ["vertical_and_slash", 1000, 6096, 0.9394539594650269], "2": ["vertical_and_slash", 1000, 6096, 0.9978679418563843], "3": ["vertical_and_slash", 1000, 6096, 0.8847857117652893], "4": ["vertical_and_slash", 1000, 6096, 0.9874283075332642], "5": ["vertical_and_slash", 30, 800, 0.9840196371078491], "6": ["vertical_and_slash", 30, 800, 0.922031819820404], "7": ["vertical_and_slash", 1000, 6096, 0.9306831359863281], "8": ["vertical_and_slash", 500, 700, 0.9805639982223511], "9": ["vertical_and_slash", 30, 800, 0.9821131825447083], "10": ["vertical_and_slash", 1000, 6096, 0.8047002553939819], "11": ["vertical_and_slash", 1000, 6096, 0.8417568206787109], "12": ["vertical_and_slash", 1000, 6096, 0.9369150400161743], "13": ["vertical_and_slash", 1000, 6096, 0.9701905250549316], "14": ["vertical_and_slash", 1000, 6096, 0.8510067462921143], "15": ["vertical_and_slash", 30, 800, 0.9547669291496277], "16": ["vertical_and_slash", 30, 800, 0.9827991127967834], "17": ["vertical_and_slash", 30, 800, 0.9525408148765564], "18": ["vertical_and_slash", 1000, 6096, 0.9154975414276123], "19": ["vertical_and_slash", 1000, 6096, 0.9758902192115784], "20": ["vertical_and_slash", 30, 800, 0.9894945621490479], "21": ["vertical_and_slash", 1000, 6096, 0.9967504143714905], "22": ["vertical_and_slash", 1000, 6096, 0.8287495374679565], "23": ["vertical_and_slash", 1000, 6096, 0.9864431023597717], "24": ["vertical_and_slash", 1000, 6096, 0.9970079064369202], "25": ["vertical_and_slash", 1000, 6096, 0.7636926770210266], "26": ["vertical_and_slash", 1000, 6096, 0.9897201657295227], "27": ["vertical_and_slash", 1000, 6096, 0.7213484048843384], "28": ["vertical_and_slash", 1000, 6096, 0.9925035834312439], "29": ["vertical_and_slash", 100, 800, 0.84375], "30": ["vertical_and_slash", 1000, 6096, 0.9683845639228821], "31": ["vertical_and_slash", 1000, 6096, 0.9849256873130798], "32": ["vertical_and_slash", 1000, 6096, 0.993407666683197], "33": ["vertical_and_slash", 1000, 6096, 0.9124467372894287], "34": ["vertical_and_slash", 1000, 6096, 0.9935197234153748], "35": ["vertical_and_slash", 30, 800, 0.9710790514945984], "36": ["vertical_and_slash", 30, 800, 0.7514054775238037], "37": ["vertical_and_slash", 30, 800, 0.952148973941803], "38": ["vertical_and_slash", 30, 800, 0.9674367904663086], "39": ["vertical_and_slash", 30, 800, 0.8270906805992126]}, {"0": ["vertical_and_slash", 30, 800, 0.9795775413513184], "1": ["vertical_and_slash", 30, 800, 0.9572718739509583], "2": ["vertical_and_slash", 30, 800, 0.9913018345832825], "3": ["vertical_and_slash", 30, 800, 0.9849529266357422], "4": ["vertical_and_slash", 30, 800, 0.9820216298103333], "5": ["vertical_and_slash", 1000, 6096, 0.9514557719230652], "6": ["vertical_and_slash", 1000, 6096, 0.973755419254303], "7": ["vertical_and_slash", 1000, 6096, 0.9838699698448181], "8": ["vertical_and_slash", 1000, 6096, 0.9535142779350281], "9": ["vertical_and_slash", 1000, 6096, 0.9678695797920227], "10": ["vertical_and_slash", 1000, 6096, 0.9828231930732727], "11": ["vertical_and_slash", 1000, 6096, 0.9351344704627991], "12": ["vertical_and_slash", 1000, 6096, 0.9376811981201172], "13": ["vertical_and_slash", 1000, 6096, 0.9833859205245972], "14": ["vertical_and_slash", 1000, 6096, 0.9327714443206787], "15": ["vertical_and_slash", 1000, 6096, 0.8042546510696411], "16": ["vertical_and_slash", 1000, 6096, 0.9626521468162537], "17": ["vertical_and_slash", 1000, 6096, 0.9401219487190247], "18": ["vertical_and_slash", 1000, 6096, 0.9630477428436279], "19": ["vertical_and_slash", 1000, 6096, 0.9426794052124023], "20": ["vertical_and_slash", 1000, 6096, 0.9575771689414978], "21": ["vertical_and_slash", 1000, 6096, 0.9775762557983398], "22": ["vertical_and_slash", 1000, 6096, 0.9962829351425171], "23": ["vertical_and_slash", 1000, 6096, 0.858562707901001], "24": ["vertical_and_slash", 1000, 6096, 0.9900478720664978], "25": ["vertical_and_slash", 1000, 6096, 0.9729108214378357], "26": ["vertical_and_slash", 1000, 6096, 0.9739375114440918], "27": ["vertical_and_slash", 1000, 6096, 0.9919418692588806], "28": ["vertical_and_slash", 1000, 6096, 0.8652782440185547], "29": ["vertical_and_slash", 1000, 6096, 0.9772907495498657], "30": ["vertical_and_slash", 30, 800, 0.9841402173042297], "31": ["vertical_and_slash", 1000, 6096, 0.957828164100647], "32": ["vertical_and_slash", 1000, 6096, 0.9010491967201233], "33": ["vertical_and_slash", 30, 800, 0.9860021471977234], "34": ["vertical_and_slash", 1000, 6096, 0.9834269285202026], "35": ["vertical_and_slash", 1000, 6096, 0.9435318112373352], "36": ["vertical_and_slash", 1000, 6096, 0.9915613532066345], "37": ["vertical_and_slash", 1000, 6096, 0.951927900314331], "38": ["vertical_and_slash", 1000, 6096, 0.9964062571525574], "39": ["vertical_and_slash", 1000, 6096, 0.9952526092529297]}, {"0": ["vertical_and_slash", 30, 800, 0.990652322769165], "1": ["vertical_and_slash", 30, 800, 0.9645078182220459], "2": ["vertical_and_slash", 30, 800, 0.9845086336135864], "3": ["vertical_and_slash", 30, 800, 0.9822641611099243], "4": ["vertical_and_slash", 30, 800, 0.9631287455558777], "5": ["vertical_and_slash", 30, 800, 0.970310628414154], "6": ["vertical_and_slash", 30, 800, 0.9824251532554626], "7": ["vertical_and_slash", 1000, 6096, 0.8656677603721619], "8": ["vertical_and_slash", 30, 800, 0.9919641017913818], "9": ["vertical_and_slash", 30, 800, 0.8610443472862244], "10": ["vertical_and_slash", 30, 800, 0.9663066864013672], "11": ["vertical_and_slash", 30, 800, 0.9498316049575806], "12": ["vertical_and_slash", 30, 800, 0.9578716158866882], "13": ["vertical_and_slash", 30, 800, 0.9055982232093811], "14": ["vertical_and_slash", 1000, 6096, 0.6185723543167114], "15": ["vertical_and_slash", 30, 800, 0.9373590350151062], "16": ["vertical_and_slash", 30, 800, 0.9511117935180664], "17": ["vertical_and_slash", 30, 800, 0.9760279059410095], "18": ["vertical_and_slash", 30, 800, 0.928572416305542], "19": ["vertical_and_slash", 30, 800, 0.935899555683136], "20": ["vertical_and_slash", 1000, 6096, 0.94867342710495], "21": ["vertical_and_slash", 1000, 6096, 0.9328466653823853], "22": ["vertical_and_slash", 1000, 6096, 0.9769710898399353], "23": ["vertical_and_slash", 1000, 6096, 0.9397515654563904], "24": ["vertical_and_slash", 1000, 6096, 0.982000470161438], "25": ["vertical_and_slash", 1000, 6096, 0.8932710289955139], "26": ["vertical_and_slash", 1000, 6096, 0.9693018794059753], "27": ["vertical_and_slash", 1000, 6096, 0.7961830496788025], "28": ["vertical_and_slash", 100, 800, 0.96875], "29": ["vertical_and_slash", 1000, 6096, 0.9837232232093811], "30": ["vertical_and_slash", 30, 800, 0.9882417917251587], "31": ["vertical_and_slash", 30, 800, 0.9839916229248047], "32": ["vertical_and_slash", 30, 800, 0.9773934483528137], "33": ["vertical_and_slash", 30, 800, 0.955439567565918], "34": ["vertical_and_slash", 30, 800, 0.9575393199920654], "35": ["vertical_and_slash", 30, 800, 0.9488707780838013], "36": ["vertical_and_slash", 30, 800, 0.8366216421127319], "37": ["vertical_and_slash", 1000, 6096, 0.9611375331878662], "38": ["vertical_and_slash", 30, 800, 0.7818154692649841], "39": ["vertical_and_slash", 1000, 6096, 0.9646857380867004]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8421997427940369], "1": ["vertical_and_slash", 1000, 6096, 0.9195594787597656], "2": ["vertical_and_slash", 1000, 6096, 0.9914657473564148], "3": ["vertical_and_slash", 1000, 6096, 0.9117868542671204], "4": ["vertical_and_slash", 1000, 6096, 0.982209324836731], "5": ["vertical_and_slash", 1000, 6096, 0.9924704432487488], "6": ["vertical_and_slash", 1000, 6096, 0.9966213703155518], "7": ["vertical_and_slash", 1000, 6096, 0.9967604279518127], "8": ["vertical_and_slash", 1000, 6096, 0.9755880832672119], "9": ["vertical_and_slash", 1000, 6096, 0.7887389063835144], "10": ["vertical_and_slash", 1000, 6096, 0.8235934972763062], "11": ["vertical_and_slash", 30, 800, 0.9387404322624207], "12": ["vertical_and_slash", 1000, 6096, 0.7767844200134277], "13": ["vertical_and_slash", 1000, 6096, 0.833696722984314], "14": ["vertical_and_slash", 1000, 6096, 0.9542989134788513], "15": ["vertical_and_slash", 30, 800, 0.8398716449737549], "16": ["vertical_and_slash", 30, 800, 0.9588367342948914], "17": ["vertical_and_slash", 30, 800, 0.9608014225959778], "18": ["vertical_and_slash", 30, 800, 0.954466700553894], "19": ["vertical_and_slash", 30, 800, 0.9503692984580994], "20": ["vertical_and_slash", 30, 800, 0.9827157855033875], "21": ["vertical_and_slash", 1000, 6096, 0.9478830099105835], "22": ["vertical_and_slash", 30, 800, 0.957087516784668], "23": ["vertical_and_slash", 1000, 6096, 0.7680861949920654], "24": ["vertical_and_slash", 30, 800, 0.9408442378044128], "25": ["vertical_and_slash", 30, 800, 0.9528371691703796], "26": ["vertical_and_slash", 30, 800, 0.967059850692749], "27": ["vertical_and_slash", 1000, 6096, 0.837132453918457], "28": ["vertical_and_slash", 30, 800, 0.9718199968338013], "29": ["vertical_and_slash", 30, 800, 0.9803667664527893], "30": ["vertical_and_slash", 1000, 6096, 0.9799139499664307], "31": ["vertical_and_slash", 1000, 6096, 0.9892632365226746], "32": ["vertical_and_slash", 1000, 6096, 0.9763562083244324], "33": ["vertical_and_slash", 1000, 6096, 0.9632612466812134], "34": ["vertical_and_slash", 1000, 6096, 0.7675636410713196], "35": ["vertical_and_slash", 1000, 6096, 0.8246073722839355], "36": ["vertical_and_slash", 1000, 6096, 0.9805220365524292], "37": ["vertical_and_slash", 1000, 6096, 0.977497935295105], "38": ["vertical_and_slash", 1000, 6096, 0.8178223371505737], "39": ["vertical_and_slash", 1000, 6096, 0.9534328579902649]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7116466164588928], "1": ["vertical_and_slash", 30, 800, 0.9681659936904907], "2": ["vertical_and_slash", 30, 800, 0.9831739664077759], "3": ["vertical_and_slash", 30, 800, 0.9707523584365845], "4": ["vertical_and_slash", 100, 800, 0.90625], "5": ["vertical_and_slash", 30, 800, 0.9711333513259888], "6": ["vertical_and_slash", 30, 800, 0.9158774018287659], "7": ["vertical_and_slash", 1000, 6096, 0.9426547884941101], "8": ["vertical_and_slash", 30, 800, 0.9739215970039368], "9": ["vertical_and_slash", 30, 800, 0.9678557515144348], "10": ["vertical_and_slash", 30, 800, 0.8351387977600098], "11": ["vertical_and_slash", 30, 800, 0.9690917730331421], "12": ["vertical_and_slash", 30, 800, 0.977911651134491], "13": ["vertical_and_slash", 30, 800, 0.9764779210090637], "14": ["vertical_and_slash", 30, 800, 0.9674893617630005], "15": ["vertical_and_slash", 1000, 6096, 0.9927276968955994], "16": ["vertical_and_slash", 1000, 6096, 0.7348263263702393], "17": ["vertical_and_slash", 1000, 6096, 0.9930658936500549], "18": ["vertical_and_slash", 1000, 6096, 0.9806174635887146], "19": ["vertical_and_slash", 1000, 6096, 0.946500837802887], "20": ["vertical_and_slash", 1000, 6096, 0.9888765811920166], "21": ["vertical_and_slash", 1000, 6096, 0.991182267665863], "22": ["vertical_and_slash", 1000, 6096, 0.9920547008514404], "23": ["vertical_and_slash", 1000, 6096, 0.9487566947937012], "24": ["vertical_and_slash", 1000, 6096, 0.980501651763916], "25": ["vertical_and_slash", 1000, 6096, 0.49585479497909546], "26": ["vertical_and_slash", 1000, 6096, 0.9806751608848572], "27": ["vertical_and_slash", 1000, 6096, 0.9853435158729553], "28": ["vertical_and_slash", 1000, 6096, 0.8501638174057007], "29": ["vertical_and_slash", 3500, 100, 0.864895761013031], "30": ["vertical_and_slash", 1000, 6096, 0.9576624631881714], "31": ["vertical_and_slash", 1000, 6096, 0.9880027770996094], "32": ["vertical_and_slash", 1000, 6096, 0.9152712225914001], "33": ["vertical_and_slash", 1000, 6096, 0.9685536026954651], "34": ["vertical_and_slash", 1000, 6096, 0.9659848213195801], "35": ["vertical_and_slash", 1000, 6096, 0.8151991963386536], "36": ["vertical_and_slash", 1000, 6096, 0.6821165680885315], "37": ["vertical_and_slash", 1000, 6096, 0.8159635066986084], "38": ["vertical_and_slash", 1000, 6096, 0.9919643402099609], "39": ["vertical_and_slash", 1000, 6096, 0.8994265198707581]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9239840507507324], "1": ["vertical_and_slash", 30, 800, 0.963828980922699], "2": ["vertical_and_slash", 30, 800, 0.9488492608070374], "3": ["vertical_and_slash", 30, 800, 0.9425246715545654], "4": ["vertical_and_slash", 30, 800, 0.9725340604782104], "5": ["vertical_and_slash", 1000, 6096, 0.9921441674232483], "6": ["vertical_and_slash", 1000, 6096, 0.9438525438308716], "7": ["vertical_and_slash", 1000, 6096, 0.7796332240104675], "8": ["vertical_and_slash", 1000, 6096, 0.906885027885437], "9": ["vertical_and_slash", 1000, 6096, 0.9573947787284851], "10": ["vertical_and_slash", 1000, 6096, 0.9609924554824829], "11": ["vertical_and_slash", 1000, 6096, 0.6450618505477905], "12": ["vertical_and_slash", 100, 800, 0.98046875], "13": ["vertical_and_slash", 1000, 6096, 0.8576703071594238], "14": ["vertical_and_slash", 30, 800, 0.9220871329307556], "15": ["vertical_and_slash", 30, 800, 0.9404322504997253], "16": ["vertical_and_slash", 30, 800, 0.948147714138031], "17": ["vertical_and_slash", 30, 800, 0.9398475885391235], "18": ["vertical_and_slash", 30, 800, 0.9544923305511475], "19": ["vertical_and_slash", 30, 800, 0.9492413997650146], "20": ["vertical_and_slash", 1000, 6096, 0.9747434258460999], "21": ["vertical_and_slash", 1000, 6096, 0.9578630924224854], "22": ["vertical_and_slash", 1000, 6096, 0.9107673764228821], "23": ["vertical_and_slash", 1000, 6096, 0.8931920528411865], "24": ["vertical_and_slash", 1000, 6096, 0.9883597493171692], "25": ["vertical_and_slash", 1000, 6096, 0.9831750392913818], "26": ["vertical_and_slash", 1000, 6096, 0.9739941358566284], "27": ["vertical_and_slash", 1000, 6096, 0.8047122955322266], "28": ["vertical_and_slash", 1000, 6096, 0.9687877297401428], "29": ["vertical_and_slash", 1000, 6096, 0.9714540839195251], "30": ["vertical_and_slash", 30, 800, 0.9567572474479675], "31": ["vertical_and_slash", 1000, 6096, 0.7386842370033264], "32": ["vertical_and_slash", 30, 800, 0.921637237071991], "33": ["vertical_and_slash", 30, 800, 0.904146671295166], "34": ["vertical_and_slash", 1000, 6096, 0.9405030608177185], "35": ["vertical_and_slash", 1000, 6096, 0.9956269264221191], "36": ["vertical_and_slash", 1000, 6096, 0.9662848114967346], "37": ["vertical_and_slash", 1000, 6096, 0.930871307849884], "38": ["vertical_and_slash", 1000, 6096, 0.951032280921936], "39": ["vertical_and_slash", 1000, 6096, 0.9942986965179443]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8469197154045105], "1": ["vertical_and_slash", 1000, 6096, 0.9677348136901855], "2": ["vertical_and_slash", 1000, 6096, 0.8848204612731934], "3": ["vertical_and_slash", 1000, 6096, 0.9299979209899902], "4": ["vertical_and_slash", 1000, 6096, 0.9267559051513672], "5": ["vertical_and_slash", 3500, 100, 0.9263042211532593], "6": ["vertical_and_slash", 1000, 6096, 0.9754449725151062], "7": ["vertical_and_slash", 1000, 6096, 0.9071605801582336], "8": ["vertical_and_slash", 1000, 6096, 0.9544404745101929], "9": ["vertical_and_slash", 1000, 6096, 0.9658820629119873], "10": ["vertical_and_slash", 30, 800, 0.9493272304534912], "11": ["vertical_and_slash", 1000, 6096, 0.9866783618927002], "12": ["vertical_and_slash", 1000, 6096, 0.9248888492584229], "13": ["vertical_and_slash", 1000, 6096, 0.9859155416488647], "14": ["vertical_and_slash", 1000, 6096, 0.9672225117683411], "15": ["vertical_and_slash", 1000, 6096, 0.9767405390739441], "16": ["vertical_and_slash", 1000, 6096, 0.9871063828468323], "17": ["vertical_and_slash", 1000, 6096, 0.9572961926460266], "18": ["vertical_and_slash", 1000, 6096, 0.975324809551239], "19": ["vertical_and_slash", 1000, 6096, 0.785001814365387], "20": ["vertical_and_slash", 30, 800, 0.9676733613014221], "21": ["vertical_and_slash", 30, 800, 0.9762175679206848], "22": ["vertical_and_slash", 30, 800, 0.9008428454399109], "23": ["vertical_and_slash", 30, 800, 0.9798709154129028], "24": ["vertical_and_slash", 30, 800, 0.9635664224624634], "25": ["vertical_and_slash", 1000, 6096, 0.9935462474822998], "26": ["vertical_and_slash", 1000, 6096, 0.970691978931427], "27": ["vertical_and_slash", 1000, 6096, 0.8415435552597046], "28": ["vertical_and_slash", 1000, 6096, 0.8626109957695007], "29": ["vertical_and_slash", 1000, 6096, 0.9868526458740234], "30": ["vertical_and_slash", 1000, 6096, 0.98097825050354], "31": ["vertical_and_slash", 1000, 6096, 0.9956587553024292], "32": ["vertical_and_slash", 1000, 6096, 0.9693565964698792], "33": ["vertical_and_slash", 1000, 6096, 0.8007195591926575], "34": ["vertical_and_slash", 1000, 6096, 0.9587075710296631], "35": ["vertical_and_slash", 30, 800, 0.949908971786499], "36": ["vertical_and_slash", 30, 800, 0.9098973274230957], "37": ["vertical_and_slash", 30, 800, 0.9684420228004456], "38": ["vertical_and_slash", 30, 800, 0.9648360013961792], "39": ["vertical_and_slash", 1000, 6096, 0.983808696269989]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9568448066711426], "1": ["vertical_and_slash", 1000, 6096, 0.9606406092643738], "2": ["vertical_and_slash", 30, 800, 0.9910467863082886], "3": ["vertical_and_slash", 30, 800, 0.9866725206375122], "4": ["vertical_and_slash", 1000, 6096, 0.9947802424430847], "5": ["vertical_and_slash", 3500, 100, 0.931855320930481], "6": ["vertical_and_slash", 1000, 6096, 0.9850279092788696], "7": ["vertical_and_slash", 1000, 6096, 0.9946127533912659], "8": ["vertical_and_slash", 1000, 6096, 0.9711607098579407], "9": ["vertical_and_slash", 1000, 6096, 0.9354884028434753], "10": ["vertical_and_slash", 100, 800, 0.984375], "11": ["vertical_and_slash", 30, 800, 0.9421122670173645], "12": ["vertical_and_slash", 1000, 6096, 0.7091702818870544], "13": ["vertical_and_slash", 1000, 6096, 0.9323245882987976], "14": ["vertical_and_slash", 30, 800, 0.951889157295227], "15": ["vertical_and_slash", 1000, 6096, 0.7719680666923523], "16": ["vertical_and_slash", 1000, 6096, 0.8451852798461914], "17": ["vertical_and_slash", 30, 800, 0.9534217119216919], "18": ["vertical_and_slash", 30, 800, 0.9695285558700562], "19": ["vertical_and_slash", 1000, 6096, 0.9293456077575684], "20": ["vertical_and_slash", 1000, 6096, 0.9119612574577332], "21": ["vertical_and_slash", 1000, 6096, 0.8391926884651184], "22": ["vertical_and_slash", 1000, 6096, 0.9146934747695923], "23": ["vertical_and_slash", 1000, 6096, 0.9410107731819153], "24": ["vertical_and_slash", 1000, 6096, 0.9515740275382996], "25": ["vertical_and_slash", 30, 800, 0.9717320799827576], "26": ["vertical_and_slash", 1000, 6096, 0.9185013175010681], "27": ["vertical_and_slash", 1000, 6096, 0.9789855480194092], "28": ["vertical_and_slash", 1000, 6096, 0.960614800453186], "29": ["vertical_and_slash", 30, 800, 0.9707027077674866], "30": ["vertical_and_slash", 30, 800, 0.969720721244812], "31": ["vertical_and_slash", 1000, 6096, 0.9377899765968323], "32": ["vertical_and_slash", 1000, 6096, 0.9398334622383118], "33": ["vertical_and_slash", 30, 800, 0.9738936424255371], "34": ["vertical_and_slash", 30, 800, 0.8629162311553955], "35": ["vertical_and_slash", 1000, 6096, 0.7715454697608948], "36": ["vertical_and_slash", 1000, 6096, 0.9310462474822998], "37": ["vertical_and_slash", 1000, 6096, 0.9557515978813171], "38": ["vertical_and_slash", 1000, 6096, 0.7538806200027466], "39": ["vertical_and_slash", 30, 800, 0.8677986264228821]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8163169026374817], "1": ["vertical_and_slash", 1000, 6096, 0.9850555658340454], "2": ["vertical_and_slash", 1000, 6096, 0.9670072793960571], "3": ["vertical_and_slash", 1000, 6096, 0.929294228553772], "4": ["vertical_and_slash", 1000, 6096, 0.7806792259216309], "5": ["vertical_and_slash", 1000, 6096, 0.9766951203346252], "6": ["vertical_and_slash", 1000, 6096, 0.9116750955581665], "7": ["vertical_and_slash", 1000, 6096, 0.9747446775436401], "8": ["vertical_and_slash", 1000, 6096, 0.9802051782608032], "9": ["vertical_and_slash", 1000, 6096, 0.9848531484603882], "10": ["vertical_and_slash", 30, 800, 0.9214944839477539], "11": ["vertical_and_slash", 1000, 6096, 0.604124903678894], "12": ["vertical_and_slash", 30, 800, 0.863186240196228], "13": ["vertical_and_slash", 100, 800, 0.9296875], "14": ["vertical_and_slash", 1000, 6096, 0.9383278489112854], "15": ["vertical_and_slash", 1000, 6096, 0.7762932181358337], "16": ["vertical_and_slash", 1000, 6096, 0.9489500522613525], "17": ["vertical_and_slash", 30, 800, 0.9838045239448547], "18": ["vertical_and_slash", 1000, 6096, 0.96181321144104], "19": ["vertical_and_slash", 30, 800, 0.9852977991104126], "20": ["vertical_and_slash", 1000, 6096, 0.961546778678894], "21": ["vertical_and_slash", 1000, 6096, 0.9577065110206604], "22": ["vertical_and_slash", 1000, 6096, 0.9402247071266174], "23": ["vertical_and_slash", 1000, 6096, 0.5855788588523865], "24": ["vertical_and_slash", 1000, 6096, 0.985479474067688], "25": ["vertical_and_slash", 30, 800, 0.9345589280128479], "26": ["vertical_and_slash", 30, 800, 0.9480632543563843], "27": ["vertical_and_slash", 30, 800, 0.9600964188575745], "28": ["vertical_and_slash", 30, 800, 0.9069157242774963], "29": ["vertical_and_slash", 30, 800, 0.9167723655700684], "30": ["vertical_and_slash", 1000, 6096, 0.994114100933075], "31": ["vertical_and_slash", 1000, 6096, 0.9920706748962402], "32": ["vertical_and_slash", 1000, 6096, 0.9526229500770569], "33": ["vertical_and_slash", 1000, 6096, 0.9954631328582764], "34": ["vertical_and_slash", 1000, 6096, 0.9839974641799927], "35": ["vertical_and_slash", 1000, 6096, 0.9283158779144287], "36": ["vertical_and_slash", 100, 800, 0.9609375], "37": ["vertical_and_slash", 1000, 6096, 0.9449872374534607], "38": ["vertical_and_slash", 1000, 6096, 0.984140157699585], "39": ["vertical_and_slash", 1000, 6096, 0.928844153881073]}, {"0": ["vertical_and_slash", 30, 800, 0.956022322177887], "1": ["vertical_and_slash", 30, 800, 0.9728148579597473], "2": ["vertical_and_slash", 30, 800, 0.9946978688240051], "3": ["vertical_and_slash", 30, 800, 0.9781919717788696], "4": ["vertical_and_slash", 30, 800, 0.9578215479850769], "5": ["vertical_and_slash", 1000, 6096, 0.9928136467933655], "6": ["vertical_and_slash", 1000, 6096, 0.7842119336128235], "7": ["vertical_and_slash", 100, 800, 0.73828125], "8": ["vertical_and_slash", 1000, 6096, 0.9648429751396179], "9": ["vertical_and_slash", 3500, 100, 0.9454610347747803], "10": ["vertical_and_slash", 1000, 6096, 0.8240343928337097], "11": ["vertical_and_slash", 1000, 6096, 0.9756290316581726], "12": ["vertical_and_slash", 1000, 6096, 0.9762590527534485], "13": ["vertical_and_slash", 3500, 100, 0.9946258068084717], "14": ["vertical_and_slash", 1000, 6096, 0.9134165644645691], "15": ["vertical_and_slash", 30, 800, 0.9681937098503113], "16": ["vertical_and_slash", 30, 800, 0.956716775894165], "17": ["vertical_and_slash", 30, 800, 0.9905259609222412], "18": ["vertical_and_slash", 30, 800, 0.9434572458267212], "19": ["vertical_and_slash", 30, 800, 0.9553202986717224], "20": ["vertical_and_slash", 1000, 6096, 0.9751352667808533], "21": ["vertical_and_slash", 1000, 6096, 0.8248187899589539], "22": ["vertical_and_slash", 1000, 6096, 0.9221310019493103], "23": ["vertical_and_slash", 1000, 6096, 0.9534516334533691], "24": ["vertical_and_slash", 1000, 6096, 0.9711092710494995], "25": ["vertical_and_slash", 1000, 6096, 0.9627410173416138], "26": ["vertical_and_slash", 1000, 6096, 0.9804104566574097], "27": ["vertical_and_slash", 1000, 6096, 0.9620197415351868], "28": ["vertical_and_slash", 1000, 6096, 0.8752495646476746], "29": ["vertical_and_slash", 1000, 6096, 0.9156225919723511], "30": ["vertical_and_slash", 1000, 6096, 0.9774807691574097], "31": ["vertical_and_slash", 30, 800, 0.8248814940452576], "32": ["vertical_and_slash", 1000, 6096, 0.9944645762443542], "33": ["vertical_and_slash", 1000, 6096, 0.9545608162879944], "34": ["vertical_and_slash", 1000, 6096, 0.9917159676551819], "35": ["vertical_and_slash", 1000, 6096, 0.9671100378036499], "36": ["vertical_and_slash", 1000, 6096, 0.9879676103591919], "37": ["vertical_and_slash", 1000, 6096, 0.9584366083145142], "38": ["vertical_and_slash", 1000, 6096, 0.9867058396339417], "39": ["vertical_and_slash", 1000, 6096, 0.960342288017273]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9842944741249084], "1": ["vertical_and_slash", 1000, 6096, 0.9766279458999634], "2": ["vertical_and_slash", 1000, 6096, 0.9333189129829407], "3": ["vertical_and_slash", 1000, 6096, 0.9609243869781494], "4": ["vertical_and_slash", 1000, 6096, 0.99432772397995], "5": ["vertical_and_slash", 1000, 6096, 0.8947204947471619], "6": ["vertical_and_slash", 1000, 6096, 0.8098337054252625], "7": ["vertical_and_slash", 1000, 6096, 0.9216397404670715], "8": ["vertical_and_slash", 1000, 6096, 0.9420117139816284], "9": ["vertical_and_slash", 1000, 6096, 0.9228754639625549], "10": ["vertical_and_slash", 1000, 6096, 0.9693853855133057], "11": ["vertical_and_slash", 1000, 6096, 0.9566335678100586], "12": ["vertical_and_slash", 1000, 6096, 0.8755287528038025], "13": ["vertical_and_slash", 1000, 6096, 0.8937090635299683], "14": ["vertical_and_slash", 1000, 6096, 0.8967458009719849], "15": ["vertical_and_slash", 30, 800, 0.9658228754997253], "16": ["vertical_and_slash", 30, 800, 0.9821271896362305], "17": ["vertical_and_slash", 30, 800, 0.971560537815094], "18": ["vertical_and_slash", 30, 800, 0.9408385157585144], "19": ["vertical_and_slash", 30, 800, 0.9789543151855469], "20": ["vertical_and_slash", 3500, 100, 0.9717743992805481], "21": ["vertical_and_slash", 1000, 6096, 0.9687156081199646], "22": ["vertical_and_slash", 3500, 100, 0.9962037205696106], "23": ["vertical_and_slash", 1000, 6096, 0.9766265749931335], "24": ["vertical_and_slash", 3500, 100, 1.0001649856567383], "25": ["vertical_and_slash", 1000, 6096, 0.9850876331329346], "26": ["vertical_and_slash", 1000, 6096, 0.9749317765235901], "27": ["vertical_and_slash", 1000, 6096, 0.7975354790687561], "28": ["vertical_and_slash", 1000, 6096, 0.9080640077590942], "29": ["vertical_and_slash", 1000, 6096, 0.8917402625083923], "30": ["vertical_and_slash", 1000, 6096, 0.9586751461029053], "31": ["vertical_and_slash", 1000, 6096, 0.9336493015289307], "32": ["vertical_and_slash", 1000, 6096, 0.893241286277771], "33": ["vertical_and_slash", 1000, 6096, 0.6480709314346313], "34": ["vertical_and_slash", 1000, 6096, 0.6734805703163147], "35": ["vertical_and_slash", 1000, 6096, 0.9827634692192078], "36": ["vertical_and_slash", 1000, 6096, 0.9228015542030334], "37": ["vertical_and_slash", 30, 800, 0.9354973435401917], "38": ["vertical_and_slash", 1000, 6096, 0.5352135896682739], "39": ["vertical_and_slash", 1000, 6096, 0.9326971173286438]}, {"0": ["vertical_and_slash", 30, 800, 0.9222443699836731], "1": ["vertical_and_slash", 1000, 6096, 0.9600304365158081], "2": ["vertical_and_slash", 1000, 6096, 0.9810464978218079], "3": ["vertical_and_slash", 1000, 6096, 0.9153046607971191], "4": ["vertical_and_slash", 30, 800, 0.9448681473731995], "5": ["vertical_and_slash", 1000, 6096, 0.9867681860923767], "6": ["vertical_and_slash", 1000, 6096, 0.9863123893737793], "7": ["vertical_and_slash", 1000, 6096, 0.9342074394226074], "8": ["vertical_and_slash", 1000, 6096, 0.9802798628807068], "9": ["vertical_and_slash", 1000, 6096, 0.9258262515068054], "10": ["vertical_and_slash", 1000, 6096, 0.9428498148918152], "11": ["vertical_and_slash", 1000, 6096, 0.8686345815658569], "12": ["vertical_and_slash", 1000, 6096, 0.9735564589500427], "13": ["vertical_and_slash", 30, 800, 0.9408270120620728], "14": ["vertical_and_slash", 1000, 6096, 0.9534137845039368], "15": ["vertical_and_slash", 100, 800, 1.0], "16": ["vertical_and_slash", 1000, 6096, 0.8630568981170654], "17": ["vertical_and_slash", 1000, 6096, 0.8105168342590332], "18": ["vertical_and_slash", 1000, 6096, 0.9791882634162903], "19": ["vertical_and_slash", 1000, 6096, 0.9362411499023438], "20": ["vertical_and_slash", 1000, 6096, 0.9848873019218445], "21": ["vertical_and_slash", 3500, 100, 0.961299479007721], "22": ["vertical_and_slash", 1000, 6096, 0.8414344787597656], "23": ["vertical_and_slash", 1000, 6096, 0.6664743423461914], "24": ["vertical_and_slash", 1000, 6096, 0.8315247893333435], "25": ["vertical_and_slash", 1000, 6096, 0.9687249660491943], "26": ["vertical_and_slash", 100, 800, 0.9375], "27": ["vertical_and_slash", 1000, 6096, 0.9115175008773804], "28": ["vertical_and_slash", 1000, 6096, 0.9924631714820862], "29": ["vertical_and_slash", 30, 800, 0.8587192296981812], "30": ["vertical_and_slash", 1000, 6096, 0.9973868727684021], "31": ["vertical_and_slash", 1000, 6096, 0.7041625380516052], "32": ["vertical_and_slash", 1000, 6096, 0.9959512948989868], "33": ["vertical_and_slash", 1000, 6096, 0.7935265898704529], "34": ["vertical_and_slash", 1000, 6096, 0.9475358724594116], "35": ["vertical_and_slash", 1000, 6096, 0.9905371069908142], "36": ["vertical_and_slash", 1000, 6096, 0.8669290542602539], "37": ["vertical_and_slash", 1000, 6096, 0.9908829927444458], "38": ["vertical_and_slash", 1000, 6096, 0.9950791001319885], "39": ["vertical_and_slash", 1000, 6096, 0.8449252843856812]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9569027423858643], "1": ["vertical_and_slash", 1000, 6096, 0.9629860520362854], "2": ["vertical_and_slash", 1000, 6096, 0.9389455318450928], "3": ["vertical_and_slash", 1000, 6096, 0.9626391530036926], "4": ["vertical_and_slash", 1000, 6096, 0.9099515080451965], "5": ["vertical_and_slash", 1000, 6096, 0.9115425944328308], "6": ["vertical_and_slash", 1000, 6096, 0.9068926572799683], "7": ["vertical_and_slash", 1000, 6096, 0.9488498568534851], "8": ["vertical_and_slash", 1000, 6096, 0.9630889892578125], "9": ["vertical_and_slash", 1000, 6096, 0.9631537199020386], "10": ["vertical_and_slash", 1000, 6096, 0.9924753904342651], "11": ["vertical_and_slash", 1000, 6096, 0.9823078513145447], "12": ["vertical_and_slash", 1000, 6096, 0.9663612246513367], "13": ["vertical_and_slash", 1000, 6096, 0.9748730659484863], "14": ["vertical_and_slash", 1000, 6096, 0.9725997447967529], "15": ["vertical_and_slash", 1000, 6096, 0.8561187982559204], "16": ["vertical_and_slash", 1000, 6096, 0.6635807752609253], "17": ["vertical_and_slash", 1000, 6096, 0.9314156770706177], "18": ["vertical_and_slash", 1000, 6096, 0.9468251466751099], "19": ["vertical_and_slash", 1000, 6096, 0.9319336414337158], "20": ["vertical_and_slash", 1000, 6096, 0.8348208665847778], "21": ["vertical_and_slash", 30, 800, 0.9684057235717773], "22": ["vertical_and_slash", 1000, 6096, 0.9221057891845703], "23": ["vertical_and_slash", 1000, 6096, 0.9854644536972046], "24": ["vertical_and_slash", 1000, 6096, 0.902038037776947], "25": ["vertical_and_slash", 1000, 6096, 0.9763542413711548], "26": ["vertical_and_slash", 1000, 6096, 0.9299762845039368], "27": ["vertical_and_slash", 1000, 6096, 0.934795618057251], "28": ["vertical_and_slash", 1000, 6096, 0.9840686917304993], "29": ["vertical_and_slash", 1000, 6096, 0.9178242683410645], "30": ["vertical_and_slash", 30, 800, 0.7700759172439575], "31": ["vertical_and_slash", 30, 800, 0.9498158693313599], "32": ["vertical_and_slash", 30, 800, 0.9387114644050598], "33": ["vertical_and_slash", 30, 800, 0.9429164528846741], "34": ["vertical_and_slash", 30, 800, 0.9483230710029602], "35": ["vertical_and_slash", 1000, 6096, 0.9542072415351868], "36": ["vertical_and_slash", 1000, 6096, 0.963629424571991], "37": ["vertical_and_slash", 1000, 6096, 0.9545782208442688], "38": ["vertical_and_slash", 1000, 6096, 0.7960972189903259], "39": ["vertical_and_slash", 1000, 6096, 0.9678391218185425]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9275780320167542], "1": ["vertical_and_slash", 1000, 6096, 0.8796406388282776], "2": ["vertical_and_slash", 1000, 6096, 0.9466372728347778], "3": ["vertical_and_slash", 1000, 6096, 0.8535729050636292], "4": ["vertical_and_slash", 1000, 6096, 0.9869552850723267], "5": ["vertical_and_slash", 1000, 6096, 0.9682961106300354], "6": ["vertical_and_slash", 3500, 100, 0.7129173874855042], "7": ["vertical_and_slash", 1000, 6096, 0.7758964896202087], "8": ["vertical_and_slash", 1000, 6096, 0.8512514233589172], "9": ["vertical_and_slash", 1000, 6096, 0.8381565809249878], "10": ["vertical_and_slash", 1000, 6096, 0.9339500069618225], "11": ["vertical_and_slash", 1000, 6096, 0.9796152710914612], "12": ["vertical_and_slash", 1000, 6096, 0.5918328166007996], "13": ["vertical_and_slash", 1000, 6096, 0.9575576186180115], "14": ["vertical_and_slash", 1000, 6096, 0.8009395003318787], "15": ["vertical_and_slash", 1000, 6096, 0.7632763981819153], "16": ["vertical_and_slash", 1000, 6096, 0.9428332448005676], "17": ["vertical_and_slash", 1000, 6096, 0.9681451916694641], "18": ["vertical_and_slash", 1000, 6096, 0.8337690830230713], "19": ["vertical_and_slash", 1000, 6096, 0.975220799446106], "20": ["vertical_and_slash", 1000, 6096, 0.8936065435409546], "21": ["vertical_and_slash", 1000, 6096, 0.7919806241989136], "22": ["vertical_and_slash", 1000, 6096, 0.9985429048538208], "23": ["vertical_and_slash", 1000, 6096, 0.7680925130844116], "24": ["vertical_and_slash", 1000, 6096, 0.9671340584754944], "25": ["vertical_and_slash", 1000, 6096, 0.8413169980049133], "26": ["vertical_and_slash", 30, 800, 0.943064272403717], "27": ["vertical_and_slash", 1000, 6096, 0.8560418486595154], "28": ["vertical_and_slash", 1000, 6096, 0.9080421328544617], "29": ["vertical_and_slash", 1000, 6096, 0.9327840209007263], "30": ["vertical_and_slash", 1000, 6096, 0.9919955730438232], "31": ["vertical_and_slash", 1000, 6096, 0.7718262672424316], "32": ["vertical_and_slash", 1000, 6096, 0.9796733856201172], "33": ["vertical_and_slash", 1000, 6096, 0.9742487072944641], "34": ["vertical_and_slash", 1000, 6096, 0.9910967946052551], "35": ["vertical_and_slash", 1000, 6096, 0.8370533585548401], "36": ["vertical_and_slash", 1000, 6096, 0.9896460771560669], "37": ["vertical_and_slash", 1000, 6096, 0.825937032699585], "38": ["vertical_and_slash", 1000, 6096, 0.9866756200790405], "39": ["vertical_and_slash", 1000, 6096, 0.9875133633613586]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9126338958740234], "1": ["vertical_and_slash", 1000, 6096, 0.9404446482658386], "2": ["vertical_and_slash", 1000, 6096, 0.9302141666412354], "3": ["vertical_and_slash", 1000, 6096, 0.9724613428115845], "4": ["vertical_and_slash", 1000, 6096, 0.9578529596328735], "5": ["vertical_and_slash", 3500, 100, 0.7536399364471436], "6": ["vertical_and_slash", 1000, 6096, 0.8029560446739197], "7": ["vertical_and_slash", 1000, 6096, 0.8653314113616943], "8": ["vertical_and_slash", 1000, 6096, 0.9585363864898682], "9": ["vertical_and_slash", 1000, 6096, 0.7445060014724731], "10": ["vertical_and_slash", 1000, 6096, 0.9735638499259949], "11": ["vertical_and_slash", 1000, 6096, 0.9240977764129639], "12": ["vertical_and_slash", 1000, 6096, 0.9833247065544128], "13": ["vertical_and_slash", 1000, 6096, 0.9832606911659241], "14": ["vertical_and_slash", 1000, 6096, 0.9889455437660217], "15": ["vertical_and_slash", 1000, 6096, 0.9221658706665039], "16": ["vertical_and_slash", 1000, 6096, 0.926628053188324], "17": ["vertical_and_slash", 1000, 6096, 0.8759210705757141], "18": ["vertical_and_slash", 3500, 100, 0.6229612231254578], "19": ["vertical_and_slash", 3500, 100, 0.5153457522392273], "20": ["vertical_and_slash", 1000, 6096, 0.8160942792892456], "21": ["vertical_and_slash", 1000, 6096, 0.9824163317680359], "22": ["vertical_and_slash", 1000, 6096, 0.9918894171714783], "23": ["vertical_and_slash", 1000, 6096, 0.9831662178039551], "24": ["vertical_and_slash", 1000, 6096, 0.9647565484046936], "25": ["vertical_and_slash", 1000, 6096, 0.9859899878501892], "26": ["vertical_and_slash", 1000, 6096, 0.6775552034378052], "27": ["vertical_and_slash", 1000, 6096, 0.8711930513381958], "28": ["vertical_and_slash", 1000, 6096, 0.9796715378761292], "29": ["vertical_and_slash", 1000, 6096, 0.9844052791595459], "30": ["vertical_and_slash", 1000, 6096, 0.8286711573600769], "31": ["vertical_and_slash", 1000, 6096, 0.8740687966346741], "32": ["vertical_and_slash", 1000, 6096, 0.8584240674972534], "33": ["vertical_and_slash", 1000, 6096, 0.9219228029251099], "34": ["vertical_and_slash", 1000, 6096, 0.9649078249931335], "35": ["vertical_and_slash", 1000, 6096, 0.9658982753753662], "36": ["vertical_and_slash", 1000, 6096, 0.9608974456787109], "37": ["vertical_and_slash", 1000, 6096, 0.7213538885116577], "38": ["vertical_and_slash", 1000, 6096, 0.9951735734939575], "39": ["vertical_and_slash", 1000, 6096, 0.6786532998085022]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9461410641670227], "1": ["vertical_and_slash", 1000, 6096, 0.9532012343406677], "2": ["vertical_and_slash", 1000, 6096, 0.9800111651420593], "3": ["vertical_and_slash", 1000, 6096, 0.9439144730567932], "4": ["vertical_and_slash", 1000, 6096, 0.9397121071815491], "5": ["vertical_and_slash", 1000, 6096, 0.9222186207771301], "6": ["vertical_and_slash", 1000, 6096, 0.9597636461257935], "7": ["vertical_and_slash", 1000, 6096, 0.8029380440711975], "8": ["vertical_and_slash", 3500, 100, 0.904583215713501], "9": ["vertical_and_slash", 1000, 6096, 0.7271357774734497], "10": ["vertical_and_slash", 1000, 6096, 0.9745004773139954], "11": ["vertical_and_slash", 1000, 6096, 0.9568464159965515], "12": ["vertical_and_slash", 1000, 6096, 0.9731445908546448], "13": ["vertical_and_slash", 1000, 6096, 0.8927473425865173], "14": ["vertical_and_slash", 1000, 6096, 0.903340756893158], "15": ["vertical_and_slash", 1000, 6096, 0.9772834777832031], "16": ["vertical_and_slash", 1000, 6096, 0.4721975326538086], "17": ["vertical_and_slash", 1000, 6096, 0.9341620802879333], "18": ["vertical_and_slash", 1000, 6096, 0.9771110415458679], "19": ["vertical_and_slash", 1000, 6096, 0.9678651690483093], "20": ["vertical_and_slash", 1000, 6096, 0.6696259379386902], "21": ["vertical_and_slash", 1000, 6096, 0.7286709547042847], "22": ["vertical_and_slash", 1000, 6096, 0.9245709180831909], "23": ["vertical_and_slash", 1000, 6096, 0.9192846417427063], "24": ["vertical_and_slash", 1000, 6096, 0.909048318862915], "25": ["vertical_and_slash", 3500, 100, 0.9234742522239685], "26": ["vertical_and_slash", 1000, 6096, 0.9972402453422546], "27": ["vertical_and_slash", 1000, 6096, 0.983555257320404], "28": ["vertical_and_slash", 1000, 6096, 0.9403573870658875], "29": ["vertical_and_slash", 1000, 6096, 0.7982310056686401], "30": ["vertical_and_slash", 1000, 6096, 0.9643942713737488], "31": ["vertical_and_slash", 1000, 6096, 0.9539694786071777], "32": ["vertical_and_slash", 1000, 6096, 0.8778087496757507], "33": ["vertical_and_slash", 1000, 6096, 0.7737547755241394], "34": ["vertical_and_slash", 1000, 6096, 0.9319139122962952], "35": ["vertical_and_slash", 1000, 6096, 0.9437268972396851], "36": ["vertical_and_slash", 1000, 6096, 0.9042237997055054], "37": ["vertical_and_slash", 1000, 6096, 0.9556217193603516], "38": ["vertical_and_slash", 1000, 6096, 0.517886757850647], "39": ["vertical_and_slash", 1000, 6096, 0.9444556832313538]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7583876848220825], "1": ["vertical_and_slash", 1000, 6096, 0.8372395634651184], "2": ["vertical_and_slash", 1000, 6096, 0.7233264446258545], "3": ["vertical_and_slash", 1000, 6096, 0.9826096296310425], "4": ["vertical_and_slash", 1000, 6096, 0.957129180431366], "5": ["vertical_and_slash", 1000, 6096, 0.8320807218551636], "6": ["vertical_and_slash", 1000, 6096, 0.7665488719940186], "7": ["vertical_and_slash", 1000, 6096, 0.8955280780792236], "8": ["vertical_and_slash", 1000, 6096, 0.9132262468338013], "9": ["vertical_and_slash", 1000, 6096, 0.9519183039665222], "10": ["vertical_and_slash", 1000, 6096, 0.9050867557525635], "11": ["vertical_and_slash", 3500, 100, 0.8862034678459167], "12": ["vertical_and_slash", 1000, 6096, 0.9349398016929626], "13": ["vertical_and_slash", 1000, 6096, 0.9568384885787964], "14": ["vertical_and_slash", 1000, 6096, 0.8388850092887878], "15": ["vertical_and_slash", 1000, 6096, 0.8531433939933777], "16": ["vertical_and_slash", 1000, 6096, 0.9131283164024353], "17": ["vertical_and_slash", 1000, 6096, 0.9847087264060974], "18": ["vertical_and_slash", 1000, 6096, 0.8578906059265137], "19": ["vertical_and_slash", 1000, 6096, 0.987770676612854], "20": ["vertical_and_slash", 1000, 6096, 0.9564622044563293], "21": ["vertical_and_slash", 1000, 6096, 0.9055446982383728], "22": ["vertical_and_slash", 1000, 6096, 0.9263967275619507], "23": ["vertical_and_slash", 1000, 6096, 0.9814872145652771], "24": ["vertical_and_slash", 1000, 6096, 0.7596684098243713], "25": ["vertical_and_slash", 1000, 6096, 0.9385899305343628], "26": ["vertical_and_slash", 1000, 6096, 0.7582556009292603], "27": ["vertical_and_slash", 1000, 6096, 0.6863613128662109], "28": ["vertical_and_slash", 1000, 6096, 0.8657163381576538], "29": ["vertical_and_slash", 3500, 100, 0.998302161693573], "30": ["vertical_and_slash", 1000, 6096, 0.9696944952011108], "31": ["vertical_and_slash", 1000, 6096, 0.9696540236473083], "32": ["vertical_and_slash", 1000, 6096, 0.9344800710678101], "33": ["vertical_and_slash", 1000, 6096, 0.9759489297866821], "34": ["vertical_and_slash", 1000, 6096, 0.9769810438156128], "35": ["vertical_and_slash", 1000, 6096, 0.8802216649055481], "36": ["vertical_and_slash", 1000, 6096, 0.9144234657287598], "37": ["vertical_and_slash", 1000, 6096, 0.9023550152778625], "38": ["vertical_and_slash", 1000, 6096, 0.9722862839698792], "39": ["vertical_and_slash", 1000, 6096, 0.6338047981262207]}, {"0": ["vertical_and_slash", 1000, 6096, 0.5404036045074463], "1": ["vertical_and_slash", 30, 800, 0.8146374821662903], "2": ["vertical_and_slash", 1000, 6096, 0.8312078714370728], "3": ["vertical_and_slash", 1000, 6096, 0.6736167073249817], "4": ["vertical_and_slash", 1000, 6096, 0.6645781993865967], "5": ["vertical_and_slash", 1000, 6096, 0.7872524857521057], "6": ["vertical_and_slash", 1000, 6096, 0.8011279702186584], "7": ["vertical_and_slash", 1000, 6096, 0.8010679483413696], "8": ["vertical_and_slash", 1000, 6096, 0.8277146220207214], "9": ["vertical_and_slash", 1000, 6096, 0.7845017313957214], "10": ["vertical_and_slash", 1000, 6096, 0.8020737767219543], "11": ["vertical_and_slash", 1000, 6096, 0.8919050097465515], "12": ["vertical_and_slash", 1000, 6096, 0.8271636962890625], "13": ["vertical_and_slash", 1000, 6096, 0.6560327410697937], "14": ["vertical_and_slash", 1000, 6096, 0.7788934707641602], "15": ["vertical_and_slash", 1000, 6096, 0.9626566767692566], "16": ["vertical_and_slash", 1000, 6096, 0.9511610269546509], "17": ["vertical_and_slash", 1000, 6096, 0.879792332649231], "18": ["vertical_and_slash", 1000, 6096, 0.9041166305541992], "19": ["vertical_and_slash", 1000, 6096, 0.9442020058631897], "20": ["vertical_and_slash", 1000, 6096, 0.9664666652679443], "21": ["vertical_and_slash", 1000, 6096, 0.9847748279571533], "22": ["vertical_and_slash", 1000, 6096, 0.8094877004623413], "23": ["vertical_and_slash", 1000, 6096, 0.8349117636680603], "24": ["vertical_and_slash", 1000, 6096, 0.5101855993270874], "25": ["vertical_and_slash", 1000, 6096, 0.8243042230606079], "26": ["vertical_and_slash", 1000, 6096, 0.9371790885925293], "27": ["vertical_and_slash", 1000, 6096, 0.7881959676742554], "28": ["vertical_and_slash", 1000, 6096, 0.7519088387489319], "29": ["vertical_and_slash", 1000, 6096, 0.7446812391281128], "30": ["vertical_and_slash", 1000, 6096, 0.9329955577850342], "31": ["vertical_and_slash", 1000, 6096, 0.9375042915344238], "32": ["vertical_and_slash", 1000, 6096, 0.945281445980072], "33": ["vertical_and_slash", 1000, 6096, 0.963613748550415], "34": ["vertical_and_slash", 1000, 6096, 0.9500555396080017], "35": ["vertical_and_slash", 1000, 6096, 0.9299502372741699], "36": ["vertical_and_slash", 1000, 6096, 0.9306116104125977], "37": ["vertical_and_slash", 1000, 6096, 0.8076436519622803], "38": ["vertical_and_slash", 1000, 6096, 0.9540168642997742], "39": ["vertical_and_slash", 1000, 6096, 0.9528605341911316]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8973404169082642], "1": ["vertical_and_slash", 1000, 6096, 0.96928870677948], "2": ["vertical_and_slash", 1000, 6096, 0.9597116112709045], "3": ["vertical_and_slash", 1000, 6096, 0.9502582550048828], "4": ["vertical_and_slash", 1000, 6096, 0.9526681303977966], "5": ["vertical_and_slash", 1000, 6096, 0.9307422637939453], "6": ["vertical_and_slash", 1000, 6096, 0.8806929588317871], "7": ["vertical_and_slash", 1000, 6096, 0.9526936411857605], "8": ["vertical_and_slash", 1000, 6096, 0.8476760983467102], "9": ["vertical_and_slash", 1000, 6096, 0.8591998815536499], "10": ["vertical_and_slash", 1000, 6096, 0.8712834119796753], "11": ["vertical_and_slash", 1000, 6096, 0.9244551658630371], "12": ["vertical_and_slash", 1000, 6096, 0.943583607673645], "13": ["vertical_and_slash", 1000, 6096, 0.7310649156570435], "14": ["vertical_and_slash", 1000, 6096, 0.6687492728233337], "15": ["vertical_and_slash", 30, 800, 0.9778791069984436], "16": ["vertical_and_slash", 1000, 6096, 0.927803099155426], "17": ["vertical_and_slash", 1000, 6096, 0.8531599640846252], "18": ["vertical_and_slash", 30, 800, 0.9787540435791016], "19": ["vertical_and_slash", 1000, 6096, 0.9461007118225098], "20": ["vertical_and_slash", 1000, 6096, 0.9680506587028503], "21": ["vertical_and_slash", 1000, 6096, 0.8342245817184448], "22": ["vertical_and_slash", 1000, 6096, 0.9115641713142395], "23": ["vertical_and_slash", 1000, 6096, 0.7917561531066895], "24": ["vertical_and_slash", 1000, 6096, 0.9889863729476929], "25": ["vertical_and_slash", 1000, 6096, 0.6319686770439148], "26": ["vertical_and_slash", 1000, 6096, 0.7421978116035461], "27": ["vertical_and_slash", 1000, 6096, 0.7825047373771667], "28": ["vertical_and_slash", 1000, 6096, 0.9324141144752502], "29": ["vertical_and_slash", 1000, 6096, 0.8561145067214966], "30": ["vertical_and_slash", 1000, 6096, 0.930234968662262], "31": ["vertical_and_slash", 1000, 6096, 0.946926474571228], "32": ["vertical_and_slash", 1000, 6096, 0.9515626430511475], "33": ["vertical_and_slash", 1000, 6096, 0.8877192139625549], "34": ["vertical_and_slash", 1000, 6096, 0.9425865411758423], "35": ["vertical_and_slash", 1000, 6096, 0.9760922193527222], "36": ["vertical_and_slash", 1000, 6096, 0.9801028966903687], "37": ["vertical_and_slash", 1000, 6096, 0.9456671476364136], "38": ["vertical_and_slash", 1000, 6096, 0.9379025101661682], "39": ["vertical_and_slash", 1000, 6096, 0.915885329246521]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9667040109634399], "1": ["vertical_and_slash", 1000, 6096, 0.9813780188560486], "2": ["vertical_and_slash", 1000, 6096, 0.8345034122467041], "3": ["vertical_and_slash", 1000, 6096, 0.9747185111045837], "4": ["vertical_and_slash", 1000, 6096, 0.9170730113983154], "5": ["vertical_and_slash", 1000, 6096, 0.4530991315841675], "6": ["vertical_and_slash", 1000, 6096, 0.8627909421920776], "7": ["vertical_and_slash", 1000, 6096, 0.45376327633857727], "8": ["vertical_and_slash", 1000, 6096, 0.9764065146446228], "9": ["vertical_and_slash", 1000, 6096, 0.9872492551803589], "10": ["vertical_and_slash", 1000, 6096, 0.9789667725563049], "11": ["vertical_and_slash", 1000, 6096, 0.8484823703765869], "12": ["vertical_and_slash", 1000, 6096, 0.9757060408592224], "13": ["vertical_and_slash", 1000, 6096, 0.9860648512840271], "14": ["vertical_and_slash", 1000, 6096, 0.9759923219680786], "15": ["vertical_and_slash", 1000, 6096, 0.9315030574798584], "16": ["vertical_and_slash", 1000, 6096, 0.9160860180854797], "17": ["vertical_and_slash", 1000, 6096, 0.8687146902084351], "18": ["vertical_and_slash", 1000, 6096, 0.9785186052322388], "19": ["vertical_and_slash", 1000, 6096, 0.9303408265113831], "20": ["vertical_and_slash", 1000, 6096, 0.6546207070350647], "21": ["vertical_and_slash", 1000, 6096, 0.7623395919799805], "22": ["vertical_and_slash", 1000, 6096, 0.9850616455078125], "23": ["vertical_and_slash", 1000, 6096, 0.9740214347839355], "24": ["vertical_and_slash", 3500, 100, 0.9572362303733826], "25": ["vertical_and_slash", 1000, 6096, 0.9950780272483826], "26": ["vertical_and_slash", 1000, 6096, 0.8183143734931946], "27": ["vertical_and_slash", 1000, 6096, 0.9874270558357239], "28": ["vertical_and_slash", 1000, 6096, 0.5650585889816284], "29": ["vertical_and_slash", 1000, 6096, 0.9916301965713501], "30": ["vertical_and_slash", 1000, 6096, 0.9855239391326904], "31": ["vertical_and_slash", 1000, 6096, 0.9759301543235779], "32": ["vertical_and_slash", 1000, 6096, 0.9471498727798462], "33": ["vertical_and_slash", 1000, 6096, 0.7073042392730713], "34": ["vertical_and_slash", 1000, 6096, 0.8422482013702393], "35": ["vertical_and_slash", 1000, 6096, 0.9553845524787903], "36": ["vertical_and_slash", 1000, 6096, 0.9821152687072754], "37": ["vertical_and_slash", 1000, 6096, 0.9492348432540894], "38": ["vertical_and_slash", 1000, 6096, 0.9881135821342468], "39": ["vertical_and_slash", 1000, 6096, 0.8560011386871338]}, {"0": ["vertical_and_slash", 1000, 6096, 0.992989182472229], "1": ["vertical_and_slash", 1000, 6096, 0.9587697386741638], "2": ["vertical_and_slash", 1000, 6096, 0.76302170753479], "3": ["vertical_and_slash", 1000, 6096, 0.9947214126586914], "4": ["vertical_and_slash", 1000, 6096, 0.9692967534065247], "5": ["vertical_and_slash", 1000, 6096, 0.5831028819084167], "6": ["vertical_and_slash", 1000, 6096, 0.9865099191665649], "7": ["vertical_and_slash", 1000, 6096, 0.8350979685783386], "8": ["vertical_and_slash", 1000, 6096, 0.8661876916885376], "9": ["vertical_and_slash", 1000, 6096, 0.8593367338180542], "10": ["vertical_and_slash", 1000, 6096, 0.9521363377571106], "11": ["vertical_and_slash", 1000, 6096, 0.9182488322257996], "12": ["vertical_and_slash", 1000, 6096, 0.9306662678718567], "13": ["vertical_and_slash", 1000, 6096, 0.727570116519928], "14": ["vertical_and_slash", 1000, 6096, 0.7808351516723633], "15": ["vertical_and_slash", 1000, 6096, 0.8531639575958252], "16": ["vertical_and_slash", 1000, 6096, 0.8910190463066101], "17": ["vertical_and_slash", 1000, 6096, 0.9920141100883484], "18": ["vertical_and_slash", 1000, 6096, 0.9695172905921936], "19": ["vertical_and_slash", 1000, 6096, 0.7991051077842712], "20": ["vertical_and_slash", 1000, 6096, 0.8571963310241699], "21": ["vertical_and_slash", 1000, 6096, 0.9812251925468445], "22": ["vertical_and_slash", 3500, 100, 0.9388821125030518], "23": ["vertical_and_slash", 1000, 6096, 0.9160824418067932], "24": ["vertical_and_slash", 1000, 6096, 0.8488214015960693], "25": ["vertical_and_slash", 1000, 6096, 0.7251642346382141], "26": ["vertical_and_slash", 1000, 6096, 0.8850113153457642], "27": ["vertical_and_slash", 1000, 6096, 0.6028397083282471], "28": ["vertical_and_slash", 1000, 6096, 0.9765220880508423], "29": ["vertical_and_slash", 3500, 100, 0.7949627041816711], "30": ["vertical_and_slash", 1000, 6096, 0.8950635194778442], "31": ["vertical_and_slash", 1000, 6096, 0.9516297578811646], "32": ["vertical_and_slash", 1000, 6096, 0.9775606989860535], "33": ["vertical_and_slash", 1000, 6096, 0.9752926826477051], "34": ["vertical_and_slash", 1000, 6096, 0.8403225541114807], "35": ["vertical_and_slash", 1000, 6096, 0.5243161916732788], "36": ["vertical_and_slash", 1000, 6096, 0.9830748438835144], "37": ["vertical_and_slash", 1000, 6096, 0.7083969116210938], "38": ["vertical_and_slash", 1000, 6096, 0.8537058234214783], "39": ["vertical_and_slash", 1000, 6096, 0.984559178352356]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9498112797737122], "1": ["vertical_and_slash", 1000, 6096, 0.9385607242584229], "2": ["vertical_and_slash", 1000, 6096, 0.9682178497314453], "3": ["vertical_and_slash", 1000, 6096, 0.9762390851974487], "4": ["vertical_and_slash", 1000, 6096, 0.9317305684089661], "5": ["vertical_and_slash", 1000, 6096, 0.8964541554450989], "6": ["vertical_and_slash", 1000, 6096, 0.9210471510887146], "7": ["vertical_and_slash", 1000, 6096, 0.892481803894043], "8": ["vertical_and_slash", 1000, 6096, 0.9111627340316772], "9": ["vertical_and_slash", 1000, 6096, 0.7132683992385864], "10": ["vertical_and_slash", 1000, 6096, 0.8506006002426147], "11": ["vertical_and_slash", 1000, 6096, 0.9456332921981812], "12": ["vertical_and_slash", 1000, 6096, 0.9103096127510071], "13": ["vertical_and_slash", 1000, 6096, 0.9694594144821167], "14": ["vertical_and_slash", 1000, 6096, 0.9494542479515076], "15": ["vertical_and_slash", 1000, 6096, 0.918458104133606], "16": ["vertical_and_slash", 1000, 6096, 0.806506335735321], "17": ["vertical_and_slash", 1000, 6096, 0.949982762336731], "18": ["vertical_and_slash", 1000, 6096, 0.9444091320037842], "19": ["vertical_and_slash", 1000, 6096, 0.9892693161964417], "20": ["vertical_and_slash", 1000, 6096, 0.9813488721847534], "21": ["vertical_and_slash", 1000, 6096, 0.9533441066741943], "22": ["vertical_and_slash", 30, 800, 0.9715531468391418], "23": ["vertical_and_slash", 1000, 6096, 0.9920973181724548], "24": ["vertical_and_slash", 1000, 6096, 0.8998830318450928], "25": ["vertical_and_slash", 1000, 6096, 0.8387008309364319], "26": ["vertical_and_slash", 1000, 6096, 0.8852952122688293], "27": ["vertical_and_slash", 1000, 6096, 0.8994826078414917], "28": ["vertical_and_slash", 1000, 6096, 0.6278740763664246], "29": ["vertical_and_slash", 1000, 6096, 0.9539070725440979], "30": ["vertical_and_slash", 1000, 6096, 0.8053310513496399], "31": ["vertical_and_slash", 1000, 6096, 0.9502439498901367], "32": ["vertical_and_slash", 1000, 6096, 0.9649514555931091], "33": ["vertical_and_slash", 1000, 6096, 0.9739537239074707], "34": ["vertical_and_slash", 1000, 6096, 0.9094957709312439], "35": ["vertical_and_slash", 1000, 6096, 0.7689841985702515], "36": ["vertical_and_slash", 1000, 6096, 0.923215389251709], "37": ["vertical_and_slash", 1000, 6096, 0.953748345375061], "38": ["vertical_and_slash", 1000, 6096, 0.9464899301528931], "39": ["vertical_and_slash", 1000, 6096, 0.7216169834136963]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9785173535346985], "1": ["vertical_and_slash", 1000, 6096, 0.9523888230323792], "2": ["vertical_and_slash", 1000, 6096, 0.8853683471679688], "3": ["vertical_and_slash", 1000, 6096, 0.9910293221473694], "4": ["vertical_and_slash", 1000, 6096, 0.732367217540741], "5": ["vertical_and_slash", 1000, 6096, 0.9800169467926025], "6": ["vertical_and_slash", 1000, 6096, 0.9848362803459167], "7": ["vertical_and_slash", 1000, 6096, 0.9665749073028564], "8": ["vertical_and_slash", 1000, 6096, 0.9339260458946228], "9": ["vertical_and_slash", 1000, 6096, 0.8708651661872864], "10": ["vertical_and_slash", 1000, 6096, 0.9750449657440186], "11": ["vertical_and_slash", 1000, 6096, 0.9913609623908997], "12": ["vertical_and_slash", 1000, 6096, 0.6213397979736328], "13": ["vertical_and_slash", 1000, 6096, 0.9094479084014893], "14": ["vertical_and_slash", 1000, 6096, 0.9622561931610107], "15": ["vertical_and_slash", 1000, 6096, 0.879033625125885], "16": ["vertical_and_slash", 1000, 6096, 0.8723779320716858], "17": ["vertical_and_slash", 1000, 6096, 0.7780442833900452], "18": ["vertical_and_slash", 1000, 6096, 0.9763181805610657], "19": ["vertical_and_slash", 1000, 6096, 0.9381588101387024], "20": ["vertical_and_slash", 1000, 6096, 0.9476160407066345], "21": ["vertical_and_slash", 1000, 6096, 0.9625934958457947], "22": ["vertical_and_slash", 1000, 6096, 0.84126877784729], "23": ["vertical_and_slash", 1000, 6096, 0.9274910688400269], "24": ["vertical_and_slash", 1000, 6096, 0.9588722586631775], "25": ["vertical_and_slash", 1000, 6096, 0.9876656532287598], "26": ["vertical_and_slash", 1000, 6096, 0.9445842504501343], "27": ["vertical_and_slash", 1000, 6096, 0.9350199103355408], "28": ["vertical_and_slash", 1000, 6096, 0.9899476170539856], "29": ["vertical_and_slash", 1000, 6096, 0.9864524602890015], "30": ["vertical_and_slash", 1000, 6096, 0.9314517974853516], "31": ["vertical_and_slash", 1000, 6096, 0.9738015532493591], "32": ["vertical_and_slash", 1000, 6096, 0.8582550883293152], "33": ["vertical_and_slash", 1000, 6096, 0.8713118433952332], "34": ["vertical_and_slash", 1000, 6096, 0.765561044216156], "35": ["vertical_and_slash", 1000, 6096, 0.9751057624816895], "36": ["vertical_and_slash", 1000, 6096, 0.9841024875640869], "37": ["vertical_and_slash", 1000, 6096, 0.9346469640731812], "38": ["vertical_and_slash", 1000, 6096, 0.953595757484436], "39": ["vertical_and_slash", 1000, 6096, 0.9572669863700867]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9877773523330688], "1": ["vertical_and_slash", 1000, 6096, 0.9743648767471313], "2": ["vertical_and_slash", 1000, 6096, 0.9933822751045227], "3": ["vertical_and_slash", 30, 800, 0.9732765555381775], "4": ["vertical_and_slash", 1000, 6096, 0.990963876247406], "5": ["vertical_and_slash", 3500, 100, 0.8850367069244385], "6": ["vertical_and_slash", 1000, 6096, 0.9621273875236511], "7": ["vertical_and_slash", 1000, 6096, 0.9601422548294067], "8": ["vertical_and_slash", 1000, 6096, 0.8808856010437012], "9": ["vertical_and_slash", 1000, 6096, 0.8910741806030273], "10": ["vertical_and_slash", 1000, 6096, 0.9748002886772156], "11": ["vertical_and_slash", 1000, 6096, 0.9393223524093628], "12": ["vertical_and_slash", 1000, 6096, 0.591710090637207], "13": ["vertical_and_slash", 1000, 6096, 0.9120664596557617], "14": ["vertical_and_slash", 1000, 6096, 0.978336751461029], "15": ["vertical_and_slash", 1000, 6096, 0.7190841436386108], "16": ["vertical_and_slash", 1000, 6096, 0.8435068130493164], "17": ["vertical_and_slash", 1000, 6096, 0.9640789031982422], "18": ["vertical_and_slash", 1000, 6096, 0.966785192489624], "19": ["vertical_and_slash", 1000, 6096, 0.9132429957389832], "20": ["vertical_and_slash", 1000, 6096, 0.8182837963104248], "21": ["vertical_and_slash", 1000, 6096, 0.7623639702796936], "22": ["vertical_and_slash", 1000, 6096, 0.8987488150596619], "23": ["vertical_and_slash", 1000, 6096, 0.9830952882766724], "24": ["vertical_and_slash", 1000, 6096, 0.9887969493865967], "25": ["vertical_and_slash", 30, 800, 0.9828926920890808], "26": ["vertical_and_slash", 1000, 6096, 0.9549503326416016], "27": ["vertical_and_slash", 1000, 6096, 0.97527676820755], "28": ["vertical_and_slash", 1000, 6096, 0.9743751883506775], "29": ["vertical_and_slash", 1000, 6096, 0.9756240844726562], "30": ["vertical_and_slash", 1000, 6096, 0.9576348662376404], "31": ["vertical_and_slash", 1000, 6096, 0.9591131210327148], "32": ["vertical_and_slash", 1000, 6096, 0.9267314672470093], "33": ["vertical_and_slash", 1000, 6096, 0.9920305609703064], "34": ["vertical_and_slash", 1000, 6096, 0.8388007879257202], "35": ["vertical_and_slash", 1000, 6096, 0.7995970249176025], "36": ["vertical_and_slash", 1000, 6096, 0.9892436265945435], "37": ["vertical_and_slash", 1000, 6096, 0.9978095293045044], "38": ["vertical_and_slash", 1000, 6096, 0.9227545261383057], "39": ["vertical_and_slash", 1000, 6096, 0.9770871996879578]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7440822124481201], "1": ["vertical_and_slash", 1000, 6096, 0.993880033493042], "2": ["vertical_and_slash", 1000, 6096, 0.9337553381919861], "3": ["vertical_and_slash", 1000, 6096, 0.9687443375587463], "4": ["vertical_and_slash", 1000, 6096, 0.6213017106056213], "5": ["vertical_and_slash", 1000, 6096, 0.975524365901947], "6": ["vertical_and_slash", 1000, 6096, 0.838395357131958], "7": ["vertical_and_slash", 1000, 6096, 0.9376958012580872], "8": ["vertical_and_slash", 1000, 6096, 0.9541943073272705], "9": ["vertical_and_slash", 1000, 6096, 0.9509809613227844], "10": ["vertical_and_slash", 100, 800, 0.91796875], "11": ["vertical_and_slash", 1000, 6096, 0.5585521459579468], "12": ["vertical_and_slash", 30, 800, 0.9218848943710327], "13": ["vertical_and_slash", 100, 800, 0.76953125], "14": ["vertical_and_slash", 100, 800, 0.83984375], "15": ["vertical_and_slash", 1000, 6096, 0.5680922865867615], "16": ["vertical_and_slash", 1000, 6096, 0.9734892249107361], "17": ["vertical_and_slash", 1000, 6096, 0.9825004935264587], "18": ["vertical_and_slash", 1000, 6096, 0.9849228858947754], "19": ["vertical_and_slash", 1000, 6096, 0.9751198291778564], "20": ["vertical_and_slash", 1000, 6096, 0.9522469639778137], "21": ["vertical_and_slash", 1000, 6096, 0.7183928489685059], "22": ["vertical_and_slash", 1000, 6096, 0.9089836478233337], "23": ["vertical_and_slash", 1000, 6096, 0.8455095291137695], "24": ["vertical_and_slash", 1000, 6096, 0.9648702144622803], "25": ["vertical_and_slash", 100, 800, 0.78515625], "26": ["vertical_and_slash", 30, 800, 0.8204413056373596], "27": ["vertical_and_slash", 30, 800, 0.8973615169525146], "28": ["vertical_and_slash", 500, 700, 0.7824938893318176], "29": ["vertical_and_slash", 100, 800, 0.8671875], "30": ["vertical_and_slash", 1000, 6096, 0.9655125737190247], "31": ["vertical_and_slash", 1000, 6096, 0.9820924401283264], "32": ["vertical_and_slash", 1000, 6096, 0.815436601638794], "33": ["vertical_and_slash", 1000, 6096, 0.9401307702064514], "34": ["vertical_and_slash", 1000, 6096, 0.9305061101913452], "35": ["vertical_and_slash", 1000, 6096, 0.7439202070236206], "36": ["vertical_and_slash", 100, 800, 0.83984375], "37": ["vertical_and_slash", 3500, 100, 0.7189878821372986], "38": ["vertical_and_slash", 30, 800, 0.927649736404419], "39": ["vertical_and_slash", 30, 800, 0.6004115343093872]}, {"0": ["vertical_and_slash", 30, 800, 0.9464746713638306], "1": ["vertical_and_slash", 30, 800, 0.9808722138404846], "2": ["vertical_and_slash", 30, 800, 0.999489426612854], "3": ["vertical_and_slash", 30, 800, 0.9870849847793579], "4": ["vertical_and_slash", 30, 800, 0.9003432393074036], "5": ["vertical_and_slash", 1000, 6096, 0.9668323397636414], "6": ["vertical_and_slash", 1000, 6096, 0.43353334069252014], "7": ["vertical_and_slash", 1000, 6096, 0.4995494782924652], "8": ["vertical_and_slash", 1000, 6096, 0.9750786423683167], "9": ["vertical_and_slash", 1000, 6096, 0.8960717916488647], "10": ["vertical_and_slash", 1000, 6096, 0.8417829871177673], "11": ["vertical_and_slash", 1000, 6096, 0.9545395970344543], "12": ["vertical_and_slash", 1000, 6096, 0.9928241968154907], "13": ["vertical_and_slash", 1000, 6096, 0.9968456625938416], "14": ["vertical_and_slash", 1000, 6096, 0.8920348882675171], "15": ["vertical_and_slash", 30, 800, 0.9781329035758972], "16": ["vertical_and_slash", 30, 800, 0.9187394976615906], "17": ["vertical_and_slash", 30, 800, 0.9841277003288269], "18": ["vertical_and_slash", 500, 700, 0.9460309147834778], "19": ["vertical_and_slash", 100, 800, 0.9296875], "20": ["vertical_and_slash", 1000, 6096, 0.9304091334342957], "21": ["vertical_and_slash", 1000, 6096, 0.7964159846305847], "22": ["vertical_and_slash", 1000, 6096, 0.8690139651298523], "23": ["vertical_and_slash", 1000, 6096, 0.9728115797042847], "24": ["vertical_and_slash", 1000, 6096, 0.930321991443634], "25": ["vertical_and_slash", 1000, 6096, 0.8683857321739197], "26": ["vertical_and_slash", 1000, 6096, 0.9151809215545654], "27": ["vertical_and_slash", 1000, 6096, 0.7217016220092773], "28": ["vertical_and_slash", 1000, 6096, 0.8309695720672607], "29": ["vertical_and_slash", 1000, 6096, 0.6367559432983398], "30": ["vertical_and_slash", 1000, 6096, 0.9175714254379272], "31": ["vertical_and_slash", 1000, 6096, 0.8639428019523621], "32": ["vertical_and_slash", 1000, 6096, 0.9559615254402161], "33": ["vertical_and_slash", 1000, 6096, 0.8565036654472351], "34": ["vertical_and_slash", 1000, 6096, 0.8550093173980713], "35": ["vertical_and_slash", 1000, 6096, 0.6822922229766846], "36": ["vertical_and_slash", 1000, 6096, 0.9167722463607788], "37": ["vertical_and_slash", 1000, 6096, 0.8001474142074585], "38": ["vertical_and_slash", 1000, 6096, 0.7961844801902771], "39": ["vertical_and_slash", 1000, 6096, 0.702430784702301]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8447131514549255], "1": ["vertical_and_slash", 1000, 6096, 0.9335206747055054], "2": ["vertical_and_slash", 1000, 6096, 0.949216365814209], "3": ["vertical_and_slash", 1000, 6096, 0.9807286858558655], "4": ["vertical_and_slash", 1000, 6096, 0.9701210856437683], "5": ["vertical_and_slash", 1000, 6096, 0.8882132172584534], "6": ["vertical_and_slash", 1000, 6096, 0.7549582123756409], "7": ["vertical_and_slash", 1000, 6096, 0.8806354403495789], "8": ["vertical_and_slash", 1000, 6096, 0.878365695476532], "9": ["vertical_and_slash", 1000, 6096, 0.9142926931381226], "10": ["vertical_and_slash", 1000, 6096, 0.7793240547180176], "11": ["vertical_and_slash", 1000, 6096, 0.9549992680549622], "12": ["vertical_and_slash", 1000, 6096, 0.8942946195602417], "13": ["vertical_and_slash", 1000, 6096, 0.8776730895042419], "14": ["vertical_and_slash", 1000, 6096, 0.8925084471702576], "15": ["vertical_and_slash", 30, 800, 0.9601020812988281], "16": ["vertical_and_slash", 30, 800, 0.9779455065727234], "17": ["vertical_and_slash", 30, 800, 0.9560198187828064], "18": ["vertical_and_slash", 30, 800, 0.9544564485549927], "19": ["vertical_and_slash", 30, 800, 0.9807350039482117], "20": ["vertical_and_slash", 1000, 6096, 0.9856274724006653], "21": ["vertical_and_slash", 1000, 6096, 0.8860489130020142], "22": ["vertical_and_slash", 1000, 6096, 0.9941253066062927], "23": ["vertical_and_slash", 1000, 6096, 0.8598624467849731], "24": ["vertical_and_slash", 1000, 6096, 0.8567866683006287], "25": ["vertical_and_slash", 1000, 6096, 0.9226493835449219], "26": ["vertical_and_slash", 1000, 6096, 0.9911103248596191], "27": ["vertical_and_slash", 1000, 6096, 0.6395439505577087], "28": ["vertical_and_slash", 1000, 6096, 0.8359321355819702], "29": ["vertical_and_slash", 1000, 6096, 0.5952777862548828], "30": ["vertical_and_slash", 1000, 6096, 0.9595168828964233], "31": ["vertical_and_slash", 1000, 6096, 0.6467736959457397], "32": ["vertical_and_slash", 1000, 6096, 0.8990224599838257], "33": ["vertical_and_slash", 1000, 6096, 0.6207748651504517], "34": ["vertical_and_slash", 1000, 6096, 0.7427456974983215], "35": ["vertical_and_slash", 1000, 6096, 0.871558666229248], "36": ["vertical_and_slash", 1000, 6096, 0.8509306907653809], "37": ["vertical_and_slash", 1000, 6096, 0.9757904410362244], "38": ["vertical_and_slash", 1000, 6096, 0.5963435769081116], "39": ["vertical_and_slash", 1000, 6096, 0.8296099305152893]}, {"0": ["vertical_and_slash", 30, 800, 0.924420952796936], "1": ["vertical_and_slash", 30, 800, 0.9692485332489014], "2": ["vertical_and_slash", 30, 800, 0.9602552652359009], "3": ["vertical_and_slash", 30, 800, 0.9161466360092163], "4": ["vertical_and_slash", 30, 800, 0.9487520456314087], "5": ["vertical_and_slash", 1000, 6096, 0.9308497309684753], "6": ["vertical_and_slash", 100, 800, 0.953125], "7": ["vertical_and_slash", 1000, 6096, 0.7571406364440918], "8": ["vertical_and_slash", 1000, 6096, 0.9170207977294922], "9": ["vertical_and_slash", 1000, 6096, 0.7520321607589722], "10": ["vertical_and_slash", 30, 800, 0.9638548493385315], "11": ["vertical_and_slash", 1000, 6096, 0.8670021295547485], "12": ["vertical_and_slash", 30, 800, 0.974168598651886], "13": ["vertical_and_slash", 100, 800, 0.94921875], "14": ["vertical_and_slash", 100, 800, 0.9609375], "15": ["vertical_and_slash", 1000, 6096, 0.7199075222015381], "16": ["vertical_and_slash", 1000, 6096, 0.6018307209014893], "17": ["vertical_and_slash", 1000, 6096, 0.5408170223236084], "18": ["vertical_and_slash", 1000, 6096, 0.8162031173706055], "19": ["vertical_and_slash", 1000, 6096, 0.6616296172142029], "20": ["vertical_and_slash", 1000, 6096, 0.9808924198150635], "21": ["vertical_and_slash", 1000, 6096, 0.7360236048698425], "22": ["vertical_and_slash", 1000, 6096, 0.6397230625152588], "23": ["vertical_and_slash", 1000, 6096, 0.4816569685935974], "24": ["vertical_and_slash", 1000, 6096, 0.7718448638916016], "25": ["vertical_and_slash", 1000, 6096, 0.7987208366394043], "26": ["vertical_and_slash", 100, 800, 0.88671875], "27": ["vertical_and_slash", 1000, 6096, 0.79582279920578], "28": ["vertical_and_slash", 3500, 100, 0.7944478392601013], "29": ["vertical_and_slash", 1000, 6096, 0.8489640355110168], "30": ["vertical_and_slash", 1000, 6096, 0.9962453246116638], "31": ["vertical_and_slash", 1000, 6096, 0.5882614254951477], "32": ["vertical_and_slash", 1000, 6096, 0.9902698397636414], "33": ["vertical_and_slash", 1000, 6096, 0.6198349595069885], "34": ["vertical_and_slash", 1000, 6096, 0.9968931078910828], "35": ["vertical_and_slash", 1000, 6096, 0.7839594483375549], "36": ["vertical_and_slash", 1000, 6096, 0.6166943907737732], "37": ["vertical_and_slash", 1000, 6096, 0.913749098777771], "38": ["vertical_and_slash", 1000, 6096, 0.9715686440467834], "39": ["vertical_and_slash", 1000, 6096, 0.6509066224098206]}, {"0": ["vertical_and_slash", 30, 800, 0.9812823534011841], "1": ["vertical_and_slash", 500, 700, 0.9815183877944946], "2": ["vertical_and_slash", 1000, 6096, 0.9515253305435181], "3": ["vertical_and_slash", 100, 800, 0.98046875], "4": ["vertical_and_slash", 100, 800, 0.90625], "5": ["vertical_and_slash", 1000, 6096, 0.7616876363754272], "6": ["vertical_and_slash", 1000, 6096, 0.7771242260932922], "7": ["vertical_and_slash", 1000, 6096, 0.9487801790237427], "8": ["vertical_and_slash", 1000, 6096, 0.9205833077430725], "9": ["vertical_and_slash", 1000, 6096, 0.9331472516059875], "10": ["vertical_and_slash", 1000, 6096, 0.958090603351593], "11": ["vertical_and_slash", 1000, 6096, 0.9708429574966431], "12": ["vertical_and_slash", 1000, 6096, 0.9118559956550598], "13": ["vertical_and_slash", 1000, 6096, 0.9296552538871765], "14": ["vertical_and_slash", 1000, 6096, 0.8654567003250122], "15": ["vertical_and_slash", 1000, 6096, 0.8276252150535583], "16": ["vertical_and_slash", 1000, 6096, 0.6740524768829346], "17": ["vertical_and_slash", 3500, 100, 0.8264937996864319], "18": ["vertical_and_slash", 1000, 6096, 0.8605716824531555], "19": ["vertical_and_slash", 1000, 6096, 0.8528086543083191], "20": ["vertical_and_slash", 1000, 6096, 0.5141767263412476], "21": ["vertical_and_slash", 1000, 6096, 0.965161144733429], "22": ["vertical_and_slash", 1000, 6096, 0.7425130009651184], "23": ["vertical_and_slash", 1000, 6096, 0.9616262912750244], "24": ["vertical_and_slash", 100, 800, 0.65234375], "25": ["vertical_and_slash", 1000, 6096, 0.9214840531349182], "26": ["vertical_and_slash", 1000, 6096, 0.7686444520950317], "27": ["vertical_and_slash", 1000, 6096, 0.7516016960144043], "28": ["vertical_and_slash", 1000, 6096, 0.8495209217071533], "29": ["vertical_and_slash", 1000, 6096, 0.6951187252998352], "30": ["vertical_and_slash", 100, 800, 0.796875], "31": ["vertical_and_slash", 30, 800, 0.9439903497695923], "32": ["vertical_and_slash", 30, 800, 0.8737513422966003], "33": ["vertical_and_slash", 30, 800, 0.9417719841003418], "34": ["vertical_and_slash", 30, 800, 0.945483922958374], "35": ["vertical_and_slash", 1000, 6096, 0.913131594657898], "36": ["vertical_and_slash", 1000, 6096, 0.9531856775283813], "37": ["vertical_and_slash", 30, 800, 0.9488642811775208], "38": ["vertical_and_slash", 1000, 6096, 0.8265434503555298], "39": ["vertical_and_slash", 1000, 6096, 0.9376404285430908]}, {"0": ["vertical_and_slash", 3500, 100, 0.87879878282547], "1": ["vertical_and_slash", 1000, 6096, 0.8191383481025696], "2": ["vertical_and_slash", 1000, 6096, 0.870719313621521], "3": ["vertical_and_slash", 3500, 100, 0.7812142372131348], "4": ["vertical_and_slash", 1000, 6096, 0.9361242651939392], "5": ["vertical_and_slash", 1000, 6096, 0.9824923872947693], "6": ["vertical_and_slash", 1000, 6096, 0.7383518218994141], "7": ["vertical_and_slash", 1000, 6096, 0.9216713905334473], "8": ["vertical_and_slash", 1000, 6096, 0.7886953949928284], "9": ["vertical_and_slash", 3500, 100, 0.7955523133277893], "10": ["vertical_and_slash", 100, 800, 0.8984375], "11": ["vertical_and_slash", 1000, 6096, 0.8654141426086426], "12": ["vertical_and_slash", 1000, 6096, 0.5474017262458801], "13": ["vertical_and_slash", 1000, 6096, 0.9105864763259888], "14": ["vertical_and_slash", 1000, 6096, 0.8612503409385681], "15": ["vertical_and_slash", 1000, 6096, 0.7002140283584595], "16": ["vertical_and_slash", 3500, 100, 0.8318234086036682], "17": ["vertical_and_slash", 3500, 100, 0.9598749279975891], "18": ["vertical_and_slash", 1000, 6096, 0.6350190043449402], "19": ["vertical_and_slash", 3500, 100, 0.6470341086387634], "20": ["vertical_and_slash", 1000, 6096, 0.8646091818809509], "21": ["vertical_and_slash", 1000, 6096, 0.7646159529685974], "22": ["vertical_and_slash", 1000, 6096, 0.9785380363464355], "23": ["vertical_and_slash", 1000, 6096, 0.7599329948425293], "24": ["vertical_and_slash", 1000, 6096, 0.9566660523414612], "25": ["vertical_and_slash", 1000, 6096, 0.8545074462890625], "26": ["vertical_and_slash", 1000, 6096, 0.9188022613525391], "27": ["vertical_and_slash", 1000, 6096, 0.7967548966407776], "28": ["vertical_and_slash", 30, 800, 0.8574272990226746], "29": ["vertical_and_slash", 1000, 6096, 0.9180968403816223], "30": ["vertical_and_slash", 1000, 6096, 0.8457361459732056], "31": ["vertical_and_slash", 1000, 6096, 0.8510711193084717], "32": ["vertical_and_slash", 1000, 6096, 0.809702455997467], "33": ["vertical_and_slash", 1000, 6096, 0.9430550932884216], "34": ["vertical_and_slash", 1000, 6096, 0.920088529586792], "35": ["vertical_and_slash", 1000, 6096, 0.503796398639679], "36": ["vertical_and_slash", 3500, 100, 0.8240435719490051], "37": ["vertical_and_slash", 1000, 6096, 0.5422858595848083], "38": ["vertical_and_slash", 1000, 6096, 0.8895501494407654], "39": ["vertical_and_slash", 100, 800, 0.6015625]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9089637398719788], "1": ["vertical_and_slash", 1000, 6096, 0.9227147102355957], "2": ["vertical_and_slash", 1000, 6096, 0.9009232521057129], "3": ["vertical_and_slash", 1000, 6096, 0.935570240020752], "4": ["vertical_and_slash", 1000, 6096, 0.9171919822692871], "5": ["vertical_and_slash", 3500, 100, 0.8500509858131409], "6": ["vertical_and_slash", 3500, 100, 0.9246079325675964], "7": ["vertical_and_slash", 1000, 6096, 0.8473656177520752], "8": ["vertical_and_slash", 1000, 6096, 0.9044296145439148], "9": ["vertical_and_slash", 1000, 6096, 0.7988041639328003], "10": ["vertical_and_slash", 1000, 6096, 0.8162409663200378], "11": ["vertical_and_slash", 1000, 6096, 0.8220713138580322], "12": ["vertical_and_slash", 1000, 6096, 0.861254096031189], "13": ["vertical_and_slash", 1000, 6096, 0.8692833185195923], "14": ["vertical_and_slash", 1000, 6096, 0.843618631362915], "15": ["vertical_and_slash", 1000, 6096, 0.8963430523872375], "16": ["vertical_and_slash", 1000, 6096, 0.7320983409881592], "17": ["vertical_and_slash", 1000, 6096, 0.8659279942512512], "18": ["vertical_and_slash", 1000, 6096, 0.7186532616615295], "19": ["vertical_and_slash", 3500, 100, 0.5217086672782898], "20": ["vertical_and_slash", 1000, 6096, 0.9265266060829163], "21": ["vertical_and_slash", 1000, 6096, 0.9159941673278809], "22": ["vertical_and_slash", 1000, 6096, 0.9610457420349121], "23": ["vertical_and_slash", 1000, 6096, 0.9610831141471863], "24": ["vertical_and_slash", 1000, 6096, 0.9017627239227295], "25": ["vertical_and_slash", 1000, 6096, 0.66230708360672], "26": ["vertical_and_slash", 1000, 6096, 0.7435241341590881], "27": ["vertical_and_slash", 1000, 6096, 0.7380475401878357], "28": ["vertical_and_slash", 1000, 6096, 0.9504127502441406], "29": ["vertical_and_slash", 1000, 6096, 0.6413756012916565], "30": ["vertical_and_slash", 1000, 6096, 0.7754616141319275], "31": ["vertical_and_slash", 3500, 100, 0.9059407114982605], "32": ["vertical_and_slash", 1000, 6096, 0.8519901633262634], "33": ["vertical_and_slash", 1000, 6096, 0.8592864274978638], "34": ["vertical_and_slash", 1000, 6096, 0.8473048210144043], "35": ["vertical_and_slash", 1000, 6096, 0.9650058746337891], "36": ["vertical_and_slash", 1000, 6096, 0.9268276691436768], "37": ["vertical_and_slash", 1000, 6096, 0.8328496813774109], "38": ["vertical_and_slash", 3500, 100, 0.9916753768920898], "39": ["vertical_and_slash", 1000, 6096, 0.6547082662582397]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8888250589370728], "1": ["vertical_and_slash", 3500, 100, 0.855343222618103], "2": ["vertical_and_slash", 3500, 100, 0.9068136811256409], "3": ["vertical_and_slash", 1000, 6096, 0.8461047410964966], "4": ["vertical_and_slash", 1000, 6096, 0.8788946866989136], "5": ["vertical_and_slash", 1000, 6096, 0.7392531633377075], "6": ["vertical_and_slash", 1000, 6096, 0.9173570275306702], "7": ["vertical_and_slash", 1000, 6096, 0.7474656105041504], "8": ["vertical_and_slash", 1000, 6096, 0.7577494978904724], "9": ["vertical_and_slash", 1000, 6096, 0.6390885710716248], "10": ["vertical_and_slash", 3500, 100, 0.9639468789100647], "11": ["vertical_and_slash", 3500, 100, 0.9189680814743042], "12": ["vertical_and_slash", 3500, 100, 0.9438474178314209], "13": ["vertical_and_slash", 1000, 6096, 0.9111708402633667], "14": ["vertical_and_slash", 3500, 100, 0.8717405796051025], "15": ["vertical_and_slash", 1000, 6096, 0.9554483890533447], "16": ["vertical_and_slash", 3500, 100, 0.6329749822616577], "17": ["vertical_and_slash", 1000, 6096, 0.9725654721260071], "18": ["vertical_and_slash", 1000, 6096, 0.9676137566566467], "19": ["vertical_and_slash", 1000, 6096, 0.9879899024963379], "20": ["vertical_and_slash", 3500, 100, 0.8249991536140442], "21": ["vertical_and_slash", 1000, 6096, 0.79244065284729], "22": ["vertical_and_slash", 3500, 100, 0.9414427280426025], "23": ["vertical_and_slash", 3500, 100, 0.9695684313774109], "24": ["vertical_and_slash", 3500, 100, 0.9840563535690308], "25": ["vertical_and_slash", 3500, 100, 0.807819128036499], "26": ["vertical_and_slash", 3500, 100, 0.9429380893707275], "27": ["vertical_and_slash", 3500, 100, 0.9146475791931152], "28": ["vertical_and_slash", 3500, 100, 0.8862847089767456], "29": ["vertical_and_slash", 1000, 6096, 0.6960760951042175], "30": ["vertical_and_slash", 3500, 100, 0.8802631497383118], "31": ["vertical_and_slash", 3500, 100, 0.8871524333953857], "32": ["vertical_and_slash", 3500, 100, 0.9022247791290283], "33": ["vertical_and_slash", 1000, 6096, 0.8153723478317261], "34": ["vertical_and_slash", 1000, 6096, 0.8235819339752197], "35": ["vertical_and_slash", 1000, 6096, 0.876413106918335], "36": ["vertical_and_slash", 1000, 6096, 0.9887911081314087], "37": ["vertical_and_slash", 1000, 6096, 0.837972104549408], "38": ["vertical_and_slash", 1000, 6096, 0.6799946427345276], "39": ["vertical_and_slash", 1000, 6096, 0.9079701900482178]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8610691428184509], "1": ["vertical_and_slash", 1000, 6096, 0.8446958065032959], "2": ["vertical_and_slash", 1000, 6096, 0.7046154141426086], "3": ["vertical_and_slash", 1000, 6096, 0.9047994613647461], "4": ["vertical_and_slash", 1000, 6096, 0.8018071055412292], "5": ["vertical_and_slash", 1000, 6096, 0.8729837536811829], "6": ["vertical_and_slash", 1000, 6096, 0.7793084979057312], "7": ["vertical_and_slash", 1000, 6096, 0.7610324621200562], "8": ["vertical_and_slash", 1000, 6096, 0.7816492915153503], "9": ["vertical_and_slash", 1000, 6096, 0.7981202602386475], "10": ["vertical_and_slash", 1000, 6096, 0.8179529309272766], "11": ["vertical_and_slash", 1000, 6096, 0.7322229743003845], "12": ["vertical_and_slash", 3500, 100, 0.8875676393508911], "13": ["vertical_and_slash", 3500, 100, 0.8698513507843018], "14": ["vertical_and_slash", 1000, 6096, 0.818621039390564], "15": ["vertical_and_slash", 1000, 6096, 0.7754248380661011], "16": ["vertical_and_slash", 3500, 100, 0.932233452796936], "17": ["vertical_and_slash", 1000, 6096, 0.7965281009674072], "18": ["vertical_and_slash", 3500, 100, 0.7507041692733765], "19": ["vertical_and_slash", 3500, 100, 0.8889761567115784], "20": ["vertical_and_slash", 1000, 6096, 0.723667562007904], "21": ["vertical_and_slash", 1000, 6096, 0.772318959236145], "22": ["vertical_and_slash", 1000, 6096, 0.8502817749977112], "23": ["vertical_and_slash", 3500, 100, 0.8506356477737427], "24": ["vertical_and_slash", 1000, 6096, 0.6900249719619751], "25": ["vertical_and_slash", 1000, 6096, 0.7579125165939331], "26": ["vertical_and_slash", 1000, 6096, 0.6445600390434265], "27": ["vertical_and_slash", 1000, 6096, 0.7010668516159058], "28": ["vertical_and_slash", 1000, 6096, 0.5476872324943542], "29": ["vertical_and_slash", 1000, 6096, 0.9130013585090637], "30": ["vertical_and_slash", 3500, 100, 0.9140709042549133], "31": ["vertical_and_slash", 3500, 100, 0.9253854751586914], "32": ["vertical_and_slash", 3500, 100, 0.7854593992233276], "33": ["vertical_and_slash", 3500, 100, 0.9177890419960022], "34": ["vertical_and_slash", 3500, 100, 0.9165396690368652], "35": ["vertical_and_slash", 1000, 6096, 0.6149138808250427], "36": ["vertical_and_slash", 1000, 6096, 0.6326434016227722], "37": ["vertical_and_slash", 1000, 6096, 0.6847227811813354], "38": ["vertical_and_slash", 500, 700, 0.9687967300415039], "39": ["vertical_and_slash", 1000, 6096, 0.5727922916412354]}, {"0": ["vertical_and_slash", 1000, 6096, 0.6168303489685059], "1": ["vertical_and_slash", 1000, 6096, 0.9429134130477905], "2": ["vertical_and_slash", 1000, 6096, 0.8808835744857788], "3": ["vertical_and_slash", 1000, 6096, 0.8070205450057983], "4": ["vertical_and_slash", 1000, 6096, 0.9392012357711792], "5": ["vertical_and_slash", 1000, 6096, 0.8180525302886963], "6": ["vertical_and_slash", 1000, 6096, 0.8325842618942261], "7": ["vertical_and_slash", 1000, 6096, 0.8694997429847717], "8": ["vertical_and_slash", 1000, 6096, 0.8627539873123169], "9": ["vertical_and_slash", 3500, 100, 0.8552942872047424], "10": ["vertical_and_slash", 1000, 6096, 0.8416227698326111], "11": ["vertical_and_slash", 1000, 6096, 0.8877426981925964], "12": ["vertical_and_slash", 1000, 6096, 0.8901243805885315], "13": ["vertical_and_slash", 1000, 6096, 0.6808613538742065], "14": ["vertical_and_slash", 1000, 6096, 0.8443505764007568], "15": ["vertical_and_slash", 3500, 100, 0.8989042639732361], "16": ["vertical_and_slash", 3500, 100, 0.9155558943748474], "17": ["vertical_and_slash", 1000, 6096, 0.8517767786979675], "18": ["vertical_and_slash", 1000, 6096, 0.9053556323051453], "19": ["vertical_and_slash", 1000, 6096, 0.8797390460968018], "20": ["vertical_and_slash", 1000, 6096, 0.8652074933052063], "21": ["vertical_and_slash", 3500, 100, 0.7757319808006287], "22": ["vertical_and_slash", 1000, 6096, 0.6072224378585815], "23": ["vertical_and_slash", 1000, 6096, 0.622667670249939], "24": ["vertical_and_slash", 1000, 6096, 0.7220238447189331], "25": ["vertical_and_slash", 1000, 6096, 0.7487852573394775], "26": ["vertical_and_slash", 1000, 6096, 0.9272388815879822], "27": ["vertical_and_slash", 1000, 6096, 0.7045100331306458], "28": ["vertical_and_slash", 3500, 100, 0.9278358221054077], "29": ["vertical_and_slash", 1000, 6096, 0.824502170085907], "30": ["vertical_and_slash", 1000, 6096, 0.8533932566642761], "31": ["vertical_and_slash", 1000, 6096, 0.826608419418335], "32": ["vertical_and_slash", 1000, 6096, 0.8184823989868164], "33": ["vertical_and_slash", 1000, 6096, 0.9089155793190002], "34": ["vertical_and_slash", 1000, 6096, 0.8557402491569519], "35": ["vertical_and_slash", 1000, 6096, 0.9522373080253601], "36": ["vertical_and_slash", 1000, 6096, 0.9461733102798462], "37": ["vertical_and_slash", 1000, 6096, 0.9220399856567383], "38": ["vertical_and_slash", 500, 700, 0.963060736656189], "39": ["vertical_and_slash", 500, 700, 0.9666464328765869]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8595849871635437], "1": ["vertical_and_slash", 1000, 6096, 0.9113075137138367], "2": ["vertical_and_slash", 1000, 6096, 0.8909251093864441], "3": ["vertical_and_slash", 1000, 6096, 0.891718864440918], "4": ["vertical_and_slash", 1000, 6096, 0.9092548489570618], "5": ["vertical_and_slash", 1000, 6096, 0.8561556935310364], "6": ["vertical_and_slash", 1000, 6096, 0.935042679309845], "7": ["vertical_and_slash", 1000, 6096, 0.9168699383735657], "8": ["vertical_and_slash", 1000, 6096, 0.8745187520980835], "9": ["vertical_and_slash", 1000, 6096, 0.7852041125297546], "10": ["vertical_and_slash", 1000, 6096, 0.926249623298645], "11": ["vertical_and_slash", 1000, 6096, 0.9412109851837158], "12": ["vertical_and_slash", 1000, 6096, 0.9069701433181763], "13": ["vertical_and_slash", 1000, 6096, 0.7985873818397522], "14": ["vertical_and_slash", 1000, 6096, 0.7528146505355835], "15": ["vertical_and_slash", 100, 750, 0.9888298511505127], "16": ["vertical_and_slash", 100, 750, 0.9778513312339783], "17": ["vertical_and_slash", 100, 750, 0.9131882786750793], "18": ["vertical_and_slash", 100, 750, 0.9821464419364929], "19": ["vertical_and_slash", 100, 750, 0.9508930444717407], "20": ["vertical_and_slash", 1000, 6096, 0.991197943687439], "21": ["vertical_and_slash", 1000, 6096, 0.7553969621658325], "22": ["vertical_and_slash", 1000, 6096, 0.9710606932640076], "23": ["vertical_and_slash", 1000, 6096, 0.8567930459976196], "24": ["vertical_and_slash", 1000, 6096, 0.9812710285186768], "25": ["vertical_and_slash", 1000, 6096, 0.5911232233047485], "26": ["vertical_and_slash", 1000, 6096, 0.6812305450439453], "27": ["vertical_and_slash", 1000, 6096, 0.8658008575439453], "28": ["vertical_and_slash", 1000, 6096, 0.9260698556900024], "29": ["vertical_and_slash", 1000, 6096, 0.6459757685661316], "30": ["vertical_and_slash", 1000, 6096, 0.8536379933357239], "31": ["vertical_and_slash", 1000, 6096, 0.8730137348175049], "32": ["vertical_and_slash", 1000, 6096, 0.8916029930114746], "33": ["vertical_and_slash", 1000, 6096, 0.8575713038444519], "34": ["vertical_and_slash", 1000, 6096, 0.8883116841316223], "35": ["vertical_and_slash", 1000, 6096, 0.9106683731079102], "36": ["vertical_and_slash", 500, 700, 0.9239460825920105], "37": ["vertical_and_slash", 1000, 6096, 0.8642502427101135], "38": ["vertical_and_slash", 1000, 6096, 0.8508860468864441], "39": ["vertical_and_slash", 1000, 6096, 0.8504559993743896]}, {"0": ["vertical_and_slash", 1000, 6096, 0.928598940372467], "1": ["vertical_and_slash", 1000, 6096, 0.9514740705490112], "2": ["vertical_and_slash", 3500, 100, 0.6018969416618347], "3": ["vertical_and_slash", 1000, 6096, 0.9011023640632629], "4": ["vertical_and_slash", 1000, 6096, 0.9049363732337952], "5": ["vertical_and_slash", 1000, 6096, 0.5831546187400818], "6": ["vertical_and_slash", 1000, 6096, 0.6291581392288208], "7": ["vertical_and_slash", 1000, 6096, 0.6943855881690979], "8": ["vertical_and_slash", 3500, 100, 0.7936410903930664], "9": ["vertical_and_slash", 1000, 6096, 0.9547888040542603], "10": ["vertical_and_slash", 1000, 6096, 0.9203049540519714], "11": ["vertical_and_slash", 1000, 6096, 0.7689617872238159], "12": ["vertical_and_slash", 3500, 100, 0.9462082982063293], "13": ["vertical_and_slash", 3500, 100, 0.900578498840332], "14": ["vertical_and_slash", 1000, 6096, 0.8473742008209229], "15": ["vertical_and_slash", 1000, 6096, 0.8825194239616394], "16": ["vertical_and_slash", 1000, 6096, 0.7928072214126587], "17": ["vertical_and_slash", 1000, 6096, 0.8306358456611633], "18": ["vertical_and_slash", 3500, 100, 0.914435863494873], "19": ["vertical_and_slash", 1000, 6096, 0.8838346600532532], "20": ["vertical_and_slash", 1000, 6096, 0.6888552308082581], "21": ["vertical_and_slash", 1000, 6096, 0.691002368927002], "22": ["vertical_and_slash", 1000, 6096, 0.9155510663986206], "23": ["vertical_and_slash", 1000, 6096, 0.9830425381660461], "24": ["vertical_and_slash", 3500, 100, 0.808506190776825], "25": ["vertical_and_slash", 1000, 6096, 0.9096055626869202], "26": ["vertical_and_slash", 1000, 6096, 0.6489323973655701], "27": ["vertical_and_slash", 500, 700, 0.9520954489707947], "28": ["vertical_and_slash", 1000, 6096, 0.677455484867096], "29": ["vertical_and_slash", 3500, 100, 0.8781428933143616], "30": ["vertical_and_slash", 1000, 6096, 0.7629638910293579], "31": ["vertical_and_slash", 3500, 100, 0.9847500920295715], "32": ["vertical_and_slash", 3500, 100, 0.7792467474937439], "33": ["vertical_and_slash", 1000, 6096, 0.6722946763038635], "34": ["vertical_and_slash", 1000, 6096, 0.7787414193153381], "35": ["vertical_and_slash", 1000, 6096, 0.8654088973999023], "36": ["vertical_and_slash", 1000, 6096, 0.9169813394546509], "37": ["vertical_and_slash", 1000, 6096, 0.7859824895858765], "38": ["vertical_and_slash", 500, 700, 0.9154335260391235], "39": ["vertical_and_slash", 1000, 6096, 0.902272641658783]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9538297653198242], "1": ["vertical_and_slash", 1000, 6096, 0.8714397549629211], "2": ["vertical_and_slash", 1000, 6096, 0.9289631843566895], "3": ["vertical_and_slash", 1000, 6096, 0.948509931564331], "4": ["vertical_and_slash", 1000, 6096, 0.8876335024833679], "5": ["vertical_and_slash", 1000, 6096, 0.728018581867218], "6": ["vertical_and_slash", 1000, 6096, 0.6807158589363098], "7": ["vertical_and_slash", 1000, 6096, 0.6307185292243958], "8": ["vertical_and_slash", 1000, 6096, 0.8759902119636536], "9": ["vertical_and_slash", 1000, 6096, 0.7629941701889038], "10": ["vertical_and_slash", 1000, 6096, 0.8320733904838562], "11": ["vertical_and_slash", 1000, 6096, 0.8799092769622803], "12": ["vertical_and_slash", 3500, 100, 0.8882818222045898], "13": ["vertical_and_slash", 1000, 6096, 0.8095906972885132], "14": ["vertical_and_slash", 1000, 6096, 0.7315647006034851], "15": ["vertical_and_slash", 1000, 6096, 0.7610905170440674], "16": ["vertical_and_slash", 1000, 6096, 0.8651058673858643], "17": ["vertical_and_slash", 3500, 100, 0.8648301959037781], "18": ["vertical_and_slash", 3500, 100, 0.8400357961654663], "19": ["vertical_and_slash", 1000, 6096, 0.9424635171890259], "20": ["vertical_and_slash", 1000, 6096, 0.5718932747840881], "21": ["vertical_and_slash", 3500, 100, 0.8907543420791626], "22": ["vertical_and_slash", 1000, 6096, 0.7895660400390625], "23": ["vertical_and_slash", 1000, 6096, 0.800386905670166], "24": ["vertical_and_slash", 1000, 6096, 0.851990282535553], "25": ["vertical_and_slash", 1000, 6096, 0.722142219543457], "26": ["vertical_and_slash", 1000, 6096, 0.578690767288208], "27": ["vertical_and_slash", 1000, 6096, 0.6095626354217529], "28": ["vertical_and_slash", 1000, 6096, 0.843013346195221], "29": ["vertical_and_slash", 1000, 6096, 0.6143648028373718], "30": ["vertical_and_slash", 1000, 6096, 0.9290770292282104], "31": ["vertical_and_slash", 1000, 6096, 0.8583142161369324], "32": ["vertical_and_slash", 1000, 6096, 0.8778369426727295], "33": ["vertical_and_slash", 1000, 6096, 0.8803922533988953], "34": ["vertical_and_slash", 1000, 6096, 0.7695662379264832], "35": ["vertical_and_slash", 1000, 6096, 0.7221435308456421], "36": ["vertical_and_slash", 3500, 100, 0.9601407051086426], "37": ["vertical_and_slash", 1000, 6096, 0.647994875907898], "38": ["vertical_and_slash", 3500, 100, 0.6290419101715088], "39": ["vertical_and_slash", 500, 700, 0.9407769441604614]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8786086440086365], "1": ["vertical_and_slash", 1000, 6096, 0.877822756767273], "2": ["vertical_and_slash", 1000, 6096, 0.8740938901901245], "3": ["vertical_and_slash", 1000, 6096, 0.9181730151176453], "4": ["vertical_and_slash", 1000, 6096, 0.870254635810852], "5": ["vertical_and_slash", 1000, 6096, 0.9360779523849487], "6": ["vertical_and_slash", 1000, 6096, 0.8691596984863281], "7": ["vertical_and_slash", 1000, 6096, 0.8499957323074341], "8": ["vertical_and_slash", 1000, 6096, 0.879431962966919], "9": ["vertical_and_slash", 1000, 6096, 0.7160205841064453], "10": ["vertical_and_slash", 1000, 6096, 0.7681173086166382], "11": ["vertical_and_slash", 3500, 100, 0.9478723406791687], "12": ["vertical_and_slash", 1000, 6096, 0.8769731521606445], "13": ["vertical_and_slash", 3500, 100, 0.975791335105896], "14": ["vertical_and_slash", 3500, 100, 0.9614863395690918], "15": ["vertical_and_slash", 1000, 6096, 0.9712170958518982], "16": ["vertical_and_slash", 1000, 6096, 0.7974410057067871], "17": ["vertical_and_slash", 1000, 6096, 0.9465020298957825], "18": ["vertical_and_slash", 1000, 6096, 0.8897132277488708], "19": ["vertical_and_slash", 500, 700, 0.9544727802276611], "20": ["vertical_and_slash", 3500, 100, 0.9202619194984436], "21": ["vertical_and_slash", 1000, 6096, 0.8902014493942261], "22": ["vertical_and_slash", 3500, 100, 0.916659414768219], "23": ["vertical_and_slash", 3500, 100, 0.9507204294204712], "24": ["vertical_and_slash", 1000, 6096, 0.853146493434906], "25": ["vertical_and_slash", 1000, 6096, 0.6662266254425049], "26": ["vertical_and_slash", 3500, 100, 0.9046615362167358], "27": ["vertical_and_slash", 1000, 6096, 0.8523986339569092], "28": ["vertical_and_slash", 1000, 6096, 0.6567543148994446], "29": ["vertical_and_slash", 1000, 6096, 0.8362465500831604], "30": ["vertical_and_slash", 1000, 6096, 0.9547703862190247], "31": ["vertical_and_slash", 3500, 100, 0.9265106320381165], "32": ["vertical_and_slash", 3500, 100, 0.9286931157112122], "33": ["vertical_and_slash", 500, 700, 0.9366575479507446], "34": ["vertical_and_slash", 3500, 100, 0.9634543061256409], "35": ["vertical_and_slash", 1000, 6096, 0.8615919351577759], "36": ["vertical_and_slash", 1000, 6096, 0.914324164390564], "37": ["vertical_and_slash", 1000, 6096, 0.9080424308776855], "38": ["vertical_and_slash", 1000, 6096, 0.7817762494087219], "39": ["vertical_and_slash", 1000, 6096, 0.8243421912193298]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9372972846031189], "1": ["vertical_and_slash", 1000, 6096, 0.9301881194114685], "2": ["vertical_and_slash", 1000, 6096, 0.8440313339233398], "3": ["vertical_and_slash", 500, 700, 0.9372088313102722], "4": ["vertical_and_slash", 1000, 6096, 0.8638481497764587], "5": ["vertical_and_slash", 3500, 100, 0.8891750574111938], "6": ["vertical_and_slash", 3500, 100, 0.9390122294425964], "7": ["vertical_and_slash", 3500, 100, 0.9635860323905945], "8": ["vertical_and_slash", 1000, 6096, 0.8151974678039551], "9": ["vertical_and_slash", 1000, 6096, 0.9613214135169983], "10": ["vertical_and_slash", 3500, 100, 0.965718686580658], "11": ["vertical_and_slash", 3500, 100, 0.9784087538719177], "12": ["vertical_and_slash", 1000, 6096, 0.6775081157684326], "13": ["vertical_and_slash", 3500, 100, 0.821177065372467], "14": ["vertical_and_slash", 3500, 100, 0.9804673194885254], "15": ["vertical_and_slash", 1000, 6096, 0.862662672996521], "16": ["vertical_and_slash", 1000, 6096, 0.9011936783790588], "17": ["vertical_and_slash", 1000, 6096, 0.8729158043861389], "18": ["vertical_and_slash", 1000, 6096, 0.9261403679847717], "19": ["vertical_and_slash", 1000, 6096, 0.786307156085968], "20": ["vertical_and_slash", 1000, 6096, 0.9203805923461914], "21": ["vertical_and_slash", 1000, 6096, 0.9091859459877014], "22": ["vertical_and_slash", 1000, 6096, 0.9625690579414368], "23": ["vertical_and_slash", 1000, 6096, 0.8890878558158875], "24": ["vertical_and_slash", 1000, 6096, 0.8999782204627991], "25": ["vertical_and_slash", 500, 700, 0.9289186596870422], "26": ["vertical_and_slash", 1000, 6096, 0.8677679896354675], "27": ["vertical_and_slash", 1000, 6096, 0.8918275833129883], "28": ["vertical_and_slash", 1000, 6096, 0.8897462487220764], "29": ["vertical_and_slash", 500, 700, 0.9221864342689514], "30": ["vertical_and_slash", 1000, 6096, 0.8146488666534424], "31": ["vertical_and_slash", 3500, 100, 0.9560171961784363], "32": ["vertical_and_slash", 1000, 6096, 0.8314923644065857], "33": ["vertical_and_slash", 1000, 6096, 0.91201251745224], "34": ["vertical_and_slash", 1000, 6096, 0.784390926361084], "35": ["vertical_and_slash", 500, 700, 0.9497572183609009], "36": ["vertical_and_slash", 1000, 6096, 0.9262849688529968], "37": ["vertical_and_slash", 1000, 6096, 0.9178636074066162], "38": ["vertical_and_slash", 1000, 6096, 0.8926842212677002], "39": ["vertical_and_slash", 500, 700, 0.9519723653793335]}, {"0": ["vertical_and_slash", 3500, 100, 0.9745585322380066], "1": ["vertical_and_slash", 3500, 100, 0.941133439540863], "2": ["vertical_and_slash", 3500, 100, 0.9281198382377625], "3": ["vertical_and_slash", 1000, 6096, 0.8015121221542358], "4": ["vertical_and_slash", 1000, 6096, 0.8777785301208496], "5": ["vertical_and_slash", 1000, 6096, 0.8672139644622803], "6": ["vertical_and_slash", 3500, 100, 0.9568738341331482], "7": ["vertical_and_slash", 1000, 6096, 0.8468345403671265], "8": ["vertical_and_slash", 3500, 100, 0.9712860584259033], "9": ["vertical_and_slash", 1000, 6096, 0.858458399772644], "10": ["vertical_and_slash", 3500, 100, 0.9881042838096619], "11": ["vertical_and_slash", 1000, 6096, 0.9116703867912292], "12": ["vertical_and_slash", 1000, 6096, 0.7518170475959778], "13": ["vertical_and_slash", 3500, 100, 0.9887413382530212], "14": ["vertical_and_slash", 3500, 100, 0.9751730561256409], "15": ["vertical_and_slash", 3500, 100, 0.9454167485237122], "16": ["vertical_and_slash", 1000, 6096, 0.8143635988235474], "17": ["vertical_and_slash", 1000, 6096, 0.9019402265548706], "18": ["vertical_and_slash", 500, 700, 0.9416331648826599], "19": ["vertical_and_slash", 1000, 6096, 0.7533318400382996], "20": ["vertical_and_slash", 500, 700, 0.9662970900535583], "21": ["vertical_and_slash", 1000, 6096, 0.9537178874015808], "22": ["vertical_and_slash", 500, 700, 0.9671534299850464], "23": ["vertical_and_slash", 100, 750, 0.9642254710197449], "24": ["vertical_and_slash", 100, 750, 0.9446746706962585], "25": ["vertical_and_slash", 1000, 6096, 0.9451234340667725], "26": ["vertical_and_slash", 1000, 6096, 0.9628996849060059], "27": ["vertical_and_slash", 1000, 6096, 0.9276864528656006], "28": ["vertical_and_slash", 1000, 6096, 0.9416473507881165], "29": ["vertical_and_slash", 1000, 6096, 0.9367255568504333], "30": ["vertical_and_slash", 1000, 6096, 0.8017884492874146], "31": ["vertical_and_slash", 1000, 6096, 0.9409653544425964], "32": ["vertical_and_slash", 1000, 6096, 0.8920343518257141], "33": ["vertical_and_slash", 1000, 6096, 0.9287262558937073], "34": ["vertical_and_slash", 1000, 6096, 0.8628440499305725], "35": ["vertical_and_slash", 3500, 100, 0.9227964282035828], "36": ["vertical_and_slash", 1000, 6096, 0.8852428793907166], "37": ["vertical_and_slash", 3500, 100, 0.9330180883407593], "38": ["vertical_and_slash", 1000, 6096, 0.9051350951194763], "39": ["vertical_and_slash", 1000, 6096, 0.8826189637184143]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9019501805305481], "1": ["vertical_and_slash", 3500, 100, 0.971427321434021], "2": ["vertical_and_slash", 1000, 6096, 0.8628111481666565], "3": ["vertical_and_slash", 500, 700, 0.9345933794975281], "4": ["vertical_and_slash", 3500, 100, 0.9272739291191101], "5": ["vertical_and_slash", 1000, 6096, 0.8541984558105469], "6": ["vertical_and_slash", 1000, 6096, 0.9373757839202881], "7": ["vertical_and_slash", 1000, 6096, 0.9490547180175781], "8": ["vertical_and_slash", 1000, 6096, 0.5631235241889954], "9": ["vertical_and_slash", 500, 700, 0.98533034324646], "10": ["vertical_and_slash", 1000, 6096, 0.7609000205993652], "11": ["vertical_and_slash", 1000, 6096, 0.892261266708374], "12": ["vertical_and_slash", 3500, 100, 0.9560328125953674], "13": ["vertical_and_slash", 1000, 6096, 0.9221851229667664], "14": ["vertical_and_slash", 3500, 100, 0.9347955584526062], "15": ["vertical_and_slash", 3500, 100, 0.9621464014053345], "16": ["vertical_and_slash", 1000, 6096, 0.8444939851760864], "17": ["vertical_and_slash", 3500, 100, 0.9578442573547363], "18": ["vertical_and_slash", 3500, 100, 0.9601399898529053], "19": ["vertical_and_slash", 1000, 6096, 0.8907353281974792], "20": ["vertical_and_slash", 3500, 100, 0.9794067144393921], "21": ["vertical_and_slash", 1000, 6096, 0.9552299976348877], "22": ["vertical_and_slash", 1000, 6096, 0.9247802495956421], "23": ["vertical_and_slash", 3500, 100, 0.9390078783035278], "24": ["vertical_and_slash", 1000, 6096, 0.9636818170547485], "25": ["vertical_and_slash", 1000, 6096, 0.8349655866622925], "26": ["vertical_and_slash", 1000, 6096, 0.590559184551239], "27": ["vertical_and_slash", 1000, 6096, 0.7694618105888367], "28": ["vertical_and_slash", 1000, 6096, 0.9326425194740295], "29": ["vertical_and_slash", 3500, 100, 0.9810479283332825], "30": ["vertical_and_slash", 3500, 100, 0.9273333549499512], "31": ["vertical_and_slash", 1000, 6096, 0.888212263584137], "32": ["vertical_and_slash", 1000, 6096, 0.9045199751853943], "33": ["vertical_and_slash", 3500, 100, 0.9616050720214844], "34": ["vertical_and_slash", 3500, 100, 0.9245432615280151], "35": ["vertical_and_slash", 3500, 100, 0.97602778673172], "36": ["vertical_and_slash", 3500, 100, 0.8930020928382874], "37": ["vertical_and_slash", 3500, 100, 0.9419819116592407], "38": ["vertical_and_slash", 3500, 100, 0.9391080737113953], "39": ["vertical_and_slash", 3500, 100, 0.965675413608551]}, {"0": ["vertical_and_slash", 3500, 100, 0.8696004748344421], "1": ["vertical_and_slash", 1000, 6096, 0.7622347474098206], "2": ["vertical_and_slash", 1000, 6096, 0.9849997758865356], "3": ["vertical_and_slash", 3500, 100, 0.9606914520263672], "4": ["vertical_and_slash", 3500, 100, 0.967926025390625], "5": ["vertical_and_slash", 3500, 100, 0.9554319381713867], "6": ["vertical_and_slash", 3500, 100, 0.8885374069213867], "7": ["vertical_and_slash", 3500, 100, 0.9941527247428894], "8": ["vertical_and_slash", 1000, 6096, 0.9297124147415161], "9": ["vertical_and_slash", 1000, 6096, 0.8034824728965759], "10": ["vertical_and_slash", 1000, 6096, 0.8601251244544983], "11": ["vertical_and_slash", 3500, 100, 0.9689736366271973], "12": ["vertical_and_slash", 3500, 100, 0.9319722652435303], "13": ["vertical_and_slash", 1000, 6096, 0.87053382396698], "14": ["vertical_and_slash", 3500, 100, 0.9439869523048401], "15": ["vertical_and_slash", 3500, 100, 0.8385700583457947], "16": ["vertical_and_slash", 3500, 100, 0.9966137409210205], "17": ["vertical_and_slash", 3500, 100, 0.9295799136161804], "18": ["vertical_and_slash", 3500, 100, 0.9759792685508728], "19": ["vertical_and_slash", 3500, 100, 0.9818010330200195], "20": ["vertical_and_slash", 1000, 6096, 0.7765905857086182], "21": ["vertical_and_slash", 1000, 6096, 0.8171805143356323], "22": ["vertical_and_slash", 1000, 6096, 0.8729115128517151], "23": ["vertical_and_slash", 1000, 6096, 0.7390596866607666], "24": ["vertical_and_slash", 1000, 6096, 0.6736255288124084], "25": ["vertical_and_slash", 3500, 100, 0.9740850925445557], "26": ["vertical_and_slash", 3500, 100, 0.9523630738258362], "27": ["vertical_and_slash", 3500, 100, 0.8162307143211365], "28": ["vertical_and_slash", 3500, 100, 0.985908031463623], "29": ["vertical_and_slash", 1000, 6096, 0.8229836225509644], "30": ["vertical_and_slash", 1000, 6096, 0.8897380232810974], "31": ["vertical_and_slash", 1000, 6096, 0.7226291298866272], "32": ["vertical_and_slash", 3500, 100, 0.9191795587539673], "33": ["vertical_and_slash", 1000, 6096, 0.6296677589416504], "34": ["vertical_and_slash", 500, 700, 0.9439336061477661], "35": ["vertical_and_slash", 3500, 100, 0.9444857239723206], "36": ["vertical_and_slash", 3500, 100, 0.9750229716300964], "37": ["vertical_and_slash", 1000, 6096, 0.8393720388412476], "38": ["vertical_and_slash", 1000, 6096, 0.8966403603553772], "39": ["vertical_and_slash", 3500, 100, 0.9563556909561157]}, {"0": ["vertical_and_slash", 100, 750, 0.9727997183799744], "1": ["vertical_and_slash", 100, 750, 0.9760631918907166], "2": ["vertical_and_slash", 100, 750, 0.9901638031005859], "3": ["vertical_and_slash", 100, 750, 0.9979047775268555], "4": ["vertical_and_slash", 100, 750, 0.9934083819389343], "5": ["vertical_and_slash", 1000, 6096, 0.9349896907806396], "6": ["vertical_and_slash", 500, 700, 0.9688169360160828], "7": ["vertical_and_slash", 1000, 6096, 0.9043138027191162], "8": ["vertical_and_slash", 100, 750, 0.9783809185028076], "9": ["vertical_and_slash", 3500, 100, 0.9335293769836426], "10": ["vertical_and_slash", 3500, 100, 0.9083223342895508], "11": ["vertical_and_slash", 3500, 100, 0.9095141291618347], "12": ["vertical_and_slash", 3500, 100, 0.9678698778152466], "13": ["vertical_and_slash", 3500, 100, 0.9273633360862732], "14": ["vertical_and_slash", 3500, 100, 0.9476717114448547], "15": ["vertical_and_slash", 3500, 100, 0.9323228597640991], "16": ["vertical_and_slash", 3500, 100, 0.9487809538841248], "17": ["vertical_and_slash", 3500, 100, 0.9877986311912537], "18": ["vertical_and_slash", 3500, 100, 0.9692082405090332], "19": ["vertical_and_slash", 1000, 6096, 0.9506139755249023], "20": ["vertical_and_slash", 1000, 6096, 0.9314926862716675], "21": ["vertical_and_slash", 1000, 6096, 0.7856042981147766], "22": ["vertical_and_slash", 1000, 6096, 0.9547144770622253], "23": ["vertical_and_slash", 1000, 6096, 0.9324890375137329], "24": ["vertical_and_slash", 1000, 6096, 0.8976241946220398], "25": ["vertical_and_slash", 3500, 100, 0.9842115044593811], "26": ["vertical_and_slash", 1000, 6096, 0.878645122051239], "27": ["vertical_and_slash", 3500, 100, 0.9562796950340271], "28": ["vertical_and_slash", 1000, 6096, 0.8640088438987732], "29": ["vertical_and_slash", 3500, 100, 0.9560847282409668], "30": ["vertical_and_slash", 500, 700, 0.941033124923706], "31": ["vertical_and_slash", 3500, 100, 0.9562534689903259], "32": ["vertical_and_slash", 1000, 6096, 0.9094836711883545], "33": ["vertical_and_slash", 500, 700, 0.8485914468765259], "34": ["vertical_and_slash", 1000, 6096, 0.8254481554031372], "35": ["vertical_and_slash", 1000, 6096, 0.9200180172920227], "36": ["vertical_and_slash", 1000, 6096, 0.7888291478157043], "37": ["vertical_and_slash", 500, 700, 0.9375855922698975], "38": ["vertical_and_slash", 1000, 6096, 0.9138672947883606], "39": ["vertical_and_slash", 1000, 6096, 0.9367169141769409]}, {"0": ["vertical_and_slash", 3500, 100, 0.9935808181762695], "1": ["vertical_and_slash", 500, 700, 0.9782410860061646], "2": ["vertical_and_slash", 500, 700, 0.9817525148391724], "3": ["vertical_and_slash", 3500, 100, 0.9866502285003662], "4": ["vertical_and_slash", 100, 750, 0.9663071036338806], "5": ["vertical_and_slash", 1000, 6096, 0.8200854063034058], "6": ["vertical_and_slash", 3500, 100, 0.9745064973831177], "7": ["vertical_and_slash", 3500, 100, 0.9574806094169617], "8": ["vertical_and_slash", 1000, 6096, 0.9059184789657593], "9": ["vertical_and_slash", 1000, 6096, 0.8054295778274536], "10": ["vertical_and_slash", 1000, 6096, 0.7761107683181763], "11": ["vertical_and_slash", 3500, 100, 0.9036723375320435], "12": ["vertical_and_slash", 1000, 6096, 0.8378755450248718], "13": ["vertical_and_slash", 1000, 6096, 0.879318118095398], "14": ["vertical_and_slash", 1000, 6096, 0.8305036425590515], "15": ["vertical_and_slash", 3500, 100, 0.960605263710022], "16": ["vertical_and_slash", 3500, 100, 0.9530232548713684], "17": ["vertical_and_slash", 3500, 100, 0.9683519005775452], "18": ["vertical_and_slash", 3500, 100, 0.9917226433753967], "19": ["vertical_and_slash", 3500, 100, 0.9557647705078125], "20": ["vertical_and_slash", 3500, 100, 0.8312321901321411], "21": ["vertical_and_slash", 3500, 100, 0.951707661151886], "22": ["vertical_and_slash", 3500, 100, 0.9560086131095886], "23": ["vertical_and_slash", 1000, 6096, 0.8589605689048767], "24": ["vertical_and_slash", 1000, 6096, 0.7082435488700867], "25": ["vertical_and_slash", 3500, 100, 0.9742854237556458], "26": ["vertical_and_slash", 3500, 100, 0.9749569892883301], "27": ["vertical_and_slash", 3500, 100, 0.9936811327934265], "28": ["vertical_and_slash", 3500, 100, 0.9898296594619751], "29": ["vertical_and_slash", 1000, 6096, 0.9012367725372314], "30": ["vertical_and_slash", 1000, 6096, 0.8048471212387085], "31": ["vertical_and_slash", 1000, 6096, 0.9322119951248169], "32": ["vertical_and_slash", 1000, 6096, 0.6638270020484924], "33": ["vertical_and_slash", 1000, 6096, 0.8314067125320435], "34": ["vertical_and_slash", 3500, 100, 0.9236209988594055], "35": ["vertical_and_slash", 3500, 100, 0.8939440846443176], "36": ["vertical_and_slash", 1000, 6096, 0.860835075378418], "37": ["vertical_and_slash", 1000, 6096, 0.8533241748809814], "38": ["vertical_and_slash", 3500, 100, 0.9218117594718933], "39": ["vertical_and_slash", 3500, 100, 0.8755636811256409]}, {"0": ["vertical_and_slash", 3500, 100, 0.9364347457885742], "1": ["vertical_and_slash", 3500, 100, 0.990091860294342], "2": ["vertical_and_slash", 3500, 100, 0.9873543381690979], "3": ["vertical_and_slash", 3500, 100, 0.9812934398651123], "4": ["vertical_and_slash", 1000, 6096, 0.8529465794563293], "5": ["vertical_and_slash", 3500, 100, 0.916752278804779], "6": ["vertical_and_slash", 1000, 6096, 0.7806752324104309], "7": ["vertical_and_slash", 3500, 100, 0.9052515625953674], "8": ["vertical_and_slash", 500, 700, 0.9749282002449036], "9": ["vertical_and_slash", 1000, 6096, 0.8711175322532654], "10": ["vertical_and_slash", 3500, 100, 0.9878072738647461], "11": ["vertical_and_slash", 3500, 100, 0.9445090293884277], "12": ["vertical_and_slash", 3500, 100, 0.980056881904602], "13": ["vertical_and_slash", 1000, 6096, 0.806226372718811], "14": ["vertical_and_slash", 3500, 100, 0.9588730931282043], "15": ["vertical_and_slash", 1000, 6096, 0.8812461495399475], "16": ["vertical_and_slash", 3500, 100, 0.8597861528396606], "17": ["vertical_and_slash", 1000, 6096, 0.7975826859474182], "18": ["vertical_and_slash", 1000, 6096, 0.8434271812438965], "19": ["vertical_and_slash", 1000, 6096, 0.8067223429679871], "20": ["vertical_and_slash", 3500, 100, 0.9826947450637817], "21": ["vertical_and_slash", 3500, 100, 0.9752951264381409], "22": ["vertical_and_slash", 3500, 100, 0.9772754907608032], "23": ["vertical_and_slash", 3500, 100, 0.9378178715705872], "24": ["vertical_and_slash", 3500, 100, 0.9705775380134583], "25": ["vertical_and_slash", 3500, 100, 0.9430608153343201], "26": ["vertical_and_slash", 3500, 100, 0.9534416794776917], "27": ["vertical_and_slash", 3500, 100, 0.9730291962623596], "28": ["vertical_and_slash", 500, 700, 0.9423494935035706], "29": ["vertical_and_slash", 3500, 100, 0.9605511426925659], "30": ["vertical_and_slash", 3500, 100, 0.9846211671829224], "31": ["vertical_and_slash", 1000, 6096, 0.6972063779830933], "32": ["vertical_and_slash", 3500, 100, 0.9382303953170776], "33": ["vertical_and_slash", 500, 700, 0.9814949035644531], "34": ["vertical_and_slash", 3500, 100, 0.9611619114875793], "35": ["vertical_and_slash", 1000, 6096, 0.915047287940979], "36": ["vertical_and_slash", 3500, 100, 0.9643358588218689], "37": ["vertical_and_slash", 3500, 100, 0.9483014345169067], "38": ["vertical_and_slash", 1000, 6096, 0.8745499849319458], "39": ["vertical_and_slash", 3500, 100, 0.9817555546760559]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7416021227836609], "1": ["vertical_and_slash", 1000, 6096, 0.8031586408615112], "2": ["vertical_and_slash", 1000, 6096, 0.7160335779190063], "3": ["vertical_and_slash", 1000, 6096, 0.840573251247406], "4": ["vertical_and_slash", 1000, 6096, 0.8590536117553711], "5": ["vertical_and_slash", 3500, 100, 0.9801639914512634], "6": ["vertical_and_slash", 3500, 100, 0.9893338680267334], "7": ["vertical_and_slash", 3500, 100, 0.9892603158950806], "8": ["vertical_and_slash", 3500, 100, 0.9961360692977905], "9": ["vertical_and_slash", 3500, 100, 0.9665273427963257], "10": ["vertical_and_slash", 500, 700, 0.9774555563926697], "11": ["vertical_and_slash", 500, 700, 0.9766563177108765], "12": ["vertical_and_slash", 3500, 100, 0.9337731003761292], "13": ["vertical_and_slash", 100, 750, 0.9394619464874268], "14": ["vertical_and_slash", 500, 700, 0.972790002822876], "15": ["vertical_and_slash", 3500, 100, 0.9930166602134705], "16": ["vertical_and_slash", 3500, 100, 0.9693329334259033], "17": ["vertical_and_slash", 3500, 100, 0.9369997978210449], "18": ["vertical_and_slash", 3500, 100, 0.9571195840835571], "19": ["vertical_and_slash", 500, 700, 0.9677093029022217], "20": ["vertical_and_slash", 1000, 6096, 0.8809376358985901], "21": ["vertical_and_slash", 3500, 100, 0.9389232993125916], "22": ["vertical_and_slash", 1000, 6096, 0.8355110287666321], "23": ["vertical_and_slash", 3500, 100, 0.9436038136482239], "24": ["vertical_and_slash", 3500, 100, 0.9544028639793396], "25": ["vertical_and_slash", 1000, 6096, 0.7780212163925171], "26": ["vertical_and_slash", 3500, 100, 0.9186717867851257], "27": ["vertical_and_slash", 3500, 100, 0.9746288657188416], "28": ["vertical_and_slash", 3500, 100, 0.9940743446350098], "29": ["vertical_and_slash", 3500, 100, 0.99359530210495], "30": ["vertical_and_slash", 1000, 6096, 0.925224781036377], "31": ["vertical_and_slash", 1000, 6096, 0.7253524661064148], "32": ["vertical_and_slash", 3500, 100, 0.9109766483306885], "33": ["vertical_and_slash", 3500, 100, 0.9156817197799683], "34": ["vertical_and_slash", 3500, 100, 0.9592355489730835], "35": ["vertical_and_slash", 1000, 6096, 0.9267030358314514], "36": ["vertical_and_slash", 3500, 100, 0.9564041495323181], "37": ["vertical_and_slash", 3500, 100, 0.9605346322059631], "38": ["vertical_and_slash", 1000, 6096, 0.8525487184524536], "39": ["vertical_and_slash", 1000, 6096, 0.9380912184715271]}, {"0": ["vertical_and_slash", 3500, 100, 0.9650010466575623], "1": ["vertical_and_slash", 1000, 6096, 0.8923070430755615], "2": ["vertical_and_slash", 3500, 100, 0.9199210405349731], "3": ["vertical_and_slash", 3500, 100, 0.9097593426704407], "4": ["vertical_and_slash", 3500, 100, 0.9252224564552307], "5": ["vertical_and_slash", 1000, 6096, 0.9744053483009338], "6": ["vertical_and_slash", 500, 700, 0.9834085702896118], "7": ["vertical_and_slash", 3500, 100, 0.967046856880188], "8": ["vertical_and_slash", 3500, 100, 0.9899594187736511], "9": ["vertical_and_slash", 500, 700, 0.9899400472640991], "10": ["vertical_and_slash", 3500, 100, 0.7678192853927612], "11": ["vertical_and_slash", 3500, 100, 0.7591788172721863], "12": ["vertical_and_slash", 3500, 100, 0.9374876618385315], "13": ["vertical_and_slash", 1000, 6096, 0.7690331935882568], "14": ["vertical_and_slash", 1000, 6096, 0.7213966846466064], "15": ["vertical_and_slash", 3500, 100, 0.9874593615531921], "16": ["vertical_and_slash", 3500, 100, 0.9963500499725342], "17": ["vertical_and_slash", 3500, 100, 0.995738685131073], "18": ["vertical_and_slash", 500, 700, 0.977127730846405], "19": ["vertical_and_slash", 3500, 100, 0.9932492971420288], "20": ["vertical_and_slash", 1000, 6096, 0.7257412075996399], "21": ["vertical_and_slash", 500, 700, 0.9480133652687073], "22": ["vertical_and_slash", 1000, 6096, 0.8787633776664734], "23": ["vertical_and_slash", 3500, 100, 0.9736138582229614], "24": ["vertical_and_slash", 1000, 6096, 0.7746095061302185], "25": ["vertical_and_slash", 3500, 100, 0.9820363521575928], "26": ["vertical_and_slash", 3500, 100, 0.9846170544624329], "27": ["vertical_and_slash", 3500, 100, 0.9875494241714478], "28": ["vertical_and_slash", 1000, 6096, 0.9172033071517944], "29": ["vertical_and_slash", 3500, 100, 0.9849454164505005], "30": ["vertical_and_slash", 3500, 100, 0.9763356447219849], "31": ["vertical_and_slash", 3500, 100, 0.9779592156410217], "32": ["vertical_and_slash", 3500, 100, 0.9957728981971741], "33": ["vertical_and_slash", 3500, 100, 0.9786238670349121], "34": ["vertical_and_slash", 3500, 100, 0.9956552386283875], "35": ["vertical_and_slash", 500, 700, 0.9829931855201721], "36": ["vertical_and_slash", 3500, 100, 0.9724671840667725], "37": ["vertical_and_slash", 500, 700, 0.9679861068725586], "38": ["vertical_and_slash", 500, 700, 0.9703396558761597], "39": ["vertical_and_slash", 3500, 100, 0.9341856837272644]}, {"0": ["vertical_and_slash", 3500, 100, 0.9708288908004761], "1": ["vertical_and_slash", 3500, 100, 0.9593217372894287], "2": ["vertical_and_slash", 3500, 100, 0.9272744059562683], "3": ["vertical_and_slash", 3500, 100, 0.9370599985122681], "4": ["vertical_and_slash", 3500, 100, 0.9821864366531372], "5": ["vertical_and_slash", 3500, 100, 0.996902346611023], "6": ["vertical_and_slash", 3500, 100, 0.9990786910057068], "7": ["vertical_and_slash", 3500, 100, 0.9983267784118652], "8": ["vertical_and_slash", 3500, 100, 0.9958170652389526], "9": ["vertical_and_slash", 3500, 100, 0.9953048825263977], "10": ["vertical_and_slash", 1000, 6096, 0.8206984400749207], "11": ["vertical_and_slash", 3500, 100, 0.8549410700798035], "12": ["vertical_and_slash", 3500, 100, 0.9161165952682495], "13": ["vertical_and_slash", 1000, 6096, 0.8110862374305725], "14": ["vertical_and_slash", 3500, 100, 0.8722493648529053], "15": ["vertical_and_slash", 1000, 6096, 0.7264871597290039], "16": ["vertical_and_slash", 500, 700, 0.957619309425354], "17": ["vertical_and_slash", 3500, 100, 0.6611396670341492], "18": ["vertical_and_slash", 1000, 6096, 0.921381950378418], "19": ["vertical_and_slash", 1000, 6096, 0.9744597673416138], "20": ["vertical_and_slash", 3500, 100, 0.9954834580421448], "21": ["vertical_and_slash", 3500, 100, 0.9977924823760986], "22": ["vertical_and_slash", 3500, 100, 0.99921715259552], "23": ["vertical_and_slash", 3500, 100, 0.9901573061943054], "24": ["vertical_and_slash", 3500, 100, 0.9842782616615295], "25": ["vertical_and_slash", 3500, 100, 0.9109574556350708], "26": ["vertical_and_slash", 3500, 100, 0.9913878440856934], "27": ["vertical_and_slash", 3500, 100, 0.9497086405754089], "28": ["vertical_and_slash", 3500, 100, 0.8343267440795898], "29": ["vertical_and_slash", 1000, 6096, 0.7446168065071106], "30": ["vertical_and_slash", 1000, 6096, 0.8945479393005371], "31": ["vertical_and_slash", 1000, 6096, 0.9325368404388428], "32": ["vertical_and_slash", 3500, 100, 0.9821228384971619], "33": ["vertical_and_slash", 3500, 100, 0.8848371505737305], "34": ["vertical_and_slash", 3500, 100, 0.9547179937362671], "35": ["vertical_and_slash", 1000, 6096, 0.9000244736671448], "36": ["vertical_and_slash", 3500, 100, 0.9935563206672668], "37": ["vertical_and_slash", 3500, 100, 0.9735134840011597], "38": ["vertical_and_slash", 3500, 100, 0.8708822131156921], "39": ["vertical_and_slash", 3500, 100, 0.9512901306152344]}, {"0": ["vertical_and_slash", 3500, 100, 0.9497001767158508], "1": ["vertical_and_slash", 3500, 100, 0.9920233488082886], "2": ["vertical_and_slash", 3500, 100, 0.9681625366210938], "3": ["vertical_and_slash", 3500, 100, 0.993034839630127], "4": ["vertical_and_slash", 3500, 100, 0.9722923040390015], "5": ["vertical_and_slash", 1000, 6096, 0.9709884524345398], "6": ["vertical_and_slash", 500, 700, 0.9836205244064331], "7": ["vertical_and_slash", 500, 700, 0.9750344753265381], "8": ["vertical_and_slash", 500, 700, 0.9863587021827698], "9": ["vertical_and_slash", 500, 700, 0.973861575126648], "10": ["vertical_and_slash", 3500, 100, 0.9314413070678711], "11": ["vertical_and_slash", 3500, 100, 0.970123291015625], "12": ["vertical_and_slash", 3500, 100, 0.9753661751747131], "13": ["vertical_and_slash", 3500, 100, 0.947053074836731], "14": ["vertical_and_slash", 3500, 100, 0.9189079999923706], "15": ["vertical_and_slash", 1000, 6096, 0.945266842842102], "16": ["vertical_and_slash", 1000, 6096, 0.8546351194381714], "17": ["vertical_and_slash", 3500, 100, 0.9422704577445984], "18": ["vertical_and_slash", 3500, 100, 0.9881876707077026], "19": ["vertical_and_slash", 1000, 6096, 0.727601170539856], "20": ["vertical_and_slash", 3500, 100, 0.9709702134132385], "21": ["vertical_and_slash", 3500, 100, 0.9775145649909973], "22": ["vertical_and_slash", 3500, 100, 0.9792836904525757], "23": ["vertical_and_slash", 3500, 100, 0.9556652903556824], "24": ["vertical_and_slash", 3500, 100, 0.9718955159187317], "25": ["vertical_and_slash", 3500, 100, 0.9800054430961609], "26": ["vertical_and_slash", 3500, 100, 0.9682377576828003], "27": ["vertical_and_slash", 3500, 100, 0.9572911858558655], "28": ["vertical_and_slash", 3500, 100, 0.9611387252807617], "29": ["vertical_and_slash", 3500, 100, 0.9873570799827576], "30": ["vertical_and_slash", 3500, 100, 0.9852196574211121], "31": ["vertical_and_slash", 3500, 100, 0.9409613609313965], "32": ["vertical_and_slash", 3500, 100, 0.94648277759552], "33": ["vertical_and_slash", 3500, 100, 0.9798470735549927], "34": ["vertical_and_slash", 3500, 100, 0.9738354682922363], "35": ["vertical_and_slash", 500, 700, 0.9622628092765808], "36": ["vertical_and_slash", 100, 750, 0.9856722354888916], "37": ["vertical_and_slash", 3500, 100, 0.9922625422477722], "38": ["vertical_and_slash", 1000, 6096, 0.694831132888794], "39": ["vertical_and_slash", 3500, 100, 0.9263876676559448]}, {"0": ["vertical_and_slash", 3500, 100, 0.8842455148696899], "1": ["vertical_and_slash", 3500, 100, 0.9280725717544556], "2": ["vertical_and_slash", 3500, 100, 0.9329488277435303], "3": ["vertical_and_slash", 3500, 100, 0.9509137868881226], "4": ["vertical_and_slash", 3500, 100, 0.9451907873153687], "5": ["vertical_and_slash", 3500, 100, 0.9826501607894897], "6": ["vertical_and_slash", 3500, 100, 0.9367103576660156], "7": ["vertical_and_slash", 3500, 100, 0.9553903937339783], "8": ["vertical_and_slash", 3500, 100, 0.9862452745437622], "9": ["vertical_and_slash", 3500, 100, 0.9670647382736206], "10": ["vertical_and_slash", 500, 700, 0.9803469181060791], "11": ["vertical_and_slash", 3500, 100, 0.9597765803337097], "12": ["vertical_and_slash", 3500, 100, 0.9911244511604309], "13": ["vertical_and_slash", 1000, 6096, 0.9873899221420288], "14": ["vertical_and_slash", 3500, 100, 0.980191171169281], "15": ["vertical_and_slash", 3500, 100, 0.9408585429191589], "16": ["vertical_and_slash", 3500, 100, 0.9263893961906433], "17": ["vertical_and_slash", 1000, 6096, 0.86265629529953], "18": ["vertical_and_slash", 3500, 100, 0.9385087490081787], "19": ["vertical_and_slash", 1000, 6096, 0.8834230303764343], "20": ["vertical_and_slash", 3500, 100, 0.9851137399673462], "21": ["vertical_and_slash", 3500, 100, 0.9848038554191589], "22": ["vertical_and_slash", 3500, 100, 0.9083573222160339], "23": ["vertical_and_slash", 3500, 100, 0.9968385696411133], "24": ["vertical_and_slash", 3500, 100, 0.9846994876861572], "25": ["vertical_and_slash", 3500, 100, 0.9845128655433655], "26": ["vertical_and_slash", 3500, 100, 0.9775534868240356], "27": ["vertical_and_slash", 3500, 100, 0.9775784015655518], "28": ["vertical_and_slash", 1000, 6096, 0.8938180804252625], "29": ["vertical_and_slash", 3500, 100, 0.9783908724784851], "30": ["vertical_and_slash", 3500, 100, 0.9138945937156677], "31": ["vertical_and_slash", 1000, 6096, 0.9123159050941467], "32": ["vertical_and_slash", 3500, 100, 0.9944044351577759], "33": ["vertical_and_slash", 3500, 100, 0.9462963938713074], "34": ["vertical_and_slash", 1000, 6096, 0.862132728099823], "35": ["vertical_and_slash", 1000, 6096, 0.9158415198326111], "36": ["vertical_and_slash", 1000, 6096, 0.8865128755569458], "37": ["vertical_and_slash", 3500, 100, 0.9162105917930603], "38": ["vertical_and_slash", 1000, 6096, 0.8729876279830933], "39": ["vertical_and_slash", 3500, 100, 0.9882699847221375]}, {"0": ["vertical_and_slash", 500, 700, 0.9879748225212097], "1": ["vertical_and_slash", 500, 700, 0.9902948141098022], "2": ["vertical_and_slash", 500, 700, 0.9819856286048889], "3": ["vertical_and_slash", 500, 700, 0.9819986820220947], "4": ["vertical_and_slash", 500, 700, 0.9875563979148865], "5": ["vertical_and_slash", 3500, 100, 0.9838576912879944], "6": ["vertical_and_slash", 3500, 100, 0.9898105263710022], "7": ["vertical_and_slash", 3500, 100, 0.9868780374526978], "8": ["vertical_and_slash", 3500, 100, 0.9766300320625305], "9": ["vertical_and_slash", 3500, 100, 0.9834796190261841], "10": ["vertical_and_slash", 3500, 100, 0.9591426253318787], "11": ["vertical_and_slash", 3500, 100, 0.9897322654724121], "12": ["vertical_and_slash", 3500, 100, 0.9568406343460083], "13": ["vertical_and_slash", 3500, 100, 0.9525031447410583], "14": ["vertical_and_slash", 3500, 100, 0.9106599688529968], "15": ["vertical_and_slash", 3500, 100, 0.9425155520439148], "16": ["vertical_and_slash", 3500, 100, 0.9530122876167297], "17": ["vertical_and_slash", 3500, 100, 0.9771508574485779], "18": ["vertical_and_slash", 3500, 100, 0.9741969704627991], "19": ["vertical_and_slash", 3500, 100, 0.923008382320404], "20": ["vertical_and_slash", 3500, 100, 0.9125261902809143], "21": ["vertical_and_slash", 3500, 100, 0.8923377990722656], "22": ["vertical_and_slash", 3500, 100, 0.9263057708740234], "23": ["vertical_and_slash", 1000, 6096, 0.8776153922080994], "24": ["vertical_and_slash", 1000, 6096, 0.8203091025352478], "25": ["vertical_and_slash", 3500, 100, 0.9936240315437317], "26": ["vertical_and_slash", 3500, 100, 0.942081868648529], "27": ["vertical_and_slash", 3500, 100, 0.888495147228241], "28": ["vertical_and_slash", 1000, 6096, 0.9731393456459045], "29": ["vertical_and_slash", 3500, 100, 0.9635735750198364], "30": ["vertical_and_slash", 1000, 6096, 0.91904217004776], "31": ["vertical_and_slash", 3500, 100, 0.9578284025192261], "32": ["vertical_and_slash", 3500, 100, 0.9497632384300232], "33": ["vertical_and_slash", 1000, 6096, 0.9199178814888], "34": ["vertical_and_slash", 1000, 6096, 0.8853236436843872], "35": ["vertical_and_slash", 3500, 100, 0.9560525417327881], "36": ["vertical_and_slash", 1000, 6096, 0.8756008744239807], "37": ["vertical_and_slash", 3500, 100, 0.9646494388580322], "38": ["vertical_and_slash", 3500, 100, 0.9527634978294373], "39": ["vertical_and_slash", 1000, 6096, 0.8780484795570374]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9523205161094666], "1": ["vertical_and_slash", 1000, 6096, 0.8662764430046082], "2": ["vertical_and_slash", 1000, 6096, 0.5992761850357056], "3": ["vertical_and_slash", 1000, 6096, 0.95766681432724], "4": ["vertical_and_slash", 3500, 100, 0.9740654826164246], "5": ["vertical_and_slash", 3500, 100, 0.9881641864776611], "6": ["vertical_and_slash", 3500, 100, 0.9748408794403076], "7": ["vertical_and_slash", 3500, 100, 0.9563772082328796], "8": ["vertical_and_slash", 1000, 6096, 0.915494978427887], "9": ["vertical_and_slash", 1000, 6096, 0.952011227607727], "10": ["vertical_and_slash", 1000, 6096, 0.6574244499206543], "11": ["vertical_and_slash", 1000, 6096, 0.6184709668159485], "12": ["vertical_and_slash", 1000, 6096, 0.8954805731773376], "13": ["vertical_and_slash", 3500, 100, 0.9257462620735168], "14": ["vertical_and_slash", 1000, 6096, 0.7625937461853027], "15": ["vertical_and_slash", 1000, 6096, 0.9133007526397705], "16": ["vertical_and_slash", 3500, 100, 0.9393588304519653], "17": ["vertical_and_slash", 1000, 6096, 0.8963243961334229], "18": ["vertical_and_slash", 1000, 6096, 0.845547080039978], "19": ["vertical_and_slash", 1000, 6096, 0.6575486660003662], "20": ["vertical_and_slash", 1000, 6096, 0.9490088820457458], "21": ["vertical_and_slash", 1000, 6096, 0.8980074524879456], "22": ["vertical_and_slash", 3500, 100, 0.9819526076316833], "23": ["vertical_and_slash", 3500, 100, 0.9400089979171753], "24": ["vertical_and_slash", 3500, 100, 0.9507432579994202], "25": ["vertical_and_slash", 1000, 6096, 0.822181761264801], "26": ["vertical_and_slash", 3500, 100, 0.9620561003684998], "27": ["vertical_and_slash", 1000, 6096, 0.8968231081962585], "28": ["vertical_and_slash", 3500, 100, 0.9769387245178223], "29": ["vertical_and_slash", 1000, 6096, 0.8478685021400452], "30": ["vertical_and_slash", 3500, 100, 0.9916781187057495], "31": ["vertical_and_slash", 3500, 100, 0.9788512587547302], "32": ["vertical_and_slash", 3500, 100, 0.9427299499511719], "33": ["vertical_and_slash", 3500, 100, 0.9363141655921936], "34": ["vertical_and_slash", 1000, 6096, 0.8980393409729004], "35": ["vertical_and_slash", 3500, 100, 0.9978752136230469], "36": ["vertical_and_slash", 1000, 6096, 0.9426165819168091], "37": ["vertical_and_slash", 3500, 100, 0.9915469288825989], "38": ["vertical_and_slash", 3500, 100, 0.9859925508499146], "39": ["vertical_and_slash", 3500, 100, 0.9733128547668457]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9628710150718689], "1": ["vertical_and_slash", 3500, 100, 0.9529829621315002], "2": ["vertical_and_slash", 1000, 6096, 0.9679727554321289], "3": ["vertical_and_slash", 3500, 100, 0.9829851388931274], "4": ["vertical_and_slash", 1000, 6096, 0.9789767861366272], "5": ["vertical_and_slash", 3500, 100, 0.9898810386657715], "6": ["vertical_and_slash", 1000, 6096, 0.9400345087051392], "7": ["vertical_and_slash", 1000, 6096, 0.9785208702087402], "8": ["vertical_and_slash", 1000, 6096, 0.8987895250320435], "9": ["vertical_and_slash", 1000, 6096, 0.8352503180503845], "10": ["vertical_and_slash", 1000, 6096, 0.7941118478775024], "11": ["vertical_and_slash", 1000, 6096, 0.9484315514564514], "12": ["vertical_and_slash", 1000, 6096, 0.9898168444633484], "13": ["vertical_and_slash", 1000, 6096, 0.9301355481147766], "14": ["vertical_and_slash", 1000, 6096, 0.9786749482154846], "15": ["vertical_and_slash", 1000, 6096, 0.9778074622154236], "16": ["vertical_and_slash", 3500, 100, 0.821785032749176], "17": ["vertical_and_slash", 1000, 6096, 0.7722371816635132], "18": ["vertical_and_slash", 1000, 6096, 0.9331441521644592], "19": ["vertical_and_slash", 1000, 6096, 0.9132733941078186], "20": ["vertical_and_slash", 1000, 6096, 0.9364298582077026], "21": ["vertical_and_slash", 3500, 100, 0.9224213361740112], "22": ["vertical_and_slash", 3500, 100, 0.9180915355682373], "23": ["vertical_and_slash", 1000, 6096, 0.9354913234710693], "24": ["vertical_and_slash", 3500, 100, 0.9481636881828308], "25": ["vertical_and_slash", 3500, 100, 0.921407163143158], "26": ["vertical_and_slash", 3500, 100, 0.9571909308433533], "27": ["vertical_and_slash", 3500, 100, 0.9619804620742798], "28": ["vertical_and_slash", 3500, 100, 0.96356201171875], "29": ["vertical_and_slash", 1000, 6096, 0.9814285039901733], "30": ["vertical_and_slash", 3500, 100, 0.9877622127532959], "31": ["vertical_and_slash", 1000, 6096, 0.9692251086235046], "32": ["vertical_and_slash", 1000, 6096, 0.9132075905799866], "33": ["vertical_and_slash", 1000, 6096, 0.9415092468261719], "34": ["vertical_and_slash", 3500, 100, 0.9885796904563904], "35": ["vertical_and_slash", 1000, 6096, 0.9537850022315979], "36": ["vertical_and_slash", 1000, 6096, 0.9696974158287048], "37": ["vertical_and_slash", 1000, 6096, 0.9127564430236816], "38": ["vertical_and_slash", 1000, 6096, 0.9566832184791565], "39": ["vertical_and_slash", 1000, 6096, 0.9499053955078125]}, {"0": ["vertical_and_slash", 1000, 6096, 0.6549110412597656], "1": ["vertical_and_slash", 1000, 6096, 0.683172345161438], "2": ["vertical_and_slash", 1000, 6096, 0.9119673371315002], "3": ["vertical_and_slash", 1000, 6096, 0.7323319315910339], "4": ["vertical_and_slash", 3500, 100, 0.7934186458587646], "5": ["vertical_and_slash", 1000, 6096, 0.873832106590271], "6": ["vertical_and_slash", 1000, 6096, 0.91180819272995], "7": ["vertical_and_slash", 1000, 6096, 0.9243932366371155], "8": ["vertical_and_slash", 1000, 6096, 0.9201204776763916], "9": ["vertical_and_slash", 1000, 6096, 0.9386066198348999], "10": ["vertical_and_slash", 1000, 6096, 0.9288856983184814], "11": ["vertical_and_slash", 1000, 6096, 0.8880561590194702], "12": ["vertical_and_slash", 1000, 6096, 0.9283311367034912], "13": ["vertical_and_slash", 3500, 100, 0.9597868323326111], "14": ["vertical_and_slash", 1000, 6096, 0.7063172459602356], "15": ["vertical_and_slash", 30, 800, 0.9916436076164246], "16": ["vertical_and_slash", 30, 800, 0.9909761548042297], "17": ["vertical_and_slash", 500, 700, 0.9670186638832092], "18": ["vertical_and_slash", 30, 800, 0.9206994771957397], "19": ["vertical_and_slash", 30, 800, 0.9967113137245178], "20": ["vertical_and_slash", 1000, 6096, 0.7543046474456787], "21": ["vertical_and_slash", 1000, 6096, 0.9320887923240662], "22": ["vertical_and_slash", 1000, 6096, 0.7732936143875122], "23": ["vertical_and_slash", 1000, 6096, 0.8612430095672607], "24": ["vertical_and_slash", 1000, 6096, 0.9027860760688782], "25": ["vertical_and_slash", 1000, 6096, 0.6559157967567444], "26": ["vertical_and_slash", 1000, 6096, 0.7605172395706177], "27": ["vertical_and_slash", 1000, 6096, 0.6479364633560181], "28": ["vertical_and_slash", 1000, 6096, 0.5812984704971313], "29": ["vertical_and_slash", 1000, 6096, 0.5665915608406067], "30": ["vertical_and_slash", 1000, 6096, 0.8588692545890808], "31": ["vertical_and_slash", 1000, 6096, 0.7422910332679749], "32": ["vertical_and_slash", 1000, 6096, 0.7186200022697449], "33": ["vertical_and_slash", 1000, 6096, 0.5696796774864197], "34": ["vertical_and_slash", 1000, 6096, 0.8439114093780518], "35": ["vertical_and_slash", 1000, 6096, 0.8851191997528076], "36": ["vertical_and_slash", 1000, 6096, 0.9613783955574036], "37": ["vertical_and_slash", 1000, 6096, 0.8653088212013245], "38": ["vertical_and_slash", 3500, 100, 0.9742029309272766], "39": ["vertical_and_slash", 1000, 6096, 0.9662774801254272]}, {"0": ["vertical_and_slash", 1000, 6096, 0.5538235306739807], "1": ["vertical_and_slash", 1000, 6096, 0.7072745561599731], "2": ["vertical_and_slash", 1000, 6096, 0.5786392688751221], "3": ["vertical_and_slash", 1000, 6096, 0.7442421913146973], "4": ["vertical_and_slash", 1000, 6096, 0.8589051365852356], "5": ["vertical_and_slash", 1000, 6096, 0.7389683723449707], "6": ["vertical_and_slash", 1000, 6096, 0.8320224285125732], "7": ["vertical_and_slash", 1000, 6096, 0.8044398427009583], "8": ["vertical_and_slash", 1000, 6096, 0.7850395441055298], "9": ["vertical_and_slash", 1000, 6096, 0.7028798460960388], "10": ["vertical_and_slash", 1000, 6096, 0.6812921762466431], "11": ["vertical_and_slash", 1000, 6096, 0.7796890139579773], "12": ["vertical_and_slash", 1000, 6096, 0.8026842474937439], "13": ["vertical_and_slash", 1000, 6096, 0.7551661729812622], "14": ["vertical_and_slash", 1000, 6096, 0.7467015981674194], "15": ["vertical_and_slash", 1000, 6096, 0.7323325276374817], "16": ["vertical_and_slash", 1000, 6096, 0.7640058398246765], "17": ["vertical_and_slash", 1000, 6096, 0.6481032371520996], "18": ["vertical_and_slash", 1000, 6096, 0.8524098992347717], "19": ["vertical_and_slash", 1000, 6096, 0.7246680855751038], "20": ["vertical_and_slash", 1000, 6096, 0.7793782353401184], "21": ["vertical_and_slash", 1000, 6096, 0.8856425881385803], "22": ["vertical_and_slash", 1000, 6096, 0.7227007746696472], "23": ["vertical_and_slash", 1000, 6096, 0.7910833358764648], "24": ["vertical_and_slash", 1000, 6096, 0.917560338973999], "25": ["vertical_and_slash", 1000, 6096, 0.6789739727973938], "26": ["vertical_and_slash", 1000, 6096, 0.7089481353759766], "27": ["vertical_and_slash", 1000, 6096, 0.6027281284332275], "28": ["vertical_and_slash", 1000, 6096, 0.6316034197807312], "29": ["vertical_and_slash", 1000, 6096, 0.7578167915344238], "30": ["vertical_and_slash", 1000, 6096, 0.8632583022117615], "31": ["vertical_and_slash", 1000, 6096, 0.9441259503364563], "32": ["vertical_and_slash", 1000, 6096, 0.754093587398529], "33": ["vertical_and_slash", 1000, 6096, 0.9082859754562378], "34": ["vertical_and_slash", 1000, 6096, 0.6905302405357361], "35": ["vertical_and_slash", 1000, 6096, 0.9823283553123474], "36": ["vertical_and_slash", 1000, 6096, 0.8287659287452698], "37": ["vertical_and_slash", 1000, 6096, 0.9960132241249084], "38": ["vertical_and_slash", 1000, 6096, 0.8146095275878906], "39": ["vertical_and_slash", 1000, 6096, 0.9862313866615295]}]
diff --git a/minference/configs/Qwen2.5_3B_flex_0.90.json b/minference/configs/Qwen2.5_3B_flex_0.90.json
new file mode 100644
index 00000000..c60d8409
--- /dev/null
+++ b/minference/configs/Qwen2.5_3B_flex_0.90.json
@@ -0,0 +1,616 @@
+[
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]
+ },
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.9, 4096], [0.9, 4096], 0.0]
+ }
+]
diff --git a/minference/configs/Qwen2.5_3B_flex_0.95.json b/minference/configs/Qwen2.5_3B_flex_0.95.json
new file mode 100644
index 00000000..709507c4
--- /dev/null
+++ b/minference/configs/Qwen2.5_3B_flex_0.95.json
@@ -0,0 +1,616 @@
+[
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]
+ },
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]},
+ {
+ "0": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "1": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "2": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "3": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "4": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "5": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "6": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "7": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "8": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "9": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "10": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "11": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "12": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "13": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "14": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0],
+ "15": ["flex_vertical_and_slash", [0.95, 4096], [0.95, 4096], 0.0]
+ }
+]
diff --git a/minference/configs/Qwen2.5_3B_kv_out_v32_fit_o_best_pattern.json b/minference/configs/Qwen2.5_3B_kv_out_v32_fit_o_best_pattern.json
new file mode 100644
index 00000000..538bb7ec
--- /dev/null
+++ b/minference/configs/Qwen2.5_3B_kv_out_v32_fit_o_best_pattern.json
@@ -0,0 +1,3530 @@
+[
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9872207641601562
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9784929752349854
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7595849633216858
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5381054878234863
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9863664507865906
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9912353157997131
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7160804867744446
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9073030352592468
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9161261916160583
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9784228205680847
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9789554476737976
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7867575883865356
+ ],
+ "12": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 1.0000020265579224
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9307636618614197
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6895971298217773
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7491968870162964
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9951784610748291
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9995638132095337
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9881429076194763
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9951925277709961
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.928062379360199
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9995735883712769
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7747997045516968
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9893845319747925
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9328423738479614
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7227432131767273
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6669939160346985
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.955822765827179
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6157850623130798
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8225603103637695
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6094294786453247
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7056097388267517
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8943619728088379
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6963416337966919
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9629008173942566
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.866447389125824
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5282332897186279
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9982369542121887
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9979943633079529
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9979172945022583
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.942166268825531
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9923297166824341
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9751147031784058
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8978350758552551
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8243312239646912
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9721394181251526
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.93731689453125
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9794054627418518
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7129239439964294
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9804909229278564
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9835291504859924
+ ],
+ "3": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9893701076507568
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9409563541412354
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8059223890304565
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6498631238937378
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.984248697757721
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7962363362312317
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.868658721446991
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8754917979240417
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8955696821212769
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9082641005516052
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8178426623344421
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8291682004928589
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8030994534492493
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9872316122055054
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9957523345947266
+ ],
+ "2": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9542893171310425
+ ],
+ "3": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9896659255027771
+ ],
+ "4": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9950734376907349
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8253392577171326
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8497561812400818
+ ],
+ "7": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9906441569328308
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9813032746315002
+ ],
+ "9": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9744712114334106
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.809856116771698
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9735696911811829
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8704012036323547
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.965289294719696
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9198070168495178
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9896268844604492
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9742131233215332
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9894583821296692
+ ],
+ "2": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9873966574668884
+ ],
+ "3": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9833617210388184
+ ],
+ "4": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9105245471000671
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.734259843826294
+ ],
+ "6": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9877724051475525
+ ],
+ "7": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9896732568740845
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8814374208450317
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9992178678512573
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9980509877204895
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9092496037483215
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8476247191429138
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9594801664352417
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8697033524513245
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9915944933891296
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.95703125
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5575676560401917
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9087390303611755
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9765625
+ ],
+ "5": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9809911847114563
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5668226480484009
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.978988766670227
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9430385828018188
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.7421875
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9403589963912964
+ ],
+ "11": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9878966808319092
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9328376650810242
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.8343550562858582
+ ],
+ "14": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.959410548210144
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9758256673812866
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.933525562286377
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6686467528343201
+ ],
+ "2": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9911661744117737
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5714142322540283
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8095535635948181
+ ],
+ "5": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9295337796211243
+ ],
+ "6": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.965142011642456
+ ],
+ "7": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.984375
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6681234240531921
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8945913910865784
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6786034107208252
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9802227020263672
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.97265625
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9555180072784424
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9749921560287476
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9921875
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8633137345314026
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9670861959457397
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8114507794380188
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9675626158714294
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9122037291526794
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9735010862350464
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9719548225402832
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9510305523872375
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.72588711977005
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9162058234214783
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.924541175365448
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9764450788497925
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9914652705192566
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7905170321464539
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9880024790763855
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9844828844070435
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9420890212059021
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.986860454082489
+ ],
+ "2": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9866741299629211
+ ],
+ "3": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.7924719452857971
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.890625
+ ],
+ "5": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9799174666404724
+ ],
+ "6": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9916488528251648
+ ],
+ "7": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.993992030620575
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9875913262367249
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.980666995048523
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9952725172042847
+ ],
+ "11": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9930256605148315
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9982009530067444
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9283435940742493
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9926015138626099
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9950297474861145
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.989981472492218
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9963133335113525
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9934775233268738
+ ],
+ "3": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9720687866210938
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.957918107509613
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8732808828353882
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9542519450187683
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9934373497962952
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9688447713851929
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9413594007492065
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9796752333641052
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9452784657478333
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9403716921806335
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9615768790245056
+ ],
+ "14": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.760350227355957
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8651421666145325
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9908298254013062
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.818602979183197
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9621649980545044
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7571771144866943
+ ],
+ "4": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9852563738822937
+ ],
+ "5": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9928317070007324
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7078589797019958
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9356410503387451
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8330709338188171
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9761743545532227
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9746711254119873
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9730107188224792
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9346276521682739
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9564436674118042
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9630159735679626
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9073519706726074
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8938577175140381
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9779987931251526
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9370880126953125
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8135497570037842
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9516273736953735
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.874685525894165
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8642981648445129
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9647448658943176
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7395755052566528
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9181990027427673
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7171043157577515
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8813474774360657
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9861239790916443
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9770793318748474
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9610176086425781
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9338920712471008
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9416564106941223
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.991210401058197
+ ],
+ "2": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9882159233093262
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.736187219619751
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8281543254852295
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9498741030693054
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9149094820022583
+ ],
+ "7": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9877381920814514
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.963042140007019
+ ],
+ "9": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9886980652809143
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9890977740287781
+ ],
+ "11": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9927736520767212
+ ],
+ "12": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.992214024066925
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9973360300064087
+ ],
+ "14": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.7854554653167725
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9945513606071472
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9653095602989197
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8165479898452759
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9046997427940369
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8763824701309204
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8182893991470337
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9363685846328735
+ ],
+ "6": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.734375
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9775119423866272
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9895726442337036
+ ],
+ "9": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9737070202827454
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.996068000793457
+ ],
+ "11": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.997230589389801
+ ],
+ "12": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9761516451835632
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9949500560760498
+ ],
+ "14": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9936166405677795
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9869099855422974
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9887930154800415
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9879409670829773
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.957123875617981
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.98828125
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.4647713899612427
+ ],
+ "5": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9909580945968628
+ ],
+ "6": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9757564067840576
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5521421432495117
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7780187129974365
+ ],
+ "9": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9887256026268005
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9927332401275635
+ ],
+ "11": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9805054664611816
+ ],
+ "12": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9525687098503113
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9362225532531738
+ ],
+ "14": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9488365054130554
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9525135159492493
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9934394955635071
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9532470703125
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9188738465309143
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9849047660827637
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9228449463844299
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9765625
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9707450866699219
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9929892420768738
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.964901864528656
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.911367654800415
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9818339943885803
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9837478399276733
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9615333080291748
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9666763544082642
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9545288681983948
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9649417400360107
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9158878326416016
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9285635948181152
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9884502291679382
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8363761901855469
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9531059265136719
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9884499907493591
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9524633884429932
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9358732104301453
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8582175374031067
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.3922925889492035
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5313135385513306
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.953125
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8960450291633606
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.90234375
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6443539261817932
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.829773485660553
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5914504528045654
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8983972668647766
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.930306077003479
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8418871164321899
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9140625
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8820360898971558
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7236220836639404
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5575733184814453
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9963138699531555
+ ],
+ "9": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9883040189743042
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9783397912979126
+ ],
+ "11": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9933704733848572
+ ],
+ "12": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9880709648132324
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9847649931907654
+ ],
+ "14": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9817938804626465
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9038569927215576
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9423895478248596
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.99609375
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9898338913917542
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8854114413261414
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9820103049278259
+ ],
+ "5": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.98828125
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6622527837753296
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5836654901504517
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7537979483604431
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9120598435401917
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9560258388519287
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 1.0
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9937338829040527
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9111098051071167
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9436591863632202
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9429888129234314
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.8515625
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.620391845703125
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6681154370307922
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9479513764381409
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5289033651351929
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.43187281489372253
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8812884092330933
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.79611736536026
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9973558783531189
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.734375
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6380698680877686
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9609375
+ ],
+ "12": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.8359375
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8900502324104309
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.8984375
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7215483784675598
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8019538521766663
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8427147269248962
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.6292986273765564
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9548527002334595
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8857505321502686
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8712131381034851
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8540765643119812
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5264020562171936
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8968150615692139
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6485419273376465
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8069987893104553
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8020429015159607
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8054234981536865
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.725652813911438
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.46037647128105164
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9644275903701782
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7860593199729919
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8588574528694153
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9157812595367432
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8626066446304321
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8797851800918579
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7836940884590149
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8759902715682983
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8079484701156616
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.7734375
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.80859375
+ ],
+ "10": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.91796875
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.93359375
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5005961656570435
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8044103384017944
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.6477628946304321
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5467575192451477
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9432184100151062
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9644391536712646
+ ],
+ "2": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.83203125
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9638855457305908
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9378725290298462
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8560249209403992
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9667811989784241
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9581833481788635
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9911800622940063
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.7890625
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9970740675926208
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.9453125
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8719268441200256
+ ],
+ "13": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9973757863044739
+ ],
+ "14": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9817723631858826
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9943931102752686
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7833037376403809
+ ],
+ "1": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.985275149345398
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8808413147926331
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6342004537582397
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.6783415675163269
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8690900206565857
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9161447286605835
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9136660099029541
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9813501834869385
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9898714423179626
+ ],
+ "10": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9806197285652161
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8513939380645752
+ ],
+ "12": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9925402402877808
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.98046875
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9726189374923706
+ ],
+ "15": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9964751601219177
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8871311545372009
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6332476735115051
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6504296660423279
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8371340036392212
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5707467198371887
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.49299511313438416
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6483507752418518
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9506110548973083
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8480803966522217
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.82421875
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9129166007041931
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5975048542022705
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8599441647529602
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.94140625
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7465125918388367
+ ],
+ "15": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9268635511398315
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9838484525680542
+ ],
+ "1": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.96875
+ ],
+ "2": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9804697632789612
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9868910908699036
+ ],
+ "4": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.953125
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8383246660232544
+ ],
+ "6": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.97265625
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7157800793647766
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9093014001846313
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.82421875
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.951353132724762
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.98046875
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.5795325040817261
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9869730472564697
+ ],
+ "14": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.953125
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6627390384674072
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9301889538764954
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9623850584030151
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.925979495048523
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9278422594070435
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9768559336662292
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9585739374160767
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9396020174026489
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8997651934623718
+ ],
+ "8": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 1.000001072883606
+ ],
+ "9": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.89453125
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9783903360366821
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9423955678939819
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6527382135391235
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9103218913078308
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.829939067363739
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.982806921005249
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9865686893463135
+ ],
+ "1": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9607634544372559
+ ],
+ "2": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.9966762065887451
+ ],
+ "3": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.75
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8039277791976929
+ ],
+ "5": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9704546332359314
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9315423965454102
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.8952665328979492
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8250367045402527
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.953521728515625
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8773703575134277
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9780517816543579
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9766848683357239
+ ],
+ "13": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9704862236976624
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9836187958717346
+ ],
+ "15": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9668617844581604
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9616305828094482
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9781763553619385
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9224084615707397
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9049281477928162
+ ],
+ "4": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9796241521835327
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9083701968193054
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9923230409622192
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9486480951309204
+ ],
+ "8": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9276089668273926
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9481350183486938
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9749733805656433
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9456705451011658
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9752734303474426
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.97265625
+ ],
+ "14": [
+ "vertical_and_slash",
+ 30,
+ 800,
+ 0.955278754234314
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9254546761512756
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9746971726417542
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9719505310058594
+ ],
+ "2": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8964815735816956
+ ],
+ "3": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8442646265029907
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9568673968315125
+ ],
+ "5": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.94114750623703
+ ],
+ "6": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.96875
+ ],
+ "7": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9370027780532837
+ ],
+ "8": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.91015625
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9786306023597717
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7740182876586914
+ ],
+ "11": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9646586179733276
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.832630455493927
+ ],
+ "13": [
+ "vertical_and_slash",
+ 100,
+ 800,
+ 0.94921875
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.5829520225524902
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9215387105941772
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9220553040504456
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9111120700836182
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8448940515518188
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9479627013206482
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9360203146934509
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.806208074092865
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8499483466148376
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9351169466972351
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.887176513671875
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9972283244132996
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7344715595245361
+ ],
+ "11": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.9290981888771057
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9191303849220276
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9607530832290649
+ ],
+ "14": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6650398373603821
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9041045308113098
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9288094639778137
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.922483503818512
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8560412526130676
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9515807628631592
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.910995364189148
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9611515402793884
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9130395650863647
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8693947196006775
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9321247935295105
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7624196410179138
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9113157391548157
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8822183012962341
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.940976083278656
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9429124593734741
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9557855129241943
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7963366508483887
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9815316796302795
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9569784998893738
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9643719792366028
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9694581627845764
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9616712927818298
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9564018249511719
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9240798354148865
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9618653059005737
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9393341541290283
+ ],
+ "9": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9590299129486084
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9623062014579773
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9482530355453491
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9658593535423279
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9724211096763611
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9585490822792053
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9729295969009399
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9484567046165466
+ ],
+ "1": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7411888241767883
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8387667536735535
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7810403108596802
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8578725457191467
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8868502974510193
+ ],
+ "6": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.7327648401260376
+ ],
+ "7": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9077197313308716
+ ],
+ "8": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9110754728317261
+ ],
+ "9": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.8923226594924927
+ ],
+ "10": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7206286191940308
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8617163300514221
+ ],
+ "12": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8827745914459229
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.7372896075248718
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9529178142547607
+ ],
+ "15": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.8600607514381409
+ ]
+ },
+ {
+ "0": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9317379593849182
+ ],
+ "1": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9660685658454895
+ ],
+ "2": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.932490885257721
+ ],
+ "3": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9033127427101135
+ ],
+ "4": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9372511506080627
+ ],
+ "5": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9416565299034119
+ ],
+ "6": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9347256422042847
+ ],
+ "7": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9515694379806519
+ ],
+ "8": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9580468535423279
+ ],
+ "9": [
+ "vertical_and_slash",
+ 500,
+ 700,
+ 0.9769356846809387
+ ],
+ "10": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.6504788398742676
+ ],
+ "11": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9479199051856995
+ ],
+ "12": [
+ "vertical_and_slash",
+ 1000,
+ 6096,
+ 0.9721158146858215
+ ],
+ "13": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9033265113830566
+ ],
+ "14": [
+ "vertical_and_slash",
+ 3500,
+ 100,
+ 0.9400415420532227
+ ],
+ "15": [
+ "vertical_and_slash",
+ 100,
+ 750,
+ 0.8886585831642151
+ ]
+ }
+]
diff --git a/minference/configs/Qwen2.5_72B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json b/minference/configs/Qwen2.5_72B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json
new file mode 100644
index 00000000..1f126c1d
--- /dev/null
+++ b/minference/configs/Qwen2.5_72B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json
@@ -0,0 +1 @@
+[{"0": ["vertical_and_slash", 1000, 6096, 0.986763596534729], "1": ["vertical_and_slash", 3500, 100, 0.7140933275222778], "2": ["vertical_and_slash", 1000, 6096, 0.7862353324890137], "3": ["vertical_and_slash", 1000, 6096, 0.7542210817337036], "4": ["vertical_and_slash", 1000, 6096, 0.793908417224884], "5": ["vertical_and_slash", 3500, 100, 0.7664338946342468], "6": ["vertical_and_slash", 1000, 6096, 0.938113272190094], "7": ["vertical_and_slash", 1000, 6096, 0.9212206602096558], "8": ["vertical_and_slash", 30, 800, 0.9966272115707397], "9": ["vertical_and_slash", 1000, 6096, 0.5321428179740906], "10": ["vertical_and_slash", 1000, 6096, 0.6569603085517883], "11": ["vertical_and_slash", 1000, 6096, 0.747432291507721], "12": ["vertical_and_slash", 1000, 6096, 0.9345226287841797], "13": ["vertical_and_slash", 1000, 6096, 0.9543651938438416], "14": ["vertical_and_slash", 30, 800, 0.9444361925125122], "15": ["vertical_and_slash", 30, 800, 0.972910463809967], "16": ["vertical_and_slash", 1000, 6096, 0.841227650642395], "17": ["vertical_and_slash", 30, 800, 0.9615581631660461], "18": ["vertical_and_slash", 30, 800, 0.9983838796615601], "19": ["vertical_and_slash", 1000, 6096, 0.7703010439872742], "20": ["vertical_and_slash", 1000, 6096, 0.7838487029075623], "21": ["vertical_and_slash", 30, 800, 0.95079505443573], "22": ["vertical_and_slash", 1000, 6096, 0.8664155006408691], "23": ["vertical_and_slash", 1000, 6096, 0.8369813561439514], "24": ["vertical_and_slash", 1000, 6096, 0.9633749723434448], "25": ["vertical_and_slash", 1000, 6096, 0.8773102760314941], "26": ["vertical_and_slash", 1000, 6096, 0.6255043745040894], "27": ["vertical_and_slash", 30, 800, 0.9913763403892517], "28": ["vertical_and_slash", 30, 800, 0.9040997624397278], "29": ["vertical_and_slash", 30, 800, 0.9142972826957703], "30": ["vertical_and_slash", 1000, 6096, 0.9019200205802917], "31": ["vertical_and_slash", 1000, 6096, 0.7566367983818054], "32": ["vertical_and_slash", 30, 800, 0.8736827373504639], "33": ["vertical_and_slash", 30, 800, 0.9961656928062439], "34": ["vertical_and_slash", 30, 800, 0.9342387318611145], "35": ["vertical_and_slash", 1000, 6096, 0.9501726627349854], "36": ["vertical_and_slash", 1000, 6096, 0.8005950450897217], "37": ["vertical_and_slash", 1000, 6096, 0.9785274267196655], "38": ["vertical_and_slash", 1000, 6096, 0.9777876734733582], "39": ["vertical_and_slash", 30, 800, 0.9579474925994873], "40": ["vertical_and_slash", 1000, 6096, 0.6405143141746521], "41": ["vertical_and_slash", 1000, 6096, 0.8845492005348206], "42": ["vertical_and_slash", 30, 800, 0.9580627679824829], "43": ["vertical_and_slash", 1000, 6096, 0.7913761734962463], "44": ["vertical_and_slash", 30, 800, 0.9864246249198914], "45": ["vertical_and_slash", 30, 800, 0.9946540594100952], "46": ["vertical_and_slash", 30, 800, 0.8702406287193298], "47": ["vertical_and_slash", 1000, 6096, 0.7772166728973389], "48": ["vertical_and_slash", 30, 800, 0.999313473701477], "49": ["vertical_and_slash", 1000, 6096, 0.9145946502685547], "50": ["vertical_and_slash", 1000, 6096, 0.8665878772735596], "51": ["vertical_and_slash", 1000, 6096, 0.8761577606201172], "52": ["vertical_and_slash", 30, 800, 0.9761298894882202], "53": ["vertical_and_slash", 1000, 6096, 0.9706177711486816], "54": ["vertical_and_slash", 1000, 6096, 0.8301690220832825], "55": ["vertical_and_slash", 1000, 6096, 0.734950840473175], "56": ["vertical_and_slash", 30, 800, 0.9488434791564941], "57": ["vertical_and_slash", 1000, 6096, 0.9202470779418945], "58": ["vertical_and_slash", 1000, 6096, 0.9603468775749207], "59": ["vertical_and_slash", 1000, 6096, 0.9749668836593628], "60": ["vertical_and_slash", 1000, 6096, 0.841002345085144], "61": ["vertical_and_slash", 1000, 6096, 0.955847978591919], "62": ["vertical_and_slash", 1000, 6096, 0.6739946007728577], "63": ["vertical_and_slash", 1000, 6096, 0.8918765187263489]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8600797653198242], "1": ["vertical_and_slash", 1000, 6096, 0.9149024486541748], "2": ["vertical_and_slash", 1000, 6096, 0.992446780204773], "3": ["vertical_and_slash", 1000, 6096, 0.7697340846061707], "4": ["vertical_and_slash", 3500, 100, 0.9469043612480164], "5": ["vertical_and_slash", 1000, 6096, 0.9232993125915527], "6": ["vertical_and_slash", 1000, 6096, 0.7438302636146545], "7": ["vertical_and_slash", 3500, 100, 0.9997028112411499], "8": ["vertical_and_slash", 3500, 100, 0.6634463667869568], "9": ["vertical_and_slash", 1000, 6096, 0.548510730266571], "10": ["vertical_and_slash", 100, 800, 1.0], "11": ["vertical_and_slash", 1000, 6096, 0.9637598395347595], "12": ["vertical_and_slash", 1000, 6096, 0.604372501373291], "13": ["vertical_and_slash", 1000, 6096, 0.5091693997383118], "14": ["vertical_and_slash", 1000, 6096, 0.5983718633651733], "15": ["vertical_and_slash", 1000, 6096, 0.816099226474762], "16": ["vertical_and_slash", 1000, 6096, 0.6020280718803406], "17": ["vertical_and_slash", 1000, 6096, 0.4753745198249817], "18": ["vertical_and_slash", 1000, 6096, 0.5231419801712036], "19": ["vertical_and_slash", 1000, 6096, 0.9284146428108215], "20": ["vertical_and_slash", 1000, 6096, 0.5161747336387634], "21": ["vertical_and_slash", 1000, 6096, 0.669787585735321], "22": ["vertical_and_slash", 1000, 6096, 0.5908206701278687], "23": ["vertical_and_slash", 1000, 6096, 0.7178939580917358], "24": ["vertical_and_slash", 30, 800, 1.000000238418579], "25": ["vertical_and_slash", 1000, 6096, 0.8569574952125549], "26": ["vertical_and_slash", 100, 800, 1.0], "27": ["vertical_and_slash", 1000, 6096, 0.9943471550941467], "28": ["vertical_and_slash", 100, 800, 1.0], "29": ["vertical_and_slash", 1000, 6096, 0.4888390600681305], "30": ["vertical_and_slash", 1000, 6096, 0.586074709892273], "31": ["vertical_and_slash", 1000, 6096, 0.5146514773368835], "32": ["vertical_and_slash", 1000, 6096, 0.511432409286499], "33": ["vertical_and_slash", 3500, 100, 0.5248153209686279], "34": ["vertical_and_slash", 1000, 6096, 0.5974372029304504], "35": ["vertical_and_slash", 1000, 6096, 0.5372591614723206], "36": ["vertical_and_slash", 1000, 6096, 0.7440279126167297], "37": ["vertical_and_slash", 1000, 6096, 0.7847241759300232], "38": ["vertical_and_slash", 3500, 100, 0.4936659336090088], "39": ["vertical_and_slash", 1000, 6096, 0.6469670534133911], "40": ["vertical_and_slash", 30, 800, 0.9722318649291992], "41": ["vertical_and_slash", 1000, 6096, 0.9024332165718079], "42": ["vertical_and_slash", 1000, 6096, 0.9396247863769531], "43": ["vertical_and_slash", 1000, 6096, 0.6701458096504211], "44": ["vertical_and_slash", 1000, 6096, 0.8596237301826477], "45": ["vertical_and_slash", 1000, 6096, 0.8594532608985901], "46": ["vertical_and_slash", 1000, 6096, 0.690616250038147], "47": ["vertical_and_slash", 1000, 6096, 0.9994174242019653], "48": ["vertical_and_slash", 1000, 6096, 0.5930584669113159], "49": ["vertical_and_slash", 1000, 6096, 0.8100162148475647], "50": ["vertical_and_slash", 100, 800, 1.0], "51": ["vertical_and_slash", 1000, 6096, 0.937853991985321], "52": ["vertical_and_slash", 30, 800, 0.7609145045280457], "53": ["vertical_and_slash", 100, 800, 1.0], "54": ["vertical_and_slash", 1000, 6096, 0.8485128283500671], "55": ["vertical_and_slash", 1000, 6096, 0.5608847737312317], "56": ["vertical_and_slash", 1000, 6096, 0.5249639749526978], "57": ["vertical_and_slash", 1000, 6096, 0.5957784652709961], "58": ["vertical_and_slash", 1000, 6096, 0.6829463839530945], "59": ["vertical_and_slash", 1000, 6096, 0.6245880126953125], "60": ["vertical_and_slash", 1000, 6096, 0.6184428930282593], "61": ["vertical_and_slash", 3500, 100, 0.8330410122871399], "62": ["vertical_and_slash", 1000, 6096, 0.7333449125289917], "63": ["vertical_and_slash", 1000, 6096, 0.5741525292396545]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4041958749294281], "1": ["vertical_and_slash", 1000, 6096, 0.4288022518157959], "2": ["vertical_and_slash", 1000, 6096, 0.38879987597465515], "3": ["vertical_and_slash", 1000, 6096, 0.44822436571121216], "4": ["vertical_and_slash", 1000, 6096, 0.3995395004749298], "5": ["vertical_and_slash", 1000, 6096, 0.4042370319366455], "6": ["vertical_and_slash", 1000, 6096, 0.4278094470500946], "7": ["vertical_and_slash", 1000, 6096, 0.3750866949558258], "8": ["vertical_and_slash", 1000, 6096, 0.41708192229270935], "9": ["vertical_and_slash", 1000, 6096, 0.4521547853946686], "10": ["vertical_and_slash", 1000, 6096, 0.42176344990730286], "11": ["vertical_and_slash", 1000, 6096, 0.4709431529045105], "12": ["vertical_and_slash", 1000, 6096, 0.45459985733032227], "13": ["vertical_and_slash", 1000, 6096, 0.44034892320632935], "14": ["vertical_and_slash", 1000, 6096, 0.43497079610824585], "15": ["vertical_and_slash", 1000, 6096, 0.4768989086151123], "16": ["vertical_and_slash", 1000, 6096, 0.4137873947620392], "17": ["vertical_and_slash", 1000, 6096, 0.38921570777893066], "18": ["vertical_and_slash", 1000, 6096, 0.38240620493888855], "19": ["vertical_and_slash", 1000, 6096, 0.3751950263977051], "20": ["vertical_and_slash", 1000, 6096, 0.3789471387863159], "21": ["vertical_and_slash", 1000, 6096, 0.4172891080379486], "22": ["vertical_and_slash", 1000, 6096, 0.40558770298957825], "23": ["vertical_and_slash", 1000, 6096, 0.42451077699661255], "24": ["vertical_and_slash", 1000, 6096, 0.40912389755249023], "25": ["vertical_and_slash", 1000, 6096, 0.38438430428504944], "26": ["vertical_and_slash", 1000, 6096, 0.40075090527534485], "27": ["vertical_and_slash", 1000, 6096, 0.4216001331806183], "28": ["vertical_and_slash", 1000, 6096, 0.3953193426132202], "29": ["vertical_and_slash", 1000, 6096, 0.403439462184906], "30": ["vertical_and_slash", 1000, 6096, 0.39651015400886536], "31": ["vertical_and_slash", 1000, 6096, 0.39794445037841797], "32": ["vertical_and_slash", 1000, 6096, 0.49248290061950684], "33": ["vertical_and_slash", 1000, 6096, 0.47064074873924255], "34": ["vertical_and_slash", 1000, 6096, 0.3961598575115204], "35": ["vertical_and_slash", 1000, 6096, 0.45812463760375977], "36": ["vertical_and_slash", 1000, 6096, 0.4338022768497467], "37": ["vertical_and_slash", 1000, 6096, 0.4352731704711914], "38": ["vertical_and_slash", 1000, 6096, 0.43466365337371826], "39": ["vertical_and_slash", 1000, 6096, 0.39760395884513855], "40": ["vertical_and_slash", 1000, 6096, 0.4140770137310028], "41": ["vertical_and_slash", 1000, 6096, 0.4386835992336273], "42": ["vertical_and_slash", 1000, 6096, 0.3910137116909027], "43": ["vertical_and_slash", 1000, 6096, 0.40568116307258606], "44": ["vertical_and_slash", 1000, 6096, 0.4361551105976105], "45": ["vertical_and_slash", 1000, 6096, 0.4012598395347595], "46": ["vertical_and_slash", 1000, 6096, 0.40161868929862976], "47": ["vertical_and_slash", 1000, 6096, 0.36730197072029114], "48": ["vertical_and_slash", 3500, 100, 0.4014899730682373], "49": ["vertical_and_slash", 1000, 6096, 0.4110775291919708], "50": ["vertical_and_slash", 1000, 6096, 0.35850822925567627], "51": ["vertical_and_slash", 3500, 100, 0.3726404905319214], "52": ["vertical_and_slash", 1000, 6096, 0.38408586382865906], "53": ["vertical_and_slash", 1000, 6096, 0.4002585709095001], "54": ["vertical_and_slash", 1000, 6096, 0.40460702776908875], "55": ["vertical_and_slash", 1000, 6096, 0.3873746991157532], "56": ["vertical_and_slash", 1000, 6096, 0.4512594938278198], "57": ["vertical_and_slash", 1000, 6096, 0.40316182374954224], "58": ["vertical_and_slash", 1000, 6096, 0.4321412146091461], "59": ["vertical_and_slash", 1000, 6096, 0.438848614692688], "60": ["vertical_and_slash", 1000, 6096, 0.38420918583869934], "61": ["vertical_and_slash", 1000, 6096, 0.4201686978340149], "62": ["vertical_and_slash", 1000, 6096, 0.43859177827835083], "63": ["vertical_and_slash", 1000, 6096, 0.39987435936927795]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4120781719684601], "1": ["vertical_and_slash", 1000, 6096, 0.4049403965473175], "2": ["vertical_and_slash", 1000, 6096, 0.41704297065734863], "3": ["vertical_and_slash", 1000, 6096, 0.4473341405391693], "4": ["vertical_and_slash", 1000, 6096, 0.387154221534729], "5": ["vertical_and_slash", 1000, 6096, 0.4092889428138733], "6": ["vertical_and_slash", 1000, 6096, 0.4282684624195099], "7": ["vertical_and_slash", 1000, 6096, 0.3927796185016632], "8": ["vertical_and_slash", 1000, 6096, 0.4124581217765808], "9": ["vertical_and_slash", 1000, 6096, 0.394416481256485], "10": ["vertical_and_slash", 1000, 6096, 0.4161239564418793], "11": ["vertical_and_slash", 1000, 6096, 0.39539748430252075], "12": ["vertical_and_slash", 1000, 6096, 0.38030242919921875], "13": ["vertical_and_slash", 1000, 6096, 0.4086754024028778], "14": ["vertical_and_slash", 1000, 6096, 0.37117940187454224], "15": ["vertical_and_slash", 1000, 6096, 0.38984131813049316], "16": ["vertical_and_slash", 1000, 6096, 0.4113542139530182], "17": ["vertical_and_slash", 1000, 6096, 0.36597442626953125], "18": ["vertical_and_slash", 1000, 6096, 0.36368462443351746], "19": ["vertical_and_slash", 1000, 6096, 0.43721628189086914], "20": ["vertical_and_slash", 1000, 6096, 0.3953881859779358], "21": ["vertical_and_slash", 1000, 6096, 0.4188574552536011], "22": ["vertical_and_slash", 1000, 6096, 0.3903310000896454], "23": ["vertical_and_slash", 1000, 6096, 0.3981025815010071], "24": ["vertical_and_slash", 1000, 6096, 0.4154193699359894], "25": ["vertical_and_slash", 1000, 6096, 0.43782466650009155], "26": ["vertical_and_slash", 1000, 6096, 0.44133397936820984], "27": ["vertical_and_slash", 1000, 6096, 0.3884759843349457], "28": ["vertical_and_slash", 1000, 6096, 0.41656237840652466], "29": ["vertical_and_slash", 1000, 6096, 0.4492548108100891], "30": ["vertical_and_slash", 1000, 6096, 0.46502360701560974], "31": ["vertical_and_slash", 1000, 6096, 0.39306503534317017], "32": ["vertical_and_slash", 1000, 6096, 0.3928229510784149], "33": ["vertical_and_slash", 1000, 6096, 0.3963738679885864], "34": ["vertical_and_slash", 1000, 6096, 0.3766452670097351], "35": ["vertical_and_slash", 1000, 6096, 0.38968878984451294], "36": ["vertical_and_slash", 1000, 6096, 0.3910392224788666], "37": ["vertical_and_slash", 1000, 6096, 0.38393229246139526], "38": ["vertical_and_slash", 1000, 6096, 0.38191699981689453], "39": ["vertical_and_slash", 1000, 6096, 0.39090079069137573], "40": ["vertical_and_slash", 1000, 6096, 0.3946172595024109], "41": ["vertical_and_slash", 1000, 6096, 0.4093622863292694], "42": ["vertical_and_slash", 1000, 6096, 0.4223671853542328], "43": ["vertical_and_slash", 1000, 6096, 0.40350762009620667], "44": ["vertical_and_slash", 1000, 6096, 0.42207562923431396], "45": ["vertical_and_slash", 1000, 6096, 0.3837047219276428], "46": ["vertical_and_slash", 1000, 6096, 0.3966195583343506], "47": ["vertical_and_slash", 1000, 6096, 0.37062761187553406], "48": ["vertical_and_slash", 1000, 6096, 0.40876197814941406], "49": ["vertical_and_slash", 1000, 6096, 0.38863226771354675], "50": ["vertical_and_slash", 1000, 6096, 0.39661023020744324], "51": ["vertical_and_slash", 1000, 6096, 0.3953996002674103], "52": ["vertical_and_slash", 1000, 6096, 0.3921317160129547], "53": ["vertical_and_slash", 1000, 6096, 0.4220465123653412], "54": ["vertical_and_slash", 1000, 6096, 0.39931660890579224], "55": ["vertical_and_slash", 1000, 6096, 0.3869324028491974], "56": ["vertical_and_slash", 1000, 6096, 0.4541178047657013], "57": ["vertical_and_slash", 1000, 6096, 0.4964122772216797], "58": ["vertical_and_slash", 1000, 6096, 0.5667144060134888], "59": ["vertical_and_slash", 1000, 6096, 0.4997110962867737], "60": ["vertical_and_slash", 1000, 6096, 0.44353070855140686], "61": ["vertical_and_slash", 1000, 6096, 0.47550979256629944], "62": ["vertical_and_slash", 1000, 6096, 0.4235854744911194], "63": ["vertical_and_slash", 1000, 6096, 0.6300308704376221]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3687804937362671], "1": ["vertical_and_slash", 1000, 6096, 0.3683833181858063], "2": ["vertical_and_slash", 1000, 6096, 0.36776039004325867], "3": ["vertical_and_slash", 1000, 6096, 0.3644953966140747], "4": ["vertical_and_slash", 1000, 6096, 0.3699450194835663], "5": ["vertical_and_slash", 1000, 6096, 0.3698665201663971], "6": ["vertical_and_slash", 1000, 6096, 0.3606824278831482], "7": ["vertical_and_slash", 1000, 6096, 0.36323097348213196], "8": ["vertical_and_slash", 1000, 6096, 0.3998467028141022], "9": ["vertical_and_slash", 1000, 6096, 0.37394946813583374], "10": ["vertical_and_slash", 1000, 6096, 0.3822495639324188], "11": ["vertical_and_slash", 1000, 6096, 0.40018007159233093], "12": ["vertical_and_slash", 1000, 6096, 0.3875938653945923], "13": ["vertical_and_slash", 1000, 6096, 0.37365344166755676], "14": ["vertical_and_slash", 1000, 6096, 0.3766206204891205], "15": ["vertical_and_slash", 1000, 6096, 0.3744458854198456], "16": ["vertical_and_slash", 1000, 6096, 0.35772523283958435], "17": ["vertical_and_slash", 1000, 6096, 0.353110134601593], "18": ["vertical_and_slash", 1000, 6096, 0.3462507426738739], "19": ["vertical_and_slash", 1000, 6096, 0.34944555163383484], "20": ["vertical_and_slash", 1000, 6096, 0.3481333553791046], "21": ["vertical_and_slash", 1000, 6096, 0.3893139362335205], "22": ["vertical_and_slash", 1000, 6096, 0.3527008891105652], "23": ["vertical_and_slash", 1000, 6096, 0.3705199956893921], "24": ["vertical_and_slash", 1000, 6096, 0.3721528649330139], "25": ["vertical_and_slash", 1000, 6096, 0.40079692006111145], "26": ["vertical_and_slash", 1000, 6096, 0.3921458125114441], "27": ["vertical_and_slash", 1000, 6096, 0.369745671749115], "28": ["vertical_and_slash", 1000, 6096, 0.36842891573905945], "29": ["vertical_and_slash", 1000, 6096, 0.37343984842300415], "30": ["vertical_and_slash", 1000, 6096, 0.3679276704788208], "31": ["vertical_and_slash", 1000, 6096, 0.3649451434612274], "32": ["vertical_and_slash", 1000, 6096, 0.3556336462497711], "33": ["vertical_and_slash", 1000, 6096, 0.37033388018608093], "34": ["vertical_and_slash", 1000, 6096, 0.3668055236339569], "35": ["vertical_and_slash", 1000, 6096, 0.3601125180721283], "36": ["vertical_and_slash", 1000, 6096, 0.3769352436065674], "37": ["vertical_and_slash", 1000, 6096, 0.3698027431964874], "38": ["vertical_and_slash", 1000, 6096, 0.36268505454063416], "39": ["vertical_and_slash", 1000, 6096, 0.37460237741470337], "40": ["vertical_and_slash", 1000, 6096, 0.4107082188129425], "41": ["vertical_and_slash", 1000, 6096, 0.40250012278556824], "42": ["vertical_and_slash", 1000, 6096, 0.42437753081321716], "43": ["vertical_and_slash", 1000, 6096, 0.38515716791152954], "44": ["vertical_and_slash", 1000, 6096, 0.401065856218338], "45": ["vertical_and_slash", 1000, 6096, 0.3984107971191406], "46": ["vertical_and_slash", 1000, 6096, 0.4534350037574768], "47": ["vertical_and_slash", 1000, 6096, 0.3914206027984619], "48": ["vertical_and_slash", 1000, 6096, 0.3602662980556488], "49": ["vertical_and_slash", 1000, 6096, 0.3774281442165375], "50": ["vertical_and_slash", 1000, 6096, 0.3682078719139099], "51": ["vertical_and_slash", 1000, 6096, 0.3723204731941223], "52": ["vertical_and_slash", 1000, 6096, 0.4006882607936859], "53": ["vertical_and_slash", 1000, 6096, 0.4119054079055786], "54": ["vertical_and_slash", 1000, 6096, 0.3734219968318939], "55": ["vertical_and_slash", 1000, 6096, 0.35755786299705505], "56": ["vertical_and_slash", 1000, 6096, 0.37784868478775024], "57": ["vertical_and_slash", 1000, 6096, 0.3806329071521759], "58": ["vertical_and_slash", 1000, 6096, 0.3713044226169586], "59": ["vertical_and_slash", 1000, 6096, 0.36325496435165405], "60": ["vertical_and_slash", 1000, 6096, 0.3864855468273163], "61": ["vertical_and_slash", 1000, 6096, 0.38987165689468384], "62": ["vertical_and_slash", 1000, 6096, 0.3959130644798279], "63": ["vertical_and_slash", 1000, 6096, 0.3735930025577545]}, {"0": ["vertical_and_slash", 1000, 6096, 0.35193315148353577], "1": ["vertical_and_slash", 1000, 6096, 0.353921502828598], "2": ["vertical_and_slash", 1000, 6096, 0.36509883403778076], "3": ["vertical_and_slash", 1000, 6096, 0.3552262783050537], "4": ["vertical_and_slash", 1000, 6096, 0.3622760772705078], "5": ["vertical_and_slash", 1000, 6096, 0.37722522020339966], "6": ["vertical_and_slash", 1000, 6096, 0.35453855991363525], "7": ["vertical_and_slash", 1000, 6096, 0.37932705879211426], "8": ["vertical_and_slash", 1000, 6096, 0.3874770700931549], "9": ["vertical_and_slash", 1000, 6096, 0.38981902599334717], "10": ["vertical_and_slash", 1000, 6096, 0.3672013580799103], "11": ["vertical_and_slash", 1000, 6096, 0.3982259929180145], "12": ["vertical_and_slash", 1000, 6096, 0.4060782492160797], "13": ["vertical_and_slash", 1000, 6096, 0.36526384949684143], "14": ["vertical_and_slash", 1000, 6096, 0.39726534485816956], "15": ["vertical_and_slash", 1000, 6096, 0.3617215156555176], "16": ["vertical_and_slash", 1000, 6096, 0.37340569496154785], "17": ["vertical_and_slash", 1000, 6096, 0.358815461397171], "18": ["vertical_and_slash", 1000, 6096, 0.3781873285770416], "19": ["vertical_and_slash", 1000, 6096, 0.39544662833213806], "20": ["vertical_and_slash", 1000, 6096, 0.3846625089645386], "21": ["vertical_and_slash", 1000, 6096, 0.3690948784351349], "22": ["vertical_and_slash", 1000, 6096, 0.3911183476448059], "23": ["vertical_and_slash", 1000, 6096, 0.4010193943977356], "24": ["vertical_and_slash", 1000, 6096, 0.37464869022369385], "25": ["vertical_and_slash", 1000, 6096, 0.37647002935409546], "26": ["vertical_and_slash", 1000, 6096, 0.35834017395973206], "27": ["vertical_and_slash", 1000, 6096, 0.3526521623134613], "28": ["vertical_and_slash", 1000, 6096, 0.3663484752178192], "29": ["vertical_and_slash", 1000, 6096, 0.3903806209564209], "30": ["vertical_and_slash", 1000, 6096, 0.3801640272140503], "31": ["vertical_and_slash", 1000, 6096, 0.3729323446750641], "32": ["vertical_and_slash", 1000, 6096, 0.3582921028137207], "33": ["vertical_and_slash", 1000, 6096, 0.35345545411109924], "34": ["vertical_and_slash", 1000, 6096, 0.3651476502418518], "35": ["vertical_and_slash", 1000, 6096, 0.3837742507457733], "36": ["vertical_and_slash", 1000, 6096, 0.3710636496543884], "37": ["vertical_and_slash", 1000, 6096, 0.3697567880153656], "38": ["vertical_and_slash", 1000, 6096, 0.3639008402824402], "39": ["vertical_and_slash", 1000, 6096, 0.35927391052246094], "40": ["vertical_and_slash", 1000, 6096, 0.3988194763660431], "41": ["vertical_and_slash", 1000, 6096, 0.3703271746635437], "42": ["vertical_and_slash", 1000, 6096, 0.3572511374950409], "43": ["vertical_and_slash", 1000, 6096, 0.4027239978313446], "44": ["vertical_and_slash", 1000, 6096, 0.41506385803222656], "45": ["vertical_and_slash", 1000, 6096, 0.3802480101585388], "46": ["vertical_and_slash", 1000, 6096, 0.4014234244823456], "47": ["vertical_and_slash", 1000, 6096, 0.37650254368782043], "48": ["vertical_and_slash", 1000, 6096, 0.36603590846061707], "49": ["vertical_and_slash", 1000, 6096, 0.3636768162250519], "50": ["vertical_and_slash", 1000, 6096, 0.3982546925544739], "51": ["vertical_and_slash", 1000, 6096, 0.35878393054008484], "52": ["vertical_and_slash", 1000, 6096, 0.38334256410598755], "53": ["vertical_and_slash", 1000, 6096, 0.3590209186077118], "54": ["vertical_and_slash", 1000, 6096, 0.3550921082496643], "55": ["vertical_and_slash", 1000, 6096, 0.36828282475471497], "56": ["vertical_and_slash", 1000, 6096, 0.36769700050354004], "57": ["vertical_and_slash", 1000, 6096, 0.38214558362960815], "58": ["vertical_and_slash", 1000, 6096, 0.35956481099128723], "59": ["vertical_and_slash", 1000, 6096, 0.35584548115730286], "60": ["vertical_and_slash", 1000, 6096, 0.3647584021091461], "61": ["vertical_and_slash", 1000, 6096, 0.3559703528881073], "62": ["vertical_and_slash", 1000, 6096, 0.37051278352737427], "63": ["vertical_and_slash", 1000, 6096, 0.3557678163051605]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3532901108264923], "1": ["vertical_and_slash", 1000, 6096, 0.38135263323783875], "2": ["vertical_and_slash", 1000, 6096, 0.36646366119384766], "3": ["vertical_and_slash", 1000, 6096, 0.3518042266368866], "4": ["vertical_and_slash", 1000, 6096, 0.35741838812828064], "5": ["vertical_and_slash", 1000, 6096, 0.36469531059265137], "6": ["vertical_and_slash", 1000, 6096, 0.3591649532318115], "7": ["vertical_and_slash", 1000, 6096, 0.36881232261657715], "8": ["vertical_and_slash", 1000, 6096, 0.3507535457611084], "9": ["vertical_and_slash", 1000, 6096, 0.36329349875450134], "10": ["vertical_and_slash", 1000, 6096, 0.37838804721832275], "11": ["vertical_and_slash", 1000, 6096, 0.38584667444229126], "12": ["vertical_and_slash", 1000, 6096, 0.41800791025161743], "13": ["vertical_and_slash", 1000, 6096, 0.3703075349330902], "14": ["vertical_and_slash", 1000, 6096, 0.4014433026313782], "15": ["vertical_and_slash", 1000, 6096, 0.38102585077285767], "16": ["vertical_and_slash", 1000, 6096, 0.36637404561042786], "17": ["vertical_and_slash", 1000, 6096, 0.3792068064212799], "18": ["vertical_and_slash", 1000, 6096, 0.37271493673324585], "19": ["vertical_and_slash", 1000, 6096, 0.3685390055179596], "20": ["vertical_and_slash", 1000, 6096, 0.3965989947319031], "21": ["vertical_and_slash", 1000, 6096, 0.378986120223999], "22": ["vertical_and_slash", 1000, 6096, 0.3768484890460968], "23": ["vertical_and_slash", 1000, 6096, 0.36243176460266113], "24": ["vertical_and_slash", 1000, 6096, 0.3578113913536072], "25": ["vertical_and_slash", 1000, 6096, 0.34768110513687134], "26": ["vertical_and_slash", 1000, 6096, 0.3537325859069824], "27": ["vertical_and_slash", 1000, 6096, 0.37451034784317017], "28": ["vertical_and_slash", 1000, 6096, 0.3547874987125397], "29": ["vertical_and_slash", 1000, 6096, 0.3574942350387573], "30": ["vertical_and_slash", 1000, 6096, 0.3711892068386078], "31": ["vertical_and_slash", 1000, 6096, 0.3567763864994049], "32": ["vertical_and_slash", 1000, 6096, 0.36786508560180664], "33": ["vertical_and_slash", 1000, 6096, 0.3623276352882385], "34": ["vertical_and_slash", 1000, 6096, 0.362504780292511], "35": ["vertical_and_slash", 1000, 6096, 0.39003536105155945], "36": ["vertical_and_slash", 1000, 6096, 0.3681637644767761], "37": ["vertical_and_slash", 1000, 6096, 0.3628450930118561], "38": ["vertical_and_slash", 1000, 6096, 0.38495102524757385], "39": ["vertical_and_slash", 1000, 6096, 0.3732006847858429], "40": ["vertical_and_slash", 1000, 6096, 0.35328197479248047], "41": ["vertical_and_slash", 1000, 6096, 0.35349389910697937], "42": ["vertical_and_slash", 1000, 6096, 0.3712894916534424], "43": ["vertical_and_slash", 1000, 6096, 0.3607247471809387], "44": ["vertical_and_slash", 1000, 6096, 0.35484495759010315], "45": ["vertical_and_slash", 1000, 6096, 0.35517847537994385], "46": ["vertical_and_slash", 1000, 6096, 0.3687111735343933], "47": ["vertical_and_slash", 1000, 6096, 0.3455841541290283], "48": ["vertical_and_slash", 1000, 6096, 0.3754482567310333], "49": ["vertical_and_slash", 1000, 6096, 0.35592541098594666], "50": ["vertical_and_slash", 1000, 6096, 0.3906378746032715], "51": ["vertical_and_slash", 1000, 6096, 0.35268643498420715], "52": ["vertical_and_slash", 1000, 6096, 0.35632285475730896], "53": ["vertical_and_slash", 1000, 6096, 0.40458643436431885], "54": ["vertical_and_slash", 1000, 6096, 0.3745880126953125], "55": ["vertical_and_slash", 1000, 6096, 0.3996974229812622], "56": ["vertical_and_slash", 1000, 6096, 0.37907955050468445], "57": ["vertical_and_slash", 1000, 6096, 0.38706812262535095], "58": ["vertical_and_slash", 1000, 6096, 0.3671049177646637], "59": ["vertical_and_slash", 1000, 6096, 0.3834826350212097], "60": ["vertical_and_slash", 1000, 6096, 0.36931970715522766], "61": ["vertical_and_slash", 1000, 6096, 0.37901273369789124], "62": ["vertical_and_slash", 1000, 6096, 0.3717547655105591], "63": ["vertical_and_slash", 1000, 6096, 0.38920164108276367]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3529539108276367], "1": ["vertical_and_slash", 1000, 6096, 0.3496830463409424], "2": ["vertical_and_slash", 1000, 6096, 0.36271557211875916], "3": ["vertical_and_slash", 1000, 6096, 0.41897669434547424], "4": ["vertical_and_slash", 1000, 6096, 0.35896649956703186], "5": ["vertical_and_slash", 1000, 6096, 0.35333800315856934], "6": ["vertical_and_slash", 1000, 6096, 0.3868519365787506], "7": ["vertical_and_slash", 1000, 6096, 0.3927587866783142], "8": ["vertical_and_slash", 1000, 6096, 0.347497820854187], "9": ["vertical_and_slash", 1000, 6096, 0.4024461805820465], "10": ["vertical_and_slash", 1000, 6096, 0.34516391158103943], "11": ["vertical_and_slash", 1000, 6096, 0.3674961030483246], "12": ["vertical_and_slash", 1000, 6096, 0.3602040410041809], "13": ["vertical_and_slash", 1000, 6096, 0.3682680130004883], "14": ["vertical_and_slash", 1000, 6096, 0.3747952878475189], "15": ["vertical_and_slash", 1000, 6096, 0.349829763174057], "16": ["vertical_and_slash", 1000, 6096, 0.38579973578453064], "17": ["vertical_and_slash", 1000, 6096, 0.39973369240760803], "18": ["vertical_and_slash", 1000, 6096, 0.3897417187690735], "19": ["vertical_and_slash", 1000, 6096, 0.38260629773139954], "20": ["vertical_and_slash", 1000, 6096, 0.3835524320602417], "21": ["vertical_and_slash", 1000, 6096, 0.37144187092781067], "22": ["vertical_and_slash", 1000, 6096, 0.37775373458862305], "23": ["vertical_and_slash", 1000, 6096, 0.36544933915138245], "24": ["vertical_and_slash", 1000, 6096, 0.3618592917919159], "25": ["vertical_and_slash", 1000, 6096, 0.3671645522117615], "26": ["vertical_and_slash", 1000, 6096, 0.3736668825149536], "27": ["vertical_and_slash", 1000, 6096, 0.37035930156707764], "28": ["vertical_and_slash", 1000, 6096, 0.3625697195529938], "29": ["vertical_and_slash", 1000, 6096, 0.37495195865631104], "30": ["vertical_and_slash", 1000, 6096, 0.3653300404548645], "31": ["vertical_and_slash", 1000, 6096, 0.3693877160549164], "32": ["vertical_and_slash", 1000, 6096, 0.3742973804473877], "33": ["vertical_and_slash", 1000, 6096, 0.3558279871940613], "34": ["vertical_and_slash", 1000, 6096, 0.36033040285110474], "35": ["vertical_and_slash", 1000, 6096, 0.35329508781433105], "36": ["vertical_and_slash", 1000, 6096, 0.3588336408138275], "37": ["vertical_and_slash", 1000, 6096, 0.3592824935913086], "38": ["vertical_and_slash", 1000, 6096, 0.3497498035430908], "39": ["vertical_and_slash", 1000, 6096, 0.35655200481414795], "40": ["vertical_and_slash", 1000, 6096, 0.38204360008239746], "41": ["vertical_and_slash", 1000, 6096, 0.3829220235347748], "42": ["vertical_and_slash", 1000, 6096, 0.37619295716285706], "43": ["vertical_and_slash", 1000, 6096, 0.38316547870635986], "44": ["vertical_and_slash", 1000, 6096, 0.3797628581523895], "45": ["vertical_and_slash", 1000, 6096, 0.36968994140625], "46": ["vertical_and_slash", 1000, 6096, 0.38430213928222656], "47": ["vertical_and_slash", 1000, 6096, 0.37873637676239014], "48": ["vertical_and_slash", 1000, 6096, 0.3514203131198883], "49": ["vertical_and_slash", 1000, 6096, 0.3704606592655182], "50": ["vertical_and_slash", 1000, 6096, 0.3647375702857971], "51": ["vertical_and_slash", 1000, 6096, 0.3655928373336792], "52": ["vertical_and_slash", 1000, 6096, 0.362859845161438], "53": ["vertical_and_slash", 1000, 6096, 0.35914677381515503], "54": ["vertical_and_slash", 1000, 6096, 0.35413849353790283], "55": ["vertical_and_slash", 1000, 6096, 0.359306663274765], "56": ["vertical_and_slash", 1000, 6096, 0.35788848996162415], "57": ["vertical_and_slash", 1000, 6096, 0.3677782118320465], "58": ["vertical_and_slash", 1000, 6096, 0.37344685196876526], "59": ["vertical_and_slash", 1000, 6096, 0.37803226709365845], "60": ["vertical_and_slash", 1000, 6096, 0.35460543632507324], "61": ["vertical_and_slash", 1000, 6096, 0.38045749068260193], "62": ["vertical_and_slash", 1000, 6096, 0.35204017162323], "63": ["vertical_and_slash", 1000, 6096, 0.36404532194137573]}, {"0": ["vertical_and_slash", 1000, 6096, 0.36929890513420105], "1": ["vertical_and_slash", 1000, 6096, 0.37554123997688293], "2": ["vertical_and_slash", 1000, 6096, 0.3657001554965973], "3": ["vertical_and_slash", 1000, 6096, 0.36816641688346863], "4": ["vertical_and_slash", 1000, 6096, 0.3725087642669678], "5": ["vertical_and_slash", 1000, 6096, 0.3759603798389435], "6": ["vertical_and_slash", 1000, 6096, 0.3760131597518921], "7": ["vertical_and_slash", 1000, 6096, 0.3656710088253021], "8": ["vertical_and_slash", 1000, 6096, 0.3777766823768616], "9": ["vertical_and_slash", 1000, 6096, 0.362973153591156], "10": ["vertical_and_slash", 1000, 6096, 0.37836745381355286], "11": ["vertical_and_slash", 1000, 6096, 0.38642123341560364], "12": ["vertical_and_slash", 1000, 6096, 0.37316152453422546], "13": ["vertical_and_slash", 1000, 6096, 0.3688306212425232], "14": ["vertical_and_slash", 1000, 6096, 0.3733440041542053], "15": ["vertical_and_slash", 1000, 6096, 0.3746086657047272], "16": ["vertical_and_slash", 1000, 6096, 0.35812750458717346], "17": ["vertical_and_slash", 1000, 6096, 0.3903712332248688], "18": ["vertical_and_slash", 1000, 6096, 0.353098601102829], "19": ["vertical_and_slash", 1000, 6096, 0.3480265438556671], "20": ["vertical_and_slash", 1000, 6096, 0.36427316069602966], "21": ["vertical_and_slash", 1000, 6096, 0.339932918548584], "22": ["vertical_and_slash", 1000, 6096, 0.3678346872329712], "23": ["vertical_and_slash", 1000, 6096, 0.3651609718799591], "24": ["vertical_and_slash", 1000, 6096, 0.3723788857460022], "25": ["vertical_and_slash", 1000, 6096, 0.3574349880218506], "26": ["vertical_and_slash", 1000, 6096, 0.3430638015270233], "27": ["vertical_and_slash", 1000, 6096, 0.3597646951675415], "28": ["vertical_and_slash", 1000, 6096, 0.36389702558517456], "29": ["vertical_and_slash", 1000, 6096, 0.36069783568382263], "30": ["vertical_and_slash", 1000, 6096, 0.362699419260025], "31": ["vertical_and_slash", 1000, 6096, 0.3624710142612457], "32": ["vertical_and_slash", 1000, 6096, 0.3652339577674866], "33": ["vertical_and_slash", 1000, 6096, 0.39067044854164124], "34": ["vertical_and_slash", 1000, 6096, 0.38539108633995056], "35": ["vertical_and_slash", 1000, 6096, 0.3761095404624939], "36": ["vertical_and_slash", 1000, 6096, 0.36962950229644775], "37": ["vertical_and_slash", 1000, 6096, 0.3847278654575348], "38": ["vertical_and_slash", 1000, 6096, 0.3722037374973297], "39": ["vertical_and_slash", 1000, 6096, 0.3574404716491699], "40": ["vertical_and_slash", 1000, 6096, 0.39785826206207275], "41": ["vertical_and_slash", 1000, 6096, 0.35344940423965454], "42": ["vertical_and_slash", 1000, 6096, 0.3442263901233673], "43": ["vertical_and_slash", 1000, 6096, 0.3728048801422119], "44": ["vertical_and_slash", 1000, 6096, 0.3635236918926239], "45": ["vertical_and_slash", 1000, 6096, 0.3618640601634979], "46": ["vertical_and_slash", 1000, 6096, 0.3929799199104309], "47": ["vertical_and_slash", 1000, 6096, 0.3645145893096924], "48": ["vertical_and_slash", 1000, 6096, 0.35935088992118835], "49": ["vertical_and_slash", 1000, 6096, 0.3580968379974365], "50": ["vertical_and_slash", 1000, 6096, 0.36632975935935974], "51": ["vertical_and_slash", 1000, 6096, 0.35363373160362244], "52": ["vertical_and_slash", 1000, 6096, 0.3659893274307251], "53": ["vertical_and_slash", 1000, 6096, 0.35278451442718506], "54": ["vertical_and_slash", 1000, 6096, 0.35445141792297363], "55": ["vertical_and_slash", 1000, 6096, 0.3566558063030243], "56": ["vertical_and_slash", 1000, 6096, 0.3710847496986389], "57": ["vertical_and_slash", 1000, 6096, 0.36688947677612305], "58": ["vertical_and_slash", 1000, 6096, 0.3898160457611084], "59": ["vertical_and_slash", 1000, 6096, 0.37338986992836], "60": ["vertical_and_slash", 1000, 6096, 0.360798180103302], "61": ["vertical_and_slash", 1000, 6096, 0.36318597197532654], "62": ["vertical_and_slash", 1000, 6096, 0.35747671127319336], "63": ["vertical_and_slash", 1000, 6096, 0.38194599747657776]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3535512387752533], "1": ["vertical_and_slash", 1000, 6096, 0.3480595648288727], "2": ["vertical_and_slash", 1000, 6096, 0.34950900077819824], "3": ["vertical_and_slash", 1000, 6096, 0.3547428548336029], "4": ["vertical_and_slash", 1000, 6096, 0.3581652343273163], "5": ["vertical_and_slash", 1000, 6096, 0.3593714237213135], "6": ["vertical_and_slash", 1000, 6096, 0.35747116804122925], "7": ["vertical_and_slash", 1000, 6096, 0.3527491092681885], "8": ["vertical_and_slash", 1000, 6096, 0.3625282049179077], "9": ["vertical_and_slash", 1000, 6096, 0.3436722457408905], "10": ["vertical_and_slash", 1000, 6096, 0.35478731989860535], "11": ["vertical_and_slash", 1000, 6096, 0.3507758677005768], "12": ["vertical_and_slash", 1000, 6096, 0.3547520935535431], "13": ["vertical_and_slash", 1000, 6096, 0.3579652011394501], "14": ["vertical_and_slash", 1000, 6096, 0.35553398728370667], "15": ["vertical_and_slash", 1000, 6096, 0.3558042645454407], "16": ["vertical_and_slash", 1000, 6096, 0.3365512192249298], "17": ["vertical_and_slash", 1000, 6096, 0.35680466890335083], "18": ["vertical_and_slash", 1000, 6096, 0.3639698624610901], "19": ["vertical_and_slash", 1000, 6096, 0.36201977729797363], "20": ["vertical_and_slash", 1000, 6096, 0.34974798560142517], "21": ["vertical_and_slash", 1000, 6096, 0.35077014565467834], "22": ["vertical_and_slash", 1000, 6096, 0.33943888545036316], "23": ["vertical_and_slash", 1000, 6096, 0.3484926223754883], "24": ["vertical_and_slash", 1000, 6096, 0.3522002100944519], "25": ["vertical_and_slash", 1000, 6096, 0.3455331027507782], "26": ["vertical_and_slash", 1000, 6096, 0.34702378511428833], "27": ["vertical_and_slash", 1000, 6096, 0.34924083948135376], "28": ["vertical_and_slash", 1000, 6096, 0.34830522537231445], "29": ["vertical_and_slash", 1000, 6096, 0.35064786672592163], "30": ["vertical_and_slash", 1000, 6096, 0.3482474088668823], "31": ["vertical_and_slash", 1000, 6096, 0.3518823981285095], "32": ["vertical_and_slash", 1000, 6096, 0.3622961640357971], "33": ["vertical_and_slash", 1000, 6096, 0.35955193638801575], "34": ["vertical_and_slash", 1000, 6096, 0.35990986227989197], "35": ["vertical_and_slash", 1000, 6096, 0.36092880368232727], "36": ["vertical_and_slash", 1000, 6096, 0.3613496422767639], "37": ["vertical_and_slash", 1000, 6096, 0.3586430251598358], "38": ["vertical_and_slash", 1000, 6096, 0.35780060291290283], "39": ["vertical_and_slash", 1000, 6096, 0.3576725423336029], "40": ["vertical_and_slash", 1000, 6096, 0.35927027463912964], "41": ["vertical_and_slash", 1000, 6096, 0.34418782591819763], "42": ["vertical_and_slash", 1000, 6096, 0.3556368350982666], "43": ["vertical_and_slash", 1000, 6096, 0.3435509204864502], "44": ["vertical_and_slash", 1000, 6096, 0.35085248947143555], "45": ["vertical_and_slash", 1000, 6096, 0.358532190322876], "46": ["vertical_and_slash", 1000, 6096, 0.34741392731666565], "47": ["vertical_and_slash", 1000, 6096, 0.34923669695854187], "48": ["vertical_and_slash", 1000, 6096, 0.3487474322319031], "49": ["vertical_and_slash", 1000, 6096, 0.35124242305755615], "50": ["vertical_and_slash", 1000, 6096, 0.3527771830558777], "51": ["vertical_and_slash", 1000, 6096, 0.3587794005870819], "52": ["vertical_and_slash", 1000, 6096, 0.35044148564338684], "53": ["vertical_and_slash", 1000, 6096, 0.3562122881412506], "54": ["vertical_and_slash", 1000, 6096, 0.3525117337703705], "55": ["vertical_and_slash", 1000, 6096, 0.3532715439796448], "56": ["vertical_and_slash", 1000, 6096, 0.36014536023139954], "57": ["vertical_and_slash", 1000, 6096, 0.3516746163368225], "58": ["vertical_and_slash", 1000, 6096, 0.35211464762687683], "59": ["vertical_and_slash", 1000, 6096, 0.3465120792388916], "60": ["vertical_and_slash", 1000, 6096, 0.35673806071281433], "61": ["vertical_and_slash", 1000, 6096, 0.34775635600090027], "62": ["vertical_and_slash", 1000, 6096, 0.3506435453891754], "63": ["vertical_and_slash", 1000, 6096, 0.3551265597343445]}, {"0": ["vertical_and_slash", 1000, 6096, 0.34709396958351135], "1": ["vertical_and_slash", 1000, 6096, 0.3521941602230072], "2": ["vertical_and_slash", 1000, 6096, 0.3605175316333771], "3": ["vertical_and_slash", 1000, 6096, 0.3478003442287445], "4": ["vertical_and_slash", 1000, 6096, 0.34613776206970215], "5": ["vertical_and_slash", 1000, 6096, 0.35352757573127747], "6": ["vertical_and_slash", 1000, 6096, 0.34626641869544983], "7": ["vertical_and_slash", 1000, 6096, 0.37169456481933594], "8": ["vertical_and_slash", 1000, 6096, 0.3522755205631256], "9": ["vertical_and_slash", 1000, 6096, 0.35593920946121216], "10": ["vertical_and_slash", 1000, 6096, 0.35087335109710693], "11": ["vertical_and_slash", 1000, 6096, 0.359891802072525], "12": ["vertical_and_slash", 1000, 6096, 0.34439340233802795], "13": ["vertical_and_slash", 1000, 6096, 0.3619743287563324], "14": ["vertical_and_slash", 1000, 6096, 0.3653984069824219], "15": ["vertical_and_slash", 1000, 6096, 0.3684181272983551], "16": ["vertical_and_slash", 1000, 6096, 0.35193508863449097], "17": ["vertical_and_slash", 1000, 6096, 0.35361143946647644], "18": ["vertical_and_slash", 1000, 6096, 0.35692527890205383], "19": ["vertical_and_slash", 1000, 6096, 0.3556109666824341], "20": ["vertical_and_slash", 1000, 6096, 0.36100152134895325], "21": ["vertical_and_slash", 1000, 6096, 0.3639155626296997], "22": ["vertical_and_slash", 1000, 6096, 0.34481197595596313], "23": ["vertical_and_slash", 1000, 6096, 0.3503955006599426], "24": ["vertical_and_slash", 1000, 6096, 0.37193241715431213], "25": ["vertical_and_slash", 1000, 6096, 0.3565455377101898], "26": ["vertical_and_slash", 1000, 6096, 0.3774701654911041], "27": ["vertical_and_slash", 1000, 6096, 0.3734953999519348], "28": ["vertical_and_slash", 1000, 6096, 0.376298725605011], "29": ["vertical_and_slash", 1000, 6096, 0.3786798119544983], "30": ["vertical_and_slash", 1000, 6096, 0.3707925081253052], "31": ["vertical_and_slash", 1000, 6096, 0.34591153264045715], "32": ["vertical_and_slash", 1000, 6096, 0.3568251132965088], "33": ["vertical_and_slash", 1000, 6096, 0.35756754875183105], "34": ["vertical_and_slash", 1000, 6096, 0.35799741744995117], "35": ["vertical_and_slash", 1000, 6096, 0.3571697771549225], "36": ["vertical_and_slash", 1000, 6096, 0.36090871691703796], "37": ["vertical_and_slash", 1000, 6096, 0.3597864806652069], "38": ["vertical_and_slash", 1000, 6096, 0.3616034984588623], "39": ["vertical_and_slash", 1000, 6096, 0.35575440526008606], "40": ["vertical_and_slash", 1000, 6096, 0.36424720287323], "41": ["vertical_and_slash", 1000, 6096, 0.3635222315788269], "42": ["vertical_and_slash", 1000, 6096, 0.3657201826572418], "43": ["vertical_and_slash", 1000, 6096, 0.36285966634750366], "44": ["vertical_and_slash", 1000, 6096, 0.3523986041545868], "45": ["vertical_and_slash", 1000, 6096, 0.3582127094268799], "46": ["vertical_and_slash", 1000, 6096, 0.35391566157341003], "47": ["vertical_and_slash", 1000, 6096, 0.3588290810585022], "48": ["vertical_and_slash", 1000, 6096, 0.3464396893978119], "49": ["vertical_and_slash", 1000, 6096, 0.3626839816570282], "50": ["vertical_and_slash", 1000, 6096, 0.3521279990673065], "51": ["vertical_and_slash", 1000, 6096, 0.34418630599975586], "52": ["vertical_and_slash", 1000, 6096, 0.347763329744339], "53": ["vertical_and_slash", 1000, 6096, 0.35166117548942566], "54": ["vertical_and_slash", 1000, 6096, 0.34576016664505005], "55": ["vertical_and_slash", 1000, 6096, 0.3604962229728699], "56": ["vertical_and_slash", 1000, 6096, 0.3628078103065491], "57": ["vertical_and_slash", 1000, 6096, 0.36260947585105896], "58": ["vertical_and_slash", 1000, 6096, 0.35946157574653625], "59": ["vertical_and_slash", 1000, 6096, 0.36163049936294556], "60": ["vertical_and_slash", 1000, 6096, 0.36154288053512573], "61": ["vertical_and_slash", 1000, 6096, 0.35513773560523987], "62": ["vertical_and_slash", 1000, 6096, 0.35982367396354675], "63": ["vertical_and_slash", 1000, 6096, 0.3561977744102478]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3557223379611969], "1": ["vertical_and_slash", 1000, 6096, 0.34601590037345886], "2": ["vertical_and_slash", 1000, 6096, 0.3523191809654236], "3": ["vertical_and_slash", 1000, 6096, 0.3615344166755676], "4": ["vertical_and_slash", 1000, 6096, 0.36122584342956543], "5": ["vertical_and_slash", 1000, 6096, 0.34895530343055725], "6": ["vertical_and_slash", 1000, 6096, 0.3464120626449585], "7": ["vertical_and_slash", 1000, 6096, 0.34759384393692017], "8": ["vertical_and_slash", 1000, 6096, 0.37067872285842896], "9": ["vertical_and_slash", 1000, 6096, 0.35137054324150085], "10": ["vertical_and_slash", 1000, 6096, 0.37144532799720764], "11": ["vertical_and_slash", 1000, 6096, 0.3733764588832855], "12": ["vertical_and_slash", 1000, 6096, 0.3811398148536682], "13": ["vertical_and_slash", 1000, 6096, 0.35406625270843506], "14": ["vertical_and_slash", 1000, 6096, 0.3508550524711609], "15": ["vertical_and_slash", 1000, 6096, 0.3513963222503662], "16": ["vertical_and_slash", 1000, 6096, 0.35570335388183594], "17": ["vertical_and_slash", 1000, 6096, 0.35880956053733826], "18": ["vertical_and_slash", 1000, 6096, 0.3616129755973816], "19": ["vertical_and_slash", 1000, 6096, 0.3597695827484131], "20": ["vertical_and_slash", 1000, 6096, 0.3512013554573059], "21": ["vertical_and_slash", 1000, 6096, 0.3535090982913971], "22": ["vertical_and_slash", 1000, 6096, 0.35621553659439087], "23": ["vertical_and_slash", 1000, 6096, 0.3497322201728821], "24": ["vertical_and_slash", 1000, 6096, 0.39522382616996765], "25": ["vertical_and_slash", 1000, 6096, 0.36795228719711304], "26": ["vertical_and_slash", 1000, 6096, 0.3678871989250183], "27": ["vertical_and_slash", 1000, 6096, 0.37098070979118347], "28": ["vertical_and_slash", 1000, 6096, 0.36182698607444763], "29": ["vertical_and_slash", 1000, 6096, 0.3710789978504181], "30": ["vertical_and_slash", 1000, 6096, 0.3709219992160797], "31": ["vertical_and_slash", 1000, 6096, 0.3704236149787903], "32": ["vertical_and_slash", 1000, 6096, 0.34234103560447693], "33": ["vertical_and_slash", 1000, 6096, 0.365505188703537], "34": ["vertical_and_slash", 1000, 6096, 0.3594202995300293], "35": ["vertical_and_slash", 1000, 6096, 0.36977869272232056], "36": ["vertical_and_slash", 1000, 6096, 0.3647242486476898], "37": ["vertical_and_slash", 1000, 6096, 0.36128872632980347], "38": ["vertical_and_slash", 1000, 6096, 0.34396445751190186], "39": ["vertical_and_slash", 1000, 6096, 0.36205241084098816], "40": ["vertical_and_slash", 1000, 6096, 0.3789259195327759], "41": ["vertical_and_slash", 1000, 6096, 0.38027358055114746], "42": ["vertical_and_slash", 1000, 6096, 0.38024842739105225], "43": ["vertical_and_slash", 1000, 6096, 0.3813612461090088], "44": ["vertical_and_slash", 1000, 6096, 0.3754211664199829], "45": ["vertical_and_slash", 1000, 6096, 0.3635827600955963], "46": ["vertical_and_slash", 1000, 6096, 0.3719649910926819], "47": ["vertical_and_slash", 1000, 6096, 0.3726890981197357], "48": ["vertical_and_slash", 1000, 6096, 0.3511112630367279], "49": ["vertical_and_slash", 1000, 6096, 0.3756664991378784], "50": ["vertical_and_slash", 1000, 6096, 0.3488610088825226], "51": ["vertical_and_slash", 1000, 6096, 0.37098345160484314], "52": ["vertical_and_slash", 1000, 6096, 0.36379075050354004], "53": ["vertical_and_slash", 1000, 6096, 0.3445033133029938], "54": ["vertical_and_slash", 1000, 6096, 0.33977729082107544], "55": ["vertical_and_slash", 1000, 6096, 0.3472400903701782], "56": ["vertical_and_slash", 1000, 6096, 0.35762330889701843], "57": ["vertical_and_slash", 1000, 6096, 0.35039278864860535], "58": ["vertical_and_slash", 1000, 6096, 0.36492598056793213], "59": ["vertical_and_slash", 1000, 6096, 0.36909225583076477], "60": ["vertical_and_slash", 1000, 6096, 0.3584064245223999], "61": ["vertical_and_slash", 1000, 6096, 0.3579098582267761], "62": ["vertical_and_slash", 1000, 6096, 0.3535459041595459], "63": ["vertical_and_slash", 1000, 6096, 0.36612752079963684]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3803720772266388], "1": ["vertical_and_slash", 1000, 6096, 0.37587398290634155], "2": ["vertical_and_slash", 1000, 6096, 0.3657146096229553], "3": ["vertical_and_slash", 1000, 6096, 0.38036075234413147], "4": ["vertical_and_slash", 1000, 6096, 0.3764478266239166], "5": ["vertical_and_slash", 1000, 6096, 0.37084871530532837], "6": ["vertical_and_slash", 1000, 6096, 0.36484453082084656], "7": ["vertical_and_slash", 1000, 6096, 0.37118613719940186], "8": ["vertical_and_slash", 1000, 6096, 0.38117992877960205], "9": ["vertical_and_slash", 1000, 6096, 0.34890997409820557], "10": ["vertical_and_slash", 1000, 6096, 0.36594924330711365], "11": ["vertical_and_slash", 1000, 6096, 0.36983510851860046], "12": ["vertical_and_slash", 1000, 6096, 0.37269458174705505], "13": ["vertical_and_slash", 1000, 6096, 0.3559139370918274], "14": ["vertical_and_slash", 1000, 6096, 0.3847796618938446], "15": ["vertical_and_slash", 1000, 6096, 0.36133089661598206], "16": ["vertical_and_slash", 1000, 6096, 0.3423650860786438], "17": ["vertical_and_slash", 1000, 6096, 0.3490035831928253], "18": ["vertical_and_slash", 1000, 6096, 0.34170082211494446], "19": ["vertical_and_slash", 1000, 6096, 0.3557381331920624], "20": ["vertical_and_slash", 1000, 6096, 0.35008713603019714], "21": ["vertical_and_slash", 1000, 6096, 0.35975366830825806], "22": ["vertical_and_slash", 1000, 6096, 0.3633672297000885], "23": ["vertical_and_slash", 1000, 6096, 0.35727372765541077], "24": ["vertical_and_slash", 1000, 6096, 0.35365545749664307], "25": ["vertical_and_slash", 1000, 6096, 0.3538397252559662], "26": ["vertical_and_slash", 1000, 6096, 0.3607936203479767], "27": ["vertical_and_slash", 1000, 6096, 0.34959518909454346], "28": ["vertical_and_slash", 1000, 6096, 0.35172247886657715], "29": ["vertical_and_slash", 1000, 6096, 0.3521333634853363], "30": ["vertical_and_slash", 1000, 6096, 0.35162240266799927], "31": ["vertical_and_slash", 1000, 6096, 0.36081263422966003], "32": ["vertical_and_slash", 1000, 6096, 0.3531310260295868], "33": ["vertical_and_slash", 1000, 6096, 0.3504440188407898], "34": ["vertical_and_slash", 1000, 6096, 0.3657781481742859], "35": ["vertical_and_slash", 1000, 6096, 0.35016027092933655], "36": ["vertical_and_slash", 1000, 6096, 0.35467785596847534], "37": ["vertical_and_slash", 1000, 6096, 0.3583979308605194], "38": ["vertical_and_slash", 1000, 6096, 0.36364051699638367], "39": ["vertical_and_slash", 1000, 6096, 0.3591064214706421], "40": ["vertical_and_slash", 1000, 6096, 0.36861851811408997], "41": ["vertical_and_slash", 1000, 6096, 0.3590241074562073], "42": ["vertical_and_slash", 1000, 6096, 0.3634859621524811], "43": ["vertical_and_slash", 1000, 6096, 0.3658461272716522], "44": ["vertical_and_slash", 1000, 6096, 0.3584599792957306], "45": ["vertical_and_slash", 1000, 6096, 0.3624853193759918], "46": ["vertical_and_slash", 1000, 6096, 0.36734816431999207], "47": ["vertical_and_slash", 1000, 6096, 0.36034080386161804], "48": ["vertical_and_slash", 1000, 6096, 0.3693649172782898], "49": ["vertical_and_slash", 1000, 6096, 0.372521311044693], "50": ["vertical_and_slash", 1000, 6096, 0.363821804523468], "51": ["vertical_and_slash", 1000, 6096, 0.34809747338294983], "52": ["vertical_and_slash", 1000, 6096, 0.36811456084251404], "53": ["vertical_and_slash", 1000, 6096, 0.3685130774974823], "54": ["vertical_and_slash", 1000, 6096, 0.37969711422920227], "55": ["vertical_and_slash", 1000, 6096, 0.35556817054748535], "56": ["vertical_and_slash", 1000, 6096, 0.3496045768260956], "57": ["vertical_and_slash", 1000, 6096, 0.35584303736686707], "58": ["vertical_and_slash", 1000, 6096, 0.34843549132347107], "59": ["vertical_and_slash", 1000, 6096, 0.34239205718040466], "60": ["vertical_and_slash", 1000, 6096, 0.3568454682826996], "61": ["vertical_and_slash", 1000, 6096, 0.3513108789920807], "62": ["vertical_and_slash", 1000, 6096, 0.3492681384086609], "63": ["vertical_and_slash", 1000, 6096, 0.35074925422668457]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3513393998146057], "1": ["vertical_and_slash", 1000, 6096, 0.35606300830841064], "2": ["vertical_and_slash", 1000, 6096, 0.35309872031211853], "3": ["vertical_and_slash", 1000, 6096, 0.3647685945034027], "4": ["vertical_and_slash", 1000, 6096, 0.3674863874912262], "5": ["vertical_and_slash", 1000, 6096, 0.37165868282318115], "6": ["vertical_and_slash", 1000, 6096, 0.3607330322265625], "7": ["vertical_and_slash", 1000, 6096, 0.36060094833374023], "8": ["vertical_and_slash", 1000, 6096, 0.3527091443538666], "9": ["vertical_and_slash", 1000, 6096, 0.3477655351161957], "10": ["vertical_and_slash", 1000, 6096, 0.35406893491744995], "11": ["vertical_and_slash", 1000, 6096, 0.35119807720184326], "12": ["vertical_and_slash", 1000, 6096, 0.3516154885292053], "13": ["vertical_and_slash", 1000, 6096, 0.35277697443962097], "14": ["vertical_and_slash", 1000, 6096, 0.35743504762649536], "15": ["vertical_and_slash", 1000, 6096, 0.34924325346946716], "16": ["vertical_and_slash", 1000, 6096, 0.3527683615684509], "17": ["vertical_and_slash", 1000, 6096, 0.3564055562019348], "18": ["vertical_and_slash", 1000, 6096, 0.3555549681186676], "19": ["vertical_and_slash", 1000, 6096, 0.35300979018211365], "20": ["vertical_and_slash", 1000, 6096, 0.36123451590538025], "21": ["vertical_and_slash", 1000, 6096, 0.35576820373535156], "22": ["vertical_and_slash", 1000, 6096, 0.3579852283000946], "23": ["vertical_and_slash", 1000, 6096, 0.3587549328804016], "24": ["vertical_and_slash", 1000, 6096, 0.36870983242988586], "25": ["vertical_and_slash", 1000, 6096, 0.3629941940307617], "26": ["vertical_and_slash", 1000, 6096, 0.35745251178741455], "27": ["vertical_and_slash", 1000, 6096, 0.35884106159210205], "28": ["vertical_and_slash", 1000, 6096, 0.3484226167201996], "29": ["vertical_and_slash", 1000, 6096, 0.3555735647678375], "30": ["vertical_and_slash", 1000, 6096, 0.3638070523738861], "31": ["vertical_and_slash", 1000, 6096, 0.35605165362358093], "32": ["vertical_and_slash", 1000, 6096, 0.36056336760520935], "33": ["vertical_and_slash", 1000, 6096, 0.34521055221557617], "34": ["vertical_and_slash", 1000, 6096, 0.35597333312034607], "35": ["vertical_and_slash", 1000, 6096, 0.3583168089389801], "36": ["vertical_and_slash", 1000, 6096, 0.35792532563209534], "37": ["vertical_and_slash", 1000, 6096, 0.3473590016365051], "38": ["vertical_and_slash", 1000, 6096, 0.350521981716156], "39": ["vertical_and_slash", 1000, 6096, 0.35414308309555054], "40": ["vertical_and_slash", 1000, 6096, 0.37257319688796997], "41": ["vertical_and_slash", 1000, 6096, 0.3452824056148529], "42": ["vertical_and_slash", 1000, 6096, 0.37493354082107544], "43": ["vertical_and_slash", 1000, 6096, 0.3684462010860443], "44": ["vertical_and_slash", 1000, 6096, 0.3659411072731018], "45": ["vertical_and_slash", 1000, 6096, 0.35485702753067017], "46": ["vertical_and_slash", 1000, 6096, 0.3829544484615326], "47": ["vertical_and_slash", 1000, 6096, 0.36950036883354187], "48": ["vertical_and_slash", 1000, 6096, 0.3656962215900421], "49": ["vertical_and_slash", 1000, 6096, 0.3545924723148346], "50": ["vertical_and_slash", 1000, 6096, 0.36071106791496277], "51": ["vertical_and_slash", 1000, 6096, 0.3538132905960083], "52": ["vertical_and_slash", 1000, 6096, 0.35370439291000366], "53": ["vertical_and_slash", 1000, 6096, 0.36327919363975525], "54": ["vertical_and_slash", 1000, 6096, 0.35711315274238586], "55": ["vertical_and_slash", 1000, 6096, 0.37136968970298767], "56": ["vertical_and_slash", 1000, 6096, 0.35655084252357483], "57": ["vertical_and_slash", 1000, 6096, 0.3431124985218048], "58": ["vertical_and_slash", 1000, 6096, 0.3624011278152466], "59": ["vertical_and_slash", 1000, 6096, 0.36383911967277527], "60": ["vertical_and_slash", 1000, 6096, 0.3574076294898987], "61": ["vertical_and_slash", 1000, 6096, 0.35906749963760376], "62": ["vertical_and_slash", 1000, 6096, 0.35693785548210144], "63": ["vertical_and_slash", 1000, 6096, 0.36621585488319397]}, {"0": ["vertical_and_slash", 1000, 6096, 0.34903019666671753], "1": ["vertical_and_slash", 1000, 6096, 0.3506462574005127], "2": ["vertical_and_slash", 1000, 6096, 0.3569040298461914], "3": ["vertical_and_slash", 1000, 6096, 0.3449978828430176], "4": ["vertical_and_slash", 1000, 6096, 0.35051506757736206], "5": ["vertical_and_slash", 1000, 6096, 0.3500886857509613], "6": ["vertical_and_slash", 1000, 6096, 0.340676486492157], "7": ["vertical_and_slash", 1000, 6096, 0.3484727144241333], "8": ["vertical_and_slash", 1000, 6096, 0.3640531599521637], "9": ["vertical_and_slash", 1000, 6096, 0.3652477562427521], "10": ["vertical_and_slash", 1000, 6096, 0.3660964369773865], "11": ["vertical_and_slash", 1000, 6096, 0.36396461725234985], "12": ["vertical_and_slash", 1000, 6096, 0.36479032039642334], "13": ["vertical_and_slash", 1000, 6096, 0.3604509234428406], "14": ["vertical_and_slash", 1000, 6096, 0.36944666504859924], "15": ["vertical_and_slash", 1000, 6096, 0.3622523248195648], "16": ["vertical_and_slash", 1000, 6096, 0.35571497678756714], "17": ["vertical_and_slash", 1000, 6096, 0.34845587611198425], "18": ["vertical_and_slash", 1000, 6096, 0.34988847374916077], "19": ["vertical_and_slash", 1000, 6096, 0.35124966502189636], "20": ["vertical_and_slash", 1000, 6096, 0.3555211126804352], "21": ["vertical_and_slash", 1000, 6096, 0.35228851437568665], "22": ["vertical_and_slash", 1000, 6096, 0.35259345173835754], "23": ["vertical_and_slash", 1000, 6096, 0.3488306999206543], "24": ["vertical_and_slash", 1000, 6096, 0.3545505106449127], "25": ["vertical_and_slash", 1000, 6096, 0.35056498646736145], "26": ["vertical_and_slash", 1000, 6096, 0.3536752462387085], "27": ["vertical_and_slash", 1000, 6096, 0.3555887043476105], "28": ["vertical_and_slash", 1000, 6096, 0.3472338318824768], "29": ["vertical_and_slash", 1000, 6096, 0.35904428362846375], "30": ["vertical_and_slash", 1000, 6096, 0.36850932240486145], "31": ["vertical_and_slash", 1000, 6096, 0.3602719306945801], "32": ["vertical_and_slash", 1000, 6096, 0.3644832968711853], "33": ["vertical_and_slash", 1000, 6096, 0.352567195892334], "34": ["vertical_and_slash", 1000, 6096, 0.3533124625682831], "35": ["vertical_and_slash", 1000, 6096, 0.35424908995628357], "36": ["vertical_and_slash", 1000, 6096, 0.3527776002883911], "37": ["vertical_and_slash", 1000, 6096, 0.35954561829566956], "38": ["vertical_and_slash", 1000, 6096, 0.3649601936340332], "39": ["vertical_and_slash", 1000, 6096, 0.3520076870918274], "40": ["vertical_and_slash", 1000, 6096, 0.35159939527511597], "41": ["vertical_and_slash", 1000, 6096, 0.34718045592308044], "42": ["vertical_and_slash", 1000, 6096, 0.3627654016017914], "43": ["vertical_and_slash", 1000, 6096, 0.37681740522384644], "44": ["vertical_and_slash", 1000, 6096, 0.357644647359848], "45": ["vertical_and_slash", 1000, 6096, 0.3583258390426636], "46": ["vertical_and_slash", 1000, 6096, 0.34899386763572693], "47": ["vertical_and_slash", 1000, 6096, 0.3514038026332855], "48": ["vertical_and_slash", 1000, 6096, 0.3616923689842224], "49": ["vertical_and_slash", 1000, 6096, 0.35973915457725525], "50": ["vertical_and_slash", 1000, 6096, 0.3535781502723694], "51": ["vertical_and_slash", 1000, 6096, 0.3484903573989868], "52": ["vertical_and_slash", 1000, 6096, 0.37762388586997986], "53": ["vertical_and_slash", 1000, 6096, 0.36778703331947327], "54": ["vertical_and_slash", 1000, 6096, 0.3514412045478821], "55": ["vertical_and_slash", 1000, 6096, 0.3491678535938263], "56": ["vertical_and_slash", 1000, 6096, 0.3541613221168518], "57": ["vertical_and_slash", 1000, 6096, 0.3583398163318634], "58": ["vertical_and_slash", 1000, 6096, 0.3617793619632721], "59": ["vertical_and_slash", 1000, 6096, 0.36384809017181396], "60": ["vertical_and_slash", 1000, 6096, 0.363248735666275], "61": ["vertical_and_slash", 1000, 6096, 0.3609289526939392], "62": ["vertical_and_slash", 1000, 6096, 0.35897397994995117], "63": ["vertical_and_slash", 1000, 6096, 0.3772814869880676]}, {"0": ["vertical_and_slash", 1000, 6096, 0.35607340931892395], "1": ["vertical_and_slash", 1000, 6096, 0.3529040515422821], "2": ["vertical_and_slash", 1000, 6096, 0.35278064012527466], "3": ["vertical_and_slash", 1000, 6096, 0.3474818766117096], "4": ["vertical_and_slash", 1000, 6096, 0.3468207120895386], "5": ["vertical_and_slash", 1000, 6096, 0.34663957357406616], "6": ["vertical_and_slash", 1000, 6096, 0.35009580850601196], "7": ["vertical_and_slash", 1000, 6096, 0.3479567766189575], "8": ["vertical_and_slash", 1000, 6096, 0.35957765579223633], "9": ["vertical_and_slash", 1000, 6096, 0.36788466572761536], "10": ["vertical_and_slash", 1000, 6096, 0.36980488896369934], "11": ["vertical_and_slash", 1000, 6096, 0.3684939742088318], "12": ["vertical_and_slash", 1000, 6096, 0.3743396997451782], "13": ["vertical_and_slash", 1000, 6096, 0.37344875931739807], "14": ["vertical_and_slash", 1000, 6096, 0.37275004386901855], "15": ["vertical_and_slash", 1000, 6096, 0.36919358372688293], "16": ["vertical_and_slash", 1000, 6096, 0.3615153729915619], "17": ["vertical_and_slash", 1000, 6096, 0.3612813353538513], "18": ["vertical_and_slash", 1000, 6096, 0.3561270236968994], "19": ["vertical_and_slash", 1000, 6096, 0.3560386598110199], "20": ["vertical_and_slash", 1000, 6096, 0.3579501211643219], "21": ["vertical_and_slash", 1000, 6096, 0.35965263843536377], "22": ["vertical_and_slash", 1000, 6096, 0.360552042722702], "23": ["vertical_and_slash", 1000, 6096, 0.36328279972076416], "24": ["vertical_and_slash", 1000, 6096, 0.3695860207080841], "25": ["vertical_and_slash", 1000, 6096, 0.3701823651790619], "26": ["vertical_and_slash", 1000, 6096, 0.3576812148094177], "27": ["vertical_and_slash", 1000, 6096, 0.35480397939682007], "28": ["vertical_and_slash", 1000, 6096, 0.35394591093063354], "29": ["vertical_and_slash", 1000, 6096, 0.3512093424797058], "30": ["vertical_and_slash", 1000, 6096, 0.3581388592720032], "31": ["vertical_and_slash", 1000, 6096, 0.350754976272583], "32": ["vertical_and_slash", 1000, 6096, 0.36395764350891113], "33": ["vertical_and_slash", 1000, 6096, 0.3530648350715637], "34": ["vertical_and_slash", 1000, 6096, 0.35249653458595276], "35": ["vertical_and_slash", 1000, 6096, 0.3534713089466095], "36": ["vertical_and_slash", 1000, 6096, 0.3611924946308136], "37": ["vertical_and_slash", 1000, 6096, 0.37173303961753845], "38": ["vertical_and_slash", 1000, 6096, 0.3460114002227783], "39": ["vertical_and_slash", 1000, 6096, 0.36525270342826843], "40": ["vertical_and_slash", 1000, 6096, 0.3656397759914398], "41": ["vertical_and_slash", 1000, 6096, 0.3683567941188812], "42": ["vertical_and_slash", 1000, 6096, 0.35285574197769165], "43": ["vertical_and_slash", 1000, 6096, 0.3531464636325836], "44": ["vertical_and_slash", 1000, 6096, 0.35900798439979553], "45": ["vertical_and_slash", 1000, 6096, 0.35990047454833984], "46": ["vertical_and_slash", 1000, 6096, 0.3644724190235138], "47": ["vertical_and_slash", 1000, 6096, 0.3629564344882965], "48": ["vertical_and_slash", 1000, 6096, 0.348752498626709], "49": ["vertical_and_slash", 1000, 6096, 0.3486400842666626], "50": ["vertical_and_slash", 1000, 6096, 0.34622979164123535], "51": ["vertical_and_slash", 1000, 6096, 0.35368436574935913], "52": ["vertical_and_slash", 1000, 6096, 0.35160955786705017], "53": ["vertical_and_slash", 1000, 6096, 0.3615584373474121], "54": ["vertical_and_slash", 1000, 6096, 0.34695038199424744], "55": ["vertical_and_slash", 1000, 6096, 0.35906386375427246], "56": ["vertical_and_slash", 1000, 6096, 0.3600368797779083], "57": ["vertical_and_slash", 1000, 6096, 0.38206207752227783], "58": ["vertical_and_slash", 1000, 6096, 0.35527360439300537], "59": ["vertical_and_slash", 1000, 6096, 0.356175035238266], "60": ["vertical_and_slash", 1000, 6096, 0.35742658376693726], "61": ["vertical_and_slash", 1000, 6096, 0.36786532402038574], "62": ["vertical_and_slash", 1000, 6096, 0.35248926281929016], "63": ["vertical_and_slash", 1000, 6096, 0.384623646736145]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3554617762565613], "1": ["vertical_and_slash", 1000, 6096, 0.3714543879032135], "2": ["vertical_and_slash", 1000, 6096, 0.3706928789615631], "3": ["vertical_and_slash", 1000, 6096, 0.36101585626602173], "4": ["vertical_and_slash", 1000, 6096, 0.3659609854221344], "5": ["vertical_and_slash", 1000, 6096, 0.3514208495616913], "6": ["vertical_and_slash", 1000, 6096, 0.3626091778278351], "7": ["vertical_and_slash", 1000, 6096, 0.3628176152706146], "8": ["vertical_and_slash", 1000, 6096, 0.35697469115257263], "9": ["vertical_and_slash", 1000, 6096, 0.36394134163856506], "10": ["vertical_and_slash", 1000, 6096, 0.3637913465499878], "11": ["vertical_and_slash", 1000, 6096, 0.3683392107486725], "12": ["vertical_and_slash", 1000, 6096, 0.3774445354938507], "13": ["vertical_and_slash", 1000, 6096, 0.36002957820892334], "14": ["vertical_and_slash", 1000, 6096, 0.3459097743034363], "15": ["vertical_and_slash", 1000, 6096, 0.3705976903438568], "16": ["vertical_and_slash", 1000, 6096, 0.3592061400413513], "17": ["vertical_and_slash", 1000, 6096, 0.35401925444602966], "18": ["vertical_and_slash", 1000, 6096, 0.35159358382225037], "19": ["vertical_and_slash", 1000, 6096, 0.363981157541275], "20": ["vertical_and_slash", 1000, 6096, 0.3634207248687744], "21": ["vertical_and_slash", 1000, 6096, 0.3604191839694977], "22": ["vertical_and_slash", 1000, 6096, 0.36510586738586426], "23": ["vertical_and_slash", 1000, 6096, 0.367746502161026], "24": ["vertical_and_slash", 1000, 6096, 0.3542253077030182], "25": ["vertical_and_slash", 1000, 6096, 0.3621029853820801], "26": ["vertical_and_slash", 1000, 6096, 0.35004016757011414], "27": ["vertical_and_slash", 1000, 6096, 0.3509881794452667], "28": ["vertical_and_slash", 1000, 6096, 0.347467839717865], "29": ["vertical_and_slash", 1000, 6096, 0.3602609634399414], "30": ["vertical_and_slash", 1000, 6096, 0.35076871514320374], "31": ["vertical_and_slash", 1000, 6096, 0.3526856601238251], "32": ["vertical_and_slash", 1000, 6096, 0.3709039092063904], "33": ["vertical_and_slash", 1000, 6096, 0.3740442097187042], "34": ["vertical_and_slash", 1000, 6096, 0.3649369478225708], "35": ["vertical_and_slash", 1000, 6096, 0.3512289524078369], "36": ["vertical_and_slash", 1000, 6096, 0.36608681082725525], "37": ["vertical_and_slash", 1000, 6096, 0.37111762166023254], "38": ["vertical_and_slash", 1000, 6096, 0.3573324680328369], "39": ["vertical_and_slash", 1000, 6096, 0.3631852865219116], "40": ["vertical_and_slash", 1000, 6096, 0.36970630288124084], "41": ["vertical_and_slash", 1000, 6096, 0.35694125294685364], "42": ["vertical_and_slash", 1000, 6096, 0.3547564148902893], "43": ["vertical_and_slash", 1000, 6096, 0.35424041748046875], "44": ["vertical_and_slash", 1000, 6096, 0.3795677125453949], "45": ["vertical_and_slash", 1000, 6096, 0.35800132155418396], "46": ["vertical_and_slash", 1000, 6096, 0.36726507544517517], "47": ["vertical_and_slash", 1000, 6096, 0.358242928981781], "48": ["vertical_and_slash", 1000, 6096, 0.3658578395843506], "49": ["vertical_and_slash", 1000, 6096, 0.3659604787826538], "50": ["vertical_and_slash", 1000, 6096, 0.3804788589477539], "51": ["vertical_and_slash", 1000, 6096, 0.3613625168800354], "52": ["vertical_and_slash", 1000, 6096, 0.35710474848747253], "53": ["vertical_and_slash", 1000, 6096, 0.3663804531097412], "54": ["vertical_and_slash", 1000, 6096, 0.37274810671806335], "55": ["vertical_and_slash", 1000, 6096, 0.36319804191589355], "56": ["vertical_and_slash", 1000, 6096, 0.3472869098186493], "57": ["vertical_and_slash", 1000, 6096, 0.350168377161026], "58": ["vertical_and_slash", 1000, 6096, 0.36493030190467834], "59": ["vertical_and_slash", 1000, 6096, 0.35300344228744507], "60": ["vertical_and_slash", 1000, 6096, 0.3494935631752014], "61": ["vertical_and_slash", 1000, 6096, 0.35829678177833557], "62": ["vertical_and_slash", 1000, 6096, 0.351550817489624], "63": ["vertical_and_slash", 1000, 6096, 0.34610000252723694]}, {"0": ["vertical_and_slash", 1000, 6096, 0.34432724118232727], "1": ["vertical_and_slash", 1000, 6096, 0.3464667797088623], "2": ["vertical_and_slash", 1000, 6096, 0.3489796221256256], "3": ["vertical_and_slash", 1000, 6096, 0.34802836179733276], "4": ["vertical_and_slash", 1000, 6096, 0.3481960594654083], "5": ["vertical_and_slash", 1000, 6096, 0.35208597779273987], "6": ["vertical_and_slash", 1000, 6096, 0.3517405688762665], "7": ["vertical_and_slash", 1000, 6096, 0.35712456703186035], "8": ["vertical_and_slash", 1000, 6096, 0.35675984621047974], "9": ["vertical_and_slash", 1000, 6096, 0.3572811782360077], "10": ["vertical_and_slash", 1000, 6096, 0.3529108464717865], "11": ["vertical_and_slash", 1000, 6096, 0.35679683089256287], "12": ["vertical_and_slash", 1000, 6096, 0.3513354957103729], "13": ["vertical_and_slash", 1000, 6096, 0.35289058089256287], "14": ["vertical_and_slash", 1000, 6096, 0.35361728072166443], "15": ["vertical_and_slash", 1000, 6096, 0.35416126251220703], "16": ["vertical_and_slash", 1000, 6096, 0.3406623303890228], "17": ["vertical_and_slash", 1000, 6096, 0.35824066400527954], "18": ["vertical_and_slash", 1000, 6096, 0.35510846972465515], "19": ["vertical_and_slash", 1000, 6096, 0.3543742299079895], "20": ["vertical_and_slash", 1000, 6096, 0.3562958836555481], "21": ["vertical_and_slash", 1000, 6096, 0.3584895730018616], "22": ["vertical_and_slash", 1000, 6096, 0.36537647247314453], "23": ["vertical_and_slash", 1000, 6096, 0.3502003252506256], "24": ["vertical_and_slash", 1000, 6096, 0.3525429368019104], "25": ["vertical_and_slash", 1000, 6096, 0.34808653593063354], "26": ["vertical_and_slash", 1000, 6096, 0.35099345445632935], "27": ["vertical_and_slash", 1000, 6096, 0.3494236469268799], "28": ["vertical_and_slash", 1000, 6096, 0.3442249894142151], "29": ["vertical_and_slash", 1000, 6096, 0.34115588665008545], "30": ["vertical_and_slash", 1000, 6096, 0.3449576795101166], "31": ["vertical_and_slash", 1000, 6096, 0.3471777141094208], "32": ["vertical_and_slash", 1000, 6096, 0.35742101073265076], "33": ["vertical_and_slash", 1000, 6096, 0.3670770227909088], "34": ["vertical_and_slash", 1000, 6096, 0.35268110036849976], "35": ["vertical_and_slash", 1000, 6096, 0.3571474850177765], "36": ["vertical_and_slash", 1000, 6096, 0.36417481303215027], "37": ["vertical_and_slash", 1000, 6096, 0.35884174704551697], "38": ["vertical_and_slash", 1000, 6096, 0.3607391119003296], "39": ["vertical_and_slash", 1000, 6096, 0.3605450987815857], "40": ["vertical_and_slash", 1000, 6096, 0.37814202904701233], "41": ["vertical_and_slash", 1000, 6096, 0.3622286915779114], "42": ["vertical_and_slash", 1000, 6096, 0.3618440330028534], "43": ["vertical_and_slash", 1000, 6096, 0.3444550931453705], "44": ["vertical_and_slash", 1000, 6096, 0.36682260036468506], "45": ["vertical_and_slash", 1000, 6096, 0.3761571943759918], "46": ["vertical_and_slash", 1000, 6096, 0.37696659564971924], "47": ["vertical_and_slash", 1000, 6096, 0.37234511971473694], "48": ["vertical_and_slash", 1000, 6096, 0.3523210287094116], "49": ["vertical_and_slash", 1000, 6096, 0.3563397526741028], "50": ["vertical_and_slash", 1000, 6096, 0.3701511025428772], "51": ["vertical_and_slash", 1000, 6096, 0.3716561794281006], "52": ["vertical_and_slash", 1000, 6096, 0.36448684334754944], "53": ["vertical_and_slash", 1000, 6096, 0.36408451199531555], "54": ["vertical_and_slash", 1000, 6096, 0.358694463968277], "55": ["vertical_and_slash", 1000, 6096, 0.3660464584827423], "56": ["vertical_and_slash", 1000, 6096, 0.3532056212425232], "57": ["vertical_and_slash", 1000, 6096, 0.3427134156227112], "58": ["vertical_and_slash", 1000, 6096, 0.36748698353767395], "59": ["vertical_and_slash", 1000, 6096, 0.3588809669017792], "60": ["vertical_and_slash", 1000, 6096, 0.3541371822357178], "61": ["vertical_and_slash", 1000, 6096, 0.3503529131412506], "62": ["vertical_and_slash", 1000, 6096, 0.3510768413543701], "63": ["vertical_and_slash", 1000, 6096, 0.3574109971523285]}, {"0": ["vertical_and_slash", 1000, 6096, 0.34764280915260315], "1": ["vertical_and_slash", 1000, 6096, 0.3553025722503662], "2": ["vertical_and_slash", 1000, 6096, 0.36146795749664307], "3": ["vertical_and_slash", 1000, 6096, 0.3431667983531952], "4": ["vertical_and_slash", 1000, 6096, 0.35751697421073914], "5": ["vertical_and_slash", 1000, 6096, 0.3563578426837921], "6": ["vertical_and_slash", 1000, 6096, 0.3508179187774658], "7": ["vertical_and_slash", 1000, 6096, 0.35259363055229187], "8": ["vertical_and_slash", 1000, 6096, 0.36348098516464233], "9": ["vertical_and_slash", 1000, 6096, 0.36099356412887573], "10": ["vertical_and_slash", 1000, 6096, 0.3530459403991699], "11": ["vertical_and_slash", 1000, 6096, 0.3535866141319275], "12": ["vertical_and_slash", 1000, 6096, 0.36309298872947693], "13": ["vertical_and_slash", 1000, 6096, 0.36333709955215454], "14": ["vertical_and_slash", 1000, 6096, 0.3485874831676483], "15": ["vertical_and_slash", 1000, 6096, 0.359643816947937], "16": ["vertical_and_slash", 1000, 6096, 0.35532501339912415], "17": ["vertical_and_slash", 1000, 6096, 0.3432304859161377], "18": ["vertical_and_slash", 1000, 6096, 0.3414149284362793], "19": ["vertical_and_slash", 1000, 6096, 0.34942278265953064], "20": ["vertical_and_slash", 1000, 6096, 0.33892586827278137], "21": ["vertical_and_slash", 1000, 6096, 0.3454037010669708], "22": ["vertical_and_slash", 1000, 6096, 0.3637683391571045], "23": ["vertical_and_slash", 1000, 6096, 0.35438698530197144], "24": ["vertical_and_slash", 1000, 6096, 0.3553435206413269], "25": ["vertical_and_slash", 1000, 6096, 0.3480622470378876], "26": ["vertical_and_slash", 1000, 6096, 0.34389230608940125], "27": ["vertical_and_slash", 1000, 6096, 0.3424895107746124], "28": ["vertical_and_slash", 1000, 6096, 0.34298592805862427], "29": ["vertical_and_slash", 1000, 6096, 0.34541377425193787], "30": ["vertical_and_slash", 1000, 6096, 0.34705451130867004], "31": ["vertical_and_slash", 1000, 6096, 0.3471730351448059], "32": ["vertical_and_slash", 1000, 6096, 0.34836483001708984], "33": ["vertical_and_slash", 1000, 6096, 0.3448792099952698], "34": ["vertical_and_slash", 1000, 6096, 0.34930700063705444], "35": ["vertical_and_slash", 1000, 6096, 0.34132060408592224], "36": ["vertical_and_slash", 1000, 6096, 0.34913164377212524], "37": ["vertical_and_slash", 1000, 6096, 0.35786524415016174], "38": ["vertical_and_slash", 1000, 6096, 0.3457883894443512], "39": ["vertical_and_slash", 1000, 6096, 0.3496552109718323], "40": ["vertical_and_slash", 1000, 6096, 0.3724355101585388], "41": ["vertical_and_slash", 1000, 6096, 0.3545190691947937], "42": ["vertical_and_slash", 1000, 6096, 0.3464403450489044], "43": ["vertical_and_slash", 1000, 6096, 0.3466481864452362], "44": ["vertical_and_slash", 1000, 6096, 0.35062170028686523], "45": ["vertical_and_slash", 1000, 6096, 0.34577232599258423], "46": ["vertical_and_slash", 1000, 6096, 0.35747799277305603], "47": ["vertical_and_slash", 1000, 6096, 0.3603668510913849], "48": ["vertical_and_slash", 1000, 6096, 0.3532966375350952], "49": ["vertical_and_slash", 1000, 6096, 0.34446266293525696], "50": ["vertical_and_slash", 1000, 6096, 0.35217180848121643], "51": ["vertical_and_slash", 1000, 6096, 0.3505837917327881], "52": ["vertical_and_slash", 1000, 6096, 0.35349851846694946], "53": ["vertical_and_slash", 1000, 6096, 0.3556930422782898], "54": ["vertical_and_slash", 1000, 6096, 0.33773455023765564], "55": ["vertical_and_slash", 1000, 6096, 0.3462047278881073], "56": ["vertical_and_slash", 1000, 6096, 0.3619225323200226], "57": ["vertical_and_slash", 1000, 6096, 0.3452504575252533], "58": ["vertical_and_slash", 1000, 6096, 0.3426881730556488], "59": ["vertical_and_slash", 1000, 6096, 0.36288192868232727], "60": ["vertical_and_slash", 1000, 6096, 0.3403901159763336], "61": ["vertical_and_slash", 1000, 6096, 0.3494422733783722], "62": ["vertical_and_slash", 1000, 6096, 0.34952718019485474], "63": ["vertical_and_slash", 1000, 6096, 0.34796565771102905]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3459327816963196], "1": ["vertical_and_slash", 1000, 6096, 0.3410843014717102], "2": ["vertical_and_slash", 1000, 6096, 0.3425574004650116], "3": ["vertical_and_slash", 1000, 6096, 0.34260931611061096], "4": ["vertical_and_slash", 1000, 6096, 0.34393376111984253], "5": ["vertical_and_slash", 1000, 6096, 0.3481740951538086], "6": ["vertical_and_slash", 1000, 6096, 0.35389724373817444], "7": ["vertical_and_slash", 1000, 6096, 0.34937620162963867], "8": ["vertical_and_slash", 1000, 6096, 0.34029558300971985], "9": ["vertical_and_slash", 1000, 6096, 0.3473105728626251], "10": ["vertical_and_slash", 1000, 6096, 0.3473002016544342], "11": ["vertical_and_slash", 1000, 6096, 0.34767448902130127], "12": ["vertical_and_slash", 1000, 6096, 0.3557590842247009], "13": ["vertical_and_slash", 1000, 6096, 0.34240567684173584], "14": ["vertical_and_slash", 1000, 6096, 0.3428141176700592], "15": ["vertical_and_slash", 1000, 6096, 0.3427167236804962], "16": ["vertical_and_slash", 1000, 6096, 0.35022908449172974], "17": ["vertical_and_slash", 1000, 6096, 0.3390810191631317], "18": ["vertical_and_slash", 1000, 6096, 0.3455875813961029], "19": ["vertical_and_slash", 1000, 6096, 0.3567105233669281], "20": ["vertical_and_slash", 1000, 6096, 0.35376840829849243], "21": ["vertical_and_slash", 1000, 6096, 0.3468630611896515], "22": ["vertical_and_slash", 1000, 6096, 0.3509548306465149], "23": ["vertical_and_slash", 1000, 6096, 0.34984976053237915], "24": ["vertical_and_slash", 1000, 6096, 0.3691045045852661], "25": ["vertical_and_slash", 1000, 6096, 0.360696017742157], "26": ["vertical_and_slash", 1000, 6096, 0.3672420382499695], "27": ["vertical_and_slash", 1000, 6096, 0.35771605372428894], "28": ["vertical_and_slash", 1000, 6096, 0.36204084753990173], "29": ["vertical_and_slash", 1000, 6096, 0.3630143702030182], "30": ["vertical_and_slash", 1000, 6096, 0.3594730794429779], "31": ["vertical_and_slash", 1000, 6096, 0.36875244975090027], "32": ["vertical_and_slash", 1000, 6096, 0.3551139533519745], "33": ["vertical_and_slash", 1000, 6096, 0.3582543432712555], "34": ["vertical_and_slash", 1000, 6096, 0.3558335304260254], "35": ["vertical_and_slash", 1000, 6096, 0.3534072935581207], "36": ["vertical_and_slash", 1000, 6096, 0.3486326038837433], "37": ["vertical_and_slash", 1000, 6096, 0.353522390127182], "38": ["vertical_and_slash", 1000, 6096, 0.36096492409706116], "39": ["vertical_and_slash", 1000, 6096, 0.3469035029411316], "40": ["vertical_and_slash", 1000, 6096, 0.3635653257369995], "41": ["vertical_and_slash", 1000, 6096, 0.347879558801651], "42": ["vertical_and_slash", 1000, 6096, 0.3509703576564789], "43": ["vertical_and_slash", 1000, 6096, 0.34847190976142883], "44": ["vertical_and_slash", 1000, 6096, 0.3432976305484772], "45": ["vertical_and_slash", 1000, 6096, 0.34260478615760803], "46": ["vertical_and_slash", 1000, 6096, 0.35007143020629883], "47": ["vertical_and_slash", 1000, 6096, 0.3443031907081604], "48": ["vertical_and_slash", 1000, 6096, 0.35166993737220764], "49": ["vertical_and_slash", 1000, 6096, 0.3492168188095093], "50": ["vertical_and_slash", 1000, 6096, 0.35814428329467773], "51": ["vertical_and_slash", 1000, 6096, 0.34855377674102783], "52": ["vertical_and_slash", 1000, 6096, 0.35336723923683167], "53": ["vertical_and_slash", 1000, 6096, 0.36824896931648254], "54": ["vertical_and_slash", 1000, 6096, 0.3457004725933075], "55": ["vertical_and_slash", 1000, 6096, 0.3596688508987427], "56": ["vertical_and_slash", 1000, 6096, 0.34920698404312134], "57": ["vertical_and_slash", 1000, 6096, 0.33919116854667664], "58": ["vertical_and_slash", 1000, 6096, 0.34763914346694946], "59": ["vertical_and_slash", 1000, 6096, 0.3515216112136841], "60": ["vertical_and_slash", 1000, 6096, 0.338532418012619], "61": ["vertical_and_slash", 1000, 6096, 0.35522371530532837], "62": ["vertical_and_slash", 1000, 6096, 0.3534988760948181], "63": ["vertical_and_slash", 1000, 6096, 0.3533090651035309]}, {"0": ["vertical_and_slash", 1000, 6096, 0.35221824049949646], "1": ["vertical_and_slash", 1000, 6096, 0.3491012156009674], "2": ["vertical_and_slash", 1000, 6096, 0.3549458384513855], "3": ["vertical_and_slash", 1000, 6096, 0.37354108691215515], "4": ["vertical_and_slash", 1000, 6096, 0.3521521985530853], "5": ["vertical_and_slash", 1000, 6096, 0.348406046628952], "6": ["vertical_and_slash", 1000, 6096, 0.3605857491493225], "7": ["vertical_and_slash", 1000, 6096, 0.34810033440589905], "8": ["vertical_and_slash", 1000, 6096, 0.3510175943374634], "9": ["vertical_and_slash", 1000, 6096, 0.34756600856781006], "10": ["vertical_and_slash", 1000, 6096, 0.35219985246658325], "11": ["vertical_and_slash", 1000, 6096, 0.3623124361038208], "12": ["vertical_and_slash", 1000, 6096, 0.35287731885910034], "13": ["vertical_and_slash", 1000, 6096, 0.3463342487812042], "14": ["vertical_and_slash", 1000, 6096, 0.3427533507347107], "15": ["vertical_and_slash", 1000, 6096, 0.3465835154056549], "16": ["vertical_and_slash", 1000, 6096, 0.3532663881778717], "17": ["vertical_and_slash", 1000, 6096, 0.3501710891723633], "18": ["vertical_and_slash", 1000, 6096, 0.35840627551078796], "19": ["vertical_and_slash", 1000, 6096, 0.35697141289711], "20": ["vertical_and_slash", 1000, 6096, 0.3443632125854492], "21": ["vertical_and_slash", 1000, 6096, 0.3491933047771454], "22": ["vertical_and_slash", 1000, 6096, 0.34688007831573486], "23": ["vertical_and_slash", 1000, 6096, 0.35349640250205994], "24": ["vertical_and_slash", 1000, 6096, 0.3556443750858307], "25": ["vertical_and_slash", 1000, 6096, 0.3573903441429138], "26": ["vertical_and_slash", 1000, 6096, 0.35858777165412903], "27": ["vertical_and_slash", 1000, 6096, 0.3628442585468292], "28": ["vertical_and_slash", 1000, 6096, 0.3676150143146515], "29": ["vertical_and_slash", 1000, 6096, 0.3625625669956207], "30": ["vertical_and_slash", 1000, 6096, 0.3588719964027405], "31": ["vertical_and_slash", 1000, 6096, 0.3458872139453888], "32": ["vertical_and_slash", 1000, 6096, 0.36003202199935913], "33": ["vertical_and_slash", 1000, 6096, 0.35756027698516846], "34": ["vertical_and_slash", 1000, 6096, 0.3496818244457245], "35": ["vertical_and_slash", 1000, 6096, 0.35343384742736816], "36": ["vertical_and_slash", 1000, 6096, 0.35637956857681274], "37": ["vertical_and_slash", 1000, 6096, 0.35602492094039917], "38": ["vertical_and_slash", 1000, 6096, 0.3588762581348419], "39": ["vertical_and_slash", 1000, 6096, 0.3521784842014313], "40": ["vertical_and_slash", 1000, 6096, 0.35313889384269714], "41": ["vertical_and_slash", 1000, 6096, 0.35535916686058044], "42": ["vertical_and_slash", 1000, 6096, 0.3611198365688324], "43": ["vertical_and_slash", 1000, 6096, 0.3495047390460968], "44": ["vertical_and_slash", 1000, 6096, 0.35576221346855164], "45": ["vertical_and_slash", 1000, 6096, 0.3527154326438904], "46": ["vertical_and_slash", 1000, 6096, 0.35097241401672363], "47": ["vertical_and_slash", 1000, 6096, 0.3484652638435364], "48": ["vertical_and_slash", 1000, 6096, 0.34861332178115845], "49": ["vertical_and_slash", 1000, 6096, 0.34920671582221985], "50": ["vertical_and_slash", 1000, 6096, 0.3588216006755829], "51": ["vertical_and_slash", 1000, 6096, 0.35237979888916016], "52": ["vertical_and_slash", 1000, 6096, 0.35864996910095215], "53": ["vertical_and_slash", 1000, 6096, 0.3515474200248718], "54": ["vertical_and_slash", 1000, 6096, 0.348897784948349], "55": ["vertical_and_slash", 1000, 6096, 0.35443463921546936], "56": ["vertical_and_slash", 1000, 6096, 0.35284748673439026], "57": ["vertical_and_slash", 1000, 6096, 0.3562506437301636], "58": ["vertical_and_slash", 1000, 6096, 0.35757559537887573], "59": ["vertical_and_slash", 1000, 6096, 0.34844326972961426], "60": ["vertical_and_slash", 1000, 6096, 0.34873706102371216], "61": ["vertical_and_slash", 1000, 6096, 0.3595990538597107], "62": ["vertical_and_slash", 1000, 6096, 0.34424206614494324], "63": ["vertical_and_slash", 1000, 6096, 0.3565748631954193]}, {"0": ["vertical_and_slash", 1000, 6096, 0.36240333318710327], "1": ["vertical_and_slash", 1000, 6096, 0.36902666091918945], "2": ["vertical_and_slash", 1000, 6096, 0.3605616092681885], "3": ["vertical_and_slash", 1000, 6096, 0.358895480632782], "4": ["vertical_and_slash", 1000, 6096, 0.35324278473854065], "5": ["vertical_and_slash", 1000, 6096, 0.3498997092247009], "6": ["vertical_and_slash", 1000, 6096, 0.34801727533340454], "7": ["vertical_and_slash", 1000, 6096, 0.35900744795799255], "8": ["vertical_and_slash", 1000, 6096, 0.38232043385505676], "9": ["vertical_and_slash", 1000, 6096, 0.3772366940975189], "10": ["vertical_and_slash", 1000, 6096, 0.3777731955051422], "11": ["vertical_and_slash", 1000, 6096, 0.37771135568618774], "12": ["vertical_and_slash", 1000, 6096, 0.3814780116081238], "13": ["vertical_and_slash", 1000, 6096, 0.3840600252151489], "14": ["vertical_and_slash", 1000, 6096, 0.3865370452404022], "15": ["vertical_and_slash", 1000, 6096, 0.3705802857875824], "16": ["vertical_and_slash", 1000, 6096, 0.34367042779922485], "17": ["vertical_and_slash", 1000, 6096, 0.36026284098625183], "18": ["vertical_and_slash", 1000, 6096, 0.3525301516056061], "19": ["vertical_and_slash", 1000, 6096, 0.35771554708480835], "20": ["vertical_and_slash", 1000, 6096, 0.37957772612571716], "21": ["vertical_and_slash", 1000, 6096, 0.36459341645240784], "22": ["vertical_and_slash", 1000, 6096, 0.3618621826171875], "23": ["vertical_and_slash", 1000, 6096, 0.34835726022720337], "24": ["vertical_and_slash", 1000, 6096, 0.3629109561443329], "25": ["vertical_and_slash", 1000, 6096, 0.3537841737270355], "26": ["vertical_and_slash", 1000, 6096, 0.3658030331134796], "27": ["vertical_and_slash", 1000, 6096, 0.354430228471756], "28": ["vertical_and_slash", 1000, 6096, 0.3565593957901001], "29": ["vertical_and_slash", 1000, 6096, 0.34543269872665405], "30": ["vertical_and_slash", 1000, 6096, 0.36171960830688477], "31": ["vertical_and_slash", 1000, 6096, 0.37722378969192505], "32": ["vertical_and_slash", 1000, 6096, 0.35732731223106384], "33": ["vertical_and_slash", 1000, 6096, 0.3577975630760193], "34": ["vertical_and_slash", 1000, 6096, 0.35206735134124756], "35": ["vertical_and_slash", 1000, 6096, 0.3868968188762665], "36": ["vertical_and_slash", 1000, 6096, 0.3453362286090851], "37": ["vertical_and_slash", 1000, 6096, 0.3762916028499603], "38": ["vertical_and_slash", 1000, 6096, 0.36835888028144836], "39": ["vertical_and_slash", 1000, 6096, 0.3589845895767212], "40": ["vertical_and_slash", 1000, 6096, 0.35398104786872864], "41": ["vertical_and_slash", 1000, 6096, 0.36083513498306274], "42": ["vertical_and_slash", 1000, 6096, 0.35398516058921814], "43": ["vertical_and_slash", 1000, 6096, 0.34927141666412354], "44": ["vertical_and_slash", 1000, 6096, 0.3546198904514313], "45": ["vertical_and_slash", 1000, 6096, 0.35189634561538696], "46": ["vertical_and_slash", 1000, 6096, 0.3464338779449463], "47": ["vertical_and_slash", 1000, 6096, 0.35696941614151], "48": ["vertical_and_slash", 1000, 6096, 0.37262997031211853], "49": ["vertical_and_slash", 1000, 6096, 0.37998321652412415], "50": ["vertical_and_slash", 1000, 6096, 0.36509281396865845], "51": ["vertical_and_slash", 1000, 6096, 0.3824317157268524], "52": ["vertical_and_slash", 1000, 6096, 0.369588166475296], "53": ["vertical_and_slash", 1000, 6096, 0.36407729983329773], "54": ["vertical_and_slash", 1000, 6096, 0.37130528688430786], "55": ["vertical_and_slash", 1000, 6096, 0.36753901839256287], "56": ["vertical_and_slash", 1000, 6096, 0.3644733130931854], "57": ["vertical_and_slash", 1000, 6096, 0.34659770131111145], "58": ["vertical_and_slash", 1000, 6096, 0.357665479183197], "59": ["vertical_and_slash", 1000, 6096, 0.36396846175193787], "60": ["vertical_and_slash", 1000, 6096, 0.3799161911010742], "61": ["vertical_and_slash", 1000, 6096, 0.3769533336162567], "62": ["vertical_and_slash", 1000, 6096, 0.36944034695625305], "63": ["vertical_and_slash", 1000, 6096, 0.376398503780365]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3596423864364624], "1": ["vertical_and_slash", 1000, 6096, 0.35582417249679565], "2": ["vertical_and_slash", 1000, 6096, 0.3580166697502136], "3": ["vertical_and_slash", 1000, 6096, 0.355632483959198], "4": ["vertical_and_slash", 1000, 6096, 0.3598264753818512], "5": ["vertical_and_slash", 1000, 6096, 0.3594970107078552], "6": ["vertical_and_slash", 1000, 6096, 0.3698621094226837], "7": ["vertical_and_slash", 1000, 6096, 0.3585624694824219], "8": ["vertical_and_slash", 1000, 6096, 0.36060258746147156], "9": ["vertical_and_slash", 1000, 6096, 0.3683858811855316], "10": ["vertical_and_slash", 1000, 6096, 0.3514469563961029], "11": ["vertical_and_slash", 1000, 6096, 0.36848750710487366], "12": ["vertical_and_slash", 1000, 6096, 0.3629189133644104], "13": ["vertical_and_slash", 1000, 6096, 0.36470934748649597], "14": ["vertical_and_slash", 1000, 6096, 0.36319875717163086], "15": ["vertical_and_slash", 1000, 6096, 0.3673321008682251], "16": ["vertical_and_slash", 1000, 6096, 0.3570441007614136], "17": ["vertical_and_slash", 1000, 6096, 0.3655947148799896], "18": ["vertical_and_slash", 1000, 6096, 0.35471320152282715], "19": ["vertical_and_slash", 1000, 6096, 0.3621051013469696], "20": ["vertical_and_slash", 1000, 6096, 0.3634290397167206], "21": ["vertical_and_slash", 1000, 6096, 0.35720810294151306], "22": ["vertical_and_slash", 1000, 6096, 0.3562961518764496], "23": ["vertical_and_slash", 1000, 6096, 0.36018916964530945], "24": ["vertical_and_slash", 1000, 6096, 0.36146417260169983], "25": ["vertical_and_slash", 1000, 6096, 0.38165074586868286], "26": ["vertical_and_slash", 1000, 6096, 0.36637696623802185], "27": ["vertical_and_slash", 1000, 6096, 0.36359351873397827], "28": ["vertical_and_slash", 1000, 6096, 0.3592822551727295], "29": ["vertical_and_slash", 1000, 6096, 0.3537937104701996], "30": ["vertical_and_slash", 1000, 6096, 0.3523446023464203], "31": ["vertical_and_slash", 1000, 6096, 0.3766748011112213], "32": ["vertical_and_slash", 1000, 6096, 0.3799092471599579], "33": ["vertical_and_slash", 1000, 6096, 0.3723529577255249], "34": ["vertical_and_slash", 1000, 6096, 0.3713177442550659], "35": ["vertical_and_slash", 1000, 6096, 0.3690522015094757], "36": ["vertical_and_slash", 1000, 6096, 0.371173620223999], "37": ["vertical_and_slash", 1000, 6096, 0.37997135519981384], "38": ["vertical_and_slash", 1000, 6096, 0.36946719884872437], "39": ["vertical_and_slash", 1000, 6096, 0.37480488419532776], "40": ["vertical_and_slash", 1000, 6096, 0.3573537766933441], "41": ["vertical_and_slash", 1000, 6096, 0.36681169271469116], "42": ["vertical_and_slash", 1000, 6096, 0.35392650961875916], "43": ["vertical_and_slash", 1000, 6096, 0.3645727038383484], "44": ["vertical_and_slash", 1000, 6096, 0.35916629433631897], "45": ["vertical_and_slash", 1000, 6096, 0.3608412444591522], "46": ["vertical_and_slash", 1000, 6096, 0.35125771164894104], "47": ["vertical_and_slash", 1000, 6096, 0.35786470770835876], "48": ["vertical_and_slash", 1000, 6096, 0.36795058846473694], "49": ["vertical_and_slash", 1000, 6096, 0.36208993196487427], "50": ["vertical_and_slash", 1000, 6096, 0.3787441849708557], "51": ["vertical_and_slash", 1000, 6096, 0.3845212459564209], "52": ["vertical_and_slash", 1000, 6096, 0.35535699129104614], "53": ["vertical_and_slash", 1000, 6096, 0.3593083918094635], "54": ["vertical_and_slash", 1000, 6096, 0.38576507568359375], "55": ["vertical_and_slash", 1000, 6096, 0.3688866198062897], "56": ["vertical_and_slash", 1000, 6096, 0.35152530670166016], "57": ["vertical_and_slash", 1000, 6096, 0.35499903559684753], "58": ["vertical_and_slash", 1000, 6096, 0.34953415393829346], "59": ["vertical_and_slash", 1000, 6096, 0.35465121269226074], "60": ["vertical_and_slash", 1000, 6096, 0.35641518235206604], "61": ["vertical_and_slash", 1000, 6096, 0.35429444909095764], "62": ["vertical_and_slash", 1000, 6096, 0.34614673256874084], "63": ["vertical_and_slash", 1000, 6096, 0.34968301653862]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3838596045970917], "1": ["vertical_and_slash", 1000, 6096, 0.3740938901901245], "2": ["vertical_and_slash", 1000, 6096, 0.3786274492740631], "3": ["vertical_and_slash", 1000, 6096, 0.367709755897522], "4": ["vertical_and_slash", 1000, 6096, 0.37714603543281555], "5": ["vertical_and_slash", 1000, 6096, 0.3726128041744232], "6": ["vertical_and_slash", 1000, 6096, 0.3933922350406647], "7": ["vertical_and_slash", 1000, 6096, 0.3892150819301605], "8": ["vertical_and_slash", 1000, 6096, 0.3596142530441284], "9": ["vertical_and_slash", 1000, 6096, 0.36803463101387024], "10": ["vertical_and_slash", 1000, 6096, 0.38924840092658997], "11": ["vertical_and_slash", 1000, 6096, 0.37759968638420105], "12": ["vertical_and_slash", 1000, 6096, 0.3748927116394043], "13": ["vertical_and_slash", 1000, 6096, 0.383086234331131], "14": ["vertical_and_slash", 1000, 6096, 0.36485934257507324], "15": ["vertical_and_slash", 1000, 6096, 0.3631777763366699], "16": ["vertical_and_slash", 1000, 6096, 0.4012652039527893], "17": ["vertical_and_slash", 1000, 6096, 0.3918410837650299], "18": ["vertical_and_slash", 1000, 6096, 0.3885546624660492], "19": ["vertical_and_slash", 1000, 6096, 0.3806963562965393], "20": ["vertical_and_slash", 1000, 6096, 0.40124863386154175], "21": ["vertical_and_slash", 1000, 6096, 0.3846050798892975], "22": ["vertical_and_slash", 1000, 6096, 0.39422282576560974], "23": ["vertical_and_slash", 1000, 6096, 0.4071311354637146], "24": ["vertical_and_slash", 1000, 6096, 0.3700668215751648], "25": ["vertical_and_slash", 1000, 6096, 0.35925909876823425], "26": ["vertical_and_slash", 1000, 6096, 0.3538868725299835], "27": ["vertical_and_slash", 1000, 6096, 0.40584176778793335], "28": ["vertical_and_slash", 1000, 6096, 0.3800268769264221], "29": ["vertical_and_slash", 1000, 6096, 0.36459800601005554], "30": ["vertical_and_slash", 1000, 6096, 0.3680082857608795], "31": ["vertical_and_slash", 1000, 6096, 0.3662860095500946], "32": ["vertical_and_slash", 1000, 6096, 0.394516259431839], "33": ["vertical_and_slash", 1000, 6096, 0.39218950271606445], "34": ["vertical_and_slash", 1000, 6096, 0.3897556960582733], "35": ["vertical_and_slash", 1000, 6096, 0.3613056540489197], "36": ["vertical_and_slash", 1000, 6096, 0.374556303024292], "37": ["vertical_and_slash", 1000, 6096, 0.3828064501285553], "38": ["vertical_and_slash", 1000, 6096, 0.3691197633743286], "39": ["vertical_and_slash", 1000, 6096, 0.37013816833496094], "40": ["vertical_and_slash", 1000, 6096, 0.3794548809528351], "41": ["vertical_and_slash", 1000, 6096, 0.3988923132419586], "42": ["vertical_and_slash", 1000, 6096, 0.38295507431030273], "43": ["vertical_and_slash", 1000, 6096, 0.37717360258102417], "44": ["vertical_and_slash", 1000, 6096, 0.3743228614330292], "45": ["vertical_and_slash", 1000, 6096, 0.3910106122493744], "46": ["vertical_and_slash", 1000, 6096, 0.3862645924091339], "47": ["vertical_and_slash", 1000, 6096, 0.39562124013900757], "48": ["vertical_and_slash", 1000, 6096, 0.39821261167526245], "49": ["vertical_and_slash", 1000, 6096, 0.37693631649017334], "50": ["vertical_and_slash", 1000, 6096, 0.3745819628238678], "51": ["vertical_and_slash", 1000, 6096, 0.3649611473083496], "52": ["vertical_and_slash", 1000, 6096, 0.3873306214809418], "53": ["vertical_and_slash", 1000, 6096, 0.3824148178100586], "54": ["vertical_and_slash", 1000, 6096, 0.39456576108932495], "55": ["vertical_and_slash", 1000, 6096, 0.3824026882648468], "56": ["vertical_and_slash", 1000, 6096, 0.38055750727653503], "57": ["vertical_and_slash", 1000, 6096, 0.3834984004497528], "58": ["vertical_and_slash", 1000, 6096, 0.3823384642601013], "59": ["vertical_and_slash", 1000, 6096, 0.39128875732421875], "60": ["vertical_and_slash", 1000, 6096, 0.37547820806503296], "61": ["vertical_and_slash", 1000, 6096, 0.3749816119670868], "62": ["vertical_and_slash", 1000, 6096, 0.3682064414024353], "63": ["vertical_and_slash", 1000, 6096, 0.37944111227989197]}, {"0": ["vertical_and_slash", 1000, 6096, 0.35705509781837463], "1": ["vertical_and_slash", 1000, 6096, 0.3716038167476654], "2": ["vertical_and_slash", 1000, 6096, 0.37556740641593933], "3": ["vertical_and_slash", 1000, 6096, 0.36181002855300903], "4": ["vertical_and_slash", 1000, 6096, 0.36510223150253296], "5": ["vertical_and_slash", 1000, 6096, 0.3555774390697479], "6": ["vertical_and_slash", 1000, 6096, 0.4002254605293274], "7": ["vertical_and_slash", 1000, 6096, 0.3812215328216553], "8": ["vertical_and_slash", 1000, 6096, 0.3643884062767029], "9": ["vertical_and_slash", 1000, 6096, 0.3509656488895416], "10": ["vertical_and_slash", 1000, 6096, 0.3535201847553253], "11": ["vertical_and_slash", 1000, 6096, 0.3601274788379669], "12": ["vertical_and_slash", 1000, 6096, 0.3539707064628601], "13": ["vertical_and_slash", 1000, 6096, 0.3596813678741455], "14": ["vertical_and_slash", 1000, 6096, 0.36038196086883545], "15": ["vertical_and_slash", 1000, 6096, 0.35906335711479187], "16": ["vertical_and_slash", 1000, 6096, 0.36838752031326294], "17": ["vertical_and_slash", 1000, 6096, 0.3571229577064514], "18": ["vertical_and_slash", 1000, 6096, 0.37303799390792847], "19": ["vertical_and_slash", 1000, 6096, 0.3710329234600067], "20": ["vertical_and_slash", 1000, 6096, 0.3571045696735382], "21": ["vertical_and_slash", 1000, 6096, 0.36434778571128845], "22": ["vertical_and_slash", 1000, 6096, 0.34811994433403015], "23": ["vertical_and_slash", 1000, 6096, 0.3708297610282898], "24": ["vertical_and_slash", 1000, 6096, 0.3542211651802063], "25": ["vertical_and_slash", 1000, 6096, 0.35113218426704407], "26": ["vertical_and_slash", 1000, 6096, 0.34948691725730896], "27": ["vertical_and_slash", 1000, 6096, 0.3700411021709442], "28": ["vertical_and_slash", 1000, 6096, 0.3533724546432495], "29": ["vertical_and_slash", 1000, 6096, 0.36216479539871216], "30": ["vertical_and_slash", 1000, 6096, 0.34435147047042847], "31": ["vertical_and_slash", 1000, 6096, 0.34692859649658203], "32": ["vertical_and_slash", 1000, 6096, 0.36148643493652344], "33": ["vertical_and_slash", 1000, 6096, 0.36518511176109314], "34": ["vertical_and_slash", 1000, 6096, 0.357986718416214], "35": ["vertical_and_slash", 1000, 6096, 0.3561370372772217], "36": ["vertical_and_slash", 1000, 6096, 0.3517566919326782], "37": ["vertical_and_slash", 1000, 6096, 0.35763266682624817], "38": ["vertical_and_slash", 1000, 6096, 0.3597792387008667], "39": ["vertical_and_slash", 1000, 6096, 0.354809433221817], "40": ["vertical_and_slash", 1000, 6096, 0.36109134554862976], "41": ["vertical_and_slash", 1000, 6096, 0.35900816321372986], "42": ["vertical_and_slash", 1000, 6096, 0.3612315356731415], "43": ["vertical_and_slash", 1000, 6096, 0.3507382273674011], "44": ["vertical_and_slash", 1000, 6096, 0.35325556993484497], "45": ["vertical_and_slash", 1000, 6096, 0.3486628830432892], "46": ["vertical_and_slash", 1000, 6096, 0.36012163758277893], "47": ["vertical_and_slash", 1000, 6096, 0.36915329098701477], "48": ["vertical_and_slash", 1000, 6096, 0.35535377264022827], "49": ["vertical_and_slash", 1000, 6096, 0.357988566160202], "50": ["vertical_and_slash", 1000, 6096, 0.3592826724052429], "51": ["vertical_and_slash", 1000, 6096, 0.35916879773139954], "52": ["vertical_and_slash", 1000, 6096, 0.36451825499534607], "53": ["vertical_and_slash", 1000, 6096, 0.365312784910202], "54": ["vertical_and_slash", 1000, 6096, 0.3569882810115814], "55": ["vertical_and_slash", 1000, 6096, 0.352722704410553], "56": ["vertical_and_slash", 1000, 6096, 0.348143070936203], "57": ["vertical_and_slash", 1000, 6096, 0.35359612107276917], "58": ["vertical_and_slash", 1000, 6096, 0.3530929684638977], "59": ["vertical_and_slash", 1000, 6096, 0.35608524084091187], "60": ["vertical_and_slash", 1000, 6096, 0.3460351824760437], "61": ["vertical_and_slash", 1000, 6096, 0.35466209053993225], "62": ["vertical_and_slash", 1000, 6096, 0.36423176527023315], "63": ["vertical_and_slash", 1000, 6096, 0.3533730208873749]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3931778371334076], "1": ["vertical_and_slash", 1000, 6096, 0.3642834424972534], "2": ["vertical_and_slash", 1000, 6096, 0.3538595139980316], "3": ["vertical_and_slash", 1000, 6096, 0.36763763427734375], "4": ["vertical_and_slash", 1000, 6096, 0.368921160697937], "5": ["vertical_and_slash", 1000, 6096, 0.36902061104774475], "6": ["vertical_and_slash", 1000, 6096, 0.36862459778785706], "7": ["vertical_and_slash", 1000, 6096, 0.3720802366733551], "8": ["vertical_and_slash", 1000, 6096, 0.36858537793159485], "9": ["vertical_and_slash", 1000, 6096, 0.3816128373146057], "10": ["vertical_and_slash", 1000, 6096, 0.37401217222213745], "11": ["vertical_and_slash", 1000, 6096, 0.37748757004737854], "12": ["vertical_and_slash", 1000, 6096, 0.3699641227722168], "13": ["vertical_and_slash", 1000, 6096, 0.36546263098716736], "14": ["vertical_and_slash", 1000, 6096, 0.36817702651023865], "15": ["vertical_and_slash", 1000, 6096, 0.3684198558330536], "16": ["vertical_and_slash", 1000, 6096, 0.34794965386390686], "17": ["vertical_and_slash", 1000, 6096, 0.35444843769073486], "18": ["vertical_and_slash", 1000, 6096, 0.36965125799179077], "19": ["vertical_and_slash", 1000, 6096, 0.37177175283432007], "20": ["vertical_and_slash", 1000, 6096, 0.3600515127182007], "21": ["vertical_and_slash", 1000, 6096, 0.35639816522598267], "22": ["vertical_and_slash", 1000, 6096, 0.368266224861145], "23": ["vertical_and_slash", 1000, 6096, 0.3539508283138275], "24": ["vertical_and_slash", 1000, 6096, 0.35987046360969543], "25": ["vertical_and_slash", 1000, 6096, 0.3850284516811371], "26": ["vertical_and_slash", 1000, 6096, 0.3628278076648712], "27": ["vertical_and_slash", 1000, 6096, 0.36719945073127747], "28": ["vertical_and_slash", 1000, 6096, 0.3714434802532196], "29": ["vertical_and_slash", 1000, 6096, 0.35934627056121826], "30": ["vertical_and_slash", 1000, 6096, 0.3800221085548401], "31": ["vertical_and_slash", 1000, 6096, 0.3706747591495514], "32": ["vertical_and_slash", 1000, 6096, 0.36730533838272095], "33": ["vertical_and_slash", 1000, 6096, 0.3612070381641388], "34": ["vertical_and_slash", 1000, 6096, 0.3533485233783722], "35": ["vertical_and_slash", 1000, 6096, 0.36042720079421997], "36": ["vertical_and_slash", 1000, 6096, 0.35477471351623535], "37": ["vertical_and_slash", 1000, 6096, 0.3655409514904022], "38": ["vertical_and_slash", 1000, 6096, 0.3696565330028534], "39": ["vertical_and_slash", 1000, 6096, 0.37737277150154114], "40": ["vertical_and_slash", 1000, 6096, 0.35743799805641174], "41": ["vertical_and_slash", 1000, 6096, 0.366700679063797], "42": ["vertical_and_slash", 1000, 6096, 0.35888656973838806], "43": ["vertical_and_slash", 1000, 6096, 0.35339823365211487], "44": ["vertical_and_slash", 1000, 6096, 0.36100104451179504], "45": ["vertical_and_slash", 1000, 6096, 0.35502028465270996], "46": ["vertical_and_slash", 1000, 6096, 0.3603726923465729], "47": ["vertical_and_slash", 1000, 6096, 0.3679773509502411], "48": ["vertical_and_slash", 1000, 6096, 0.3771432042121887], "49": ["vertical_and_slash", 1000, 6096, 0.3680749833583832], "50": ["vertical_and_slash", 1000, 6096, 0.36326372623443604], "51": ["vertical_and_slash", 1000, 6096, 0.39025411009788513], "52": ["vertical_and_slash", 1000, 6096, 0.35703593492507935], "53": ["vertical_and_slash", 1000, 6096, 0.3687364161014557], "54": ["vertical_and_slash", 1000, 6096, 0.37020066380500793], "55": ["vertical_and_slash", 1000, 6096, 0.37313100695610046], "56": ["vertical_and_slash", 1000, 6096, 0.3619880676269531], "57": ["vertical_and_slash", 1000, 6096, 0.36600005626678467], "58": ["vertical_and_slash", 1000, 6096, 0.37536153197288513], "59": ["vertical_and_slash", 1000, 6096, 0.38637101650238037], "60": ["vertical_and_slash", 1000, 6096, 0.3636045455932617], "61": ["vertical_and_slash", 1000, 6096, 0.360166996717453], "62": ["vertical_and_slash", 1000, 6096, 0.37850505113601685], "63": ["vertical_and_slash", 1000, 6096, 0.36676937341690063]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3548937737941742], "1": ["vertical_and_slash", 1000, 6096, 0.36738303303718567], "2": ["vertical_and_slash", 1000, 6096, 0.3612668216228485], "3": ["vertical_and_slash", 1000, 6096, 0.36427491903305054], "4": ["vertical_and_slash", 1000, 6096, 0.3638644516468048], "5": ["vertical_and_slash", 1000, 6096, 0.35370346903800964], "6": ["vertical_and_slash", 1000, 6096, 0.3696232736110687], "7": ["vertical_and_slash", 1000, 6096, 0.36269131302833557], "8": ["vertical_and_slash", 1000, 6096, 0.36273589730262756], "9": ["vertical_and_slash", 1000, 6096, 0.3557991683483124], "10": ["vertical_and_slash", 1000, 6096, 0.36523130536079407], "11": ["vertical_and_slash", 1000, 6096, 0.35788318514823914], "12": ["vertical_and_slash", 1000, 6096, 0.3633737862110138], "13": ["vertical_and_slash", 1000, 6096, 0.35715389251708984], "14": ["vertical_and_slash", 1000, 6096, 0.3777281641960144], "15": ["vertical_and_slash", 1000, 6096, 0.36494290828704834], "16": ["vertical_and_slash", 1000, 6096, 0.3488314747810364], "17": ["vertical_and_slash", 1000, 6096, 0.3668014407157898], "18": ["vertical_and_slash", 1000, 6096, 0.35723841190338135], "19": ["vertical_and_slash", 1000, 6096, 0.3630497455596924], "20": ["vertical_and_slash", 1000, 6096, 0.36350610852241516], "21": ["vertical_and_slash", 1000, 6096, 0.36361217498779297], "22": ["vertical_and_slash", 1000, 6096, 0.36266276240348816], "23": ["vertical_and_slash", 1000, 6096, 0.35679030418395996], "24": ["vertical_and_slash", 1000, 6096, 0.3621756136417389], "25": ["vertical_and_slash", 1000, 6096, 0.35286587476730347], "26": ["vertical_and_slash", 1000, 6096, 0.3639208972454071], "27": ["vertical_and_slash", 1000, 6096, 0.3519850969314575], "28": ["vertical_and_slash", 1000, 6096, 0.3610385060310364], "29": ["vertical_and_slash", 1000, 6096, 0.36866331100463867], "30": ["vertical_and_slash", 1000, 6096, 0.36765778064727783], "31": ["vertical_and_slash", 1000, 6096, 0.3584417402744293], "32": ["vertical_and_slash", 1000, 6096, 0.36715859174728394], "33": ["vertical_and_slash", 1000, 6096, 0.36040839552879333], "34": ["vertical_and_slash", 1000, 6096, 0.3551597595214844], "35": ["vertical_and_slash", 1000, 6096, 0.35368868708610535], "36": ["vertical_and_slash", 1000, 6096, 0.3622777462005615], "37": ["vertical_and_slash", 1000, 6096, 0.3710472285747528], "38": ["vertical_and_slash", 1000, 6096, 0.35943904519081116], "39": ["vertical_and_slash", 1000, 6096, 0.36313191056251526], "40": ["vertical_and_slash", 1000, 6096, 0.3677571713924408], "41": ["vertical_and_slash", 1000, 6096, 0.3536376655101776], "42": ["vertical_and_slash", 1000, 6096, 0.36021164059638977], "43": ["vertical_and_slash", 1000, 6096, 0.3610834777355194], "44": ["vertical_and_slash", 1000, 6096, 0.3667569160461426], "45": ["vertical_and_slash", 1000, 6096, 0.358430415391922], "46": ["vertical_and_slash", 1000, 6096, 0.38498231768608093], "47": ["vertical_and_slash", 1000, 6096, 0.35477393865585327], "48": ["vertical_and_slash", 1000, 6096, 0.36701440811157227], "49": ["vertical_and_slash", 1000, 6096, 0.35579514503479004], "50": ["vertical_and_slash", 1000, 6096, 0.38987258076667786], "51": ["vertical_and_slash", 1000, 6096, 0.34733253717422485], "52": ["vertical_and_slash", 1000, 6096, 0.3638548254966736], "53": ["vertical_and_slash", 1000, 6096, 0.36266210675239563], "54": ["vertical_and_slash", 1000, 6096, 0.36965125799179077], "55": ["vertical_and_slash", 1000, 6096, 0.36654898524284363], "56": ["vertical_and_slash", 1000, 6096, 0.37689706683158875], "57": ["vertical_and_slash", 1000, 6096, 0.38585710525512695], "58": ["vertical_and_slash", 1000, 6096, 0.3647611439228058], "59": ["vertical_and_slash", 1000, 6096, 0.3626324236392975], "60": ["vertical_and_slash", 1000, 6096, 0.36868584156036377], "61": ["vertical_and_slash", 1000, 6096, 0.3690192401409149], "62": ["vertical_and_slash", 1000, 6096, 0.36659443378448486], "63": ["vertical_and_slash", 1000, 6096, 0.3758666515350342]}, {"0": ["vertical_and_slash", 1000, 6096, 0.36850395798683167], "1": ["vertical_and_slash", 1000, 6096, 0.3581145405769348], "2": ["vertical_and_slash", 1000, 6096, 0.36827608942985535], "3": ["vertical_and_slash", 1000, 6096, 0.36597394943237305], "4": ["vertical_and_slash", 1000, 6096, 0.3910580277442932], "5": ["vertical_and_slash", 1000, 6096, 0.3778153657913208], "6": ["vertical_and_slash", 1000, 6096, 0.36965054273605347], "7": ["vertical_and_slash", 1000, 6096, 0.36731308698654175], "8": ["vertical_and_slash", 1000, 6096, 0.396945595741272], "9": ["vertical_and_slash", 1000, 6096, 0.37278303503990173], "10": ["vertical_and_slash", 1000, 6096, 0.3733264207839966], "11": ["vertical_and_slash", 1000, 6096, 0.36181527376174927], "12": ["vertical_and_slash", 1000, 6096, 0.36614060401916504], "13": ["vertical_and_slash", 1000, 6096, 0.3702528774738312], "14": ["vertical_and_slash", 1000, 6096, 0.3768591284751892], "15": ["vertical_and_slash", 1000, 6096, 0.36329689621925354], "16": ["vertical_and_slash", 1000, 6096, 0.36512449383735657], "17": ["vertical_and_slash", 1000, 6096, 0.3830595314502716], "18": ["vertical_and_slash", 1000, 6096, 0.3838402032852173], "19": ["vertical_and_slash", 1000, 6096, 0.3613676130771637], "20": ["vertical_and_slash", 1000, 6096, 0.3807452619075775], "21": ["vertical_and_slash", 1000, 6096, 0.3827093243598938], "22": ["vertical_and_slash", 1000, 6096, 0.3725214898586273], "23": ["vertical_and_slash", 1000, 6096, 0.3759484887123108], "24": ["vertical_and_slash", 1000, 6096, 0.358857125043869], "25": ["vertical_and_slash", 1000, 6096, 0.3818485140800476], "26": ["vertical_and_slash", 1000, 6096, 0.3606180250644684], "27": ["vertical_and_slash", 1000, 6096, 0.36270400881767273], "28": ["vertical_and_slash", 1000, 6096, 0.3581010103225708], "29": ["vertical_and_slash", 1000, 6096, 0.3554738759994507], "30": ["vertical_and_slash", 1000, 6096, 0.3617974519729614], "31": ["vertical_and_slash", 1000, 6096, 0.35834309458732605], "32": ["vertical_and_slash", 1000, 6096, 0.38094067573547363], "33": ["vertical_and_slash", 1000, 6096, 0.3771992325782776], "34": ["vertical_and_slash", 1000, 6096, 0.3670611083507538], "35": ["vertical_and_slash", 1000, 6096, 0.36383306980133057], "36": ["vertical_and_slash", 1000, 6096, 0.36192139983177185], "37": ["vertical_and_slash", 1000, 6096, 0.3630627393722534], "38": ["vertical_and_slash", 1000, 6096, 0.37583932280540466], "39": ["vertical_and_slash", 1000, 6096, 0.400662899017334], "40": ["vertical_and_slash", 1000, 6096, 0.3679583966732025], "41": ["vertical_and_slash", 1000, 6096, 0.3823017179965973], "42": ["vertical_and_slash", 1000, 6096, 0.3929624855518341], "43": ["vertical_and_slash", 1000, 6096, 0.36404553055763245], "44": ["vertical_and_slash", 1000, 6096, 0.36244669556617737], "45": ["vertical_and_slash", 1000, 6096, 0.386248916387558], "46": ["vertical_and_slash", 1000, 6096, 0.37525251507759094], "47": ["vertical_and_slash", 1000, 6096, 0.3734144866466522], "48": ["vertical_and_slash", 1000, 6096, 0.4007520377635956], "49": ["vertical_and_slash", 1000, 6096, 0.3655022084712982], "50": ["vertical_and_slash", 1000, 6096, 0.3558310866355896], "51": ["vertical_and_slash", 1000, 6096, 0.36059245467185974], "52": ["vertical_and_slash", 1000, 6096, 0.3775298595428467], "53": ["vertical_and_slash", 1000, 6096, 0.3599264621734619], "54": ["vertical_and_slash", 1000, 6096, 0.3615720868110657], "55": ["vertical_and_slash", 1000, 6096, 0.35796263813972473], "56": ["vertical_and_slash", 1000, 6096, 0.3553612232208252], "57": ["vertical_and_slash", 1000, 6096, 0.3680879473686218], "58": ["vertical_and_slash", 1000, 6096, 0.36146068572998047], "59": ["vertical_and_slash", 1000, 6096, 0.3759180009365082], "60": ["vertical_and_slash", 1000, 6096, 0.3811896741390228], "61": ["vertical_and_slash", 1000, 6096, 0.36121442914009094], "62": ["vertical_and_slash", 1000, 6096, 0.36773768067359924], "63": ["vertical_and_slash", 1000, 6096, 0.3584331274032593]}, {"0": ["vertical_and_slash", 1000, 6096, 0.35846367478370667], "1": ["vertical_and_slash", 1000, 6096, 0.35577067732810974], "2": ["vertical_and_slash", 1000, 6096, 0.37057381868362427], "3": ["vertical_and_slash", 1000, 6096, 0.3563302159309387], "4": ["vertical_and_slash", 1000, 6096, 0.3516903519630432], "5": ["vertical_and_slash", 1000, 6096, 0.35427016019821167], "6": ["vertical_and_slash", 1000, 6096, 0.3704267740249634], "7": ["vertical_and_slash", 1000, 6096, 0.35345372557640076], "8": ["vertical_and_slash", 1000, 6096, 0.38433846831321716], "9": ["vertical_and_slash", 1000, 6096, 0.3649720251560211], "10": ["vertical_and_slash", 1000, 6096, 0.3617335557937622], "11": ["vertical_and_slash", 1000, 6096, 0.3705902099609375], "12": ["vertical_and_slash", 1000, 6096, 0.36617499589920044], "13": ["vertical_and_slash", 1000, 6096, 0.3620688021183014], "14": ["vertical_and_slash", 1000, 6096, 0.3747033178806305], "15": ["vertical_and_slash", 1000, 6096, 0.3551149368286133], "16": ["vertical_and_slash", 1000, 6096, 0.37485724687576294], "17": ["vertical_and_slash", 1000, 6096, 0.4122089147567749], "18": ["vertical_and_slash", 1000, 6096, 0.39652928709983826], "19": ["vertical_and_slash", 1000, 6096, 0.39465364813804626], "20": ["vertical_and_slash", 1000, 6096, 0.3825835585594177], "21": ["vertical_and_slash", 1000, 6096, 0.3810631334781647], "22": ["vertical_and_slash", 1000, 6096, 0.3784922957420349], "23": ["vertical_and_slash", 1000, 6096, 0.3887871205806732], "24": ["vertical_and_slash", 1000, 6096, 0.36692020297050476], "25": ["vertical_and_slash", 1000, 6096, 0.37071481347084045], "26": ["vertical_and_slash", 1000, 6096, 0.3516279458999634], "27": ["vertical_and_slash", 1000, 6096, 0.3616047203540802], "28": ["vertical_and_slash", 1000, 6096, 0.3589160740375519], "29": ["vertical_and_slash", 1000, 6096, 0.3620811402797699], "30": ["vertical_and_slash", 1000, 6096, 0.3520447313785553], "31": ["vertical_and_slash", 1000, 6096, 0.3662589192390442], "32": ["vertical_and_slash", 1000, 6096, 0.3764660656452179], "33": ["vertical_and_slash", 1000, 6096, 0.353208065032959], "34": ["vertical_and_slash", 1000, 6096, 0.35743463039398193], "35": ["vertical_and_slash", 1000, 6096, 0.3676375150680542], "36": ["vertical_and_slash", 1000, 6096, 0.35279345512390137], "37": ["vertical_and_slash", 1000, 6096, 0.356752872467041], "38": ["vertical_and_slash", 1000, 6096, 0.36274370551109314], "39": ["vertical_and_slash", 1000, 6096, 0.35527342557907104], "40": ["vertical_and_slash", 1000, 6096, 0.36079367995262146], "41": ["vertical_and_slash", 1000, 6096, 0.380588561296463], "42": ["vertical_and_slash", 1000, 6096, 0.3519578278064728], "43": ["vertical_and_slash", 1000, 6096, 0.37228891253471375], "44": ["vertical_and_slash", 1000, 6096, 0.3672102093696594], "45": ["vertical_and_slash", 1000, 6096, 0.3534199595451355], "46": ["vertical_and_slash", 1000, 6096, 0.3712427020072937], "47": ["vertical_and_slash", 1000, 6096, 0.36316147446632385], "48": ["vertical_and_slash", 1000, 6096, 0.35100939869880676], "49": ["vertical_and_slash", 1000, 6096, 0.3563114106655121], "50": ["vertical_and_slash", 1000, 6096, 0.37929487228393555], "51": ["vertical_and_slash", 1000, 6096, 0.34619879722595215], "52": ["vertical_and_slash", 1000, 6096, 0.3466967046260834], "53": ["vertical_and_slash", 1000, 6096, 0.35562488436698914], "54": ["vertical_and_slash", 1000, 6096, 0.35379159450531006], "55": ["vertical_and_slash", 1000, 6096, 0.36761674284935], "56": ["vertical_and_slash", 1000, 6096, 0.368429034948349], "57": ["vertical_and_slash", 1000, 6096, 0.3571030795574188], "58": ["vertical_and_slash", 1000, 6096, 0.3568216860294342], "59": ["vertical_and_slash", 1000, 6096, 0.3531847596168518], "60": ["vertical_and_slash", 1000, 6096, 0.36207497119903564], "61": ["vertical_and_slash", 1000, 6096, 0.36003273725509644], "62": ["vertical_and_slash", 1000, 6096, 0.35298487544059753], "63": ["vertical_and_slash", 1000, 6096, 0.3573254942893982]}, {"0": ["vertical_and_slash", 1000, 6096, 0.36988967657089233], "1": ["vertical_and_slash", 1000, 6096, 0.3786493241786957], "2": ["vertical_and_slash", 1000, 6096, 0.3722054362297058], "3": ["vertical_and_slash", 1000, 6096, 0.3686315417289734], "4": ["vertical_and_slash", 1000, 6096, 0.36418673396110535], "5": ["vertical_and_slash", 1000, 6096, 0.36363521218299866], "6": ["vertical_and_slash", 1000, 6096, 0.37099841237068176], "7": ["vertical_and_slash", 1000, 6096, 0.36587923765182495], "8": ["vertical_and_slash", 1000, 6096, 0.37431344389915466], "9": ["vertical_and_slash", 1000, 6096, 0.37673500180244446], "10": ["vertical_and_slash", 1000, 6096, 0.3711942732334137], "11": ["vertical_and_slash", 1000, 6096, 0.3502347767353058], "12": ["vertical_and_slash", 1000, 6096, 0.3726385831832886], "13": ["vertical_and_slash", 1000, 6096, 0.3668077290058136], "14": ["vertical_and_slash", 1000, 6096, 0.3682154417037964], "15": ["vertical_and_slash", 1000, 6096, 0.35592854022979736], "16": ["vertical_and_slash", 1000, 6096, 0.369729608297348], "17": ["vertical_and_slash", 1000, 6096, 0.36776936054229736], "18": ["vertical_and_slash", 1000, 6096, 0.3661026060581207], "19": ["vertical_and_slash", 1000, 6096, 0.3768647313117981], "20": ["vertical_and_slash", 1000, 6096, 0.36792081594467163], "21": ["vertical_and_slash", 1000, 6096, 0.36255598068237305], "22": ["vertical_and_slash", 1000, 6096, 0.37747922539711], "23": ["vertical_and_slash", 1000, 6096, 0.3602563142776489], "24": ["vertical_and_slash", 1000, 6096, 0.37305548787117004], "25": ["vertical_and_slash", 1000, 6096, 0.36342135071754456], "26": ["vertical_and_slash", 1000, 6096, 0.36872705817222595], "27": ["vertical_and_slash", 1000, 6096, 0.36216554045677185], "28": ["vertical_and_slash", 1000, 6096, 0.3642615079879761], "29": ["vertical_and_slash", 1000, 6096, 0.38684481382369995], "30": ["vertical_and_slash", 1000, 6096, 0.39398616552352905], "31": ["vertical_and_slash", 1000, 6096, 0.3574921786785126], "32": ["vertical_and_slash", 1000, 6096, 0.3710246682167053], "33": ["vertical_and_slash", 1000, 6096, 0.3531113862991333], "34": ["vertical_and_slash", 1000, 6096, 0.36995407938957214], "35": ["vertical_and_slash", 1000, 6096, 0.3949207365512848], "36": ["vertical_and_slash", 1000, 6096, 0.3767360746860504], "37": ["vertical_and_slash", 1000, 6096, 0.36763161420822144], "38": ["vertical_and_slash", 1000, 6096, 0.377422571182251], "39": ["vertical_and_slash", 1000, 6096, 0.3577679991722107], "40": ["vertical_and_slash", 1000, 6096, 0.3891659080982208], "41": ["vertical_and_slash", 1000, 6096, 0.35428881645202637], "42": ["vertical_and_slash", 1000, 6096, 0.35467350482940674], "43": ["vertical_and_slash", 1000, 6096, 0.35850000381469727], "44": ["vertical_and_slash", 1000, 6096, 0.367127388715744], "45": ["vertical_and_slash", 1000, 6096, 0.364938884973526], "46": ["vertical_and_slash", 1000, 6096, 0.3574575185775757], "47": ["vertical_and_slash", 1000, 6096, 0.35903337597846985], "48": ["vertical_and_slash", 1000, 6096, 0.36964285373687744], "49": ["vertical_and_slash", 1000, 6096, 0.36062365770339966], "50": ["vertical_and_slash", 1000, 6096, 0.36559319496154785], "51": ["vertical_and_slash", 1000, 6096, 0.37086179852485657], "52": ["vertical_and_slash", 1000, 6096, 0.36136606335639954], "53": ["vertical_and_slash", 1000, 6096, 0.3632615804672241], "54": ["vertical_and_slash", 1000, 6096, 0.3805314302444458], "55": ["vertical_and_slash", 1000, 6096, 0.37055110931396484], "56": ["vertical_and_slash", 1000, 6096, 0.3511624336242676], "57": ["vertical_and_slash", 1000, 6096, 0.3552010953426361], "58": ["vertical_and_slash", 1000, 6096, 0.36884501576423645], "59": ["vertical_and_slash", 1000, 6096, 0.35190796852111816], "60": ["vertical_and_slash", 1000, 6096, 0.3580698072910309], "61": ["vertical_and_slash", 1000, 6096, 0.3622550368309021], "62": ["vertical_and_slash", 1000, 6096, 0.35490208864212036], "63": ["vertical_and_slash", 1000, 6096, 0.3539677560329437]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3723457455635071], "1": ["vertical_and_slash", 1000, 6096, 0.34933480620384216], "2": ["vertical_and_slash", 1000, 6096, 0.3553128242492676], "3": ["vertical_and_slash", 1000, 6096, 0.3497467041015625], "4": ["vertical_and_slash", 1000, 6096, 0.3525950014591217], "5": ["vertical_and_slash", 1000, 6096, 0.36987364292144775], "6": ["vertical_and_slash", 1000, 6096, 0.3440614938735962], "7": ["vertical_and_slash", 1000, 6096, 0.353555828332901], "8": ["vertical_and_slash", 1000, 6096, 0.3599127233028412], "9": ["vertical_and_slash", 1000, 6096, 0.36862239241600037], "10": ["vertical_and_slash", 1000, 6096, 0.35471898317337036], "11": ["vertical_and_slash", 1000, 6096, 0.3710974156856537], "12": ["vertical_and_slash", 1000, 6096, 0.35415175557136536], "13": ["vertical_and_slash", 1000, 6096, 0.3684650659561157], "14": ["vertical_and_slash", 1000, 6096, 0.35992300510406494], "15": ["vertical_and_slash", 1000, 6096, 0.36509189009666443], "16": ["vertical_and_slash", 1000, 6096, 0.3744514286518097], "17": ["vertical_and_slash", 1000, 6096, 0.38132333755493164], "18": ["vertical_and_slash", 1000, 6096, 0.3621731996536255], "19": ["vertical_and_slash", 1000, 6096, 0.3674362599849701], "20": ["vertical_and_slash", 1000, 6096, 0.37179428339004517], "21": ["vertical_and_slash", 1000, 6096, 0.36172401905059814], "22": ["vertical_and_slash", 1000, 6096, 0.3577146530151367], "23": ["vertical_and_slash", 1000, 6096, 0.3748832941055298], "24": ["vertical_and_slash", 1000, 6096, 0.36602434515953064], "25": ["vertical_and_slash", 1000, 6096, 0.36314520239830017], "26": ["vertical_and_slash", 1000, 6096, 0.35823217034339905], "27": ["vertical_and_slash", 1000, 6096, 0.38833943009376526], "28": ["vertical_and_slash", 1000, 6096, 0.37277889251708984], "29": ["vertical_and_slash", 1000, 6096, 0.3815577030181885], "30": ["vertical_and_slash", 1000, 6096, 0.3633916676044464], "31": ["vertical_and_slash", 1000, 6096, 0.37324953079223633], "32": ["vertical_and_slash", 1000, 6096, 0.3726341724395752], "33": ["vertical_and_slash", 1000, 6096, 0.36305421590805054], "34": ["vertical_and_slash", 1000, 6096, 0.3770907521247864], "35": ["vertical_and_slash", 1000, 6096, 0.3698122799396515], "36": ["vertical_and_slash", 1000, 6096, 0.3681694269180298], "37": ["vertical_and_slash", 1000, 6096, 0.35934916138648987], "38": ["vertical_and_slash", 1000, 6096, 0.36005425453186035], "39": ["vertical_and_slash", 1000, 6096, 0.37538209557533264], "40": ["vertical_and_slash", 1000, 6096, 0.3658217191696167], "41": ["vertical_and_slash", 1000, 6096, 0.3678448796272278], "42": ["vertical_and_slash", 1000, 6096, 0.3745443522930145], "43": ["vertical_and_slash", 1000, 6096, 0.3608796000480652], "44": ["vertical_and_slash", 1000, 6096, 0.3725166320800781], "45": ["vertical_and_slash", 1000, 6096, 0.36169788241386414], "46": ["vertical_and_slash", 1000, 6096, 0.3636934161186218], "47": ["vertical_and_slash", 1000, 6096, 0.3660849630832672], "48": ["vertical_and_slash", 1000, 6096, 0.3755761682987213], "49": ["vertical_and_slash", 1000, 6096, 0.37916556000709534], "50": ["vertical_and_slash", 1000, 6096, 0.3731103837490082], "51": ["vertical_and_slash", 1000, 6096, 0.36178165674209595], "52": ["vertical_and_slash", 1000, 6096, 0.36359649896621704], "53": ["vertical_and_slash", 1000, 6096, 0.36505231261253357], "54": ["vertical_and_slash", 1000, 6096, 0.3661971390247345], "55": ["vertical_and_slash", 1000, 6096, 0.3661949038505554], "56": ["vertical_and_slash", 1000, 6096, 0.363591730594635], "57": ["vertical_and_slash", 1000, 6096, 0.3626288175582886], "58": ["vertical_and_slash", 1000, 6096, 0.3802110254764557], "59": ["vertical_and_slash", 1000, 6096, 0.3671741187572479], "60": ["vertical_and_slash", 1000, 6096, 0.353272020816803], "61": ["vertical_and_slash", 1000, 6096, 0.38400959968566895], "62": ["vertical_and_slash", 1000, 6096, 0.36862093210220337], "63": ["vertical_and_slash", 1000, 6096, 0.3596445918083191]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3707430958747864], "1": ["vertical_and_slash", 1000, 6096, 0.37086567282676697], "2": ["vertical_and_slash", 1000, 6096, 0.3676173985004425], "3": ["vertical_and_slash", 1000, 6096, 0.37377071380615234], "4": ["vertical_and_slash", 1000, 6096, 0.4047752618789673], "5": ["vertical_and_slash", 1000, 6096, 0.3648380935192108], "6": ["vertical_and_slash", 1000, 6096, 0.37241676449775696], "7": ["vertical_and_slash", 1000, 6096, 0.3725520074367523], "8": ["vertical_and_slash", 1000, 6096, 0.3546747863292694], "9": ["vertical_and_slash", 1000, 6096, 0.38987457752227783], "10": ["vertical_and_slash", 1000, 6096, 0.3584054410457611], "11": ["vertical_and_slash", 1000, 6096, 0.356178879737854], "12": ["vertical_and_slash", 1000, 6096, 0.38233277201652527], "13": ["vertical_and_slash", 1000, 6096, 0.3607275187969208], "14": ["vertical_and_slash", 1000, 6096, 0.35629308223724365], "15": ["vertical_and_slash", 1000, 6096, 0.3599489629268646], "16": ["vertical_and_slash", 1000, 6096, 0.3945033550262451], "17": ["vertical_and_slash", 1000, 6096, 0.3645462691783905], "18": ["vertical_and_slash", 1000, 6096, 0.3715648651123047], "19": ["vertical_and_slash", 1000, 6096, 0.40202438831329346], "20": ["vertical_and_slash", 1000, 6096, 0.36872684955596924], "21": ["vertical_and_slash", 1000, 6096, 0.3562847673892975], "22": ["vertical_and_slash", 1000, 6096, 0.3612552583217621], "23": ["vertical_and_slash", 1000, 6096, 0.3856153190135956], "24": ["vertical_and_slash", 1000, 6096, 0.37231454253196716], "25": ["vertical_and_slash", 1000, 6096, 0.36353084444999695], "26": ["vertical_and_slash", 1000, 6096, 0.3750617504119873], "27": ["vertical_and_slash", 1000, 6096, 0.3768419325351715], "28": ["vertical_and_slash", 1000, 6096, 0.3756144940853119], "29": ["vertical_and_slash", 1000, 6096, 0.37833476066589355], "30": ["vertical_and_slash", 1000, 6096, 0.40866655111312866], "31": ["vertical_and_slash", 1000, 6096, 0.4331510066986084], "32": ["vertical_and_slash", 1000, 6096, 0.37292832136154175], "33": ["vertical_and_slash", 1000, 6096, 0.37494567036628723], "34": ["vertical_and_slash", 1000, 6096, 0.3652471899986267], "35": ["vertical_and_slash", 1000, 6096, 0.38806092739105225], "36": ["vertical_and_slash", 1000, 6096, 0.36393195390701294], "37": ["vertical_and_slash", 1000, 6096, 0.38638973236083984], "38": ["vertical_and_slash", 1000, 6096, 0.37968358397483826], "39": ["vertical_and_slash", 1000, 6096, 0.3632664382457733], "40": ["vertical_and_slash", 1000, 6096, 0.36780208349227905], "41": ["vertical_and_slash", 1000, 6096, 0.36112621426582336], "42": ["vertical_and_slash", 1000, 6096, 0.3582766056060791], "43": ["vertical_and_slash", 1000, 6096, 0.3637661039829254], "44": ["vertical_and_slash", 1000, 6096, 0.3703443109989166], "45": ["vertical_and_slash", 1000, 6096, 0.4049914479255676], "46": ["vertical_and_slash", 1000, 6096, 0.3609428405761719], "47": ["vertical_and_slash", 1000, 6096, 0.36670637130737305], "48": ["vertical_and_slash", 1000, 6096, 0.3747187852859497], "49": ["vertical_and_slash", 1000, 6096, 0.3636295199394226], "50": ["vertical_and_slash", 1000, 6096, 0.3768143057823181], "51": ["vertical_and_slash", 1000, 6096, 0.3694286048412323], "52": ["vertical_and_slash", 1000, 6096, 0.378746896982193], "53": ["vertical_and_slash", 1000, 6096, 0.3634762167930603], "54": ["vertical_and_slash", 1000, 6096, 0.3727971315383911], "55": ["vertical_and_slash", 1000, 6096, 0.3653217852115631], "56": ["vertical_and_slash", 1000, 6096, 0.367485374212265], "57": ["vertical_and_slash", 1000, 6096, 0.3756074011325836], "58": ["vertical_and_slash", 1000, 6096, 0.3887893557548523], "59": ["vertical_and_slash", 1000, 6096, 0.3786999583244324], "60": ["vertical_and_slash", 1000, 6096, 0.3563457429409027], "61": ["vertical_and_slash", 1000, 6096, 0.3757339417934418], "62": ["vertical_and_slash", 1000, 6096, 0.36879095435142517], "63": ["vertical_and_slash", 1000, 6096, 0.3806837797164917]}, {"0": ["vertical_and_slash", 1000, 6096, 0.36706632375717163], "1": ["vertical_and_slash", 1000, 6096, 0.3655582666397095], "2": ["vertical_and_slash", 1000, 6096, 0.37614279985427856], "3": ["vertical_and_slash", 1000, 6096, 0.38118016719818115], "4": ["vertical_and_slash", 1000, 6096, 0.3987402319908142], "5": ["vertical_and_slash", 1000, 6096, 0.3925172686576843], "6": ["vertical_and_slash", 1000, 6096, 0.38840582966804504], "7": ["vertical_and_slash", 1000, 6096, 0.3706561028957367], "8": ["vertical_and_slash", 1000, 6096, 0.3779316842556], "9": ["vertical_and_slash", 1000, 6096, 0.3793015778064728], "10": ["vertical_and_slash", 1000, 6096, 0.37752217054367065], "11": ["vertical_and_slash", 1000, 6096, 0.4030648171901703], "12": ["vertical_and_slash", 1000, 6096, 0.36050185561180115], "13": ["vertical_and_slash", 1000, 6096, 0.3870128095149994], "14": ["vertical_and_slash", 1000, 6096, 0.3856786787509918], "15": ["vertical_and_slash", 1000, 6096, 0.3648412227630615], "16": ["vertical_and_slash", 1000, 6096, 0.37373417615890503], "17": ["vertical_and_slash", 1000, 6096, 0.3737295866012573], "18": ["vertical_and_slash", 1000, 6096, 0.38046690821647644], "19": ["vertical_and_slash", 1000, 6096, 0.37589219212532043], "20": ["vertical_and_slash", 1000, 6096, 0.41382068395614624], "21": ["vertical_and_slash", 1000, 6096, 0.3615049719810486], "22": ["vertical_and_slash", 1000, 6096, 0.3730368912220001], "23": ["vertical_and_slash", 1000, 6096, 0.37197166681289673], "24": ["vertical_and_slash", 1000, 6096, 0.3827100694179535], "25": ["vertical_and_slash", 1000, 6096, 0.3608359694480896], "26": ["vertical_and_slash", 1000, 6096, 0.3752790689468384], "27": ["vertical_and_slash", 1000, 6096, 0.40355929732322693], "28": ["vertical_and_slash", 1000, 6096, 0.3800477385520935], "29": ["vertical_and_slash", 1000, 6096, 0.380631685256958], "30": ["vertical_and_slash", 1000, 6096, 0.4045211672782898], "31": ["vertical_and_slash", 1000, 6096, 0.4033356010913849], "32": ["vertical_and_slash", 1000, 6096, 0.359937459230423], "33": ["vertical_and_slash", 1000, 6096, 0.37010765075683594], "34": ["vertical_and_slash", 1000, 6096, 0.3764887750148773], "35": ["vertical_and_slash", 1000, 6096, 0.36818215250968933], "36": ["vertical_and_slash", 1000, 6096, 0.37908652424812317], "37": ["vertical_and_slash", 1000, 6096, 0.4085911214351654], "38": ["vertical_and_slash", 1000, 6096, 0.3881496787071228], "39": ["vertical_and_slash", 1000, 6096, 0.360572874546051], "40": ["vertical_and_slash", 1000, 6096, 0.37943530082702637], "41": ["vertical_and_slash", 1000, 6096, 0.37614139914512634], "42": ["vertical_and_slash", 1000, 6096, 0.37029382586479187], "43": ["vertical_and_slash", 1000, 6096, 0.37189608812332153], "44": ["vertical_and_slash", 1000, 6096, 0.37779733538627625], "45": ["vertical_and_slash", 1000, 6096, 0.39761558175086975], "46": ["vertical_and_slash", 1000, 6096, 0.37077397108078003], "47": ["vertical_and_slash", 1000, 6096, 0.3797474503517151], "48": ["vertical_and_slash", 1000, 6096, 0.3868771195411682], "49": ["vertical_and_slash", 1000, 6096, 0.3618972897529602], "50": ["vertical_and_slash", 1000, 6096, 0.37080925703048706], "51": ["vertical_and_slash", 1000, 6096, 0.3660261332988739], "52": ["vertical_and_slash", 1000, 6096, 0.37270429730415344], "53": ["vertical_and_slash", 1000, 6096, 0.36772626638412476], "54": ["vertical_and_slash", 1000, 6096, 0.3666740357875824], "55": ["vertical_and_slash", 1000, 6096, 0.36910828948020935], "56": ["vertical_and_slash", 1000, 6096, 0.3645970821380615], "57": ["vertical_and_slash", 1000, 6096, 0.36695384979248047], "58": ["vertical_and_slash", 1000, 6096, 0.37162768840789795], "59": ["vertical_and_slash", 1000, 6096, 0.37216371297836304], "60": ["vertical_and_slash", 1000, 6096, 0.39008471369743347], "61": ["vertical_and_slash", 1000, 6096, 0.36079591512680054], "62": ["vertical_and_slash", 1000, 6096, 0.366362988948822], "63": ["vertical_and_slash", 1000, 6096, 0.36743342876434326]}, {"0": ["vertical_and_slash", 1000, 6096, 0.36306527256965637], "1": ["vertical_and_slash", 1000, 6096, 0.36875298619270325], "2": ["vertical_and_slash", 1000, 6096, 0.3626720905303955], "3": ["vertical_and_slash", 1000, 6096, 0.36523672938346863], "4": ["vertical_and_slash", 1000, 6096, 0.3721795082092285], "5": ["vertical_and_slash", 1000, 6096, 0.3632601201534271], "6": ["vertical_and_slash", 1000, 6096, 0.37907862663269043], "7": ["vertical_and_slash", 1000, 6096, 0.36727118492126465], "8": ["vertical_and_slash", 1000, 6096, 0.3691934645175934], "9": ["vertical_and_slash", 1000, 6096, 0.37414559721946716], "10": ["vertical_and_slash", 1000, 6096, 0.37421566247940063], "11": ["vertical_and_slash", 1000, 6096, 0.37319624423980713], "12": ["vertical_and_slash", 1000, 6096, 0.36583033204078674], "13": ["vertical_and_slash", 1000, 6096, 0.36412525177001953], "14": ["vertical_and_slash", 1000, 6096, 0.36642736196517944], "15": ["vertical_and_slash", 1000, 6096, 0.3707560896873474], "16": ["vertical_and_slash", 1000, 6096, 0.3655088245868683], "17": ["vertical_and_slash", 1000, 6096, 0.37037795782089233], "18": ["vertical_and_slash", 1000, 6096, 0.37896546721458435], "19": ["vertical_and_slash", 1000, 6096, 0.36277440190315247], "20": ["vertical_and_slash", 1000, 6096, 0.363753080368042], "21": ["vertical_and_slash", 1000, 6096, 0.3706473410129547], "22": ["vertical_and_slash", 1000, 6096, 0.4224102199077606], "23": ["vertical_and_slash", 1000, 6096, 0.3691635727882385], "24": ["vertical_and_slash", 1000, 6096, 0.3549754023551941], "25": ["vertical_and_slash", 1000, 6096, 0.3793523609638214], "26": ["vertical_and_slash", 1000, 6096, 0.38030463457107544], "27": ["vertical_and_slash", 1000, 6096, 0.3687661588191986], "28": ["vertical_and_slash", 1000, 6096, 0.3615373373031616], "29": ["vertical_and_slash", 1000, 6096, 0.37074559926986694], "30": ["vertical_and_slash", 1000, 6096, 0.39575546979904175], "31": ["vertical_and_slash", 1000, 6096, 0.36954790353775024], "32": ["vertical_and_slash", 1000, 6096, 0.4100349545478821], "33": ["vertical_and_slash", 1000, 6096, 0.3844066560268402], "34": ["vertical_and_slash", 1000, 6096, 0.38558483123779297], "35": ["vertical_and_slash", 1000, 6096, 0.38043975830078125], "36": ["vertical_and_slash", 1000, 6096, 0.3896539509296417], "37": ["vertical_and_slash", 1000, 6096, 0.38982605934143066], "38": ["vertical_and_slash", 1000, 6096, 0.3863334357738495], "39": ["vertical_and_slash", 1000, 6096, 0.3781353831291199], "40": ["vertical_and_slash", 1000, 6096, 0.37749823927879333], "41": ["vertical_and_slash", 1000, 6096, 0.37304434180259705], "42": ["vertical_and_slash", 1000, 6096, 0.3774571120738983], "43": ["vertical_and_slash", 1000, 6096, 0.37045958638191223], "44": ["vertical_and_slash", 1000, 6096, 0.3844292163848877], "45": ["vertical_and_slash", 1000, 6096, 0.3806615173816681], "46": ["vertical_and_slash", 1000, 6096, 0.3785519003868103], "47": ["vertical_and_slash", 1000, 6096, 0.3687250316143036], "48": ["vertical_and_slash", 1000, 6096, 0.3720493018627167], "49": ["vertical_and_slash", 1000, 6096, 0.38739871978759766], "50": ["vertical_and_slash", 1000, 6096, 0.373379647731781], "51": ["vertical_and_slash", 1000, 6096, 0.3726637363433838], "52": ["vertical_and_slash", 1000, 6096, 0.3591633141040802], "53": ["vertical_and_slash", 1000, 6096, 0.38414353132247925], "54": ["vertical_and_slash", 1000, 6096, 0.3833041191101074], "55": ["vertical_and_slash", 1000, 6096, 0.37687915563583374], "56": ["vertical_and_slash", 1000, 6096, 0.3612945079803467], "57": ["vertical_and_slash", 1000, 6096, 0.37004929780960083], "58": ["vertical_and_slash", 1000, 6096, 0.3824690878391266], "59": ["vertical_and_slash", 1000, 6096, 0.38277438282966614], "60": ["vertical_and_slash", 1000, 6096, 0.36277109384536743], "61": ["vertical_and_slash", 1000, 6096, 0.3694477379322052], "62": ["vertical_and_slash", 1000, 6096, 0.3580164313316345], "63": ["vertical_and_slash", 1000, 6096, 0.3602277636528015]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4039778411388397], "1": ["vertical_and_slash", 1000, 6096, 0.3840399980545044], "2": ["vertical_and_slash", 1000, 6096, 0.3989053964614868], "3": ["vertical_and_slash", 1000, 6096, 0.3916483223438263], "4": ["vertical_and_slash", 1000, 6096, 0.40573927760124207], "5": ["vertical_and_slash", 1000, 6096, 0.3942083716392517], "6": ["vertical_and_slash", 1000, 6096, 0.4069986641407013], "7": ["vertical_and_slash", 1000, 6096, 0.3920276463031769], "8": ["vertical_and_slash", 1000, 6096, 0.3992125391960144], "9": ["vertical_and_slash", 1000, 6096, 0.3895735740661621], "10": ["vertical_and_slash", 1000, 6096, 0.3886878490447998], "11": ["vertical_and_slash", 1000, 6096, 0.390937864780426], "12": ["vertical_and_slash", 1000, 6096, 0.3975723385810852], "13": ["vertical_and_slash", 1000, 6096, 0.40424004197120667], "14": ["vertical_and_slash", 1000, 6096, 0.3875581920146942], "15": ["vertical_and_slash", 1000, 6096, 0.39835602045059204], "16": ["vertical_and_slash", 1000, 6096, 0.3640047311782837], "17": ["vertical_and_slash", 1000, 6096, 0.37977293133735657], "18": ["vertical_and_slash", 1000, 6096, 0.4101698398590088], "19": ["vertical_and_slash", 1000, 6096, 0.37873947620391846], "20": ["vertical_and_slash", 1000, 6096, 0.3817698359489441], "21": ["vertical_and_slash", 1000, 6096, 0.37903347611427307], "22": ["vertical_and_slash", 1000, 6096, 0.3943164050579071], "23": ["vertical_and_slash", 1000, 6096, 0.3759208619594574], "24": ["vertical_and_slash", 1000, 6096, 0.4154088497161865], "25": ["vertical_and_slash", 1000, 6096, 0.3986920416355133], "26": ["vertical_and_slash", 1000, 6096, 0.3963497579097748], "27": ["vertical_and_slash", 1000, 6096, 0.4350094199180603], "28": ["vertical_and_slash", 1000, 6096, 0.42964354157447815], "29": ["vertical_and_slash", 1000, 6096, 0.3912089467048645], "30": ["vertical_and_slash", 1000, 6096, 0.4026780426502228], "31": ["vertical_and_slash", 1000, 6096, 0.4090764820575714], "32": ["vertical_and_slash", 1000, 6096, 0.3827900290489197], "33": ["vertical_and_slash", 1000, 6096, 0.37728604674339294], "34": ["vertical_and_slash", 1000, 6096, 0.3761049509048462], "35": ["vertical_and_slash", 1000, 6096, 0.38322049379348755], "36": ["vertical_and_slash", 1000, 6096, 0.38781628012657166], "37": ["vertical_and_slash", 1000, 6096, 0.38224077224731445], "38": ["vertical_and_slash", 1000, 6096, 0.38049909472465515], "39": ["vertical_and_slash", 1000, 6096, 0.3686068654060364], "40": ["vertical_and_slash", 1000, 6096, 0.4020129442214966], "41": ["vertical_and_slash", 1000, 6096, 0.39173197746276855], "42": ["vertical_and_slash", 1000, 6096, 0.4130908250808716], "43": ["vertical_and_slash", 1000, 6096, 0.4019201397895813], "44": ["vertical_and_slash", 1000, 6096, 0.42907246947288513], "45": ["vertical_and_slash", 1000, 6096, 0.39822593331336975], "46": ["vertical_and_slash", 1000, 6096, 0.4348330497741699], "47": ["vertical_and_slash", 1000, 6096, 0.39063209295272827], "48": ["vertical_and_slash", 1000, 6096, 0.38388970494270325], "49": ["vertical_and_slash", 1000, 6096, 0.3799162209033966], "50": ["vertical_and_slash", 1000, 6096, 0.4046593904495239], "51": ["vertical_and_slash", 1000, 6096, 0.4185912013053894], "52": ["vertical_and_slash", 1000, 6096, 0.38748830556869507], "53": ["vertical_and_slash", 1000, 6096, 0.39950403571128845], "54": ["vertical_and_slash", 1000, 6096, 0.5005795359611511], "55": ["vertical_and_slash", 1000, 6096, 0.4002453088760376], "56": ["vertical_and_slash", 1000, 6096, 0.4022608697414398], "57": ["vertical_and_slash", 1000, 6096, 0.4004478454589844], "58": ["vertical_and_slash", 1000, 6096, 0.4053657352924347], "59": ["vertical_and_slash", 1000, 6096, 0.408309668302536], "60": ["vertical_and_slash", 1000, 6096, 0.4448804259300232], "61": ["vertical_and_slash", 1000, 6096, 0.3959033489227295], "62": ["vertical_and_slash", 1000, 6096, 0.40239018201828003], "63": ["vertical_and_slash", 1000, 6096, 0.41335490345954895]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3706471920013428], "1": ["vertical_and_slash", 1000, 6096, 0.35741689801216125], "2": ["vertical_and_slash", 1000, 6096, 0.3570856750011444], "3": ["vertical_and_slash", 1000, 6096, 0.37074166536331177], "4": ["vertical_and_slash", 1000, 6096, 0.3855551481246948], "5": ["vertical_and_slash", 1000, 6096, 0.38317832350730896], "6": ["vertical_and_slash", 1000, 6096, 0.37700149416923523], "7": ["vertical_and_slash", 1000, 6096, 0.3792213797569275], "8": ["vertical_and_slash", 1000, 6096, 0.3897951543331146], "9": ["vertical_and_slash", 1000, 6096, 0.37844720482826233], "10": ["vertical_and_slash", 1000, 6096, 0.36881741881370544], "11": ["vertical_and_slash", 1000, 6096, 0.3818166255950928], "12": ["vertical_and_slash", 1000, 6096, 0.37467947602272034], "13": ["vertical_and_slash", 1000, 6096, 0.3728982210159302], "14": ["vertical_and_slash", 1000, 6096, 0.3756228983402252], "15": ["vertical_and_slash", 1000, 6096, 0.37589016556739807], "16": ["vertical_and_slash", 1000, 6096, 0.385915070772171], "17": ["vertical_and_slash", 1000, 6096, 0.3876124620437622], "18": ["vertical_and_slash", 1000, 6096, 0.38806164264678955], "19": ["vertical_and_slash", 1000, 6096, 0.3949907422065735], "20": ["vertical_and_slash", 1000, 6096, 0.38594308495521545], "21": ["vertical_and_slash", 1000, 6096, 0.3900632858276367], "22": ["vertical_and_slash", 1000, 6096, 0.3696674406528473], "23": ["vertical_and_slash", 1000, 6096, 0.3661253750324249], "24": ["vertical_and_slash", 1000, 6096, 0.39153459668159485], "25": ["vertical_and_slash", 1000, 6096, 0.37490084767341614], "26": ["vertical_and_slash", 1000, 6096, 0.3744400143623352], "27": ["vertical_and_slash", 1000, 6096, 0.3719973564147949], "28": ["vertical_and_slash", 1000, 6096, 0.3776869773864746], "29": ["vertical_and_slash", 1000, 6096, 0.36573711037635803], "30": ["vertical_and_slash", 1000, 6096, 0.37679925560951233], "31": ["vertical_and_slash", 1000, 6096, 0.37429529428482056], "32": ["vertical_and_slash", 1000, 6096, 0.40574073791503906], "33": ["vertical_and_slash", 1000, 6096, 0.4047424793243408], "34": ["vertical_and_slash", 1000, 6096, 0.3995601236820221], "35": ["vertical_and_slash", 1000, 6096, 0.4040420949459076], "36": ["vertical_and_slash", 1000, 6096, 0.3997073471546173], "37": ["vertical_and_slash", 1000, 6096, 0.4102416932582855], "38": ["vertical_and_slash", 1000, 6096, 0.3905298113822937], "39": ["vertical_and_slash", 1000, 6096, 0.44595959782600403], "40": ["vertical_and_slash", 1000, 6096, 0.3737184703350067], "41": ["vertical_and_slash", 1000, 6096, 0.3719173073768616], "42": ["vertical_and_slash", 1000, 6096, 0.3770376145839691], "43": ["vertical_and_slash", 1000, 6096, 0.3661225736141205], "44": ["vertical_and_slash", 1000, 6096, 0.3717348277568817], "45": ["vertical_and_slash", 1000, 6096, 0.3730773627758026], "46": ["vertical_and_slash", 1000, 6096, 0.36479508876800537], "47": ["vertical_and_slash", 1000, 6096, 0.3806614875793457], "48": ["vertical_and_slash", 1000, 6096, 0.36240845918655396], "49": ["vertical_and_slash", 1000, 6096, 0.3675076365470886], "50": ["vertical_and_slash", 1000, 6096, 0.3807787299156189], "51": ["vertical_and_slash", 1000, 6096, 0.37817779183387756], "52": ["vertical_and_slash", 1000, 6096, 0.37741580605506897], "53": ["vertical_and_slash", 1000, 6096, 0.367034912109375], "54": ["vertical_and_slash", 1000, 6096, 0.3636292815208435], "55": ["vertical_and_slash", 1000, 6096, 0.391410231590271], "56": ["vertical_and_slash", 1000, 6096, 0.41227975487709045], "57": ["vertical_and_slash", 1000, 6096, 0.5169029831886292], "58": ["vertical_and_slash", 1000, 6096, 0.3772544860839844], "59": ["vertical_and_slash", 1000, 6096, 0.5957722067832947], "60": ["vertical_and_slash", 1000, 6096, 0.424430787563324], "61": ["vertical_and_slash", 1000, 6096, 0.4101772904396057], "62": ["vertical_and_slash", 1000, 6096, 0.41165632009506226], "63": ["vertical_and_slash", 1000, 6096, 0.38721099495887756]}, {"0": ["vertical_and_slash", 1000, 6096, 0.37447601556777954], "1": ["vertical_and_slash", 1000, 6096, 0.4790041446685791], "2": ["vertical_and_slash", 1000, 6096, 0.40661314129829407], "3": ["vertical_and_slash", 1000, 6096, 0.4088316857814789], "4": ["vertical_and_slash", 1000, 6096, 0.4091930389404297], "5": ["vertical_and_slash", 1000, 6096, 0.4241996705532074], "6": ["vertical_and_slash", 1000, 6096, 0.400899201631546], "7": ["vertical_and_slash", 1000, 6096, 0.42722800374031067], "8": ["vertical_and_slash", 1000, 6096, 0.38910478353500366], "9": ["vertical_and_slash", 1000, 6096, 0.3791801333427429], "10": ["vertical_and_slash", 1000, 6096, 0.38338571786880493], "11": ["vertical_and_slash", 1000, 6096, 0.37486883997917175], "12": ["vertical_and_slash", 1000, 6096, 0.3823448121547699], "13": ["vertical_and_slash", 1000, 6096, 0.36912602186203003], "14": ["vertical_and_slash", 1000, 6096, 0.3896883726119995], "15": ["vertical_and_slash", 1000, 6096, 0.37409359216690063], "16": ["vertical_and_slash", 1000, 6096, 0.40094268321990967], "17": ["vertical_and_slash", 1000, 6096, 0.3888852894306183], "18": ["vertical_and_slash", 1000, 6096, 0.3863041400909424], "19": ["vertical_and_slash", 1000, 6096, 0.37142568826675415], "20": ["vertical_and_slash", 1000, 6096, 0.36776575446128845], "21": ["vertical_and_slash", 1000, 6096, 0.3767136335372925], "22": ["vertical_and_slash", 1000, 6096, 0.3742954730987549], "23": ["vertical_and_slash", 1000, 6096, 0.4436018168926239], "24": ["vertical_and_slash", 1000, 6096, 0.3708394169807434], "25": ["vertical_and_slash", 1000, 6096, 0.39653295278549194], "26": ["vertical_and_slash", 1000, 6096, 0.3875144422054291], "27": ["vertical_and_slash", 1000, 6096, 0.3972586691379547], "28": ["vertical_and_slash", 1000, 6096, 0.3860595226287842], "29": ["vertical_and_slash", 1000, 6096, 0.3874390125274658], "30": ["vertical_and_slash", 1000, 6096, 0.38409948348999023], "31": ["vertical_and_slash", 1000, 6096, 0.3878306746482849], "32": ["vertical_and_slash", 1000, 6096, 0.37347739934921265], "33": ["vertical_and_slash", 1000, 6096, 0.363208532333374], "34": ["vertical_and_slash", 1000, 6096, 0.3598812222480774], "35": ["vertical_and_slash", 1000, 6096, 0.37200987339019775], "36": ["vertical_and_slash", 1000, 6096, 0.3641003966331482], "37": ["vertical_and_slash", 1000, 6096, 0.37580639123916626], "38": ["vertical_and_slash", 1000, 6096, 0.3615226149559021], "39": ["vertical_and_slash", 1000, 6096, 0.3691566586494446], "40": ["vertical_and_slash", 1000, 6096, 0.3898833394050598], "41": ["vertical_and_slash", 1000, 6096, 0.4383479058742523], "42": ["vertical_and_slash", 1000, 6096, 0.38696521520614624], "43": ["vertical_and_slash", 1000, 6096, 0.4115564525127411], "44": ["vertical_and_slash", 1000, 6096, 0.4046195149421692], "45": ["vertical_and_slash", 1000, 6096, 0.40935882925987244], "46": ["vertical_and_slash", 1000, 6096, 0.41425347328186035], "47": ["vertical_and_slash", 1000, 6096, 0.3915831446647644], "48": ["vertical_and_slash", 1000, 6096, 0.44332489371299744], "49": ["vertical_and_slash", 1000, 6096, 0.3729479908943176], "50": ["vertical_and_slash", 1000, 6096, 0.38895633816719055], "51": ["vertical_and_slash", 1000, 6096, 0.37990084290504456], "52": ["vertical_and_slash", 1000, 6096, 0.3813694417476654], "53": ["vertical_and_slash", 1000, 6096, 0.4002147316932678], "54": ["vertical_and_slash", 1000, 6096, 0.39762377738952637], "55": ["vertical_and_slash", 1000, 6096, 0.386418879032135], "56": ["vertical_and_slash", 1000, 6096, 0.3998110592365265], "57": ["vertical_and_slash", 1000, 6096, 0.3750094771385193], "58": ["vertical_and_slash", 1000, 6096, 0.3680530786514282], "59": ["vertical_and_slash", 1000, 6096, 0.37554049491882324], "60": ["vertical_and_slash", 1000, 6096, 0.36950787901878357], "61": ["vertical_and_slash", 1000, 6096, 0.399164617061615], "62": ["vertical_and_slash", 1000, 6096, 0.3931753933429718], "63": ["vertical_and_slash", 1000, 6096, 0.3978849947452545]}, {"0": ["vertical_and_slash", 1000, 6096, 0.38595595955848694], "1": ["vertical_and_slash", 1000, 6096, 0.4035450518131256], "2": ["vertical_and_slash", 1000, 6096, 0.4002217948436737], "3": ["vertical_and_slash", 1000, 6096, 0.40864628553390503], "4": ["vertical_and_slash", 1000, 6096, 0.3645535707473755], "5": ["vertical_and_slash", 1000, 6096, 0.3845541477203369], "6": ["vertical_and_slash", 1000, 6096, 0.3912951648235321], "7": ["vertical_and_slash", 1000, 6096, 0.44991204142570496], "8": ["vertical_and_slash", 1000, 6096, 0.37015870213508606], "9": ["vertical_and_slash", 1000, 6096, 0.369625449180603], "10": ["vertical_and_slash", 1000, 6096, 0.3833763599395752], "11": ["vertical_and_slash", 1000, 6096, 0.3692909777164459], "12": ["vertical_and_slash", 1000, 6096, 0.36958950757980347], "13": ["vertical_and_slash", 1000, 6096, 0.3855385482311249], "14": ["vertical_and_slash", 1000, 6096, 0.3770018219947815], "15": ["vertical_and_slash", 1000, 6096, 0.3913373649120331], "16": ["vertical_and_slash", 1000, 6096, 0.41501563787460327], "17": ["vertical_and_slash", 1000, 6096, 0.3868281841278076], "18": ["vertical_and_slash", 1000, 6096, 0.393069326877594], "19": ["vertical_and_slash", 1000, 6096, 0.3809382915496826], "20": ["vertical_and_slash", 1000, 6096, 0.3820367455482483], "21": ["vertical_and_slash", 1000, 6096, 0.3974625766277313], "22": ["vertical_and_slash", 1000, 6096, 0.412346750497818], "23": ["vertical_and_slash", 1000, 6096, 0.3955797553062439], "24": ["vertical_and_slash", 1000, 6096, 0.39638373255729675], "25": ["vertical_and_slash", 1000, 6096, 0.373254656791687], "26": ["vertical_and_slash", 1000, 6096, 0.35620468854904175], "27": ["vertical_and_slash", 1000, 6096, 0.38046714663505554], "28": ["vertical_and_slash", 1000, 6096, 0.3757074475288391], "29": ["vertical_and_slash", 1000, 6096, 0.4051363468170166], "30": ["vertical_and_slash", 1000, 6096, 0.38987085223197937], "31": ["vertical_and_slash", 1000, 6096, 0.3777565360069275], "32": ["vertical_and_slash", 1000, 6096, 0.3967498242855072], "33": ["vertical_and_slash", 1000, 6096, 0.3695752024650574], "34": ["vertical_and_slash", 1000, 6096, 0.3612961173057556], "35": ["vertical_and_slash", 1000, 6096, 0.3760088384151459], "36": ["vertical_and_slash", 1000, 6096, 0.37277939915657043], "37": ["vertical_and_slash", 1000, 6096, 0.35879847407341003], "38": ["vertical_and_slash", 1000, 6096, 0.38178789615631104], "39": ["vertical_and_slash", 1000, 6096, 0.3704139292240143], "40": ["vertical_and_slash", 1000, 6096, 0.3891811966896057], "41": ["vertical_and_slash", 1000, 6096, 0.3769999146461487], "42": ["vertical_and_slash", 1000, 6096, 0.3725990056991577], "43": ["vertical_and_slash", 1000, 6096, 0.3769390285015106], "44": ["vertical_and_slash", 1000, 6096, 0.37641140818595886], "45": ["vertical_and_slash", 1000, 6096, 0.38110262155532837], "46": ["vertical_and_slash", 1000, 6096, 0.40392354130744934], "47": ["vertical_and_slash", 1000, 6096, 0.37084200978279114], "48": ["vertical_and_slash", 1000, 6096, 0.37346723675727844], "49": ["vertical_and_slash", 1000, 6096, 0.3927254378795624], "50": ["vertical_and_slash", 1000, 6096, 0.3728257715702057], "51": ["vertical_and_slash", 1000, 6096, 0.3679296374320984], "52": ["vertical_and_slash", 1000, 6096, 0.37344327569007874], "53": ["vertical_and_slash", 1000, 6096, 0.366904616355896], "54": ["vertical_and_slash", 1000, 6096, 0.39149734377861023], "55": ["vertical_and_slash", 1000, 6096, 0.36786317825317383], "56": ["vertical_and_slash", 1000, 6096, 0.3947741985321045], "57": ["vertical_and_slash", 1000, 6096, 0.3887660503387451], "58": ["vertical_and_slash", 1000, 6096, 0.3740643858909607], "59": ["vertical_and_slash", 1000, 6096, 0.3733753263950348], "60": ["vertical_and_slash", 1000, 6096, 0.36869099736213684], "61": ["vertical_and_slash", 1000, 6096, 0.36369210481643677], "62": ["vertical_and_slash", 1000, 6096, 0.3839704096317291], "63": ["vertical_and_slash", 1000, 6096, 0.4054853022098541]}, {"0": ["vertical_and_slash", 1000, 6096, 0.37920868396759033], "1": ["vertical_and_slash", 1000, 6096, 0.37221774458885193], "2": ["vertical_and_slash", 1000, 6096, 0.39343246817588806], "3": ["vertical_and_slash", 1000, 6096, 0.3713682293891907], "4": ["vertical_and_slash", 1000, 6096, 0.3765051066875458], "5": ["vertical_and_slash", 1000, 6096, 0.37029996514320374], "6": ["vertical_and_slash", 1000, 6096, 0.39791688323020935], "7": ["vertical_and_slash", 1000, 6096, 0.388713002204895], "8": ["vertical_and_slash", 1000, 6096, 0.37844955921173096], "9": ["vertical_and_slash", 1000, 6096, 0.40360358357429504], "10": ["vertical_and_slash", 1000, 6096, 0.3691402077674866], "11": ["vertical_and_slash", 1000, 6096, 0.3989088237285614], "12": ["vertical_and_slash", 1000, 6096, 0.37790223956108093], "13": ["vertical_and_slash", 1000, 6096, 0.37461018562316895], "14": ["vertical_and_slash", 1000, 6096, 0.39198315143585205], "15": ["vertical_and_slash", 1000, 6096, 0.3862866163253784], "16": ["vertical_and_slash", 1000, 6096, 0.3747318387031555], "17": ["vertical_and_slash", 1000, 6096, 0.375667005777359], "18": ["vertical_and_slash", 1000, 6096, 0.3736613392829895], "19": ["vertical_and_slash", 1000, 6096, 0.3563859760761261], "20": ["vertical_and_slash", 1000, 6096, 0.38991859555244446], "21": ["vertical_and_slash", 1000, 6096, 0.37189388275146484], "22": ["vertical_and_slash", 1000, 6096, 0.36875858902931213], "23": ["vertical_and_slash", 1000, 6096, 0.3678094744682312], "24": ["vertical_and_slash", 1000, 6096, 0.3549753725528717], "25": ["vertical_and_slash", 1000, 6096, 0.354531854391098], "26": ["vertical_and_slash", 1000, 6096, 0.36384543776512146], "27": ["vertical_and_slash", 1000, 6096, 0.3663633167743683], "28": ["vertical_and_slash", 1000, 6096, 0.38154923915863037], "29": ["vertical_and_slash", 1000, 6096, 0.3653130531311035], "30": ["vertical_and_slash", 1000, 6096, 0.3662702739238739], "31": ["vertical_and_slash", 1000, 6096, 0.3573468029499054], "32": ["vertical_and_slash", 1000, 6096, 0.36941322684288025], "33": ["vertical_and_slash", 1000, 6096, 0.3798534572124481], "34": ["vertical_and_slash", 1000, 6096, 0.36849015951156616], "35": ["vertical_and_slash", 1000, 6096, 0.3560854196548462], "36": ["vertical_and_slash", 1000, 6096, 0.3708340525627136], "37": ["vertical_and_slash", 1000, 6096, 0.3763386905193329], "38": ["vertical_and_slash", 1000, 6096, 0.3671092092990875], "39": ["vertical_and_slash", 1000, 6096, 0.3701918125152588], "40": ["vertical_and_slash", 1000, 6096, 0.3780665099620819], "41": ["vertical_and_slash", 1000, 6096, 0.37764742970466614], "42": ["vertical_and_slash", 1000, 6096, 0.3777092397212982], "43": ["vertical_and_slash", 1000, 6096, 0.3682219386100769], "44": ["vertical_and_slash", 1000, 6096, 0.40082645416259766], "45": ["vertical_and_slash", 1000, 6096, 0.3729124665260315], "46": ["vertical_and_slash", 1000, 6096, 0.3881452679634094], "47": ["vertical_and_slash", 1000, 6096, 0.3749277591705322], "48": ["vertical_and_slash", 1000, 6096, 0.3880806267261505], "49": ["vertical_and_slash", 1000, 6096, 0.40079692006111145], "50": ["vertical_and_slash", 1000, 6096, 0.37890687584877014], "51": ["vertical_and_slash", 1000, 6096, 0.3790092468261719], "52": ["vertical_and_slash", 1000, 6096, 0.38351261615753174], "53": ["vertical_and_slash", 1000, 6096, 0.3758959174156189], "54": ["vertical_and_slash", 1000, 6096, 0.3678317666053772], "55": ["vertical_and_slash", 1000, 6096, 0.3778243958950043], "56": ["vertical_and_slash", 1000, 6096, 0.3744249641895294], "57": ["vertical_and_slash", 1000, 6096, 0.3834068477153778], "58": ["vertical_and_slash", 1000, 6096, 0.37582066655158997], "59": ["vertical_and_slash", 1000, 6096, 0.36424726247787476], "60": ["vertical_and_slash", 1000, 6096, 0.3701358735561371], "61": ["vertical_and_slash", 1000, 6096, 0.39455196261405945], "62": ["vertical_and_slash", 1000, 6096, 0.3896448016166687], "63": ["vertical_and_slash", 1000, 6096, 0.37670162320137024]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3811589181423187], "1": ["vertical_and_slash", 1000, 6096, 0.3841848075389862], "2": ["vertical_and_slash", 1000, 6096, 0.3760468065738678], "3": ["vertical_and_slash", 1000, 6096, 0.37511754035949707], "4": ["vertical_and_slash", 1000, 6096, 0.3810584545135498], "5": ["vertical_and_slash", 1000, 6096, 0.3713425397872925], "6": ["vertical_and_slash", 1000, 6096, 0.37169769406318665], "7": ["vertical_and_slash", 1000, 6096, 0.3741718530654907], "8": ["vertical_and_slash", 1000, 6096, 0.37103790044784546], "9": ["vertical_and_slash", 1000, 6096, 0.387432336807251], "10": ["vertical_and_slash", 1000, 6096, 0.38835567235946655], "11": ["vertical_and_slash", 1000, 6096, 0.37432238459587097], "12": ["vertical_and_slash", 1000, 6096, 0.3788174092769623], "13": ["vertical_and_slash", 1000, 6096, 0.37582120299339294], "14": ["vertical_and_slash", 1000, 6096, 0.37076568603515625], "15": ["vertical_and_slash", 1000, 6096, 0.36853086948394775], "16": ["vertical_and_slash", 1000, 6096, 0.3828295171260834], "17": ["vertical_and_slash", 1000, 6096, 0.3750588297843933], "18": ["vertical_and_slash", 1000, 6096, 0.3932008147239685], "19": ["vertical_and_slash", 1000, 6096, 0.36563560366630554], "20": ["vertical_and_slash", 1000, 6096, 0.39691710472106934], "21": ["vertical_and_slash", 1000, 6096, 0.3666548728942871], "22": ["vertical_and_slash", 1000, 6096, 0.3842177093029022], "23": ["vertical_and_slash", 1000, 6096, 0.37503913044929504], "24": ["vertical_and_slash", 1000, 6096, 0.39641210436820984], "25": ["vertical_and_slash", 1000, 6096, 0.3699389696121216], "26": ["vertical_and_slash", 1000, 6096, 0.36201953887939453], "27": ["vertical_and_slash", 1000, 6096, 0.3829905092716217], "28": ["vertical_and_slash", 1000, 6096, 0.37781956791877747], "29": ["vertical_and_slash", 1000, 6096, 0.37581804394721985], "30": ["vertical_and_slash", 1000, 6096, 0.3681574761867523], "31": ["vertical_and_slash", 1000, 6096, 0.37502700090408325], "32": ["vertical_and_slash", 1000, 6096, 0.3631366491317749], "33": ["vertical_and_slash", 1000, 6096, 0.3712453246116638], "34": ["vertical_and_slash", 1000, 6096, 0.41524067521095276], "35": ["vertical_and_slash", 1000, 6096, 0.36995548009872437], "36": ["vertical_and_slash", 1000, 6096, 0.37194108963012695], "37": ["vertical_and_slash", 1000, 6096, 0.37832242250442505], "38": ["vertical_and_slash", 1000, 6096, 0.38845959305763245], "39": ["vertical_and_slash", 1000, 6096, 0.3674226403236389], "40": ["vertical_and_slash", 1000, 6096, 0.38835468888282776], "41": ["vertical_and_slash", 1000, 6096, 0.3673158288002014], "42": ["vertical_and_slash", 1000, 6096, 0.3699919283390045], "43": ["vertical_and_slash", 1000, 6096, 0.3741922378540039], "44": ["vertical_and_slash", 1000, 6096, 0.3682250380516052], "45": ["vertical_and_slash", 1000, 6096, 0.37582725286483765], "46": ["vertical_and_slash", 1000, 6096, 0.37054142355918884], "47": ["vertical_and_slash", 1000, 6096, 0.3725587725639343], "48": ["vertical_and_slash", 1000, 6096, 0.40629929304122925], "49": ["vertical_and_slash", 1000, 6096, 0.38878676295280457], "50": ["vertical_and_slash", 1000, 6096, 0.3731081783771515], "51": ["vertical_and_slash", 1000, 6096, 0.3691254258155823], "52": ["vertical_and_slash", 1000, 6096, 0.3775404095649719], "53": ["vertical_and_slash", 1000, 6096, 0.3754219710826874], "54": ["vertical_and_slash", 1000, 6096, 0.3778444528579712], "55": ["vertical_and_slash", 1000, 6096, 0.3741757273674011], "56": ["vertical_and_slash", 1000, 6096, 0.36563917994499207], "57": ["vertical_and_slash", 1000, 6096, 0.37611716985702515], "58": ["vertical_and_slash", 1000, 6096, 0.3799763023853302], "59": ["vertical_and_slash", 1000, 6096, 0.38478827476501465], "60": ["vertical_and_slash", 1000, 6096, 0.36727210879325867], "61": ["vertical_and_slash", 1000, 6096, 0.38235795497894287], "62": ["vertical_and_slash", 1000, 6096, 0.3827672004699707], "63": ["vertical_and_slash", 1000, 6096, 0.3814229369163513]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3566187918186188], "1": ["vertical_and_slash", 1000, 6096, 0.3619067668914795], "2": ["vertical_and_slash", 1000, 6096, 0.37470543384552], "3": ["vertical_and_slash", 1000, 6096, 0.35270142555236816], "4": ["vertical_and_slash", 1000, 6096, 0.3594345450401306], "5": ["vertical_and_slash", 1000, 6096, 0.3605547845363617], "6": ["vertical_and_slash", 1000, 6096, 0.3628823459148407], "7": ["vertical_and_slash", 1000, 6096, 0.3554936647415161], "8": ["vertical_and_slash", 1000, 6096, 0.36920759081840515], "9": ["vertical_and_slash", 1000, 6096, 0.3635232150554657], "10": ["vertical_and_slash", 1000, 6096, 0.3662239611148834], "11": ["vertical_and_slash", 1000, 6096, 0.3648357689380646], "12": ["vertical_and_slash", 1000, 6096, 0.40021297335624695], "13": ["vertical_and_slash", 1000, 6096, 0.3818429410457611], "14": ["vertical_and_slash", 1000, 6096, 0.376149982213974], "15": ["vertical_and_slash", 1000, 6096, 0.3606378436088562], "16": ["vertical_and_slash", 1000, 6096, 0.37280526757240295], "17": ["vertical_and_slash", 1000, 6096, 0.36760470271110535], "18": ["vertical_and_slash", 1000, 6096, 0.3810350298881531], "19": ["vertical_and_slash", 1000, 6096, 0.3643076717853546], "20": ["vertical_and_slash", 1000, 6096, 0.36931589245796204], "21": ["vertical_and_slash", 1000, 6096, 0.3787781000137329], "22": ["vertical_and_slash", 1000, 6096, 0.38158154487609863], "23": ["vertical_and_slash", 1000, 6096, 0.36768874526023865], "24": ["vertical_and_slash", 1000, 6096, 0.3652656674385071], "25": ["vertical_and_slash", 1000, 6096, 0.361856073141098], "26": ["vertical_and_slash", 1000, 6096, 0.37997981905937195], "27": ["vertical_and_slash", 1000, 6096, 0.36848005652427673], "28": ["vertical_and_slash", 1000, 6096, 0.371115118265152], "29": ["vertical_and_slash", 1000, 6096, 0.3674304485321045], "30": ["vertical_and_slash", 1000, 6096, 0.36765751242637634], "31": ["vertical_and_slash", 1000, 6096, 0.3805314302444458], "32": ["vertical_and_slash", 1000, 6096, 0.37854450941085815], "33": ["vertical_and_slash", 1000, 6096, 0.36444634199142456], "34": ["vertical_and_slash", 1000, 6096, 0.3698270916938782], "35": ["vertical_and_slash", 1000, 6096, 0.3623827397823334], "36": ["vertical_and_slash", 1000, 6096, 0.35766351222991943], "37": ["vertical_and_slash", 1000, 6096, 0.36073973774909973], "38": ["vertical_and_slash", 1000, 6096, 0.3541683852672577], "39": ["vertical_and_slash", 1000, 6096, 0.3749571740627289], "40": ["vertical_and_slash", 1000, 6096, 0.3504309058189392], "41": ["vertical_and_slash", 1000, 6096, 0.37222686409950256], "42": ["vertical_and_slash", 1000, 6096, 0.3539640009403229], "43": ["vertical_and_slash", 1000, 6096, 0.3528381288051605], "44": ["vertical_and_slash", 1000, 6096, 0.3537288010120392], "45": ["vertical_and_slash", 1000, 6096, 0.35230109095573425], "46": ["vertical_and_slash", 1000, 6096, 0.36482012271881104], "47": ["vertical_and_slash", 1000, 6096, 0.3669638931751251], "48": ["vertical_and_slash", 1000, 6096, 0.3642110228538513], "49": ["vertical_and_slash", 1000, 6096, 0.35827207565307617], "50": ["vertical_and_slash", 1000, 6096, 0.3619210422039032], "51": ["vertical_and_slash", 1000, 6096, 0.363626092672348], "52": ["vertical_and_slash", 1000, 6096, 0.3676761984825134], "53": ["vertical_and_slash", 1000, 6096, 0.36392325162887573], "54": ["vertical_and_slash", 1000, 6096, 0.37135830521583557], "55": ["vertical_and_slash", 1000, 6096, 0.3713507056236267], "56": ["vertical_and_slash", 1000, 6096, 0.3715764582157135], "57": ["vertical_and_slash", 1000, 6096, 0.3801923096179962], "58": ["vertical_and_slash", 1000, 6096, 0.3708692193031311], "59": ["vertical_and_slash", 1000, 6096, 0.4186125099658966], "60": ["vertical_and_slash", 1000, 6096, 0.3894710838794708], "61": ["vertical_and_slash", 1000, 6096, 0.3814823627471924], "62": ["vertical_and_slash", 1000, 6096, 0.3804266154766083], "63": ["vertical_and_slash", 1000, 6096, 0.3950898051261902]}, {"0": ["vertical_and_slash", 1000, 6096, 0.357938289642334], "1": ["vertical_and_slash", 1000, 6096, 0.36024388670921326], "2": ["vertical_and_slash", 1000, 6096, 0.360128253698349], "3": ["vertical_and_slash", 1000, 6096, 0.3575917184352875], "4": ["vertical_and_slash", 1000, 6096, 0.3520105481147766], "5": ["vertical_and_slash", 1000, 6096, 0.35521453619003296], "6": ["vertical_and_slash", 1000, 6096, 0.3585459291934967], "7": ["vertical_and_slash", 1000, 6096, 0.3757757544517517], "8": ["vertical_and_slash", 1000, 6096, 0.3700443506240845], "9": ["vertical_and_slash", 1000, 6096, 0.38120803236961365], "10": ["vertical_and_slash", 1000, 6096, 0.3807760179042816], "11": ["vertical_and_slash", 1000, 6096, 0.35879161953926086], "12": ["vertical_and_slash", 1000, 6096, 0.3715957701206207], "13": ["vertical_and_slash", 1000, 6096, 0.3634534180164337], "14": ["vertical_and_slash", 1000, 6096, 0.37540698051452637], "15": ["vertical_and_slash", 1000, 6096, 0.3736206293106079], "16": ["vertical_and_slash", 1000, 6096, 0.35017797350883484], "17": ["vertical_and_slash", 1000, 6096, 0.35521772503852844], "18": ["vertical_and_slash", 1000, 6096, 0.36621612310409546], "19": ["vertical_and_slash", 1000, 6096, 0.3608991503715515], "20": ["vertical_and_slash", 1000, 6096, 0.3451351225376129], "21": ["vertical_and_slash", 1000, 6096, 0.349854975938797], "22": ["vertical_and_slash", 1000, 6096, 0.35185539722442627], "23": ["vertical_and_slash", 1000, 6096, 0.35351797938346863], "24": ["vertical_and_slash", 1000, 6096, 0.35628360509872437], "25": ["vertical_and_slash", 1000, 6096, 0.3568998873233795], "26": ["vertical_and_slash", 1000, 6096, 0.36489948630332947], "27": ["vertical_and_slash", 1000, 6096, 0.3635106086730957], "28": ["vertical_and_slash", 1000, 6096, 0.35797807574272156], "29": ["vertical_and_slash", 1000, 6096, 0.3563781976699829], "30": ["vertical_and_slash", 1000, 6096, 0.36060047149658203], "31": ["vertical_and_slash", 1000, 6096, 0.3795471787452698], "32": ["vertical_and_slash", 1000, 6096, 0.35462477803230286], "33": ["vertical_and_slash", 1000, 6096, 0.355884313583374], "34": ["vertical_and_slash", 1000, 6096, 0.350188672542572], "35": ["vertical_and_slash", 1000, 6096, 0.34990882873535156], "36": ["vertical_and_slash", 1000, 6096, 0.35742005705833435], "37": ["vertical_and_slash", 1000, 6096, 0.36220425367355347], "38": ["vertical_and_slash", 1000, 6096, 0.35615041851997375], "39": ["vertical_and_slash", 1000, 6096, 0.35913610458374023], "40": ["vertical_and_slash", 1000, 6096, 0.37658798694610596], "41": ["vertical_and_slash", 1000, 6096, 0.35342302918434143], "42": ["vertical_and_slash", 1000, 6096, 0.35605353116989136], "43": ["vertical_and_slash", 1000, 6096, 0.3665013015270233], "44": ["vertical_and_slash", 1000, 6096, 0.3637731671333313], "45": ["vertical_and_slash", 1000, 6096, 0.3680705428123474], "46": ["vertical_and_slash", 1000, 6096, 0.3507547378540039], "47": ["vertical_and_slash", 1000, 6096, 0.3610736131668091], "48": ["vertical_and_slash", 1000, 6096, 0.35632461309432983], "49": ["vertical_and_slash", 1000, 6096, 0.3560921847820282], "50": ["vertical_and_slash", 1000, 6096, 0.36020880937576294], "51": ["vertical_and_slash", 1000, 6096, 0.3490670919418335], "52": ["vertical_and_slash", 1000, 6096, 0.3547857999801636], "53": ["vertical_and_slash", 1000, 6096, 0.35797232389450073], "54": ["vertical_and_slash", 1000, 6096, 0.3661348521709442], "55": ["vertical_and_slash", 1000, 6096, 0.35477980971336365], "56": ["vertical_and_slash", 1000, 6096, 0.37271708250045776], "57": ["vertical_and_slash", 1000, 6096, 0.365212082862854], "58": ["vertical_and_slash", 1000, 6096, 0.34998568892478943], "59": ["vertical_and_slash", 1000, 6096, 0.36771050095558167], "60": ["vertical_and_slash", 1000, 6096, 0.35784322023391724], "61": ["vertical_and_slash", 1000, 6096, 0.35909396409988403], "62": ["vertical_and_slash", 1000, 6096, 0.3517145812511444], "63": ["vertical_and_slash", 1000, 6096, 0.3692740499973297]}, {"0": ["vertical_and_slash", 1000, 6096, 0.3740648627281189], "1": ["vertical_and_slash", 1000, 6096, 0.37486496567726135], "2": ["vertical_and_slash", 1000, 6096, 0.3852858245372772], "3": ["vertical_and_slash", 1000, 6096, 0.37480664253234863], "4": ["vertical_and_slash", 1000, 6096, 0.3619067370891571], "5": ["vertical_and_slash", 1000, 6096, 0.3711041510105133], "6": ["vertical_and_slash", 1000, 6096, 0.3649260997772217], "7": ["vertical_and_slash", 1000, 6096, 0.3850594162940979], "8": ["vertical_and_slash", 1000, 6096, 0.37193939089775085], "9": ["vertical_and_slash", 1000, 6096, 0.3600161373615265], "10": ["vertical_and_slash", 1000, 6096, 0.3698127865791321], "11": ["vertical_and_slash", 1000, 6096, 0.3740616738796234], "12": ["vertical_and_slash", 1000, 6096, 0.36515018343925476], "13": ["vertical_and_slash", 1000, 6096, 0.3710411787033081], "14": ["vertical_and_slash", 1000, 6096, 0.37784063816070557], "15": ["vertical_and_slash", 1000, 6096, 0.35425975918769836], "16": ["vertical_and_slash", 1000, 6096, 0.3996830880641937], "17": ["vertical_and_slash", 1000, 6096, 0.3871521055698395], "18": ["vertical_and_slash", 1000, 6096, 0.37696951627731323], "19": ["vertical_and_slash", 1000, 6096, 0.3683464229106903], "20": ["vertical_and_slash", 1000, 6096, 0.38347938656806946], "21": ["vertical_and_slash", 1000, 6096, 0.3825044631958008], "22": ["vertical_and_slash", 1000, 6096, 0.38928768038749695], "23": ["vertical_and_slash", 1000, 6096, 0.38038480281829834], "24": ["vertical_and_slash", 1000, 6096, 0.3715679347515106], "25": ["vertical_and_slash", 1000, 6096, 0.38445761799812317], "26": ["vertical_and_slash", 1000, 6096, 0.37853386998176575], "27": ["vertical_and_slash", 1000, 6096, 0.41824617981910706], "28": ["vertical_and_slash", 1000, 6096, 0.38957327604293823], "29": ["vertical_and_slash", 1000, 6096, 0.37777233123779297], "30": ["vertical_and_slash", 1000, 6096, 0.38173285126686096], "31": ["vertical_and_slash", 1000, 6096, 0.3898252844810486], "32": ["vertical_and_slash", 1000, 6096, 0.3769368529319763], "33": ["vertical_and_slash", 1000, 6096, 0.36450859904289246], "34": ["vertical_and_slash", 1000, 6096, 0.35922348499298096], "35": ["vertical_and_slash", 1000, 6096, 0.3637149930000305], "36": ["vertical_and_slash", 1000, 6096, 0.3646656572818756], "37": ["vertical_and_slash", 1000, 6096, 0.35800236463546753], "38": ["vertical_and_slash", 1000, 6096, 0.37881869077682495], "39": ["vertical_and_slash", 1000, 6096, 0.3738784193992615], "40": ["vertical_and_slash", 1000, 6096, 0.36251866817474365], "41": ["vertical_and_slash", 1000, 6096, 0.36398065090179443], "42": ["vertical_and_slash", 1000, 6096, 0.3841305375099182], "43": ["vertical_and_slash", 1000, 6096, 0.3754885494709015], "44": ["vertical_and_slash", 1000, 6096, 0.3591947555541992], "45": ["vertical_and_slash", 1000, 6096, 0.3660524785518646], "46": ["vertical_and_slash", 1000, 6096, 0.36477401852607727], "47": ["vertical_and_slash", 1000, 6096, 0.38785961270332336], "48": ["vertical_and_slash", 1000, 6096, 0.37626680731773376], "49": ["vertical_and_slash", 1000, 6096, 0.37975895404815674], "50": ["vertical_and_slash", 1000, 6096, 0.38084113597869873], "51": ["vertical_and_slash", 1000, 6096, 0.3661802113056183], "52": ["vertical_and_slash", 1000, 6096, 0.357367604970932], "53": ["vertical_and_slash", 1000, 6096, 0.36629053950309753], "54": ["vertical_and_slash", 1000, 6096, 0.3700225055217743], "55": ["vertical_and_slash", 1000, 6096, 0.3759019076824188], "56": ["vertical_and_slash", 1000, 6096, 0.3702946901321411], "57": ["vertical_and_slash", 1000, 6096, 0.3601836562156677], "58": ["vertical_and_slash", 1000, 6096, 0.3662373721599579], "59": ["vertical_and_slash", 1000, 6096, 0.37751877307891846], "60": ["vertical_and_slash", 1000, 6096, 0.38427868485450745], "61": ["vertical_and_slash", 1000, 6096, 0.37322092056274414], "62": ["vertical_and_slash", 1000, 6096, 0.36712998151779175], "63": ["vertical_and_slash", 1000, 6096, 0.3592616021633148]}, {"0": ["vertical_and_slash", 1000, 6096, 0.36339646577835083], "1": ["vertical_and_slash", 1000, 6096, 0.3600403964519501], "2": ["vertical_and_slash", 1000, 6096, 0.3781537711620331], "3": ["vertical_and_slash", 1000, 6096, 0.3728213608264923], "4": ["vertical_and_slash", 1000, 6096, 0.3802107572555542], "5": ["vertical_and_slash", 1000, 6096, 0.3941442668437958], "6": ["vertical_and_slash", 1000, 6096, 0.372054785490036], "7": ["vertical_and_slash", 1000, 6096, 0.3692447543144226], "8": ["vertical_and_slash", 1000, 6096, 0.3656737208366394], "9": ["vertical_and_slash", 1000, 6096, 0.365548700094223], "10": ["vertical_and_slash", 1000, 6096, 0.36535951495170593], "11": ["vertical_and_slash", 1000, 6096, 0.39655086398124695], "12": ["vertical_and_slash", 1000, 6096, 0.3640058636665344], "13": ["vertical_and_slash", 1000, 6096, 0.3749487102031708], "14": ["vertical_and_slash", 1000, 6096, 0.36926528811454773], "15": ["vertical_and_slash", 1000, 6096, 0.3742831349372864], "16": ["vertical_and_slash", 1000, 6096, 0.36030033230781555], "17": ["vertical_and_slash", 1000, 6096, 0.36436963081359863], "18": ["vertical_and_slash", 1000, 6096, 0.37299400568008423], "19": ["vertical_and_slash", 1000, 6096, 0.37171757221221924], "20": ["vertical_and_slash", 1000, 6096, 0.3668251633644104], "21": ["vertical_and_slash", 1000, 6096, 0.360074520111084], "22": ["vertical_and_slash", 1000, 6096, 0.3724484443664551], "23": ["vertical_and_slash", 1000, 6096, 0.3665454387664795], "24": ["vertical_and_slash", 1000, 6096, 0.39180997014045715], "25": ["vertical_and_slash", 1000, 6096, 0.3832904100418091], "26": ["vertical_and_slash", 1000, 6096, 0.3707634508609772], "27": ["vertical_and_slash", 1000, 6096, 0.37063664197921753], "28": ["vertical_and_slash", 1000, 6096, 0.3716368079185486], "29": ["vertical_and_slash", 1000, 6096, 0.3782338500022888], "30": ["vertical_and_slash", 1000, 6096, 0.365977019071579], "31": ["vertical_and_slash", 1000, 6096, 0.3705078661441803], "32": ["vertical_and_slash", 1000, 6096, 0.37710925936698914], "33": ["vertical_and_slash", 1000, 6096, 0.38067111372947693], "34": ["vertical_and_slash", 1000, 6096, 0.3733297884464264], "35": ["vertical_and_slash", 1000, 6096, 0.3890914022922516], "36": ["vertical_and_slash", 1000, 6096, 0.36826762557029724], "37": ["vertical_and_slash", 1000, 6096, 0.3700847029685974], "38": ["vertical_and_slash", 1000, 6096, 0.37415599822998047], "39": ["vertical_and_slash", 1000, 6096, 0.3744021952152252], "40": ["vertical_and_slash", 1000, 6096, 0.35732969641685486], "41": ["vertical_and_slash", 1000, 6096, 0.39529746770858765], "42": ["vertical_and_slash", 1000, 6096, 0.3495636284351349], "43": ["vertical_and_slash", 1000, 6096, 0.3593502938747406], "44": ["vertical_and_slash", 1000, 6096, 0.3548181354999542], "45": ["vertical_and_slash", 1000, 6096, 0.35128769278526306], "46": ["vertical_and_slash", 1000, 6096, 0.3526619076728821], "47": ["vertical_and_slash", 1000, 6096, 0.3649129867553711], "48": ["vertical_and_slash", 1000, 6096, 0.35753485560417175], "49": ["vertical_and_slash", 1000, 6096, 0.37248465418815613], "50": ["vertical_and_slash", 1000, 6096, 0.3646085560321808], "51": ["vertical_and_slash", 1000, 6096, 0.3752126097679138], "52": ["vertical_and_slash", 1000, 6096, 0.3686138987541199], "53": ["vertical_and_slash", 1000, 6096, 0.38248470425605774], "54": ["vertical_and_slash", 1000, 6096, 0.3554113805294037], "55": ["vertical_and_slash", 1000, 6096, 0.37308669090270996], "56": ["vertical_and_slash", 1000, 6096, 0.3537604510784149], "57": ["vertical_and_slash", 1000, 6096, 0.36234408617019653], "58": ["vertical_and_slash", 1000, 6096, 0.356534481048584], "59": ["vertical_and_slash", 1000, 6096, 0.3515625], "60": ["vertical_and_slash", 1000, 6096, 0.3611590266227722], "61": ["vertical_and_slash", 1000, 6096, 0.351370245218277], "62": ["vertical_and_slash", 1000, 6096, 0.3639746606349945], "63": ["vertical_and_slash", 1000, 6096, 0.3573199510574341]}, {"0": ["vertical_and_slash", 1000, 6096, 0.38574838638305664], "1": ["vertical_and_slash", 1000, 6096, 0.39867493510246277], "2": ["vertical_and_slash", 1000, 6096, 0.3738492727279663], "3": ["vertical_and_slash", 1000, 6096, 0.38173624873161316], "4": ["vertical_and_slash", 1000, 6096, 0.39940178394317627], "5": ["vertical_and_slash", 1000, 6096, 0.38346046209335327], "6": ["vertical_and_slash", 1000, 6096, 0.3759038746356964], "7": ["vertical_and_slash", 1000, 6096, 0.37453997135162354], "8": ["vertical_and_slash", 1000, 6096, 0.42327409982681274], "9": ["vertical_and_slash", 1000, 6096, 0.40512293577194214], "10": ["vertical_and_slash", 1000, 6096, 0.35500475764274597], "11": ["vertical_and_slash", 1000, 6096, 0.3618972599506378], "12": ["vertical_and_slash", 1000, 6096, 0.3731614947319031], "13": ["vertical_and_slash", 1000, 6096, 0.3725593388080597], "14": ["vertical_and_slash", 1000, 6096, 0.37344107031822205], "15": ["vertical_and_slash", 1000, 6096, 0.37377434968948364], "16": ["vertical_and_slash", 1000, 6096, 0.3819137215614319], "17": ["vertical_and_slash", 1000, 6096, 0.3737521767616272], "18": ["vertical_and_slash", 1000, 6096, 0.3633767068386078], "19": ["vertical_and_slash", 1000, 6096, 0.35884949564933777], "20": ["vertical_and_slash", 1000, 6096, 0.35816293954849243], "21": ["vertical_and_slash", 1000, 6096, 0.36315861344337463], "22": ["vertical_and_slash", 1000, 6096, 0.37718406319618225], "23": ["vertical_and_slash", 1000, 6096, 0.3707951605319977], "24": ["vertical_and_slash", 1000, 6096, 0.38978198170661926], "25": ["vertical_and_slash", 1000, 6096, 0.38809099793434143], "26": ["vertical_and_slash", 1000, 6096, 0.43344786763191223], "27": ["vertical_and_slash", 1000, 6096, 0.3935534656047821], "28": ["vertical_and_slash", 1000, 6096, 0.4064139723777771], "29": ["vertical_and_slash", 1000, 6096, 0.4335075914859772], "30": ["vertical_and_slash", 1000, 6096, 0.43194717168807983], "31": ["vertical_and_slash", 1000, 6096, 0.41418778896331787], "32": ["vertical_and_slash", 1000, 6096, 0.4268084764480591], "33": ["vertical_and_slash", 1000, 6096, 0.36528879404067993], "34": ["vertical_and_slash", 1000, 6096, 0.38026881217956543], "35": ["vertical_and_slash", 1000, 6096, 0.379869282245636], "36": ["vertical_and_slash", 1000, 6096, 0.3884719908237457], "37": ["vertical_and_slash", 1000, 6096, 0.38413292169570923], "38": ["vertical_and_slash", 1000, 6096, 0.3849411904811859], "39": ["vertical_and_slash", 1000, 6096, 0.38790372014045715], "40": ["vertical_and_slash", 1000, 6096, 0.3804219365119934], "41": ["vertical_and_slash", 1000, 6096, 0.37322115898132324], "42": ["vertical_and_slash", 1000, 6096, 0.36112016439437866], "43": ["vertical_and_slash", 1000, 6096, 0.3840564489364624], "44": ["vertical_and_slash", 1000, 6096, 0.37754812836647034], "45": ["vertical_and_slash", 1000, 6096, 0.370213121175766], "46": ["vertical_and_slash", 1000, 6096, 0.36632460355758667], "47": ["vertical_and_slash", 1000, 6096, 0.3661336898803711], "48": ["vertical_and_slash", 1000, 6096, 0.3828354775905609], "49": ["vertical_and_slash", 1000, 6096, 0.3649117946624756], "50": ["vertical_and_slash", 1000, 6096, 0.3730873167514801], "51": ["vertical_and_slash", 1000, 6096, 0.3679453730583191], "52": ["vertical_and_slash", 1000, 6096, 0.39418232440948486], "53": ["vertical_and_slash", 1000, 6096, 0.3661491870880127], "54": ["vertical_and_slash", 1000, 6096, 0.39416685700416565], "55": ["vertical_and_slash", 1000, 6096, 0.37335261702537537], "56": ["vertical_and_slash", 1000, 6096, 0.3683229088783264], "57": ["vertical_and_slash", 1000, 6096, 0.38090837001800537], "58": ["vertical_and_slash", 1000, 6096, 0.37130388617515564], "59": ["vertical_and_slash", 1000, 6096, 0.3633832335472107], "60": ["vertical_and_slash", 1000, 6096, 0.3612724244594574], "61": ["vertical_and_slash", 1000, 6096, 0.37360474467277527], "62": ["vertical_and_slash", 1000, 6096, 0.3683355748653412], "63": ["vertical_and_slash", 1000, 6096, 0.36973321437835693]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4660157561302185], "1": ["vertical_and_slash", 1000, 6096, 0.44268858432769775], "2": ["vertical_and_slash", 1000, 6096, 0.40737831592559814], "3": ["vertical_and_slash", 1000, 6096, 0.4223536252975464], "4": ["vertical_and_slash", 1000, 6096, 0.44070756435394287], "5": ["vertical_and_slash", 1000, 6096, 0.43303588032722473], "6": ["vertical_and_slash", 1000, 6096, 0.4434970021247864], "7": ["vertical_and_slash", 1000, 6096, 0.43618330359458923], "8": ["vertical_and_slash", 1000, 6096, 0.4757910966873169], "9": ["vertical_and_slash", 1000, 6096, 0.3944772779941559], "10": ["vertical_and_slash", 1000, 6096, 0.37947404384613037], "11": ["vertical_and_slash", 1000, 6096, 0.39983874559402466], "12": ["vertical_and_slash", 1000, 6096, 0.41626355051994324], "13": ["vertical_and_slash", 1000, 6096, 0.40138667821884155], "14": ["vertical_and_slash", 1000, 6096, 0.42066308856010437], "15": ["vertical_and_slash", 1000, 6096, 0.4272059202194214], "16": ["vertical_and_slash", 1000, 6096, 0.4761974513530731], "17": ["vertical_and_slash", 1000, 6096, 0.46922603249549866], "18": ["vertical_and_slash", 1000, 6096, 0.4519406259059906], "19": ["vertical_and_slash", 1000, 6096, 0.4118191599845886], "20": ["vertical_and_slash", 1000, 6096, 0.4034320116043091], "21": ["vertical_and_slash", 1000, 6096, 0.3967684507369995], "22": ["vertical_and_slash", 1000, 6096, 0.5331102013587952], "23": ["vertical_and_slash", 1000, 6096, 0.45297539234161377], "24": ["vertical_and_slash", 1000, 6096, 0.4178527891635895], "25": ["vertical_and_slash", 1000, 6096, 0.4222751557826996], "26": ["vertical_and_slash", 1000, 6096, 0.47714754939079285], "27": ["vertical_and_slash", 1000, 6096, 0.3798837661743164], "28": ["vertical_and_slash", 1000, 6096, 0.4320160150527954], "29": ["vertical_and_slash", 1000, 6096, 0.42287904024124146], "30": ["vertical_and_slash", 1000, 6096, 0.3990868926048279], "31": ["vertical_and_slash", 1000, 6096, 0.4125703275203705], "32": ["vertical_and_slash", 1000, 6096, 0.45855456590652466], "33": ["vertical_and_slash", 1000, 6096, 0.42893120646476746], "34": ["vertical_and_slash", 1000, 6096, 0.39093878865242004], "35": ["vertical_and_slash", 1000, 6096, 0.4588657319545746], "36": ["vertical_and_slash", 1000, 6096, 0.45003220438957214], "37": ["vertical_and_slash", 1000, 6096, 0.4481702744960785], "38": ["vertical_and_slash", 1000, 6096, 0.41273245215415955], "39": ["vertical_and_slash", 1000, 6096, 0.4360661804676056], "40": ["vertical_and_slash", 1000, 6096, 0.392119824886322], "41": ["vertical_and_slash", 1000, 6096, 0.39839404821395874], "42": ["vertical_and_slash", 1000, 6096, 0.4766814410686493], "43": ["vertical_and_slash", 1000, 6096, 0.39955753087997437], "44": ["vertical_and_slash", 1000, 6096, 0.46402132511138916], "45": ["vertical_and_slash", 1000, 6096, 0.4062078297138214], "46": ["vertical_and_slash", 1000, 6096, 0.428473562002182], "47": ["vertical_and_slash", 1000, 6096, 0.3990144729614258], "48": ["vertical_and_slash", 1000, 6096, 0.4036410450935364], "49": ["vertical_and_slash", 1000, 6096, 0.41786909103393555], "50": ["vertical_and_slash", 1000, 6096, 0.40974608063697815], "51": ["vertical_and_slash", 1000, 6096, 0.4099010229110718], "52": ["vertical_and_slash", 1000, 6096, 0.4186150133609772], "53": ["vertical_and_slash", 1000, 6096, 0.38694512844085693], "54": ["vertical_and_slash", 1000, 6096, 0.4010957181453705], "55": ["vertical_and_slash", 1000, 6096, 0.3977228105068207], "56": ["vertical_and_slash", 1000, 6096, 0.41657763719558716], "57": ["vertical_and_slash", 1000, 6096, 0.4591718912124634], "58": ["vertical_and_slash", 1000, 6096, 0.3988221287727356], "59": ["vertical_and_slash", 1000, 6096, 0.43518510460853577], "60": ["vertical_and_slash", 1000, 6096, 0.39043381810188293], "61": ["vertical_and_slash", 1000, 6096, 0.404490202665329], "62": ["vertical_and_slash", 1000, 6096, 0.4715130031108856], "63": ["vertical_and_slash", 1000, 6096, 0.43431273102760315]}, {"0": ["vertical_and_slash", 1000, 6096, 0.6515300273895264], "1": ["vertical_and_slash", 1000, 6096, 0.4505981504917145], "2": ["vertical_and_slash", 1000, 6096, 0.5212909579277039], "3": ["vertical_and_slash", 1000, 6096, 0.5411262512207031], "4": ["vertical_and_slash", 1000, 6096, 0.5160713195800781], "5": ["vertical_and_slash", 1000, 6096, 0.7855145335197449], "6": ["vertical_and_slash", 1000, 6096, 0.7294418811798096], "7": ["vertical_and_slash", 1000, 6096, 0.620935320854187], "8": ["vertical_and_slash", 1000, 6096, 0.5861965417861938], "9": ["vertical_and_slash", 1000, 6096, 0.7902467846870422], "10": ["vertical_and_slash", 1000, 6096, 0.6646871566772461], "11": ["vertical_and_slash", 1000, 6096, 0.5973751544952393], "12": ["vertical_and_slash", 1000, 6096, 0.5890780687332153], "13": ["vertical_and_slash", 1000, 6096, 0.5717021226882935], "14": ["vertical_and_slash", 1000, 6096, 0.6620676517486572], "15": ["vertical_and_slash", 1000, 6096, 0.5246609449386597], "16": ["vertical_and_slash", 1000, 6096, 0.6719539165496826], "17": ["vertical_and_slash", 1000, 6096, 0.6614174246788025], "18": ["vertical_and_slash", 1000, 6096, 0.6296570301055908], "19": ["vertical_and_slash", 1000, 6096, 0.7111376523971558], "20": ["vertical_and_slash", 1000, 6096, 0.6233162879943848], "21": ["vertical_and_slash", 1000, 6096, 0.5408943891525269], "22": ["vertical_and_slash", 1000, 6096, 0.6016606688499451], "23": ["vertical_and_slash", 1000, 6096, 0.6799359321594238], "24": ["vertical_and_slash", 100, 750, 0.6610040068626404], "25": ["vertical_and_slash", 1000, 6096, 0.7889211773872375], "26": ["vertical_and_slash", 500, 700, 0.7028640508651733], "27": ["vertical_and_slash", 3500, 100, 0.9975858926773071], "28": ["vertical_and_slash", 1000, 6096, 0.5448091626167297], "29": ["vertical_and_slash", 1000, 6096, 0.5302020311355591], "30": ["vertical_and_slash", 1000, 6096, 0.5323185324668884], "31": ["vertical_and_slash", 1000, 6096, 0.5891228914260864], "32": ["vertical_and_slash", 1000, 6096, 0.6411687135696411], "33": ["vertical_and_slash", 1000, 6096, 0.5666732788085938], "34": ["vertical_and_slash", 1000, 6096, 0.5724313855171204], "35": ["vertical_and_slash", 1000, 6096, 0.5331219434738159], "36": ["vertical_and_slash", 1000, 6096, 0.7171430587768555], "37": ["vertical_and_slash", 1000, 6096, 0.7017630338668823], "38": ["vertical_and_slash", 1000, 6096, 0.49976542592048645], "39": ["vertical_and_slash", 1000, 6096, 0.6129236817359924], "40": ["vertical_and_slash", 3500, 100, 0.9475569725036621], "41": ["vertical_and_slash", 3500, 100, 0.9157208204269409], "42": ["vertical_and_slash", 1000, 6096, 0.5766180753707886], "43": ["vertical_and_slash", 3500, 100, 0.7831137776374817], "44": ["vertical_and_slash", 3500, 100, 0.9988152980804443], "45": ["vertical_and_slash", 1000, 6096, 0.6728964447975159], "46": ["vertical_and_slash", 1000, 6096, 0.6020607948303223], "47": ["vertical_and_slash", 1000, 6096, 0.8311521410942078], "48": ["vertical_and_slash", 1000, 6096, 0.728510320186615], "49": ["vertical_and_slash", 1000, 6096, 0.6043213605880737], "50": ["vertical_and_slash", 1000, 6096, 0.7433146238327026], "51": ["vertical_and_slash", 1000, 6096, 0.6159440875053406], "52": ["vertical_and_slash", 1000, 6096, 0.8119044899940491], "53": ["vertical_and_slash", 1000, 6096, 0.7101396918296814], "54": ["vertical_and_slash", 1000, 6096, 0.6002670526504517], "55": ["vertical_and_slash", 1000, 6096, 0.540912389755249], "56": ["vertical_and_slash", 1000, 6096, 0.5397825837135315], "57": ["vertical_and_slash", 1000, 6096, 0.6722782850265503], "58": ["vertical_and_slash", 1000, 6096, 0.7824200987815857], "59": ["vertical_and_slash", 1000, 6096, 0.5845274925231934], "60": ["vertical_and_slash", 1000, 6096, 0.6616315841674805], "61": ["vertical_and_slash", 1000, 6096, 0.7247171998023987], "62": ["vertical_and_slash", 1000, 6096, 0.549510657787323], "63": ["vertical_and_slash", 1000, 6096, 0.5926190614700317]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4252599775791168], "1": ["vertical_and_slash", 1000, 6096, 0.4883565306663513], "2": ["vertical_and_slash", 1000, 6096, 0.4752468168735504], "3": ["vertical_and_slash", 1000, 6096, 0.5031676888465881], "4": ["vertical_and_slash", 1000, 6096, 0.43338432908058167], "5": ["vertical_and_slash", 1000, 6096, 0.4636978805065155], "6": ["vertical_and_slash", 1000, 6096, 0.4409412145614624], "7": ["vertical_and_slash", 1000, 6096, 0.47628507018089294], "8": ["vertical_and_slash", 1000, 6096, 0.5530309081077576], "9": ["vertical_and_slash", 1000, 6096, 0.5880792140960693], "10": ["vertical_and_slash", 1000, 6096, 0.4694439172744751], "11": ["vertical_and_slash", 1000, 6096, 0.537788450717926], "12": ["vertical_and_slash", 1000, 6096, 0.5475204586982727], "13": ["vertical_and_slash", 1000, 6096, 0.6977584362030029], "14": ["vertical_and_slash", 1000, 6096, 0.48760995268821716], "15": ["vertical_and_slash", 1000, 6096, 0.5735030174255371], "16": ["vertical_and_slash", 1000, 6096, 0.6117610335350037], "17": ["vertical_and_slash", 1000, 6096, 0.5888631939888], "18": ["vertical_and_slash", 1000, 6096, 0.5204699039459229], "19": ["vertical_and_slash", 1000, 6096, 0.4763566851615906], "20": ["vertical_and_slash", 1000, 6096, 0.4791199862957001], "21": ["vertical_and_slash", 1000, 6096, 0.5428594350814819], "22": ["vertical_and_slash", 1000, 6096, 0.5583009719848633], "23": ["vertical_and_slash", 1000, 6096, 0.6372721791267395], "24": ["vertical_and_slash", 1000, 6096, 0.5033087730407715], "25": ["vertical_and_slash", 1000, 6096, 0.5379639863967896], "26": ["vertical_and_slash", 1000, 6096, 0.4355233609676361], "27": ["vertical_and_slash", 1000, 6096, 0.4637106955051422], "28": ["vertical_and_slash", 1000, 6096, 0.4392826557159424], "29": ["vertical_and_slash", 1000, 6096, 0.5123783946037292], "30": ["vertical_and_slash", 1000, 6096, 0.428566038608551], "31": ["vertical_and_slash", 1000, 6096, 0.49957188963890076], "32": ["vertical_and_slash", 1000, 6096, 0.4765026569366455], "33": ["vertical_and_slash", 1000, 6096, 0.48780757188796997], "34": ["vertical_and_slash", 1000, 6096, 0.4796866774559021], "35": ["vertical_and_slash", 1000, 6096, 0.45575737953186035], "36": ["vertical_and_slash", 1000, 6096, 0.4900350570678711], "37": ["vertical_and_slash", 1000, 6096, 0.5219666361808777], "38": ["vertical_and_slash", 1000, 6096, 0.5130321979522705], "39": ["vertical_and_slash", 1000, 6096, 0.470228374004364], "40": ["vertical_and_slash", 1000, 6096, 0.45862460136413574], "41": ["vertical_and_slash", 1000, 6096, 0.49796903133392334], "42": ["vertical_and_slash", 1000, 6096, 0.6387613415718079], "43": ["vertical_and_slash", 1000, 6096, 0.446304053068161], "44": ["vertical_and_slash", 1000, 6096, 0.49576324224472046], "45": ["vertical_and_slash", 1000, 6096, 0.6096411347389221], "46": ["vertical_and_slash", 1000, 6096, 0.4416685998439789], "47": ["vertical_and_slash", 1000, 6096, 0.5421876311302185], "48": ["vertical_and_slash", 1000, 6096, 0.4301278293132782], "49": ["vertical_and_slash", 1000, 6096, 0.45988330245018005], "50": ["vertical_and_slash", 1000, 6096, 0.45609840750694275], "51": ["vertical_and_slash", 1000, 6096, 0.4052225649356842], "52": ["vertical_and_slash", 1000, 6096, 0.4515402317047119], "53": ["vertical_and_slash", 1000, 6096, 0.47514936327934265], "54": ["vertical_and_slash", 1000, 6096, 0.4908839464187622], "55": ["vertical_and_slash", 1000, 6096, 0.5232811570167542], "56": ["vertical_and_slash", 1000, 6096, 0.4608404040336609], "57": ["vertical_and_slash", 1000, 6096, 0.5184496641159058], "58": ["vertical_and_slash", 1000, 6096, 0.5562751889228821], "59": ["vertical_and_slash", 1000, 6096, 0.48673340678215027], "60": ["vertical_and_slash", 1000, 6096, 0.4445953667163849], "61": ["vertical_and_slash", 1000, 6096, 0.5669548511505127], "62": ["vertical_and_slash", 1000, 6096, 0.4686007797718048], "63": ["vertical_and_slash", 1000, 6096, 0.5273696184158325]}, {"0": ["vertical_and_slash", 1000, 6096, 0.49603065848350525], "1": ["vertical_and_slash", 1000, 6096, 0.47548967599868774], "2": ["vertical_and_slash", 1000, 6096, 0.49186134338378906], "3": ["vertical_and_slash", 1000, 6096, 0.5280434489250183], "4": ["vertical_and_slash", 1000, 6096, 0.6219698786735535], "5": ["vertical_and_slash", 1000, 6096, 0.4586299955844879], "6": ["vertical_and_slash", 1000, 6096, 0.5682519674301147], "7": ["vertical_and_slash", 1000, 6096, 0.4918508529663086], "8": ["vertical_and_slash", 1000, 6096, 0.46008843183517456], "9": ["vertical_and_slash", 1000, 6096, 0.46560248732566833], "10": ["vertical_and_slash", 1000, 6096, 0.40793412923812866], "11": ["vertical_and_slash", 1000, 6096, 0.4280939996242523], "12": ["vertical_and_slash", 1000, 6096, 0.4262527823448181], "13": ["vertical_and_slash", 1000, 6096, 0.426309198141098], "14": ["vertical_and_slash", 1000, 6096, 0.4230140745639801], "15": ["vertical_and_slash", 1000, 6096, 0.45394617319107056], "16": ["vertical_and_slash", 1000, 6096, 0.44627854228019714], "17": ["vertical_and_slash", 1000, 6096, 0.4369026720523834], "18": ["vertical_and_slash", 1000, 6096, 0.4790937900543213], "19": ["vertical_and_slash", 1000, 6096, 0.45985525846481323], "20": ["vertical_and_slash", 1000, 6096, 0.454631507396698], "21": ["vertical_and_slash", 1000, 6096, 0.4451937675476074], "22": ["vertical_and_slash", 1000, 6096, 0.4661335349082947], "23": ["vertical_and_slash", 1000, 6096, 0.5026747584342957], "24": ["vertical_and_slash", 1000, 6096, 0.48514047265052795], "25": ["vertical_and_slash", 1000, 6096, 0.5012954473495483], "26": ["vertical_and_slash", 1000, 6096, 0.4554841220378876], "27": ["vertical_and_slash", 1000, 6096, 0.5559595227241516], "28": ["vertical_and_slash", 1000, 6096, 0.4844999313354492], "29": ["vertical_and_slash", 1000, 6096, 0.4845874011516571], "30": ["vertical_and_slash", 1000, 6096, 0.5289229154586792], "31": ["vertical_and_slash", 1000, 6096, 0.5059404969215393], "32": ["vertical_and_slash", 1000, 6096, 0.5129176378250122], "33": ["vertical_and_slash", 1000, 6096, 0.626176655292511], "34": ["vertical_and_slash", 1000, 6096, 0.43903034925460815], "35": ["vertical_and_slash", 1000, 6096, 0.5201215147972107], "36": ["vertical_and_slash", 1000, 6096, 0.5163840651512146], "37": ["vertical_and_slash", 1000, 6096, 0.47918590903282166], "38": ["vertical_and_slash", 1000, 6096, 0.5416553616523743], "39": ["vertical_and_slash", 1000, 6096, 0.5589368343353271], "40": ["vertical_and_slash", 1000, 6096, 0.48116788268089294], "41": ["vertical_and_slash", 1000, 6096, 0.5362274646759033], "42": ["vertical_and_slash", 1000, 6096, 0.5691434144973755], "43": ["vertical_and_slash", 1000, 6096, 0.5563256740570068], "44": ["vertical_and_slash", 1000, 6096, 0.4893530309200287], "45": ["vertical_and_slash", 1000, 6096, 0.5584943890571594], "46": ["vertical_and_slash", 1000, 6096, 0.5845264196395874], "47": ["vertical_and_slash", 1000, 6096, 0.5377086997032166], "48": ["vertical_and_slash", 1000, 6096, 0.4625309407711029], "49": ["vertical_and_slash", 1000, 6096, 0.5252872109413147], "50": ["vertical_and_slash", 1000, 6096, 0.5072610974311829], "51": ["vertical_and_slash", 1000, 6096, 0.44711118936538696], "52": ["vertical_and_slash", 1000, 6096, 0.44222235679626465], "53": ["vertical_and_slash", 1000, 6096, 0.43695294857025146], "54": ["vertical_and_slash", 1000, 6096, 0.4558522701263428], "55": ["vertical_and_slash", 1000, 6096, 0.4409315288066864], "56": ["vertical_and_slash", 1000, 6096, 0.4447178542613983], "57": ["vertical_and_slash", 1000, 6096, 0.4697675108909607], "58": ["vertical_and_slash", 1000, 6096, 0.6203633546829224], "59": ["vertical_and_slash", 1000, 6096, 0.46101099252700806], "60": ["vertical_and_slash", 1000, 6096, 0.43203189969062805], "61": ["vertical_and_slash", 1000, 6096, 0.5085282921791077], "62": ["vertical_and_slash", 1000, 6096, 0.5017989277839661], "63": ["vertical_and_slash", 1000, 6096, 0.47700297832489014]}, {"0": ["vertical_and_slash", 1000, 6096, 0.5508335828781128], "1": ["vertical_and_slash", 1000, 6096, 0.48519182205200195], "2": ["vertical_and_slash", 1000, 6096, 0.4807107150554657], "3": ["vertical_and_slash", 1000, 6096, 0.4761774241924286], "4": ["vertical_and_slash", 1000, 6096, 0.47014880180358887], "5": ["vertical_and_slash", 1000, 6096, 0.5103469491004944], "6": ["vertical_and_slash", 1000, 6096, 0.47286152839660645], "7": ["vertical_and_slash", 1000, 6096, 0.49109745025634766], "8": ["vertical_and_slash", 1000, 6096, 0.5270926356315613], "9": ["vertical_and_slash", 1000, 6096, 0.4805757403373718], "10": ["vertical_and_slash", 1000, 6096, 0.4837512969970703], "11": ["vertical_and_slash", 1000, 6096, 0.4716533422470093], "12": ["vertical_and_slash", 1000, 6096, 0.5143030881881714], "13": ["vertical_and_slash", 1000, 6096, 0.4525413513183594], "14": ["vertical_and_slash", 1000, 6096, 0.44194716215133667], "15": ["vertical_and_slash", 1000, 6096, 0.4889015555381775], "16": ["vertical_and_slash", 1000, 6096, 0.4962087869644165], "17": ["vertical_and_slash", 1000, 6096, 0.5432232618331909], "18": ["vertical_and_slash", 1000, 6096, 0.6017995476722717], "19": ["vertical_and_slash", 1000, 6096, 0.5581268072128296], "20": ["vertical_and_slash", 1000, 6096, 0.4742332100868225], "21": ["vertical_and_slash", 1000, 6096, 0.49977874755859375], "22": ["vertical_and_slash", 1000, 6096, 0.6924723982810974], "23": ["vertical_and_slash", 1000, 6096, 0.4914107620716095], "24": ["vertical_and_slash", 1000, 6096, 0.5957783460617065], "25": ["vertical_and_slash", 1000, 6096, 0.5231834650039673], "26": ["vertical_and_slash", 1000, 6096, 0.6044517159461975], "27": ["vertical_and_slash", 1000, 6096, 0.466342568397522], "28": ["vertical_and_slash", 1000, 6096, 0.5144947171211243], "29": ["vertical_and_slash", 1000, 6096, 0.43954920768737793], "30": ["vertical_and_slash", 1000, 6096, 0.44242915511131287], "31": ["vertical_and_slash", 1000, 6096, 0.5326108336448669], "32": ["vertical_and_slash", 1000, 6096, 0.483676016330719], "33": ["vertical_and_slash", 1000, 6096, 0.4718259572982788], "34": ["vertical_and_slash", 1000, 6096, 0.4781665802001953], "35": ["vertical_and_slash", 1000, 6096, 0.4421194791793823], "36": ["vertical_and_slash", 1000, 6096, 0.41562676429748535], "37": ["vertical_and_slash", 1000, 6096, 0.45732495188713074], "38": ["vertical_and_slash", 1000, 6096, 0.4249989986419678], "39": ["vertical_and_slash", 1000, 6096, 0.42548123002052307], "40": ["vertical_and_slash", 1000, 6096, 0.4482090473175049], "41": ["vertical_and_slash", 1000, 6096, 0.45386213064193726], "42": ["vertical_and_slash", 1000, 6096, 0.6224333047866821], "43": ["vertical_and_slash", 1000, 6096, 0.5152375102043152], "44": ["vertical_and_slash", 1000, 6096, 0.514540433883667], "45": ["vertical_and_slash", 1000, 6096, 0.46065428853034973], "46": ["vertical_and_slash", 1000, 6096, 0.4539284408092499], "47": ["vertical_and_slash", 1000, 6096, 0.4635797142982483], "48": ["vertical_and_slash", 1000, 6096, 0.6751765012741089], "49": ["vertical_and_slash", 1000, 6096, 0.6199723482131958], "50": ["vertical_and_slash", 1000, 6096, 0.5457154512405396], "51": ["vertical_and_slash", 1000, 6096, 0.5221445560455322], "52": ["vertical_and_slash", 1000, 6096, 0.5495740175247192], "53": ["vertical_and_slash", 1000, 6096, 0.5216384530067444], "54": ["vertical_and_slash", 1000, 6096, 0.5271718502044678], "55": ["vertical_and_slash", 1000, 6096, 0.48567965626716614], "56": ["vertical_and_slash", 1000, 6096, 0.5094864964485168], "57": ["vertical_and_slash", 1000, 6096, 0.5672903060913086], "58": ["vertical_and_slash", 1000, 6096, 0.5199127793312073], "59": ["vertical_and_slash", 1000, 6096, 0.47909924387931824], "60": ["vertical_and_slash", 1000, 6096, 0.4821353554725647], "61": ["vertical_and_slash", 1000, 6096, 0.5182809233665466], "62": ["vertical_and_slash", 1000, 6096, 0.47902852296829224], "63": ["vertical_and_slash", 1000, 6096, 0.5085273385047913]}, {"0": ["vertical_and_slash", 1000, 6096, 0.46411532163619995], "1": ["vertical_and_slash", 1000, 6096, 0.4814801812171936], "2": ["vertical_and_slash", 1000, 6096, 0.4919608533382416], "3": ["vertical_and_slash", 1000, 6096, 0.48692598938941956], "4": ["vertical_and_slash", 1000, 6096, 0.4608973264694214], "5": ["vertical_and_slash", 1000, 6096, 0.49146127700805664], "6": ["vertical_and_slash", 1000, 6096, 0.5505412817001343], "7": ["vertical_and_slash", 1000, 6096, 0.4926054775714874], "8": ["vertical_and_slash", 1000, 6096, 0.4249810576438904], "9": ["vertical_and_slash", 1000, 6096, 0.48160871863365173], "10": ["vertical_and_slash", 1000, 6096, 0.4928482174873352], "11": ["vertical_and_slash", 1000, 6096, 0.4588164687156677], "12": ["vertical_and_slash", 1000, 6096, 0.5219614505767822], "13": ["vertical_and_slash", 1000, 6096, 0.4350699186325073], "14": ["vertical_and_slash", 1000, 6096, 0.4663994014263153], "15": ["vertical_and_slash", 1000, 6096, 0.5028926134109497], "16": ["vertical_and_slash", 1000, 6096, 0.46177345514297485], "17": ["vertical_and_slash", 1000, 6096, 0.4776650369167328], "18": ["vertical_and_slash", 1000, 6096, 0.46503064036369324], "19": ["vertical_and_slash", 1000, 6096, 0.4174710512161255], "20": ["vertical_and_slash", 1000, 6096, 0.46418294310569763], "21": ["vertical_and_slash", 1000, 6096, 0.46166330575942993], "22": ["vertical_and_slash", 1000, 6096, 0.4569198489189148], "23": ["vertical_and_slash", 1000, 6096, 0.461004376411438], "24": ["vertical_and_slash", 1000, 6096, 0.49399131536483765], "25": ["vertical_and_slash", 1000, 6096, 0.48389819264411926], "26": ["vertical_and_slash", 1000, 6096, 0.6117018461227417], "27": ["vertical_and_slash", 1000, 6096, 0.542839765548706], "28": ["vertical_and_slash", 1000, 6096, 0.5442553162574768], "29": ["vertical_and_slash", 1000, 6096, 0.4584463834762573], "30": ["vertical_and_slash", 1000, 6096, 0.5563862323760986], "31": ["vertical_and_slash", 1000, 6096, 0.5317840576171875], "32": ["vertical_and_slash", 1000, 6096, 0.4908250868320465], "33": ["vertical_and_slash", 1000, 6096, 0.48119741678237915], "34": ["vertical_and_slash", 1000, 6096, 0.4150558114051819], "35": ["vertical_and_slash", 1000, 6096, 0.5354346632957458], "36": ["vertical_and_slash", 1000, 6096, 0.5643889307975769], "37": ["vertical_and_slash", 1000, 6096, 0.5317975878715515], "38": ["vertical_and_slash", 1000, 6096, 0.430389404296875], "39": ["vertical_and_slash", 1000, 6096, 0.4580123722553253], "40": ["vertical_and_slash", 1000, 6096, 0.5535675287246704], "41": ["vertical_and_slash", 1000, 6096, 0.5178784132003784], "42": ["vertical_and_slash", 1000, 6096, 0.5910638570785522], "43": ["vertical_and_slash", 1000, 6096, 0.5038484334945679], "44": ["vertical_and_slash", 1000, 6096, 0.5149455070495605], "45": ["vertical_and_slash", 1000, 6096, 0.499920517206192], "46": ["vertical_and_slash", 1000, 6096, 0.5273756384849548], "47": ["vertical_and_slash", 1000, 6096, 0.5767374634742737], "48": ["vertical_and_slash", 1000, 6096, 0.501274049282074], "49": ["vertical_and_slash", 1000, 6096, 0.44631439447402954], "50": ["vertical_and_slash", 1000, 6096, 0.47696930170059204], "51": ["vertical_and_slash", 1000, 6096, 0.5155672430992126], "52": ["vertical_and_slash", 1000, 6096, 0.533007800579071], "53": ["vertical_and_slash", 1000, 6096, 0.439169704914093], "54": ["vertical_and_slash", 1000, 6096, 0.4862023591995239], "55": ["vertical_and_slash", 1000, 6096, 0.4748392403125763], "56": ["vertical_and_slash", 1000, 6096, 0.4349669814109802], "57": ["vertical_and_slash", 1000, 6096, 0.4347091019153595], "58": ["vertical_and_slash", 1000, 6096, 0.5977556109428406], "59": ["vertical_and_slash", 1000, 6096, 0.4450590908527374], "60": ["vertical_and_slash", 1000, 6096, 0.4675091803073883], "61": ["vertical_and_slash", 1000, 6096, 0.4610532522201538], "62": ["vertical_and_slash", 1000, 6096, 0.4187769293785095], "63": ["vertical_and_slash", 1000, 6096, 0.44002583622932434]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4410122036933899], "1": ["vertical_and_slash", 1000, 6096, 0.4698728024959564], "2": ["vertical_and_slash", 1000, 6096, 0.4841963052749634], "3": ["vertical_and_slash", 1000, 6096, 0.5416591763496399], "4": ["vertical_and_slash", 1000, 6096, 0.46994850039482117], "5": ["vertical_and_slash", 1000, 6096, 0.46216586232185364], "6": ["vertical_and_slash", 1000, 6096, 0.4344812035560608], "7": ["vertical_and_slash", 1000, 6096, 0.4949011206626892], "8": ["vertical_and_slash", 1000, 6096, 0.5123904943466187], "9": ["vertical_and_slash", 1000, 6096, 0.49445581436157227], "10": ["vertical_and_slash", 1000, 6096, 0.397065132856369], "11": ["vertical_and_slash", 1000, 6096, 0.449197381734848], "12": ["vertical_and_slash", 1000, 6096, 0.470003604888916], "13": ["vertical_and_slash", 1000, 6096, 0.4447023272514343], "14": ["vertical_and_slash", 1000, 6096, 0.4352264404296875], "15": ["vertical_and_slash", 1000, 6096, 0.4315386116504669], "16": ["vertical_and_slash", 1000, 6096, 0.501745343208313], "17": ["vertical_and_slash", 1000, 6096, 0.506654679775238], "18": ["vertical_and_slash", 1000, 6096, 0.4476299583911896], "19": ["vertical_and_slash", 1000, 6096, 0.42433300614356995], "20": ["vertical_and_slash", 1000, 6096, 0.43641358613967896], "21": ["vertical_and_slash", 1000, 6096, 0.44634297490119934], "22": ["vertical_and_slash", 1000, 6096, 0.46055343747138977], "23": ["vertical_and_slash", 1000, 6096, 0.4448672831058502], "24": ["vertical_and_slash", 1000, 6096, 0.43385112285614014], "25": ["vertical_and_slash", 1000, 6096, 0.469549298286438], "26": ["vertical_and_slash", 1000, 6096, 0.48707136511802673], "27": ["vertical_and_slash", 1000, 6096, 0.4610847532749176], "28": ["vertical_and_slash", 1000, 6096, 0.47444581985473633], "29": ["vertical_and_slash", 1000, 6096, 0.4331929683685303], "30": ["vertical_and_slash", 1000, 6096, 0.4875403344631195], "31": ["vertical_and_slash", 1000, 6096, 0.4527733027935028], "32": ["vertical_and_slash", 1000, 6096, 0.6327832937240601], "33": ["vertical_and_slash", 1000, 6096, 0.5090292692184448], "34": ["vertical_and_slash", 1000, 6096, 0.5597342252731323], "35": ["vertical_and_slash", 1000, 6096, 0.4642238914966583], "36": ["vertical_and_slash", 1000, 6096, 0.5467671751976013], "37": ["vertical_and_slash", 1000, 6096, 0.493025004863739], "38": ["vertical_and_slash", 1000, 6096, 0.5197491645812988], "39": ["vertical_and_slash", 1000, 6096, 0.5347897410392761], "40": ["vertical_and_slash", 1000, 6096, 0.43812426924705505], "41": ["vertical_and_slash", 1000, 6096, 0.4833676517009735], "42": ["vertical_and_slash", 1000, 6096, 0.45229673385620117], "43": ["vertical_and_slash", 1000, 6096, 0.4973534941673279], "44": ["vertical_and_slash", 1000, 6096, 0.42278361320495605], "45": ["vertical_and_slash", 1000, 6096, 0.45839664340019226], "46": ["vertical_and_slash", 1000, 6096, 0.4865185618400574], "47": ["vertical_and_slash", 1000, 6096, 0.4275314509868622], "48": ["vertical_and_slash", 1000, 6096, 0.42882177233695984], "49": ["vertical_and_slash", 1000, 6096, 0.4077993631362915], "50": ["vertical_and_slash", 1000, 6096, 0.4209230840206146], "51": ["vertical_and_slash", 1000, 6096, 0.4334058463573456], "52": ["vertical_and_slash", 1000, 6096, 0.4365113377571106], "53": ["vertical_and_slash", 1000, 6096, 0.4487689435482025], "54": ["vertical_and_slash", 1000, 6096, 0.4237121641635895], "55": ["vertical_and_slash", 1000, 6096, 0.4091438055038452], "56": ["vertical_and_slash", 1000, 6096, 0.5794570446014404], "57": ["vertical_and_slash", 1000, 6096, 0.48699721693992615], "58": ["vertical_and_slash", 1000, 6096, 0.5281403064727783], "59": ["vertical_and_slash", 1000, 6096, 0.5473008751869202], "60": ["vertical_and_slash", 1000, 6096, 0.47732555866241455], "61": ["vertical_and_slash", 1000, 6096, 0.4529553949832916], "62": ["vertical_and_slash", 1000, 6096, 0.5141850709915161], "63": ["vertical_and_slash", 1000, 6096, 0.48903656005859375]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4680977165699005], "1": ["vertical_and_slash", 1000, 6096, 0.4371976852416992], "2": ["vertical_and_slash", 1000, 6096, 0.4695456326007843], "3": ["vertical_and_slash", 1000, 6096, 0.4250563681125641], "4": ["vertical_and_slash", 1000, 6096, 0.43243923783302307], "5": ["vertical_and_slash", 1000, 6096, 0.43911492824554443], "6": ["vertical_and_slash", 1000, 6096, 0.4326302409172058], "7": ["vertical_and_slash", 1000, 6096, 0.43591880798339844], "8": ["vertical_and_slash", 1000, 6096, 0.4389408826828003], "9": ["vertical_and_slash", 1000, 6096, 0.5881101489067078], "10": ["vertical_and_slash", 1000, 6096, 0.5652815103530884], "11": ["vertical_and_slash", 1000, 6096, 0.4700525999069214], "12": ["vertical_and_slash", 1000, 6096, 0.4685748815536499], "13": ["vertical_and_slash", 1000, 6096, 0.4869748055934906], "14": ["vertical_and_slash", 1000, 6096, 0.5501187443733215], "15": ["vertical_and_slash", 1000, 6096, 0.47760817408561707], "16": ["vertical_and_slash", 1000, 6096, 0.5102484822273254], "17": ["vertical_and_slash", 1000, 6096, 0.4423278272151947], "18": ["vertical_and_slash", 1000, 6096, 0.4662271738052368], "19": ["vertical_and_slash", 1000, 6096, 0.48650604486465454], "20": ["vertical_and_slash", 1000, 6096, 0.5088574290275574], "21": ["vertical_and_slash", 1000, 6096, 0.5231244564056396], "22": ["vertical_and_slash", 1000, 6096, 0.5574001669883728], "23": ["vertical_and_slash", 1000, 6096, 0.5048187971115112], "24": ["vertical_and_slash", 1000, 6096, 0.5544211268424988], "25": ["vertical_and_slash", 1000, 6096, 0.5070116519927979], "26": ["vertical_and_slash", 1000, 6096, 0.5297066569328308], "27": ["vertical_and_slash", 1000, 6096, 0.5339233875274658], "28": ["vertical_and_slash", 1000, 6096, 0.4757484495639801], "29": ["vertical_and_slash", 1000, 6096, 0.4518473446369171], "30": ["vertical_and_slash", 1000, 6096, 0.536051332950592], "31": ["vertical_and_slash", 1000, 6096, 0.49460625648498535], "32": ["vertical_and_slash", 1000, 6096, 0.4786381423473358], "33": ["vertical_and_slash", 1000, 6096, 0.4159650206565857], "34": ["vertical_and_slash", 1000, 6096, 0.4478234350681305], "35": ["vertical_and_slash", 1000, 6096, 0.43833127617836], "36": ["vertical_and_slash", 1000, 6096, 0.42480653524398804], "37": ["vertical_and_slash", 1000, 6096, 0.4351668655872345], "38": ["vertical_and_slash", 1000, 6096, 0.421877920627594], "39": ["vertical_and_slash", 1000, 6096, 0.4373645782470703], "40": ["vertical_and_slash", 1000, 6096, 0.44804298877716064], "41": ["vertical_and_slash", 1000, 6096, 0.41106289625167847], "42": ["vertical_and_slash", 1000, 6096, 0.48366662859916687], "43": ["vertical_and_slash", 1000, 6096, 0.4742499887943268], "44": ["vertical_and_slash", 1000, 6096, 0.4662328064441681], "45": ["vertical_and_slash", 1000, 6096, 0.48549675941467285], "46": ["vertical_and_slash", 1000, 6096, 0.4085937440395355], "47": ["vertical_and_slash", 1000, 6096, 0.43713751435279846], "48": ["vertical_and_slash", 1000, 6096, 0.4707960784435272], "49": ["vertical_and_slash", 1000, 6096, 0.47421640157699585], "50": ["vertical_and_slash", 1000, 6096, 0.5181362628936768], "51": ["vertical_and_slash", 1000, 6096, 0.5666000843048096], "52": ["vertical_and_slash", 1000, 6096, 0.4868500530719757], "53": ["vertical_and_slash", 1000, 6096, 0.5660496354103088], "54": ["vertical_and_slash", 1000, 6096, 0.488077312707901], "55": ["vertical_and_slash", 1000, 6096, 0.505315899848938], "56": ["vertical_and_slash", 1000, 6096, 0.3824881911277771], "57": ["vertical_and_slash", 1000, 6096, 0.45055705308914185], "58": ["vertical_and_slash", 1000, 6096, 0.46733978390693665], "59": ["vertical_and_slash", 1000, 6096, 0.4182070195674896], "60": ["vertical_and_slash", 1000, 6096, 0.4379477798938751], "61": ["vertical_and_slash", 1000, 6096, 0.4458672106266022], "62": ["vertical_and_slash", 1000, 6096, 0.4483768939971924], "63": ["vertical_and_slash", 1000, 6096, 0.4731011390686035]}, {"0": ["vertical_and_slash", 1000, 6096, 0.49868935346603394], "1": ["vertical_and_slash", 1000, 6096, 0.43684789538383484], "2": ["vertical_and_slash", 1000, 6096, 0.485123872756958], "3": ["vertical_and_slash", 1000, 6096, 0.4484354257583618], "4": ["vertical_and_slash", 1000, 6096, 0.46331787109375], "5": ["vertical_and_slash", 1000, 6096, 0.4914728105068207], "6": ["vertical_and_slash", 1000, 6096, 0.47908055782318115], "7": ["vertical_and_slash", 1000, 6096, 0.46361416578292847], "8": ["vertical_and_slash", 1000, 6096, 0.42380788922309875], "9": ["vertical_and_slash", 1000, 6096, 0.43345317244529724], "10": ["vertical_and_slash", 1000, 6096, 0.39636552333831787], "11": ["vertical_and_slash", 1000, 6096, 0.43411770462989807], "12": ["vertical_and_slash", 1000, 6096, 0.38948938250541687], "13": ["vertical_and_slash", 1000, 6096, 0.4370361864566803], "14": ["vertical_and_slash", 1000, 6096, 0.4305015206336975], "15": ["vertical_and_slash", 1000, 6096, 0.441876083612442], "16": ["vertical_and_slash", 1000, 6096, 0.559000551700592], "17": ["vertical_and_slash", 1000, 6096, 0.5019267201423645], "18": ["vertical_and_slash", 1000, 6096, 0.43639013171195984], "19": ["vertical_and_slash", 1000, 6096, 0.5084851384162903], "20": ["vertical_and_slash", 1000, 6096, 0.5436097979545593], "21": ["vertical_and_slash", 1000, 6096, 0.46610280871391296], "22": ["vertical_and_slash", 1000, 6096, 0.4841943085193634], "23": ["vertical_and_slash", 1000, 6096, 0.4670003652572632], "24": ["vertical_and_slash", 1000, 6096, 0.44832494854927063], "25": ["vertical_and_slash", 1000, 6096, 0.4518541097640991], "26": ["vertical_and_slash", 1000, 6096, 0.4015377163887024], "27": ["vertical_and_slash", 1000, 6096, 0.43146830797195435], "28": ["vertical_and_slash", 1000, 6096, 0.4164595901966095], "29": ["vertical_and_slash", 1000, 6096, 0.4055072069168091], "30": ["vertical_and_slash", 1000, 6096, 0.4328496754169464], "31": ["vertical_and_slash", 1000, 6096, 0.45635512471199036], "32": ["vertical_and_slash", 1000, 6096, 0.4604948163032532], "33": ["vertical_and_slash", 1000, 6096, 0.43708550930023193], "34": ["vertical_and_slash", 1000, 6096, 0.42541879415512085], "35": ["vertical_and_slash", 1000, 6096, 0.4194093644618988], "36": ["vertical_and_slash", 1000, 6096, 0.45281535387039185], "37": ["vertical_and_slash", 1000, 6096, 0.4280872344970703], "38": ["vertical_and_slash", 1000, 6096, 0.41747596859931946], "39": ["vertical_and_slash", 1000, 6096, 0.4374108612537384], "40": ["vertical_and_slash", 1000, 6096, 0.4656330347061157], "41": ["vertical_and_slash", 1000, 6096, 0.4495127499103546], "42": ["vertical_and_slash", 1000, 6096, 0.4701860547065735], "43": ["vertical_and_slash", 1000, 6096, 0.4698845446109772], "44": ["vertical_and_slash", 1000, 6096, 0.4743942618370056], "45": ["vertical_and_slash", 1000, 6096, 0.4253699779510498], "46": ["vertical_and_slash", 1000, 6096, 0.49538272619247437], "47": ["vertical_and_slash", 1000, 6096, 0.4604148864746094], "48": ["vertical_and_slash", 1000, 6096, 0.5269549489021301], "49": ["vertical_and_slash", 1000, 6096, 0.49753352999687195], "50": ["vertical_and_slash", 1000, 6096, 0.5316624045372009], "51": ["vertical_and_slash", 1000, 6096, 0.4844776391983032], "52": ["vertical_and_slash", 1000, 6096, 0.48804163932800293], "53": ["vertical_and_slash", 1000, 6096, 0.5560648441314697], "54": ["vertical_and_slash", 1000, 6096, 0.5279173254966736], "55": ["vertical_and_slash", 1000, 6096, 0.48062485456466675], "56": ["vertical_and_slash", 1000, 6096, 0.5288979411125183], "57": ["vertical_and_slash", 1000, 6096, 0.46650952100753784], "58": ["vertical_and_slash", 1000, 6096, 0.430527925491333], "59": ["vertical_and_slash", 1000, 6096, 0.4855365753173828], "60": ["vertical_and_slash", 1000, 6096, 0.46641477942466736], "61": ["vertical_and_slash", 1000, 6096, 0.47819194197654724], "62": ["vertical_and_slash", 1000, 6096, 0.4669392704963684], "63": ["vertical_and_slash", 1000, 6096, 0.4664957821369171]}, {"0": ["vertical_and_slash", 1000, 6096, 0.46156609058380127], "1": ["vertical_and_slash", 1000, 6096, 0.4344591498374939], "2": ["vertical_and_slash", 1000, 6096, 0.45364364981651306], "3": ["vertical_and_slash", 1000, 6096, 0.40165942907333374], "4": ["vertical_and_slash", 1000, 6096, 0.5074824094772339], "5": ["vertical_and_slash", 1000, 6096, 0.5014492869377136], "6": ["vertical_and_slash", 1000, 6096, 0.4601341784000397], "7": ["vertical_and_slash", 1000, 6096, 0.4660815894603729], "8": ["vertical_and_slash", 1000, 6096, 0.48788508772850037], "9": ["vertical_and_slash", 1000, 6096, 0.45073363184928894], "10": ["vertical_and_slash", 1000, 6096, 0.51706463098526], "11": ["vertical_and_slash", 1000, 6096, 0.4794793725013733], "12": ["vertical_and_slash", 1000, 6096, 0.46139317750930786], "13": ["vertical_and_slash", 1000, 6096, 0.48711517453193665], "14": ["vertical_and_slash", 1000, 6096, 0.5426151156425476], "15": ["vertical_and_slash", 1000, 6096, 0.5296763181686401], "16": ["vertical_and_slash", 1000, 6096, 0.42265570163726807], "17": ["vertical_and_slash", 1000, 6096, 0.42996805906295776], "18": ["vertical_and_slash", 1000, 6096, 0.5997281670570374], "19": ["vertical_and_slash", 1000, 6096, 0.4596956968307495], "20": ["vertical_and_slash", 1000, 6096, 0.4728432893753052], "21": ["vertical_and_slash", 1000, 6096, 0.4775063991546631], "22": ["vertical_and_slash", 1000, 6096, 0.4365221858024597], "23": ["vertical_and_slash", 1000, 6096, 0.4354846179485321], "24": ["vertical_and_slash", 1000, 6096, 0.46633511781692505], "25": ["vertical_and_slash", 1000, 6096, 0.4233175814151764], "26": ["vertical_and_slash", 1000, 6096, 0.41976645588874817], "27": ["vertical_and_slash", 1000, 6096, 0.4517352879047394], "28": ["vertical_and_slash", 1000, 6096, 0.44699952006340027], "29": ["vertical_and_slash", 1000, 6096, 0.45979517698287964], "30": ["vertical_and_slash", 1000, 6096, 0.45407551527023315], "31": ["vertical_and_slash", 1000, 6096, 0.45852193236351013], "32": ["vertical_and_slash", 1000, 6096, 0.5058051347732544], "33": ["vertical_and_slash", 1000, 6096, 0.42709600925445557], "34": ["vertical_and_slash", 1000, 6096, 0.4407193660736084], "35": ["vertical_and_slash", 1000, 6096, 0.4322720766067505], "36": ["vertical_and_slash", 1000, 6096, 0.4517405033111572], "37": ["vertical_and_slash", 1000, 6096, 0.423203706741333], "38": ["vertical_and_slash", 1000, 6096, 0.4180232286453247], "39": ["vertical_and_slash", 1000, 6096, 0.41504526138305664], "40": ["vertical_and_slash", 1000, 6096, 0.4676785171031952], "41": ["vertical_and_slash", 1000, 6096, 0.455748975276947], "42": ["vertical_and_slash", 1000, 6096, 0.4705861508846283], "43": ["vertical_and_slash", 1000, 6096, 0.43080949783325195], "44": ["vertical_and_slash", 1000, 6096, 0.40227770805358887], "45": ["vertical_and_slash", 1000, 6096, 0.4504108726978302], "46": ["vertical_and_slash", 1000, 6096, 0.4206928610801697], "47": ["vertical_and_slash", 1000, 6096, 0.427991658449173], "48": ["vertical_and_slash", 1000, 6096, 0.40786105394363403], "49": ["vertical_and_slash", 1000, 6096, 0.4938378930091858], "50": ["vertical_and_slash", 1000, 6096, 0.44073113799095154], "51": ["vertical_and_slash", 1000, 6096, 0.4573996365070343], "52": ["vertical_and_slash", 1000, 6096, 0.44514530897140503], "53": ["vertical_and_slash", 1000, 6096, 0.419162392616272], "54": ["vertical_and_slash", 1000, 6096, 0.44647255539894104], "55": ["vertical_and_slash", 1000, 6096, 0.4442127048969269], "56": ["vertical_and_slash", 1000, 6096, 0.4511460065841675], "57": ["vertical_and_slash", 1000, 6096, 0.4671826660633087], "58": ["vertical_and_slash", 1000, 6096, 0.4602128863334656], "59": ["vertical_and_slash", 1000, 6096, 0.3823090195655823], "60": ["vertical_and_slash", 1000, 6096, 0.41939136385917664], "61": ["vertical_and_slash", 1000, 6096, 0.47402676939964294], "62": ["vertical_and_slash", 1000, 6096, 0.42607003450393677], "63": ["vertical_and_slash", 1000, 6096, 0.4505433440208435]}, {"0": ["vertical_and_slash", 1000, 6096, 0.5006622672080994], "1": ["vertical_and_slash", 1000, 6096, 0.46405261754989624], "2": ["vertical_and_slash", 1000, 6096, 0.45316213369369507], "3": ["vertical_and_slash", 1000, 6096, 0.5376506447792053], "4": ["vertical_and_slash", 1000, 6096, 0.43944159150123596], "5": ["vertical_and_slash", 1000, 6096, 0.44613298773765564], "6": ["vertical_and_slash", 1000, 6096, 0.49151816964149475], "7": ["vertical_and_slash", 1000, 6096, 0.4590199887752533], "8": ["vertical_and_slash", 1000, 6096, 0.46354150772094727], "9": ["vertical_and_slash", 1000, 6096, 0.44742056727409363], "10": ["vertical_and_slash", 1000, 6096, 0.4852762818336487], "11": ["vertical_and_slash", 1000, 6096, 0.4336296319961548], "12": ["vertical_and_slash", 1000, 6096, 0.485228031873703], "13": ["vertical_and_slash", 1000, 6096, 0.4717806875705719], "14": ["vertical_and_slash", 1000, 6096, 0.43996620178222656], "15": ["vertical_and_slash", 1000, 6096, 0.46323850750923157], "16": ["vertical_and_slash", 1000, 6096, 0.4487152099609375], "17": ["vertical_and_slash", 1000, 6096, 0.5211811661720276], "18": ["vertical_and_slash", 1000, 6096, 0.4828346371650696], "19": ["vertical_and_slash", 1000, 6096, 0.5952524542808533], "20": ["vertical_and_slash", 1000, 6096, 0.4820915162563324], "21": ["vertical_and_slash", 1000, 6096, 0.4538187086582184], "22": ["vertical_and_slash", 1000, 6096, 0.4793757200241089], "23": ["vertical_and_slash", 1000, 6096, 0.5379461646080017], "24": ["vertical_and_slash", 1000, 6096, 0.4987999200820923], "25": ["vertical_and_slash", 1000, 6096, 0.45517730712890625], "26": ["vertical_and_slash", 1000, 6096, 0.46436721086502075], "27": ["vertical_and_slash", 1000, 6096, 0.4353807866573334], "28": ["vertical_and_slash", 1000, 6096, 0.4843997061252594], "29": ["vertical_and_slash", 1000, 6096, 0.5093401074409485], "30": ["vertical_and_slash", 1000, 6096, 0.4451083540916443], "31": ["vertical_and_slash", 1000, 6096, 0.44662216305732727], "32": ["vertical_and_slash", 1000, 6096, 0.4040859639644623], "33": ["vertical_and_slash", 1000, 6096, 0.46042072772979736], "34": ["vertical_and_slash", 1000, 6096, 0.46055611968040466], "35": ["vertical_and_slash", 1000, 6096, 0.42913317680358887], "36": ["vertical_and_slash", 1000, 6096, 0.3893626928329468], "37": ["vertical_and_slash", 1000, 6096, 0.408761203289032], "38": ["vertical_and_slash", 1000, 6096, 0.4394263029098511], "39": ["vertical_and_slash", 1000, 6096, 0.43646544218063354], "40": ["vertical_and_slash", 1000, 6096, 0.5609836578369141], "41": ["vertical_and_slash", 1000, 6096, 0.43525248765945435], "42": ["vertical_and_slash", 1000, 6096, 0.5020953416824341], "43": ["vertical_and_slash", 1000, 6096, 0.4100808799266815], "44": ["vertical_and_slash", 1000, 6096, 0.5318180322647095], "45": ["vertical_and_slash", 1000, 6096, 0.4413049519062042], "46": ["vertical_and_slash", 1000, 6096, 0.47434812784194946], "47": ["vertical_and_slash", 1000, 6096, 0.44107258319854736], "48": ["vertical_and_slash", 1000, 6096, 0.44639524817466736], "49": ["vertical_and_slash", 1000, 6096, 0.4767860770225525], "50": ["vertical_and_slash", 1000, 6096, 0.4389037489891052], "51": ["vertical_and_slash", 1000, 6096, 0.4821619391441345], "52": ["vertical_and_slash", 1000, 6096, 0.4760032296180725], "53": ["vertical_and_slash", 1000, 6096, 0.4596281051635742], "54": ["vertical_and_slash", 1000, 6096, 0.4296860694885254], "55": ["vertical_and_slash", 1000, 6096, 0.45710304379463196], "56": ["vertical_and_slash", 1000, 6096, 0.4868467450141907], "57": ["vertical_and_slash", 1000, 6096, 0.4491821825504303], "58": ["vertical_and_slash", 1000, 6096, 0.45621252059936523], "59": ["vertical_and_slash", 1000, 6096, 0.4795650541782379], "60": ["vertical_and_slash", 1000, 6096, 0.5382686257362366], "61": ["vertical_and_slash", 1000, 6096, 0.500622570514679], "62": ["vertical_and_slash", 1000, 6096, 0.4879606366157532], "63": ["vertical_and_slash", 1000, 6096, 0.480704665184021]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4610425531864166], "1": ["vertical_and_slash", 1000, 6096, 0.49995917081832886], "2": ["vertical_and_slash", 1000, 6096, 0.5112566947937012], "3": ["vertical_and_slash", 1000, 6096, 0.48804163932800293], "4": ["vertical_and_slash", 1000, 6096, 0.4305706024169922], "5": ["vertical_and_slash", 1000, 6096, 0.4577021896839142], "6": ["vertical_and_slash", 1000, 6096, 0.4743388891220093], "7": ["vertical_and_slash", 1000, 6096, 0.4689671993255615], "8": ["vertical_and_slash", 1000, 6096, 0.49951285123825073], "9": ["vertical_and_slash", 1000, 6096, 0.4941416084766388], "10": ["vertical_and_slash", 1000, 6096, 0.5463333129882812], "11": ["vertical_and_slash", 1000, 6096, 0.6222777366638184], "12": ["vertical_and_slash", 1000, 6096, 0.5937525629997253], "13": ["vertical_and_slash", 1000, 6096, 0.5389970541000366], "14": ["vertical_and_slash", 1000, 6096, 0.6881392002105713], "15": ["vertical_and_slash", 1000, 6096, 0.5356760621070862], "16": ["vertical_and_slash", 1000, 6096, 0.44537773728370667], "17": ["vertical_and_slash", 1000, 6096, 0.47778087854385376], "18": ["vertical_and_slash", 1000, 6096, 0.4210891127586365], "19": ["vertical_and_slash", 1000, 6096, 0.4378519654273987], "20": ["vertical_and_slash", 1000, 6096, 0.4415459930896759], "21": ["vertical_and_slash", 1000, 6096, 0.49283453822135925], "22": ["vertical_and_slash", 1000, 6096, 0.45049038529396057], "23": ["vertical_and_slash", 1000, 6096, 0.41443753242492676], "24": ["vertical_and_slash", 1000, 6096, 0.4621303975582123], "25": ["vertical_and_slash", 1000, 6096, 0.462938129901886], "26": ["vertical_and_slash", 1000, 6096, 0.4876759946346283], "27": ["vertical_and_slash", 1000, 6096, 0.4517706036567688], "28": ["vertical_and_slash", 1000, 6096, 0.46042361855506897], "29": ["vertical_and_slash", 1000, 6096, 0.43416768312454224], "30": ["vertical_and_slash", 1000, 6096, 0.4509710669517517], "31": ["vertical_and_slash", 1000, 6096, 0.4585767984390259], "32": ["vertical_and_slash", 1000, 6096, 0.4582922160625458], "33": ["vertical_and_slash", 1000, 6096, 0.4833015501499176], "34": ["vertical_and_slash", 1000, 6096, 0.4756855368614197], "35": ["vertical_and_slash", 1000, 6096, 0.5436107516288757], "36": ["vertical_and_slash", 1000, 6096, 0.4973475933074951], "37": ["vertical_and_slash", 1000, 6096, 0.4956595003604889], "38": ["vertical_and_slash", 1000, 6096, 0.5446509122848511], "39": ["vertical_and_slash", 1000, 6096, 0.49477577209472656], "40": ["vertical_and_slash", 1000, 6096, 0.49619248509407043], "41": ["vertical_and_slash", 1000, 6096, 0.46806007623672485], "42": ["vertical_and_slash", 1000, 6096, 0.4280075132846832], "43": ["vertical_and_slash", 1000, 6096, 0.4434117376804352], "44": ["vertical_and_slash", 1000, 6096, 0.5141025185585022], "45": ["vertical_and_slash", 1000, 6096, 0.49044564366340637], "46": ["vertical_and_slash", 1000, 6096, 0.4777011275291443], "47": ["vertical_and_slash", 1000, 6096, 0.4780564606189728], "48": ["vertical_and_slash", 1000, 6096, 0.46662136912345886], "49": ["vertical_and_slash", 1000, 6096, 0.44851696491241455], "50": ["vertical_and_slash", 1000, 6096, 0.5457106232643127], "51": ["vertical_and_slash", 1000, 6096, 0.43583062291145325], "52": ["vertical_and_slash", 1000, 6096, 0.47212931513786316], "53": ["vertical_and_slash", 1000, 6096, 0.4564628303050995], "54": ["vertical_and_slash", 1000, 6096, 0.5045322775840759], "55": ["vertical_and_slash", 1000, 6096, 0.499168336391449], "56": ["vertical_and_slash", 1000, 6096, 0.436750590801239], "57": ["vertical_and_slash", 1000, 6096, 0.4143075942993164], "58": ["vertical_and_slash", 1000, 6096, 0.4040338099002838], "59": ["vertical_and_slash", 1000, 6096, 0.46886804699897766], "60": ["vertical_and_slash", 1000, 6096, 0.4203106164932251], "61": ["vertical_and_slash", 1000, 6096, 0.4341118335723877], "62": ["vertical_and_slash", 1000, 6096, 0.4342438578605652], "63": ["vertical_and_slash", 1000, 6096, 0.4707201421260834]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4871036112308502], "1": ["vertical_and_slash", 1000, 6096, 0.4440644085407257], "2": ["vertical_and_slash", 1000, 6096, 0.4700782895088196], "3": ["vertical_and_slash", 1000, 6096, 0.46281448006629944], "4": ["vertical_and_slash", 1000, 6096, 0.49472811818122864], "5": ["vertical_and_slash", 1000, 6096, 0.45801684260368347], "6": ["vertical_and_slash", 1000, 6096, 0.4247463345527649], "7": ["vertical_and_slash", 1000, 6096, 0.453382283449173], "8": ["vertical_and_slash", 1000, 6096, 0.45166850090026855], "9": ["vertical_and_slash", 1000, 6096, 0.44272664189338684], "10": ["vertical_and_slash", 1000, 6096, 0.44253090023994446], "11": ["vertical_and_slash", 1000, 6096, 0.39399054646492004], "12": ["vertical_and_slash", 1000, 6096, 0.3937581181526184], "13": ["vertical_and_slash", 1000, 6096, 0.4613962471485138], "14": ["vertical_and_slash", 1000, 6096, 0.4740557074546814], "15": ["vertical_and_slash", 1000, 6096, 0.4162762463092804], "16": ["vertical_and_slash", 1000, 6096, 0.5162456035614014], "17": ["vertical_and_slash", 1000, 6096, 0.4732455909252167], "18": ["vertical_and_slash", 1000, 6096, 0.4727972745895386], "19": ["vertical_and_slash", 1000, 6096, 0.4828619360923767], "20": ["vertical_and_slash", 1000, 6096, 0.442760705947876], "21": ["vertical_and_slash", 1000, 6096, 0.47842422127723694], "22": ["vertical_and_slash", 1000, 6096, 0.48131659626960754], "23": ["vertical_and_slash", 1000, 6096, 0.45729491114616394], "24": ["vertical_and_slash", 1000, 6096, 0.4555695950984955], "25": ["vertical_and_slash", 1000, 6096, 0.4792913794517517], "26": ["vertical_and_slash", 1000, 6096, 0.49888163805007935], "27": ["vertical_and_slash", 1000, 6096, 0.4676705300807953], "28": ["vertical_and_slash", 1000, 6096, 0.4547538161277771], "29": ["vertical_and_slash", 1000, 6096, 0.4233119785785675], "30": ["vertical_and_slash", 1000, 6096, 0.47922998666763306], "31": ["vertical_and_slash", 1000, 6096, 0.436598002910614], "32": ["vertical_and_slash", 1000, 6096, 0.469208300113678], "33": ["vertical_and_slash", 1000, 6096, 0.4815141260623932], "34": ["vertical_and_slash", 1000, 6096, 0.4648880958557129], "35": ["vertical_and_slash", 1000, 6096, 0.4749490022659302], "36": ["vertical_and_slash", 1000, 6096, 0.46467679738998413], "37": ["vertical_and_slash", 1000, 6096, 0.4545235335826874], "38": ["vertical_and_slash", 1000, 6096, 0.4838399887084961], "39": ["vertical_and_slash", 1000, 6096, 0.48400357365608215], "40": ["vertical_and_slash", 1000, 6096, 0.4289005398750305], "41": ["vertical_and_slash", 1000, 6096, 0.44258639216423035], "42": ["vertical_and_slash", 1000, 6096, 0.46961772441864014], "43": ["vertical_and_slash", 1000, 6096, 0.4575331211090088], "44": ["vertical_and_slash", 1000, 6096, 0.45828476548194885], "45": ["vertical_and_slash", 1000, 6096, 0.4459514915943146], "46": ["vertical_and_slash", 1000, 6096, 0.44108614325523376], "47": ["vertical_and_slash", 1000, 6096, 0.42896515130996704], "48": ["vertical_and_slash", 1000, 6096, 0.4458617568016052], "49": ["vertical_and_slash", 1000, 6096, 0.4411483108997345], "50": ["vertical_and_slash", 1000, 6096, 0.443783164024353], "51": ["vertical_and_slash", 1000, 6096, 0.4898652136325836], "52": ["vertical_and_slash", 1000, 6096, 0.4821060299873352], "53": ["vertical_and_slash", 1000, 6096, 0.5022719502449036], "54": ["vertical_and_slash", 1000, 6096, 0.44981300830841064], "55": ["vertical_and_slash", 1000, 6096, 0.4964408278465271], "56": ["vertical_and_slash", 1000, 6096, 0.4587104916572571], "57": ["vertical_and_slash", 1000, 6096, 0.40922877192497253], "58": ["vertical_and_slash", 1000, 6096, 0.4486827552318573], "59": ["vertical_and_slash", 1000, 6096, 0.4549761116504669], "60": ["vertical_and_slash", 1000, 6096, 0.4746684432029724], "61": ["vertical_and_slash", 1000, 6096, 0.46824130415916443], "62": ["vertical_and_slash", 1000, 6096, 0.4197424352169037], "63": ["vertical_and_slash", 1000, 6096, 0.4324394166469574]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4464409351348877], "1": ["vertical_and_slash", 1000, 6096, 0.44726434350013733], "2": ["vertical_and_slash", 1000, 6096, 0.43623343110084534], "3": ["vertical_and_slash", 1000, 6096, 0.4100121259689331], "4": ["vertical_and_slash", 1000, 6096, 0.44499504566192627], "5": ["vertical_and_slash", 1000, 6096, 0.4254415035247803], "6": ["vertical_and_slash", 1000, 6096, 0.43571239709854126], "7": ["vertical_and_slash", 1000, 6096, 0.41138631105422974], "8": ["vertical_and_slash", 1000, 6096, 0.5077680945396423], "9": ["vertical_and_slash", 1000, 6096, 0.5145558714866638], "10": ["vertical_and_slash", 1000, 6096, 0.4979165494441986], "11": ["vertical_and_slash", 1000, 6096, 0.46764636039733887], "12": ["vertical_and_slash", 1000, 6096, 0.43496280908584595], "13": ["vertical_and_slash", 1000, 6096, 0.5552638173103333], "14": ["vertical_and_slash", 1000, 6096, 0.44382771849632263], "15": ["vertical_and_slash", 1000, 6096, 0.45148763060569763], "16": ["vertical_and_slash", 1000, 6096, 0.42927852272987366], "17": ["vertical_and_slash", 1000, 6096, 0.43927478790283203], "18": ["vertical_and_slash", 1000, 6096, 0.43774962425231934], "19": ["vertical_and_slash", 1000, 6096, 0.3907265365123749], "20": ["vertical_and_slash", 1000, 6096, 0.42041918635368347], "21": ["vertical_and_slash", 1000, 6096, 0.39021292328834534], "22": ["vertical_and_slash", 1000, 6096, 0.4048554003238678], "23": ["vertical_and_slash", 1000, 6096, 0.4424504041671753], "24": ["vertical_and_slash", 1000, 6096, 0.44060394167900085], "25": ["vertical_and_slash", 1000, 6096, 0.47901421785354614], "26": ["vertical_and_slash", 1000, 6096, 0.48109138011932373], "27": ["vertical_and_slash", 1000, 6096, 0.4754808247089386], "28": ["vertical_and_slash", 1000, 6096, 0.47412535548210144], "29": ["vertical_and_slash", 1000, 6096, 0.45830145478248596], "30": ["vertical_and_slash", 1000, 6096, 0.4998073875904083], "31": ["vertical_and_slash", 1000, 6096, 0.4560244679450989], "32": ["vertical_and_slash", 1000, 6096, 0.42667490243911743], "33": ["vertical_and_slash", 1000, 6096, 0.4230283498764038], "34": ["vertical_and_slash", 1000, 6096, 0.41141417622566223], "35": ["vertical_and_slash", 1000, 6096, 0.43961217999458313], "36": ["vertical_and_slash", 1000, 6096, 0.4182392358779907], "37": ["vertical_and_slash", 1000, 6096, 0.4724932312965393], "38": ["vertical_and_slash", 1000, 6096, 0.39434540271759033], "39": ["vertical_and_slash", 1000, 6096, 0.39312228560447693], "40": ["vertical_and_slash", 1000, 6096, 0.45065808296203613], "41": ["vertical_and_slash", 1000, 6096, 0.4716014266014099], "42": ["vertical_and_slash", 1000, 6096, 0.4860571026802063], "43": ["vertical_and_slash", 1000, 6096, 0.4002953767776489], "44": ["vertical_and_slash", 1000, 6096, 0.4269581437110901], "45": ["vertical_and_slash", 1000, 6096, 0.42445188760757446], "46": ["vertical_and_slash", 1000, 6096, 0.4120684266090393], "47": ["vertical_and_slash", 1000, 6096, 0.48267993330955505], "48": ["vertical_and_slash", 1000, 6096, 0.5203260779380798], "49": ["vertical_and_slash", 1000, 6096, 0.4621511697769165], "50": ["vertical_and_slash", 1000, 6096, 0.46635761857032776], "51": ["vertical_and_slash", 1000, 6096, 0.5125889182090759], "52": ["vertical_and_slash", 1000, 6096, 0.4715118408203125], "53": ["vertical_and_slash", 1000, 6096, 0.5192175507545471], "54": ["vertical_and_slash", 1000, 6096, 0.48767170310020447], "55": ["vertical_and_slash", 1000, 6096, 0.48188334703445435], "56": ["vertical_and_slash", 1000, 6096, 0.42499253153800964], "57": ["vertical_and_slash", 1000, 6096, 0.45263782143592834], "58": ["vertical_and_slash", 1000, 6096, 0.4479575753211975], "59": ["vertical_and_slash", 1000, 6096, 0.46581268310546875], "60": ["vertical_and_slash", 1000, 6096, 0.47035205364227295], "61": ["vertical_and_slash", 1000, 6096, 0.4550620913505554], "62": ["vertical_and_slash", 1000, 6096, 0.47533509135246277], "63": ["vertical_and_slash", 1000, 6096, 0.43259280920028687]}, {"0": ["vertical_and_slash", 1000, 6096, 0.41196390986442566], "1": ["vertical_and_slash", 1000, 6096, 0.49067917466163635], "2": ["vertical_and_slash", 1000, 6096, 0.4619360566139221], "3": ["vertical_and_slash", 1000, 6096, 0.46897298097610474], "4": ["vertical_and_slash", 1000, 6096, 0.45634278655052185], "5": ["vertical_and_slash", 1000, 6096, 0.4341055452823639], "6": ["vertical_and_slash", 1000, 6096, 0.4876362085342407], "7": ["vertical_and_slash", 1000, 6096, 0.4685698449611664], "8": ["vertical_and_slash", 1000, 6096, 0.4344838261604309], "9": ["vertical_and_slash", 1000, 6096, 0.4717770516872406], "10": ["vertical_and_slash", 1000, 6096, 0.4596409797668457], "11": ["vertical_and_slash", 1000, 6096, 0.42682644724845886], "12": ["vertical_and_slash", 1000, 6096, 0.4774504601955414], "13": ["vertical_and_slash", 1000, 6096, 0.4251762926578522], "14": ["vertical_and_slash", 1000, 6096, 0.49037012457847595], "15": ["vertical_and_slash", 1000, 6096, 0.4333910346031189], "16": ["vertical_and_slash", 1000, 6096, 0.43163245916366577], "17": ["vertical_and_slash", 1000, 6096, 0.4194749891757965], "18": ["vertical_and_slash", 1000, 6096, 0.41633814573287964], "19": ["vertical_and_slash", 1000, 6096, 0.4252642095088959], "20": ["vertical_and_slash", 1000, 6096, 0.4419412314891815], "21": ["vertical_and_slash", 1000, 6096, 0.45909765362739563], "22": ["vertical_and_slash", 1000, 6096, 0.46222731471061707], "23": ["vertical_and_slash", 1000, 6096, 0.46412402391433716], "24": ["vertical_and_slash", 1000, 6096, 0.4632507562637329], "25": ["vertical_and_slash", 1000, 6096, 0.4524560272693634], "26": ["vertical_and_slash", 1000, 6096, 0.4381413459777832], "27": ["vertical_and_slash", 1000, 6096, 0.43589267134666443], "28": ["vertical_and_slash", 1000, 6096, 0.43646979331970215], "29": ["vertical_and_slash", 1000, 6096, 0.45901456475257874], "30": ["vertical_and_slash", 1000, 6096, 0.4098890721797943], "31": ["vertical_and_slash", 1000, 6096, 0.42721879482269287], "32": ["vertical_and_slash", 1000, 6096, 0.44780540466308594], "33": ["vertical_and_slash", 1000, 6096, 0.42450201511383057], "34": ["vertical_and_slash", 1000, 6096, 0.4172991216182709], "35": ["vertical_and_slash", 1000, 6096, 0.39206284284591675], "36": ["vertical_and_slash", 1000, 6096, 0.4081953465938568], "37": ["vertical_and_slash", 1000, 6096, 0.4493129849433899], "38": ["vertical_and_slash", 1000, 6096, 0.42046698927879333], "39": ["vertical_and_slash", 1000, 6096, 0.4381744861602783], "40": ["vertical_and_slash", 1000, 6096, 0.4188445210456848], "41": ["vertical_and_slash", 1000, 6096, 0.4710225760936737], "42": ["vertical_and_slash", 1000, 6096, 0.41677844524383545], "43": ["vertical_and_slash", 1000, 6096, 0.41065770387649536], "44": ["vertical_and_slash", 1000, 6096, 0.5095037221908569], "45": ["vertical_and_slash", 1000, 6096, 0.46741998195648193], "46": ["vertical_and_slash", 1000, 6096, 0.4201884865760803], "47": ["vertical_and_slash", 1000, 6096, 0.446139395236969], "48": ["vertical_and_slash", 1000, 6096, 0.4009939730167389], "49": ["vertical_and_slash", 1000, 6096, 0.445808470249176], "50": ["vertical_and_slash", 1000, 6096, 0.41517871618270874], "51": ["vertical_and_slash", 1000, 6096, 0.4320959150791168], "52": ["vertical_and_slash", 1000, 6096, 0.42084017395973206], "53": ["vertical_and_slash", 1000, 6096, 0.4376211166381836], "54": ["vertical_and_slash", 1000, 6096, 0.40491870045661926], "55": ["vertical_and_slash", 1000, 6096, 0.42636844515800476], "56": ["vertical_and_slash", 1000, 6096, 0.39353469014167786], "57": ["vertical_and_slash", 1000, 6096, 0.4346638321876526], "58": ["vertical_and_slash", 1000, 6096, 0.43779733777046204], "59": ["vertical_and_slash", 1000, 6096, 0.4394199252128601], "60": ["vertical_and_slash", 1000, 6096, 0.4796709716320038], "61": ["vertical_and_slash", 1000, 6096, 0.3951183259487152], "62": ["vertical_and_slash", 1000, 6096, 0.40380311012268066], "63": ["vertical_and_slash", 1000, 6096, 0.4174964129924774]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4239487946033478], "1": ["vertical_and_slash", 1000, 6096, 0.4351397156715393], "2": ["vertical_and_slash", 1000, 6096, 0.41296911239624023], "3": ["vertical_and_slash", 1000, 6096, 0.4516132175922394], "4": ["vertical_and_slash", 1000, 6096, 0.425337553024292], "5": ["vertical_and_slash", 1000, 6096, 0.4211907386779785], "6": ["vertical_and_slash", 1000, 6096, 0.4304128885269165], "7": ["vertical_and_slash", 1000, 6096, 0.46903693675994873], "8": ["vertical_and_slash", 1000, 6096, 0.43601086735725403], "9": ["vertical_and_slash", 1000, 6096, 0.4114161729812622], "10": ["vertical_and_slash", 1000, 6096, 0.4085327684879303], "11": ["vertical_and_slash", 1000, 6096, 0.4591577351093292], "12": ["vertical_and_slash", 1000, 6096, 0.40370404720306396], "13": ["vertical_and_slash", 1000, 6096, 0.44846436381340027], "14": ["vertical_and_slash", 1000, 6096, 0.40686100721359253], "15": ["vertical_and_slash", 1000, 6096, 0.4023570716381073], "16": ["vertical_and_slash", 1000, 6096, 0.4111619293689728], "17": ["vertical_and_slash", 1000, 6096, 0.4005275368690491], "18": ["vertical_and_slash", 1000, 6096, 0.40477803349494934], "19": ["vertical_and_slash", 1000, 6096, 0.417398065328598], "20": ["vertical_and_slash", 1000, 6096, 0.404429167509079], "21": ["vertical_and_slash", 1000, 6096, 0.4438396692276001], "22": ["vertical_and_slash", 1000, 6096, 0.3886646032333374], "23": ["vertical_and_slash", 1000, 6096, 0.39642104506492615], "24": ["vertical_and_slash", 1000, 6096, 0.4205353856086731], "25": ["vertical_and_slash", 1000, 6096, 0.4857335686683655], "26": ["vertical_and_slash", 1000, 6096, 0.42725813388824463], "27": ["vertical_and_slash", 1000, 6096, 0.43865305185317993], "28": ["vertical_and_slash", 1000, 6096, 0.40018174052238464], "29": ["vertical_and_slash", 1000, 6096, 0.4311603903770447], "30": ["vertical_and_slash", 1000, 6096, 0.43331217765808105], "31": ["vertical_and_slash", 1000, 6096, 0.41610822081565857], "32": ["vertical_and_slash", 1000, 6096, 0.47987785935401917], "33": ["vertical_and_slash", 1000, 6096, 0.4603978991508484], "34": ["vertical_and_slash", 1000, 6096, 0.43094581365585327], "35": ["vertical_and_slash", 1000, 6096, 0.4697044789791107], "36": ["vertical_and_slash", 1000, 6096, 0.5069623589515686], "37": ["vertical_and_slash", 1000, 6096, 0.430965393781662], "38": ["vertical_and_slash", 1000, 6096, 0.47608134150505066], "39": ["vertical_and_slash", 1000, 6096, 0.43306058645248413], "40": ["vertical_and_slash", 1000, 6096, 0.5272747874259949], "41": ["vertical_and_slash", 1000, 6096, 0.48420649766921997], "42": ["vertical_and_slash", 1000, 6096, 0.4559733271598816], "43": ["vertical_and_slash", 1000, 6096, 0.4907423257827759], "44": ["vertical_and_slash", 1000, 6096, 0.49108800292015076], "45": ["vertical_and_slash", 1000, 6096, 0.4035692512989044], "46": ["vertical_and_slash", 1000, 6096, 0.4610893130302429], "47": ["vertical_and_slash", 1000, 6096, 0.43595078587532043], "48": ["vertical_and_slash", 1000, 6096, 0.4787057340145111], "49": ["vertical_and_slash", 1000, 6096, 0.44304853677749634], "50": ["vertical_and_slash", 1000, 6096, 0.46307796239852905], "51": ["vertical_and_slash", 1000, 6096, 0.44755324721336365], "52": ["vertical_and_slash", 1000, 6096, 0.4295732378959656], "53": ["vertical_and_slash", 1000, 6096, 0.44709932804107666], "54": ["vertical_and_slash", 1000, 6096, 0.4485390782356262], "55": ["vertical_and_slash", 1000, 6096, 0.454115092754364], "56": ["vertical_and_slash", 1000, 6096, 0.40697693824768066], "57": ["vertical_and_slash", 1000, 6096, 0.41237378120422363], "58": ["vertical_and_slash", 1000, 6096, 0.4081058204174042], "59": ["vertical_and_slash", 1000, 6096, 0.43974828720092773], "60": ["vertical_and_slash", 1000, 6096, 0.4016929864883423], "61": ["vertical_and_slash", 1000, 6096, 0.40429458022117615], "62": ["vertical_and_slash", 1000, 6096, 0.4192523956298828], "63": ["vertical_and_slash", 1000, 6096, 0.428290456533432]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4101574718952179], "1": ["vertical_and_slash", 1000, 6096, 0.4577615559101105], "2": ["vertical_and_slash", 1000, 6096, 0.4077059328556061], "3": ["vertical_and_slash", 1000, 6096, 0.4107079803943634], "4": ["vertical_and_slash", 1000, 6096, 0.4415101706981659], "5": ["vertical_and_slash", 1000, 6096, 0.4108622670173645], "6": ["vertical_and_slash", 1000, 6096, 0.4050958454608917], "7": ["vertical_and_slash", 1000, 6096, 0.3791305422782898], "8": ["vertical_and_slash", 1000, 6096, 0.49950700998306274], "9": ["vertical_and_slash", 1000, 6096, 0.4336501657962799], "10": ["vertical_and_slash", 1000, 6096, 0.4386191666126251], "11": ["vertical_and_slash", 1000, 6096, 0.45302891731262207], "12": ["vertical_and_slash", 1000, 6096, 0.4744378626346588], "13": ["vertical_and_slash", 1000, 6096, 0.4481927454471588], "14": ["vertical_and_slash", 1000, 6096, 0.46455684304237366], "15": ["vertical_and_slash", 1000, 6096, 0.42508944869041443], "16": ["vertical_and_slash", 1000, 6096, 0.4307916760444641], "17": ["vertical_and_slash", 1000, 6096, 0.4309004247188568], "18": ["vertical_and_slash", 1000, 6096, 0.3975801467895508], "19": ["vertical_and_slash", 1000, 6096, 0.4193505346775055], "20": ["vertical_and_slash", 1000, 6096, 0.3982695937156677], "21": ["vertical_and_slash", 1000, 6096, 0.3875119984149933], "22": ["vertical_and_slash", 1000, 6096, 0.39722633361816406], "23": ["vertical_and_slash", 1000, 6096, 0.43693411350250244], "24": ["vertical_and_slash", 1000, 6096, 0.39304590225219727], "25": ["vertical_and_slash", 1000, 6096, 0.4171867072582245], "26": ["vertical_and_slash", 1000, 6096, 0.41079092025756836], "27": ["vertical_and_slash", 1000, 6096, 0.40743306279182434], "28": ["vertical_and_slash", 1000, 6096, 0.394339919090271], "29": ["vertical_and_slash", 1000, 6096, 0.41892820596694946], "30": ["vertical_and_slash", 1000, 6096, 0.40570348501205444], "31": ["vertical_and_slash", 1000, 6096, 0.40508630871772766], "32": ["vertical_and_slash", 1000, 6096, 0.44171246886253357], "33": ["vertical_and_slash", 1000, 6096, 0.4105490446090698], "34": ["vertical_and_slash", 1000, 6096, 0.43791526556015015], "35": ["vertical_and_slash", 1000, 6096, 0.4373561143875122], "36": ["vertical_and_slash", 1000, 6096, 0.4033970832824707], "37": ["vertical_and_slash", 1000, 6096, 0.41454461216926575], "38": ["vertical_and_slash", 1000, 6096, 0.4561547636985779], "39": ["vertical_and_slash", 1000, 6096, 0.4326895773410797], "40": ["vertical_and_slash", 1000, 6096, 0.42113184928894043], "41": ["vertical_and_slash", 1000, 6096, 0.4186099171638489], "42": ["vertical_and_slash", 1000, 6096, 0.415260374546051], "43": ["vertical_and_slash", 1000, 6096, 0.4058596193790436], "44": ["vertical_and_slash", 1000, 6096, 0.4201618432998657], "45": ["vertical_and_slash", 1000, 6096, 0.40581250190734863], "46": ["vertical_and_slash", 1000, 6096, 0.4296327233314514], "47": ["vertical_and_slash", 1000, 6096, 0.41733217239379883], "48": ["vertical_and_slash", 1000, 6096, 0.4185822606086731], "49": ["vertical_and_slash", 1000, 6096, 0.43070054054260254], "50": ["vertical_and_slash", 1000, 6096, 0.41008901596069336], "51": ["vertical_and_slash", 1000, 6096, 0.43363747000694275], "52": ["vertical_and_slash", 1000, 6096, 0.40455758571624756], "53": ["vertical_and_slash", 1000, 6096, 0.4250848591327667], "54": ["vertical_and_slash", 1000, 6096, 0.4156292974948883], "55": ["vertical_and_slash", 1000, 6096, 0.4007231295108795], "56": ["vertical_and_slash", 1000, 6096, 0.4558859169483185], "57": ["vertical_and_slash", 1000, 6096, 0.47333818674087524], "58": ["vertical_and_slash", 1000, 6096, 0.43588149547576904], "59": ["vertical_and_slash", 1000, 6096, 0.40850475430488586], "60": ["vertical_and_slash", 1000, 6096, 0.43398308753967285], "61": ["vertical_and_slash", 1000, 6096, 0.44212424755096436], "62": ["vertical_and_slash", 1000, 6096, 0.4408489465713501], "63": ["vertical_and_slash", 1000, 6096, 0.42384544014930725]}, {"0": ["vertical_and_slash", 1000, 6096, 0.44959524273872375], "1": ["vertical_and_slash", 1000, 6096, 0.40397071838378906], "2": ["vertical_and_slash", 1000, 6096, 0.4183576703071594], "3": ["vertical_and_slash", 1000, 6096, 0.3997814953327179], "4": ["vertical_and_slash", 1000, 6096, 0.40042009949684143], "5": ["vertical_and_slash", 1000, 6096, 0.41261377930641174], "6": ["vertical_and_slash", 1000, 6096, 0.4164784848690033], "7": ["vertical_and_slash", 1000, 6096, 0.41687101125717163], "8": ["vertical_and_slash", 1000, 6096, 0.4323374032974243], "9": ["vertical_and_slash", 1000, 6096, 0.4002479612827301], "10": ["vertical_and_slash", 1000, 6096, 0.4433388113975525], "11": ["vertical_and_slash", 1000, 6096, 0.44976067543029785], "12": ["vertical_and_slash", 1000, 6096, 0.3911975920200348], "13": ["vertical_and_slash", 1000, 6096, 0.39400991797447205], "14": ["vertical_and_slash", 1000, 6096, 0.41410118341445923], "15": ["vertical_and_slash", 1000, 6096, 0.4281277060508728], "16": ["vertical_and_slash", 1000, 6096, 0.42150166630744934], "17": ["vertical_and_slash", 1000, 6096, 0.43334466218948364], "18": ["vertical_and_slash", 1000, 6096, 0.4204079210758209], "19": ["vertical_and_slash", 1000, 6096, 0.40655770897865295], "20": ["vertical_and_slash", 1000, 6096, 0.4883589446544647], "21": ["vertical_and_slash", 1000, 6096, 0.43792545795440674], "22": ["vertical_and_slash", 1000, 6096, 0.416965126991272], "23": ["vertical_and_slash", 1000, 6096, 0.3963960111141205], "24": ["vertical_and_slash", 1000, 6096, 0.44691526889801025], "25": ["vertical_and_slash", 1000, 6096, 0.4580703377723694], "26": ["vertical_and_slash", 1000, 6096, 0.4538658559322357], "27": ["vertical_and_slash", 1000, 6096, 0.4247162938117981], "28": ["vertical_and_slash", 1000, 6096, 0.43145743012428284], "29": ["vertical_and_slash", 1000, 6096, 0.40788334608078003], "30": ["vertical_and_slash", 1000, 6096, 0.4160279333591461], "31": ["vertical_and_slash", 1000, 6096, 0.4436606168746948], "32": ["vertical_and_slash", 1000, 6096, 0.41131365299224854], "33": ["vertical_and_slash", 1000, 6096, 0.43534308671951294], "34": ["vertical_and_slash", 1000, 6096, 0.41851377487182617], "35": ["vertical_and_slash", 1000, 6096, 0.4113036096096039], "36": ["vertical_and_slash", 1000, 6096, 0.4192235469818115], "37": ["vertical_and_slash", 1000, 6096, 0.4183359146118164], "38": ["vertical_and_slash", 1000, 6096, 0.4008432924747467], "39": ["vertical_and_slash", 1000, 6096, 0.4386296272277832], "40": ["vertical_and_slash", 1000, 6096, 0.4255584478378296], "41": ["vertical_and_slash", 1000, 6096, 0.41084498167037964], "42": ["vertical_and_slash", 1000, 6096, 0.43078991770744324], "43": ["vertical_and_slash", 1000, 6096, 0.4380718767642975], "44": ["vertical_and_slash", 1000, 6096, 0.4195430278778076], "45": ["vertical_and_slash", 1000, 6096, 0.41243594884872437], "46": ["vertical_and_slash", 1000, 6096, 0.43243858218193054], "47": ["vertical_and_slash", 1000, 6096, 0.4173465669155121], "48": ["vertical_and_slash", 1000, 6096, 0.4212799668312073], "49": ["vertical_and_slash", 1000, 6096, 0.4459543526172638], "50": ["vertical_and_slash", 1000, 6096, 0.4229510724544525], "51": ["vertical_and_slash", 1000, 6096, 0.4106156826019287], "52": ["vertical_and_slash", 1000, 6096, 0.41930636763572693], "53": ["vertical_and_slash", 1000, 6096, 0.3940839171409607], "54": ["vertical_and_slash", 1000, 6096, 0.39987513422966003], "55": ["vertical_and_slash", 1000, 6096, 0.4136069715023041], "56": ["vertical_and_slash", 1000, 6096, 0.401793509721756], "57": ["vertical_and_slash", 1000, 6096, 0.3943578600883484], "58": ["vertical_and_slash", 1000, 6096, 0.3962498903274536], "59": ["vertical_and_slash", 1000, 6096, 0.38980820775032043], "60": ["vertical_and_slash", 1000, 6096, 0.4121648967266083], "61": ["vertical_and_slash", 1000, 6096, 0.4019972085952759], "62": ["vertical_and_slash", 1000, 6096, 0.39886045455932617], "63": ["vertical_and_slash", 1000, 6096, 0.42420724034309387]}, {"0": ["vertical_and_slash", 1000, 6096, 0.48364296555519104], "1": ["vertical_and_slash", 1000, 6096, 0.4717230200767517], "2": ["vertical_and_slash", 1000, 6096, 0.452895849943161], "3": ["vertical_and_slash", 1000, 6096, 0.4522700607776642], "4": ["vertical_and_slash", 1000, 6096, 0.5154377818107605], "5": ["vertical_and_slash", 1000, 6096, 0.49225249886512756], "6": ["vertical_and_slash", 1000, 6096, 0.43799397349357605], "7": ["vertical_and_slash", 1000, 6096, 0.4751964807510376], "8": ["vertical_and_slash", 1000, 6096, 0.4342237710952759], "9": ["vertical_and_slash", 1000, 6096, 0.42858144640922546], "10": ["vertical_and_slash", 1000, 6096, 0.4604540169239044], "11": ["vertical_and_slash", 1000, 6096, 0.4530663788318634], "12": ["vertical_and_slash", 1000, 6096, 0.41591793298721313], "13": ["vertical_and_slash", 1000, 6096, 0.43644267320632935], "14": ["vertical_and_slash", 1000, 6096, 0.43991270661354065], "15": ["vertical_and_slash", 1000, 6096, 0.4879067838191986], "16": ["vertical_and_slash", 1000, 6096, 0.45472514629364014], "17": ["vertical_and_slash", 1000, 6096, 0.4594883322715759], "18": ["vertical_and_slash", 1000, 6096, 0.4516313672065735], "19": ["vertical_and_slash", 1000, 6096, 0.4207311272621155], "20": ["vertical_and_slash", 1000, 6096, 0.4796852171421051], "21": ["vertical_and_slash", 1000, 6096, 0.48815667629241943], "22": ["vertical_and_slash", 1000, 6096, 0.4447852075099945], "23": ["vertical_and_slash", 1000, 6096, 0.41200879216194153], "24": ["vertical_and_slash", 1000, 6096, 0.4587388336658478], "25": ["vertical_and_slash", 1000, 6096, 0.4690956771373749], "26": ["vertical_and_slash", 1000, 6096, 0.44069719314575195], "27": ["vertical_and_slash", 1000, 6096, 0.4631904065608978], "28": ["vertical_and_slash", 1000, 6096, 0.43032166361808777], "29": ["vertical_and_slash", 1000, 6096, 0.46667057275772095], "30": ["vertical_and_slash", 1000, 6096, 0.4422731101512909], "31": ["vertical_and_slash", 1000, 6096, 0.5016435980796814], "32": ["vertical_and_slash", 1000, 6096, 0.4675638675689697], "33": ["vertical_and_slash", 1000, 6096, 0.48116132616996765], "34": ["vertical_and_slash", 1000, 6096, 0.4740670919418335], "35": ["vertical_and_slash", 1000, 6096, 0.4829050898551941], "36": ["vertical_and_slash", 1000, 6096, 0.4375820457935333], "37": ["vertical_and_slash", 1000, 6096, 0.48593977093696594], "38": ["vertical_and_slash", 1000, 6096, 0.44426506757736206], "39": ["vertical_and_slash", 1000, 6096, 0.40366020798683167], "40": ["vertical_and_slash", 1000, 6096, 0.40848782658576965], "41": ["vertical_and_slash", 1000, 6096, 0.4562687873840332], "42": ["vertical_and_slash", 1000, 6096, 0.43066683411598206], "43": ["vertical_and_slash", 1000, 6096, 0.4031464457511902], "44": ["vertical_and_slash", 1000, 6096, 0.46236732602119446], "45": ["vertical_and_slash", 1000, 6096, 0.42977461218833923], "46": ["vertical_and_slash", 1000, 6096, 0.4209780693054199], "47": ["vertical_and_slash", 1000, 6096, 0.4478985667228699], "48": ["vertical_and_slash", 1000, 6096, 0.45017704367637634], "49": ["vertical_and_slash", 1000, 6096, 0.49236157536506653], "50": ["vertical_and_slash", 1000, 6096, 0.5808568596839905], "51": ["vertical_and_slash", 1000, 6096, 0.4486314654350281], "52": ["vertical_and_slash", 1000, 6096, 0.46910712122917175], "53": ["vertical_and_slash", 1000, 6096, 0.45368051528930664], "54": ["vertical_and_slash", 1000, 6096, 0.45691919326782227], "55": ["vertical_and_slash", 1000, 6096, 0.4526861608028412], "56": ["vertical_and_slash", 1000, 6096, 0.3945328891277313], "57": ["vertical_and_slash", 1000, 6096, 0.45485904812812805], "58": ["vertical_and_slash", 1000, 6096, 0.39875221252441406], "59": ["vertical_and_slash", 1000, 6096, 0.398979127407074], "60": ["vertical_and_slash", 1000, 6096, 0.3915409743785858], "61": ["vertical_and_slash", 1000, 6096, 0.4436233937740326], "62": ["vertical_and_slash", 1000, 6096, 0.44332996010780334], "63": ["vertical_and_slash", 1000, 6096, 0.4192809462547302]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4083382487297058], "1": ["vertical_and_slash", 1000, 6096, 0.45214566588401794], "2": ["vertical_and_slash", 1000, 6096, 0.46128031611442566], "3": ["vertical_and_slash", 1000, 6096, 0.461419016122818], "4": ["vertical_and_slash", 1000, 6096, 0.46537184715270996], "5": ["vertical_and_slash", 1000, 6096, 0.45843201875686646], "6": ["vertical_and_slash", 1000, 6096, 0.436462938785553], "7": ["vertical_and_slash", 1000, 6096, 0.4044104814529419], "8": ["vertical_and_slash", 1000, 6096, 0.4010699689388275], "9": ["vertical_and_slash", 1000, 6096, 0.4149378836154938], "10": ["vertical_and_slash", 1000, 6096, 0.41411200165748596], "11": ["vertical_and_slash", 1000, 6096, 0.4252440631389618], "12": ["vertical_and_slash", 1000, 6096, 0.3909092843532562], "13": ["vertical_and_slash", 1000, 6096, 0.40964949131011963], "14": ["vertical_and_slash", 1000, 6096, 0.43248191475868225], "15": ["vertical_and_slash", 1000, 6096, 0.4018459916114807], "16": ["vertical_and_slash", 1000, 6096, 0.47511592507362366], "17": ["vertical_and_slash", 1000, 6096, 0.4394337236881256], "18": ["vertical_and_slash", 1000, 6096, 0.469129353761673], "19": ["vertical_and_slash", 1000, 6096, 0.4710557162761688], "20": ["vertical_and_slash", 1000, 6096, 0.5133346915245056], "21": ["vertical_and_slash", 1000, 6096, 0.49758636951446533], "22": ["vertical_and_slash", 1000, 6096, 0.48919305205345154], "23": ["vertical_and_slash", 1000, 6096, 0.4646253287792206], "24": ["vertical_and_slash", 1000, 6096, 0.46295323967933655], "25": ["vertical_and_slash", 1000, 6096, 0.4469587504863739], "26": ["vertical_and_slash", 1000, 6096, 0.46789979934692383], "27": ["vertical_and_slash", 1000, 6096, 0.4575258791446686], "28": ["vertical_and_slash", 1000, 6096, 0.4206456243991852], "29": ["vertical_and_slash", 1000, 6096, 0.4639497697353363], "30": ["vertical_and_slash", 1000, 6096, 0.43697389960289], "31": ["vertical_and_slash", 1000, 6096, 0.39967435598373413], "32": ["vertical_and_slash", 1000, 6096, 0.39794400334358215], "33": ["vertical_and_slash", 1000, 6096, 0.4049685597419739], "34": ["vertical_and_slash", 1000, 6096, 0.46176981925964355], "35": ["vertical_and_slash", 1000, 6096, 0.3953578472137451], "36": ["vertical_and_slash", 1000, 6096, 0.3959680497646332], "37": ["vertical_and_slash", 1000, 6096, 0.3959308862686157], "38": ["vertical_and_slash", 1000, 6096, 0.40508049726486206], "39": ["vertical_and_slash", 1000, 6096, 0.5294240117073059], "40": ["vertical_and_slash", 1000, 6096, 0.4242344796657562], "41": ["vertical_and_slash", 1000, 6096, 0.4055561423301697], "42": ["vertical_and_slash", 1000, 6096, 0.43623894453048706], "43": ["vertical_and_slash", 1000, 6096, 0.4488736391067505], "44": ["vertical_and_slash", 1000, 6096, 0.41467463970184326], "45": ["vertical_and_slash", 1000, 6096, 0.4662145674228668], "46": ["vertical_and_slash", 1000, 6096, 0.44489923119544983], "47": ["vertical_and_slash", 1000, 6096, 0.4684538245201111], "48": ["vertical_and_slash", 1000, 6096, 0.4628884196281433], "49": ["vertical_and_slash", 1000, 6096, 0.47114959359169006], "50": ["vertical_and_slash", 1000, 6096, 0.47233450412750244], "51": ["vertical_and_slash", 1000, 6096, 0.4758494794368744], "52": ["vertical_and_slash", 1000, 6096, 0.47417914867401123], "53": ["vertical_and_slash", 1000, 6096, 0.4466601312160492], "54": ["vertical_and_slash", 1000, 6096, 0.4647086262702942], "55": ["vertical_and_slash", 1000, 6096, 0.44375768303871155], "56": ["vertical_and_slash", 1000, 6096, 0.4040372967720032], "57": ["vertical_and_slash", 1000, 6096, 0.4092615842819214], "58": ["vertical_and_slash", 1000, 6096, 0.44290173053741455], "59": ["vertical_and_slash", 1000, 6096, 0.42179983854293823], "60": ["vertical_and_slash", 1000, 6096, 0.4124796390533447], "61": ["vertical_and_slash", 1000, 6096, 0.42501330375671387], "62": ["vertical_and_slash", 1000, 6096, 0.44264912605285645], "63": ["vertical_and_slash", 1000, 6096, 0.4255794584751129]}, {"0": ["vertical_and_slash", 1000, 6096, 0.42147547006607056], "1": ["vertical_and_slash", 1000, 6096, 0.41203203797340393], "2": ["vertical_and_slash", 1000, 6096, 0.4681873023509979], "3": ["vertical_and_slash", 1000, 6096, 0.4572085738182068], "4": ["vertical_and_slash", 1000, 6096, 0.3959766924381256], "5": ["vertical_and_slash", 1000, 6096, 0.45085418224334717], "6": ["vertical_and_slash", 1000, 6096, 0.4325563609600067], "7": ["vertical_and_slash", 1000, 6096, 0.4148317873477936], "8": ["vertical_and_slash", 1000, 6096, 0.40095576643943787], "9": ["vertical_and_slash", 1000, 6096, 0.423039972782135], "10": ["vertical_and_slash", 1000, 6096, 0.40223005414009094], "11": ["vertical_and_slash", 1000, 6096, 0.408540815114975], "12": ["vertical_and_slash", 1000, 6096, 0.41916194558143616], "13": ["vertical_and_slash", 1000, 6096, 0.3993696868419647], "14": ["vertical_and_slash", 1000, 6096, 0.44539397954940796], "15": ["vertical_and_slash", 1000, 6096, 0.42680105566978455], "16": ["vertical_and_slash", 1000, 6096, 0.4311271905899048], "17": ["vertical_and_slash", 1000, 6096, 0.4777764678001404], "18": ["vertical_and_slash", 1000, 6096, 0.4622744023799896], "19": ["vertical_and_slash", 1000, 6096, 0.4316270053386688], "20": ["vertical_and_slash", 1000, 6096, 0.46070656180381775], "21": ["vertical_and_slash", 1000, 6096, 0.4462578296661377], "22": ["vertical_and_slash", 1000, 6096, 0.43893542885780334], "23": ["vertical_and_slash", 1000, 6096, 0.4467122256755829], "24": ["vertical_and_slash", 1000, 6096, 0.46702101826667786], "25": ["vertical_and_slash", 1000, 6096, 0.4734901487827301], "26": ["vertical_and_slash", 1000, 6096, 0.4426150321960449], "27": ["vertical_and_slash", 1000, 6096, 0.43851664662361145], "28": ["vertical_and_slash", 1000, 6096, 0.47790661454200745], "29": ["vertical_and_slash", 1000, 6096, 0.482031911611557], "30": ["vertical_and_slash", 1000, 6096, 0.5285704135894775], "31": ["vertical_and_slash", 1000, 6096, 0.4808852970600128], "32": ["vertical_and_slash", 1000, 6096, 0.4501112103462219], "33": ["vertical_and_slash", 1000, 6096, 0.4097900390625], "34": ["vertical_and_slash", 1000, 6096, 0.4423898458480835], "35": ["vertical_and_slash", 1000, 6096, 0.42996037006378174], "36": ["vertical_and_slash", 1000, 6096, 0.4312116801738739], "37": ["vertical_and_slash", 1000, 6096, 0.48136240243911743], "38": ["vertical_and_slash", 1000, 6096, 0.41975006461143494], "39": ["vertical_and_slash", 1000, 6096, 0.43328291177749634], "40": ["vertical_and_slash", 1000, 6096, 0.40401166677474976], "41": ["vertical_and_slash", 1000, 6096, 0.41780921816825867], "42": ["vertical_and_slash", 1000, 6096, 0.40782302618026733], "43": ["vertical_and_slash", 1000, 6096, 0.42197301983833313], "44": ["vertical_and_slash", 1000, 6096, 0.42206668853759766], "45": ["vertical_and_slash", 1000, 6096, 0.44842952489852905], "46": ["vertical_and_slash", 1000, 6096, 0.39938899874687195], "47": ["vertical_and_slash", 1000, 6096, 0.42117616534233093], "48": ["vertical_and_slash", 1000, 6096, 0.42615175247192383], "49": ["vertical_and_slash", 1000, 6096, 0.48588621616363525], "50": ["vertical_and_slash", 1000, 6096, 0.4490658938884735], "51": ["vertical_and_slash", 1000, 6096, 0.47492700815200806], "52": ["vertical_and_slash", 1000, 6096, 0.5028776526451111], "53": ["vertical_and_slash", 1000, 6096, 0.4417460262775421], "54": ["vertical_and_slash", 1000, 6096, 0.5020673871040344], "55": ["vertical_and_slash", 1000, 6096, 0.45033106207847595], "56": ["vertical_and_slash", 1000, 6096, 0.5146317481994629], "57": ["vertical_and_slash", 1000, 6096, 0.41560080647468567], "58": ["vertical_and_slash", 1000, 6096, 0.42193955183029175], "59": ["vertical_and_slash", 1000, 6096, 0.4381254315376282], "60": ["vertical_and_slash", 1000, 6096, 0.4416804015636444], "61": ["vertical_and_slash", 1000, 6096, 0.48629769682884216], "62": ["vertical_and_slash", 1000, 6096, 0.4229067862033844], "63": ["vertical_and_slash", 1000, 6096, 0.42772412300109863]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4584697484970093], "1": ["vertical_and_slash", 1000, 6096, 0.46616891026496887], "2": ["vertical_and_slash", 1000, 6096, 0.41038748621940613], "3": ["vertical_and_slash", 1000, 6096, 0.4120330810546875], "4": ["vertical_and_slash", 1000, 6096, 0.40631571412086487], "5": ["vertical_and_slash", 1000, 6096, 0.4552481174468994], "6": ["vertical_and_slash", 1000, 6096, 0.4068601429462433], "7": ["vertical_and_slash", 1000, 6096, 0.4478835165500641], "8": ["vertical_and_slash", 1000, 6096, 0.4137802720069885], "9": ["vertical_and_slash", 1000, 6096, 0.41817671060562134], "10": ["vertical_and_slash", 1000, 6096, 0.42421311140060425], "11": ["vertical_and_slash", 1000, 6096, 0.41684532165527344], "12": ["vertical_and_slash", 1000, 6096, 0.4350496232509613], "13": ["vertical_and_slash", 1000, 6096, 0.4171273410320282], "14": ["vertical_and_slash", 1000, 6096, 0.42235636711120605], "15": ["vertical_and_slash", 1000, 6096, 0.40064737200737], "16": ["vertical_and_slash", 1000, 6096, 0.40872976183891296], "17": ["vertical_and_slash", 1000, 6096, 0.44691795110702515], "18": ["vertical_and_slash", 1000, 6096, 0.43124058842658997], "19": ["vertical_and_slash", 1000, 6096, 0.42339736223220825], "20": ["vertical_and_slash", 1000, 6096, 0.4259387254714966], "21": ["vertical_and_slash", 1000, 6096, 0.42283204197883606], "22": ["vertical_and_slash", 1000, 6096, 0.4318123161792755], "23": ["vertical_and_slash", 1000, 6096, 0.44762444496154785], "24": ["vertical_and_slash", 1000, 6096, 0.41051214933395386], "25": ["vertical_and_slash", 1000, 6096, 0.40274983644485474], "26": ["vertical_and_slash", 1000, 6096, 0.4487321674823761], "27": ["vertical_and_slash", 1000, 6096, 0.39229485392570496], "28": ["vertical_and_slash", 1000, 6096, 0.4172550439834595], "29": ["vertical_and_slash", 1000, 6096, 0.4370187222957611], "30": ["vertical_and_slash", 1000, 6096, 0.42838963866233826], "31": ["vertical_and_slash", 1000, 6096, 0.4138513207435608], "32": ["vertical_and_slash", 1000, 6096, 0.4259270429611206], "33": ["vertical_and_slash", 1000, 6096, 0.46567222476005554], "34": ["vertical_and_slash", 1000, 6096, 0.3927738666534424], "35": ["vertical_and_slash", 1000, 6096, 0.39695343375205994], "36": ["vertical_and_slash", 1000, 6096, 0.41959208250045776], "37": ["vertical_and_slash", 1000, 6096, 0.4277595281600952], "38": ["vertical_and_slash", 1000, 6096, 0.4403259754180908], "39": ["vertical_and_slash", 1000, 6096, 0.4374598264694214], "40": ["vertical_and_slash", 1000, 6096, 0.4446956217288971], "41": ["vertical_and_slash", 1000, 6096, 0.3925056755542755], "42": ["vertical_and_slash", 1000, 6096, 0.4351813793182373], "43": ["vertical_and_slash", 1000, 6096, 0.4478727877140045], "44": ["vertical_and_slash", 1000, 6096, 0.4127780497074127], "45": ["vertical_and_slash", 1000, 6096, 0.44412708282470703], "46": ["vertical_and_slash", 1000, 6096, 0.3877170979976654], "47": ["vertical_and_slash", 1000, 6096, 0.4123915135860443], "48": ["vertical_and_slash", 1000, 6096, 0.46444058418273926], "49": ["vertical_and_slash", 1000, 6096, 0.4853903353214264], "50": ["vertical_and_slash", 1000, 6096, 0.5146200060844421], "51": ["vertical_and_slash", 1000, 6096, 0.4353112280368805], "52": ["vertical_and_slash", 1000, 6096, 0.4590053856372833], "53": ["vertical_and_slash", 1000, 6096, 0.4303146004676819], "54": ["vertical_and_slash", 1000, 6096, 0.5191827416419983], "55": ["vertical_and_slash", 1000, 6096, 0.4200748801231384], "56": ["vertical_and_slash", 1000, 6096, 0.4660356938838959], "57": ["vertical_and_slash", 1000, 6096, 0.5328012704849243], "58": ["vertical_and_slash", 1000, 6096, 0.47039514780044556], "59": ["vertical_and_slash", 1000, 6096, 0.46823129057884216], "60": ["vertical_and_slash", 1000, 6096, 0.4646708369255066], "61": ["vertical_and_slash", 1000, 6096, 0.4534791111946106], "62": ["vertical_and_slash", 1000, 6096, 0.4965309202671051], "63": ["vertical_and_slash", 1000, 6096, 0.44491276144981384]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4043881893157959], "1": ["vertical_and_slash", 1000, 6096, 0.4173121750354767], "2": ["vertical_and_slash", 1000, 6096, 0.4095896780490875], "3": ["vertical_and_slash", 1000, 6096, 0.40488138794898987], "4": ["vertical_and_slash", 1000, 6096, 0.39857208728790283], "5": ["vertical_and_slash", 1000, 6096, 0.3990212380886078], "6": ["vertical_and_slash", 1000, 6096, 0.4020608961582184], "7": ["vertical_and_slash", 1000, 6096, 0.45248809456825256], "8": ["vertical_and_slash", 1000, 6096, 0.4245346486568451], "9": ["vertical_and_slash", 1000, 6096, 0.3950469493865967], "10": ["vertical_and_slash", 1000, 6096, 0.40338242053985596], "11": ["vertical_and_slash", 1000, 6096, 0.4220753014087677], "12": ["vertical_and_slash", 1000, 6096, 0.42855024337768555], "13": ["vertical_and_slash", 1000, 6096, 0.46150583028793335], "14": ["vertical_and_slash", 1000, 6096, 0.41535064578056335], "15": ["vertical_and_slash", 1000, 6096, 0.3809932470321655], "16": ["vertical_and_slash", 1000, 6096, 0.41546621918678284], "17": ["vertical_and_slash", 1000, 6096, 0.4050697088241577], "18": ["vertical_and_slash", 1000, 6096, 0.40115588903427124], "19": ["vertical_and_slash", 1000, 6096, 0.3866824805736542], "20": ["vertical_and_slash", 1000, 6096, 0.3827419877052307], "21": ["vertical_and_slash", 1000, 6096, 0.39036694169044495], "22": ["vertical_and_slash", 1000, 6096, 0.4219423234462738], "23": ["vertical_and_slash", 1000, 6096, 0.40016454458236694], "24": ["vertical_and_slash", 1000, 6096, 0.44786909222602844], "25": ["vertical_and_slash", 1000, 6096, 0.43695586919784546], "26": ["vertical_and_slash", 1000, 6096, 0.4668352007865906], "27": ["vertical_and_slash", 1000, 6096, 0.40734604001045227], "28": ["vertical_and_slash", 1000, 6096, 0.42964407801628113], "29": ["vertical_and_slash", 1000, 6096, 0.437633752822876], "30": ["vertical_and_slash", 1000, 6096, 0.4141658842563629], "31": ["vertical_and_slash", 1000, 6096, 0.424523264169693], "32": ["vertical_and_slash", 1000, 6096, 0.40085065364837646], "33": ["vertical_and_slash", 1000, 6096, 0.41611167788505554], "34": ["vertical_and_slash", 1000, 6096, 0.40371477603912354], "35": ["vertical_and_slash", 1000, 6096, 0.40979868173599243], "36": ["vertical_and_slash", 1000, 6096, 0.37632742524147034], "37": ["vertical_and_slash", 1000, 6096, 0.4153359532356262], "38": ["vertical_and_slash", 1000, 6096, 0.39626869559288025], "39": ["vertical_and_slash", 1000, 6096, 0.4658893942832947], "40": ["vertical_and_slash", 1000, 6096, 0.4303628206253052], "41": ["vertical_and_slash", 1000, 6096, 0.49544447660446167], "42": ["vertical_and_slash", 1000, 6096, 0.4331376552581787], "43": ["vertical_and_slash", 1000, 6096, 0.4377516508102417], "44": ["vertical_and_slash", 1000, 6096, 0.42366859316825867], "45": ["vertical_and_slash", 1000, 6096, 0.4472139775753021], "46": ["vertical_and_slash", 1000, 6096, 0.4474920630455017], "47": ["vertical_and_slash", 1000, 6096, 0.41395819187164307], "48": ["vertical_and_slash", 1000, 6096, 0.40072521567344666], "49": ["vertical_and_slash", 1000, 6096, 0.4071871042251587], "50": ["vertical_and_slash", 1000, 6096, 0.40712592005729675], "51": ["vertical_and_slash", 1000, 6096, 0.3931507468223572], "52": ["vertical_and_slash", 1000, 6096, 0.39640992879867554], "53": ["vertical_and_slash", 1000, 6096, 0.3992505967617035], "54": ["vertical_and_slash", 1000, 6096, 0.3958287835121155], "55": ["vertical_and_slash", 1000, 6096, 0.4423298239707947], "56": ["vertical_and_slash", 1000, 6096, 0.39003536105155945], "57": ["vertical_and_slash", 1000, 6096, 0.43981343507766724], "58": ["vertical_and_slash", 1000, 6096, 0.4286176264286041], "59": ["vertical_and_slash", 1000, 6096, 0.5141983032226562], "60": ["vertical_and_slash", 1000, 6096, 0.4228653013706207], "61": ["vertical_and_slash", 1000, 6096, 0.4017927944660187], "62": ["vertical_and_slash", 1000, 6096, 0.42662766575813293], "63": ["vertical_and_slash", 1000, 6096, 0.4551727771759033]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4101772606372833], "1": ["vertical_and_slash", 1000, 6096, 0.45215198397636414], "2": ["vertical_and_slash", 1000, 6096, 0.4378770887851715], "3": ["vertical_and_slash", 1000, 6096, 0.4565732479095459], "4": ["vertical_and_slash", 1000, 6096, 0.4757949113845825], "5": ["vertical_and_slash", 1000, 6096, 0.4960455894470215], "6": ["vertical_and_slash", 1000, 6096, 0.45826244354248047], "7": ["vertical_and_slash", 1000, 6096, 0.4590229392051697], "8": ["vertical_and_slash", 1000, 6096, 0.4033694565296173], "9": ["vertical_and_slash", 1000, 6096, 0.4126805067062378], "10": ["vertical_and_slash", 1000, 6096, 0.3863148093223572], "11": ["vertical_and_slash", 1000, 6096, 0.42539238929748535], "12": ["vertical_and_slash", 1000, 6096, 0.4168001413345337], "13": ["vertical_and_slash", 1000, 6096, 0.38667619228363037], "14": ["vertical_and_slash", 1000, 6096, 0.4207625389099121], "15": ["vertical_and_slash", 1000, 6096, 0.4098585844039917], "16": ["vertical_and_slash", 1000, 6096, 0.4023457467556], "17": ["vertical_and_slash", 1000, 6096, 0.4122420847415924], "18": ["vertical_and_slash", 1000, 6096, 0.42438074946403503], "19": ["vertical_and_slash", 1000, 6096, 0.4265388250350952], "20": ["vertical_and_slash", 1000, 6096, 0.4095523953437805], "21": ["vertical_and_slash", 1000, 6096, 0.4229103922843933], "22": ["vertical_and_slash", 1000, 6096, 0.40301379561424255], "23": ["vertical_and_slash", 1000, 6096, 0.38633987307548523], "24": ["vertical_and_slash", 1000, 6096, 0.39182931184768677], "25": ["vertical_and_slash", 1000, 6096, 0.3753899931907654], "26": ["vertical_and_slash", 1000, 6096, 0.37671300768852234], "27": ["vertical_and_slash", 1000, 6096, 0.43426939845085144], "28": ["vertical_and_slash", 1000, 6096, 0.4032061696052551], "29": ["vertical_and_slash", 1000, 6096, 0.3800085186958313], "30": ["vertical_and_slash", 1000, 6096, 0.41766825318336487], "31": ["vertical_and_slash", 1000, 6096, 0.43816980719566345], "32": ["vertical_and_slash", 1000, 6096, 0.4106159508228302], "33": ["vertical_and_slash", 1000, 6096, 0.43140092492103577], "34": ["vertical_and_slash", 1000, 6096, 0.40499770641326904], "35": ["vertical_and_slash", 1000, 6096, 0.3806232511997223], "36": ["vertical_and_slash", 1000, 6096, 0.3875381350517273], "37": ["vertical_and_slash", 1000, 6096, 0.4168725311756134], "38": ["vertical_and_slash", 1000, 6096, 0.4007999897003174], "39": ["vertical_and_slash", 1000, 6096, 0.5003949403762817], "40": ["vertical_and_slash", 1000, 6096, 0.44368794560432434], "41": ["vertical_and_slash", 1000, 6096, 0.4310685992240906], "42": ["vertical_and_slash", 1000, 6096, 0.4623230993747711], "43": ["vertical_and_slash", 1000, 6096, 0.4325317144393921], "44": ["vertical_and_slash", 1000, 6096, 0.49024340510368347], "45": ["vertical_and_slash", 1000, 6096, 0.42193228006362915], "46": ["vertical_and_slash", 1000, 6096, 0.45666995644569397], "47": ["vertical_and_slash", 1000, 6096, 0.4615112543106079], "48": ["vertical_and_slash", 1000, 6096, 0.4018436074256897], "49": ["vertical_and_slash", 1000, 6096, 0.41064509749412537], "50": ["vertical_and_slash", 1000, 6096, 0.4044787883758545], "51": ["vertical_and_slash", 1000, 6096, 0.40013518929481506], "52": ["vertical_and_slash", 1000, 6096, 0.4124900996685028], "53": ["vertical_and_slash", 1000, 6096, 0.43888017535209656], "54": ["vertical_and_slash", 1000, 6096, 0.43000340461730957], "55": ["vertical_and_slash", 1000, 6096, 0.6813428997993469], "56": ["vertical_and_slash", 1000, 6096, 0.44261133670806885], "57": ["vertical_and_slash", 1000, 6096, 0.4170132577419281], "58": ["vertical_and_slash", 1000, 6096, 0.42587220668792725], "59": ["vertical_and_slash", 1000, 6096, 0.4067803621292114], "60": ["vertical_and_slash", 1000, 6096, 0.4122718572616577], "61": ["vertical_and_slash", 1000, 6096, 0.4301850497722626], "62": ["vertical_and_slash", 1000, 6096, 0.4275870621204376], "63": ["vertical_and_slash", 1000, 6096, 0.45231565833091736]}, {"0": ["vertical_and_slash", 1000, 6096, 0.44697433710098267], "1": ["vertical_and_slash", 1000, 6096, 0.4607485830783844], "2": ["vertical_and_slash", 1000, 6096, 0.44853660464286804], "3": ["vertical_and_slash", 1000, 6096, 0.43875864148139954], "4": ["vertical_and_slash", 1000, 6096, 0.42582377791404724], "5": ["vertical_and_slash", 1000, 6096, 0.4604422450065613], "6": ["vertical_and_slash", 1000, 6096, 0.4579061269760132], "7": ["vertical_and_slash", 1000, 6096, 0.4410973787307739], "8": ["vertical_and_slash", 1000, 6096, 0.45419272780418396], "9": ["vertical_and_slash", 1000, 6096, 0.46266111731529236], "10": ["vertical_and_slash", 1000, 6096, 0.4489307701587677], "11": ["vertical_and_slash", 1000, 6096, 0.509620726108551], "12": ["vertical_and_slash", 1000, 6096, 0.44351205229759216], "13": ["vertical_and_slash", 1000, 6096, 0.44235724210739136], "14": ["vertical_and_slash", 1000, 6096, 0.4517609477043152], "15": ["vertical_and_slash", 1000, 6096, 0.46620190143585205], "16": ["vertical_and_slash", 1000, 6096, 0.4668675661087036], "17": ["vertical_and_slash", 1000, 6096, 0.43357986211776733], "18": ["vertical_and_slash", 1000, 6096, 0.4629114866256714], "19": ["vertical_and_slash", 1000, 6096, 0.4538978338241577], "20": ["vertical_and_slash", 1000, 6096, 0.4220212399959564], "21": ["vertical_and_slash", 1000, 6096, 0.41660019755363464], "22": ["vertical_and_slash", 1000, 6096, 0.47515133023262024], "23": ["vertical_and_slash", 1000, 6096, 0.4418085515499115], "24": ["vertical_and_slash", 1000, 6096, 0.383597731590271], "25": ["vertical_and_slash", 1000, 6096, 0.4070056080818176], "26": ["vertical_and_slash", 1000, 6096, 0.404597669839859], "27": ["vertical_and_slash", 1000, 6096, 0.39642176032066345], "28": ["vertical_and_slash", 1000, 6096, 0.3832813799381256], "29": ["vertical_and_slash", 1000, 6096, 0.3867214322090149], "30": ["vertical_and_slash", 1000, 6096, 0.41958194971084595], "31": ["vertical_and_slash", 1000, 6096, 0.40713247656822205], "32": ["vertical_and_slash", 1000, 6096, 0.4551275968551636], "33": ["vertical_and_slash", 1000, 6096, 0.40838247537612915], "34": ["vertical_and_slash", 1000, 6096, 0.42449069023132324], "35": ["vertical_and_slash", 1000, 6096, 0.42248961329460144], "36": ["vertical_and_slash", 1000, 6096, 0.48490288853645325], "37": ["vertical_and_slash", 1000, 6096, 0.433091938495636], "38": ["vertical_and_slash", 1000, 6096, 0.44063800573349], "39": ["vertical_and_slash", 1000, 6096, 0.4442283809185028], "40": ["vertical_and_slash", 1000, 6096, 0.4911304712295532], "41": ["vertical_and_slash", 1000, 6096, 0.475352019071579], "42": ["vertical_and_slash", 1000, 6096, 0.4779019057750702], "43": ["vertical_and_slash", 1000, 6096, 0.45615696907043457], "44": ["vertical_and_slash", 1000, 6096, 0.4562494158744812], "45": ["vertical_and_slash", 1000, 6096, 0.48661932349205017], "46": ["vertical_and_slash", 1000, 6096, 0.4426514208316803], "47": ["vertical_and_slash", 1000, 6096, 0.4993079900741577], "48": ["vertical_and_slash", 1000, 6096, 0.4666630029678345], "49": ["vertical_and_slash", 1000, 6096, 0.47171393036842346], "50": ["vertical_and_slash", 1000, 6096, 0.4370881915092468], "51": ["vertical_and_slash", 1000, 6096, 0.4621376395225525], "52": ["vertical_and_slash", 1000, 6096, 0.48471084237098694], "53": ["vertical_and_slash", 1000, 6096, 0.4245794415473938], "54": ["vertical_and_slash", 1000, 6096, 0.4353333115577698], "55": ["vertical_and_slash", 1000, 6096, 0.4354815185070038], "56": ["vertical_and_slash", 1000, 6096, 0.44448697566986084], "57": ["vertical_and_slash", 1000, 6096, 0.4014750123023987], "58": ["vertical_and_slash", 1000, 6096, 0.43126317858695984], "59": ["vertical_and_slash", 1000, 6096, 0.3952622711658478], "60": ["vertical_and_slash", 1000, 6096, 0.46373337507247925], "61": ["vertical_and_slash", 1000, 6096, 0.404558002948761], "62": ["vertical_and_slash", 1000, 6096, 0.4081554710865021], "63": ["vertical_and_slash", 1000, 6096, 0.4124828577041626]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4351811110973358], "1": ["vertical_and_slash", 1000, 6096, 0.3981532156467438], "2": ["vertical_and_slash", 1000, 6096, 0.42058178782463074], "3": ["vertical_and_slash", 1000, 6096, 0.40448155999183655], "4": ["vertical_and_slash", 1000, 6096, 0.41487765312194824], "5": ["vertical_and_slash", 1000, 6096, 0.3999534547328949], "6": ["vertical_and_slash", 1000, 6096, 0.45424216985702515], "7": ["vertical_and_slash", 1000, 6096, 0.4069691002368927], "8": ["vertical_and_slash", 1000, 6096, 0.41979601979255676], "9": ["vertical_and_slash", 1000, 6096, 0.4352521300315857], "10": ["vertical_and_slash", 1000, 6096, 0.4270220696926117], "11": ["vertical_and_slash", 1000, 6096, 0.4273131787776947], "12": ["vertical_and_slash", 1000, 6096, 0.4640975296497345], "13": ["vertical_and_slash", 1000, 6096, 0.4364737272262573], "14": ["vertical_and_slash", 1000, 6096, 0.4030042290687561], "15": ["vertical_and_slash", 1000, 6096, 0.423104852437973], "16": ["vertical_and_slash", 1000, 6096, 0.42416995763778687], "17": ["vertical_and_slash", 1000, 6096, 0.4346545338630676], "18": ["vertical_and_slash", 1000, 6096, 0.44395214319229126], "19": ["vertical_and_slash", 1000, 6096, 0.4027864336967468], "20": ["vertical_and_slash", 1000, 6096, 0.44964757561683655], "21": ["vertical_and_slash", 1000, 6096, 0.4317897856235504], "22": ["vertical_and_slash", 1000, 6096, 0.4401680827140808], "23": ["vertical_and_slash", 1000, 6096, 0.43819960951805115], "24": ["vertical_and_slash", 1000, 6096, 0.5060697197914124], "25": ["vertical_and_slash", 1000, 6096, 0.47234803438186646], "26": ["vertical_and_slash", 1000, 6096, 0.4685342311859131], "27": ["vertical_and_slash", 1000, 6096, 0.41296109557151794], "28": ["vertical_and_slash", 1000, 6096, 0.4412449598312378], "29": ["vertical_and_slash", 1000, 6096, 0.44436922669410706], "30": ["vertical_and_slash", 1000, 6096, 0.46993914246559143], "31": ["vertical_and_slash", 1000, 6096, 0.44301149249076843], "32": ["vertical_and_slash", 1000, 6096, 0.403775691986084], "33": ["vertical_and_slash", 1000, 6096, 0.4313740134239197], "34": ["vertical_and_slash", 1000, 6096, 0.38116422295570374], "35": ["vertical_and_slash", 1000, 6096, 0.43086186051368713], "36": ["vertical_and_slash", 1000, 6096, 0.4278552532196045], "37": ["vertical_and_slash", 1000, 6096, 0.4122614562511444], "38": ["vertical_and_slash", 1000, 6096, 0.4042198956012726], "39": ["vertical_and_slash", 1000, 6096, 0.44961830973625183], "40": ["vertical_and_slash", 1000, 6096, 0.4176432192325592], "41": ["vertical_and_slash", 1000, 6096, 0.40186458826065063], "42": ["vertical_and_slash", 1000, 6096, 0.3841889500617981], "43": ["vertical_and_slash", 1000, 6096, 0.4189383387565613], "44": ["vertical_and_slash", 1000, 6096, 0.38606399297714233], "45": ["vertical_and_slash", 1000, 6096, 0.40241703391075134], "46": ["vertical_and_slash", 1000, 6096, 0.3996281623840332], "47": ["vertical_and_slash", 1000, 6096, 0.40117841958999634], "48": ["vertical_and_slash", 1000, 6096, 0.40223342180252075], "49": ["vertical_and_slash", 1000, 6096, 0.3943672776222229], "50": ["vertical_and_slash", 1000, 6096, 0.4139846861362457], "51": ["vertical_and_slash", 1000, 6096, 0.3711702823638916], "52": ["vertical_and_slash", 1000, 6096, 0.41200658679008484], "53": ["vertical_and_slash", 1000, 6096, 0.3865068256855011], "54": ["vertical_and_slash", 1000, 6096, 0.43875691294670105], "55": ["vertical_and_slash", 1000, 6096, 0.42653921246528625], "56": ["vertical_and_slash", 1000, 6096, 0.44719335436820984], "57": ["vertical_and_slash", 1000, 6096, 0.4334162473678589], "58": ["vertical_and_slash", 1000, 6096, 0.40671905875205994], "59": ["vertical_and_slash", 1000, 6096, 0.4527854025363922], "60": ["vertical_and_slash", 1000, 6096, 0.43385353684425354], "61": ["vertical_and_slash", 1000, 6096, 0.45996612310409546], "62": ["vertical_and_slash", 1000, 6096, 0.46519333124160767], "63": ["vertical_and_slash", 1000, 6096, 0.4571540951728821]}, {"0": ["vertical_and_slash", 1000, 6096, 0.40718746185302734], "1": ["vertical_and_slash", 1000, 6096, 0.4038862884044647], "2": ["vertical_and_slash", 1000, 6096, 0.4440782368183136], "3": ["vertical_and_slash", 1000, 6096, 0.4117133915424347], "4": ["vertical_and_slash", 1000, 6096, 0.4189381003379822], "5": ["vertical_and_slash", 1000, 6096, 0.4250599145889282], "6": ["vertical_and_slash", 1000, 6096, 0.40343791246414185], "7": ["vertical_and_slash", 3500, 100, 0.9445822238922119], "8": ["vertical_and_slash", 1000, 6096, 0.36314883828163147], "9": ["vertical_and_slash", 1000, 6096, 0.3939807713031769], "10": ["vertical_and_slash", 1000, 6096, 0.39417216181755066], "11": ["vertical_and_slash", 1000, 6096, 0.3964526057243347], "12": ["vertical_and_slash", 1000, 6096, 0.37263479828834534], "13": ["vertical_and_slash", 1000, 6096, 0.386431485414505], "14": ["vertical_and_slash", 1000, 6096, 0.37345343828201294], "15": ["vertical_and_slash", 1000, 6096, 0.4052848517894745], "16": ["vertical_and_slash", 1000, 6096, 0.45257389545440674], "17": ["vertical_and_slash", 1000, 6096, 0.47269120812416077], "18": ["vertical_and_slash", 1000, 6096, 0.4498786926269531], "19": ["vertical_and_slash", 1000, 6096, 0.4285966157913208], "20": ["vertical_and_slash", 1000, 6096, 0.44905272126197815], "21": ["vertical_and_slash", 1000, 6096, 0.4617483913898468], "22": ["vertical_and_slash", 1000, 6096, 0.44625189900398254], "23": ["vertical_and_slash", 1000, 6096, 0.4690914750099182], "24": ["vertical_and_slash", 1000, 6096, 0.4239480793476105], "25": ["vertical_and_slash", 1000, 6096, 0.4318782389163971], "26": ["vertical_and_slash", 1000, 6096, 0.37758392095565796], "27": ["vertical_and_slash", 1000, 6096, 0.4179005026817322], "28": ["vertical_and_slash", 1000, 6096, 0.3984597325325012], "29": ["vertical_and_slash", 1000, 6096, 0.42307016253471375], "30": ["vertical_and_slash", 1000, 6096, 0.40798643231391907], "31": ["vertical_and_slash", 1000, 6096, 0.4217320382595062], "32": ["vertical_and_slash", 1000, 6096, 0.4085003137588501], "33": ["vertical_and_slash", 1000, 6096, 0.3773442506790161], "34": ["vertical_and_slash", 1000, 6096, 0.38419532775878906], "35": ["vertical_and_slash", 1000, 6096, 0.38318905234336853], "36": ["vertical_and_slash", 1000, 6096, 0.38485437631607056], "37": ["vertical_and_slash", 1000, 6096, 0.38473278284072876], "38": ["vertical_and_slash", 1000, 6096, 0.3786332309246063], "39": ["vertical_and_slash", 1000, 6096, 0.3728543817996979], "40": ["vertical_and_slash", 1000, 6096, 0.48820143938064575], "41": ["vertical_and_slash", 1000, 6096, 0.4573459327220917], "42": ["vertical_and_slash", 1000, 6096, 0.4666195213794708], "43": ["vertical_and_slash", 1000, 6096, 0.4641812741756439], "44": ["vertical_and_slash", 1000, 6096, 0.4558063745498657], "45": ["vertical_and_slash", 1000, 6096, 0.46042171120643616], "46": ["vertical_and_slash", 1000, 6096, 0.4454670548439026], "47": ["vertical_and_slash", 1000, 6096, 0.47812163829803467], "48": ["vertical_and_slash", 1000, 6096, 0.48090553283691406], "49": ["vertical_and_slash", 1000, 6096, 0.481012225151062], "50": ["vertical_and_slash", 1000, 6096, 0.40375110507011414], "51": ["vertical_and_slash", 1000, 6096, 0.42435672879219055], "52": ["vertical_and_slash", 1000, 6096, 0.4466771185398102], "53": ["vertical_and_slash", 1000, 6096, 0.4886696934700012], "54": ["vertical_and_slash", 1000, 6096, 0.4548218250274658], "55": ["vertical_and_slash", 1000, 6096, 0.4734651446342468], "56": ["vertical_and_slash", 1000, 6096, 0.46207866072654724], "57": ["vertical_and_slash", 1000, 6096, 0.44902878999710083], "58": ["vertical_and_slash", 1000, 6096, 0.4795519709587097], "59": ["vertical_and_slash", 1000, 6096, 0.45527422428131104], "60": ["vertical_and_slash", 1000, 6096, 0.4471772015094757], "61": ["vertical_and_slash", 1000, 6096, 0.45790204405784607], "62": ["vertical_and_slash", 1000, 6096, 0.4293295443058014], "63": ["vertical_and_slash", 1000, 6096, 0.4505779445171356]}, {"0": ["vertical_and_slash", 1000, 6096, 0.39857083559036255], "1": ["vertical_and_slash", 1000, 6096, 0.4103509783744812], "2": ["vertical_and_slash", 1000, 6096, 0.43639466166496277], "3": ["vertical_and_slash", 1000, 6096, 0.4126514792442322], "4": ["vertical_and_slash", 1000, 6096, 0.41147199273109436], "5": ["vertical_and_slash", 1000, 6096, 0.45295777916908264], "6": ["vertical_and_slash", 1000, 6096, 0.4433559775352478], "7": ["vertical_and_slash", 1000, 6096, 0.4024285078048706], "8": ["vertical_and_slash", 1000, 6096, 0.3950817584991455], "9": ["vertical_and_slash", 1000, 6096, 0.4034844934940338], "10": ["vertical_and_slash", 1000, 6096, 0.38807550072669983], "11": ["vertical_and_slash", 1000, 6096, 0.38301098346710205], "12": ["vertical_and_slash", 1000, 6096, 0.3886299133300781], "13": ["vertical_and_slash", 1000, 6096, 0.4265405535697937], "14": ["vertical_and_slash", 1000, 6096, 0.41502171754837036], "15": ["vertical_and_slash", 1000, 6096, 0.4060073494911194], "16": ["vertical_and_slash", 1000, 6096, 0.42001980543136597], "17": ["vertical_and_slash", 1000, 6096, 0.3945963382720947], "18": ["vertical_and_slash", 1000, 6096, 0.4144432246685028], "19": ["vertical_and_slash", 1000, 6096, 0.41850578784942627], "20": ["vertical_and_slash", 1000, 6096, 0.4031118154525757], "21": ["vertical_and_slash", 1000, 6096, 0.39632436633110046], "22": ["vertical_and_slash", 1000, 6096, 0.4040523171424866], "23": ["vertical_and_slash", 1000, 6096, 0.41261595487594604], "24": ["vertical_and_slash", 1000, 6096, 0.4244656264781952], "25": ["vertical_and_slash", 1000, 6096, 0.4182716906070709], "26": ["vertical_and_slash", 1000, 6096, 0.4234496057033539], "27": ["vertical_and_slash", 1000, 6096, 0.44754424691200256], "28": ["vertical_and_slash", 1000, 6096, 0.4120115637779236], "29": ["vertical_and_slash", 1000, 6096, 0.3861342668533325], "30": ["vertical_and_slash", 1000, 6096, 0.41802895069122314], "31": ["vertical_and_slash", 1000, 6096, 0.4266071319580078], "32": ["vertical_and_slash", 1000, 6096, 0.48436301946640015], "33": ["vertical_and_slash", 1000, 6096, 0.4572465121746063], "34": ["vertical_and_slash", 1000, 6096, 0.4271185100078583], "35": ["vertical_and_slash", 1000, 6096, 0.4697634279727936], "36": ["vertical_and_slash", 1000, 6096, 0.4746943414211273], "37": ["vertical_and_slash", 1000, 6096, 0.4438468813896179], "38": ["vertical_and_slash", 1000, 6096, 0.4875548779964447], "39": ["vertical_and_slash", 1000, 6096, 0.40887153148651123], "40": ["vertical_and_slash", 1000, 6096, 0.3892485797405243], "41": ["vertical_and_slash", 1000, 6096, 0.3806789219379425], "42": ["vertical_and_slash", 1000, 6096, 0.3928752541542053], "43": ["vertical_and_slash", 1000, 6096, 0.38539451360702515], "44": ["vertical_and_slash", 1000, 6096, 0.39261287450790405], "45": ["vertical_and_slash", 1000, 6096, 0.396320641040802], "46": ["vertical_and_slash", 1000, 6096, 0.3840222656726837], "47": ["vertical_and_slash", 1000, 6096, 0.3855746388435364], "48": ["vertical_and_slash", 1000, 6096, 0.442661315202713], "49": ["vertical_and_slash", 1000, 6096, 0.43800032138824463], "50": ["vertical_and_slash", 1000, 6096, 0.45664283633232117], "51": ["vertical_and_slash", 1000, 6096, 0.41562509536743164], "52": ["vertical_and_slash", 1000, 6096, 0.42730820178985596], "53": ["vertical_and_slash", 1000, 6096, 0.4285511076450348], "54": ["vertical_and_slash", 1000, 6096, 0.44418779015541077], "55": ["vertical_and_slash", 1000, 6096, 0.4460558593273163], "56": ["vertical_and_slash", 1000, 6096, 0.45465680956840515], "57": ["vertical_and_slash", 1000, 6096, 0.44334807991981506], "58": ["vertical_and_slash", 1000, 6096, 0.40696513652801514], "59": ["vertical_and_slash", 1000, 6096, 0.4347396194934845], "60": ["vertical_and_slash", 1000, 6096, 0.4170531630516052], "61": ["vertical_and_slash", 1000, 6096, 0.41781049966812134], "62": ["vertical_and_slash", 1000, 6096, 0.43386784195899963], "63": ["vertical_and_slash", 1000, 6096, 0.4100179076194763]}, {"0": ["vertical_and_slash", 1000, 6096, 0.45123299956321716], "1": ["vertical_and_slash", 1000, 6096, 0.4095185101032257], "2": ["vertical_and_slash", 1000, 6096, 0.40505924820899963], "3": ["vertical_and_slash", 1000, 6096, 0.3849453926086426], "4": ["vertical_and_slash", 1000, 6096, 0.4201718270778656], "5": ["vertical_and_slash", 1000, 6096, 0.39727768301963806], "6": ["vertical_and_slash", 1000, 6096, 0.3974360525608063], "7": ["vertical_and_slash", 1000, 6096, 0.41699373722076416], "8": ["vertical_and_slash", 1000, 6096, 0.3744014501571655], "9": ["vertical_and_slash", 1000, 6096, 0.38205116987228394], "10": ["vertical_and_slash", 1000, 6096, 0.3889963626861572], "11": ["vertical_and_slash", 1000, 6096, 0.3945786952972412], "12": ["vertical_and_slash", 1000, 6096, 0.37313205003738403], "13": ["vertical_and_slash", 1000, 6096, 0.3829169273376465], "14": ["vertical_and_slash", 1000, 6096, 0.3846512734889984], "15": ["vertical_and_slash", 1000, 6096, 0.40607815980911255], "16": ["vertical_and_slash", 1000, 6096, 0.3788847327232361], "17": ["vertical_and_slash", 1000, 6096, 0.4088612198829651], "18": ["vertical_and_slash", 1000, 6096, 0.3701470196247101], "19": ["vertical_and_slash", 1000, 6096, 0.4099588096141815], "20": ["vertical_and_slash", 1000, 6096, 0.3862540125846863], "21": ["vertical_and_slash", 1000, 6096, 0.3728336691856384], "22": ["vertical_and_slash", 1000, 6096, 0.3909212648868561], "23": ["vertical_and_slash", 1000, 6096, 0.3847512900829315], "24": ["vertical_and_slash", 1000, 6096, 0.4101769030094147], "25": ["vertical_and_slash", 1000, 6096, 0.43904662132263184], "26": ["vertical_and_slash", 1000, 6096, 0.44531306624412537], "27": ["vertical_and_slash", 1000, 6096, 0.40623241662979126], "28": ["vertical_and_slash", 1000, 6096, 0.42200684547424316], "29": ["vertical_and_slash", 1000, 6096, 0.41637739539146423], "30": ["vertical_and_slash", 1000, 6096, 0.44864195585250854], "31": ["vertical_and_slash", 1000, 6096, 0.441709965467453], "32": ["vertical_and_slash", 1000, 6096, 0.4173974096775055], "33": ["vertical_and_slash", 1000, 6096, 0.429293692111969], "34": ["vertical_and_slash", 1000, 6096, 0.41661274433135986], "35": ["vertical_and_slash", 1000, 6096, 0.4141969382762909], "36": ["vertical_and_slash", 1000, 6096, 0.4228658676147461], "37": ["vertical_and_slash", 1000, 6096, 0.41671058535575867], "38": ["vertical_and_slash", 1000, 6096, 0.4203051030635834], "39": ["vertical_and_slash", 1000, 6096, 0.4380231201648712], "40": ["vertical_and_slash", 1000, 6096, 0.4442541301250458], "41": ["vertical_and_slash", 1000, 6096, 0.4027573764324188], "42": ["vertical_and_slash", 1000, 6096, 0.3897315561771393], "43": ["vertical_and_slash", 1000, 6096, 0.4889545738697052], "44": ["vertical_and_slash", 3500, 100, 0.9080018401145935], "45": ["vertical_and_slash", 1000, 6096, 0.424771785736084], "46": ["vertical_and_slash", 1000, 6096, 0.41788142919540405], "47": ["vertical_and_slash", 1000, 6096, 0.43710553646087646], "48": ["vertical_and_slash", 1000, 6096, 0.3814886808395386], "49": ["vertical_and_slash", 1000, 6096, 0.39274469017982483], "50": ["vertical_and_slash", 1000, 6096, 0.3772584795951843], "51": ["vertical_and_slash", 1000, 6096, 0.37337997555732727], "52": ["vertical_and_slash", 1000, 6096, 0.3813028633594513], "53": ["vertical_and_slash", 1000, 6096, 0.3770943284034729], "54": ["vertical_and_slash", 1000, 6096, 0.3933960795402527], "55": ["vertical_and_slash", 1000, 6096, 0.3717898428440094], "56": ["vertical_and_slash", 1000, 6096, 0.42294055223464966], "57": ["vertical_and_slash", 1000, 6096, 0.40006473660469055], "58": ["vertical_and_slash", 1000, 6096, 0.44513818621635437], "59": ["vertical_and_slash", 1000, 6096, 0.4027887284755707], "60": ["vertical_and_slash", 1000, 6096, 0.4225102365016937], "61": ["vertical_and_slash", 1000, 6096, 0.4044889211654663], "62": ["vertical_and_slash", 1000, 6096, 0.414350688457489], "63": ["vertical_and_slash", 1000, 6096, 0.42536458373069763]}, {"0": ["vertical_and_slash", 1000, 6096, 0.46021974086761475], "1": ["vertical_and_slash", 1000, 6096, 0.4453316926956177], "2": ["vertical_and_slash", 1000, 6096, 0.429535835981369], "3": ["vertical_and_slash", 1000, 6096, 0.43480899930000305], "4": ["vertical_and_slash", 1000, 6096, 0.41186216473579407], "5": ["vertical_and_slash", 1000, 6096, 0.4454421401023865], "6": ["vertical_and_slash", 1000, 6096, 0.4061136543750763], "7": ["vertical_and_slash", 1000, 6096, 0.3940480947494507], "8": ["vertical_and_slash", 1000, 6096, 0.46371760964393616], "9": ["vertical_and_slash", 1000, 6096, 0.4541304111480713], "10": ["vertical_and_slash", 1000, 6096, 0.48032745718955994], "11": ["vertical_and_slash", 1000, 6096, 0.47879964113235474], "12": ["vertical_and_slash", 1000, 6096, 0.46804067492485046], "13": ["vertical_and_slash", 1000, 6096, 0.46731066703796387], "14": ["vertical_and_slash", 1000, 6096, 0.4543973505496979], "15": ["vertical_and_slash", 1000, 6096, 0.4150257408618927], "16": ["vertical_and_slash", 1000, 6096, 0.4893783628940582], "17": ["vertical_and_slash", 1000, 6096, 0.43723565340042114], "18": ["vertical_and_slash", 1000, 6096, 0.4348812401294708], "19": ["vertical_and_slash", 1000, 6096, 0.3872233033180237], "20": ["vertical_and_slash", 1000, 6096, 0.4312712252140045], "21": ["vertical_and_slash", 1000, 6096, 0.4226430654525757], "22": ["vertical_and_slash", 1000, 6096, 0.4524467885494232], "23": ["vertical_and_slash", 1000, 6096, 0.4165063500404358], "24": ["vertical_and_slash", 1000, 6096, 0.43630945682525635], "25": ["vertical_and_slash", 1000, 6096, 0.4542234241962433], "26": ["vertical_and_slash", 1000, 6096, 0.44751906394958496], "27": ["vertical_and_slash", 1000, 6096, 0.4559422731399536], "28": ["vertical_and_slash", 1000, 6096, 0.4684887230396271], "29": ["vertical_and_slash", 1000, 6096, 0.4103529751300812], "30": ["vertical_and_slash", 1000, 6096, 0.5165730714797974], "31": ["vertical_and_slash", 1000, 6096, 0.47106608748435974], "32": ["vertical_and_slash", 1000, 6096, 0.47161316871643066], "33": ["vertical_and_slash", 1000, 6096, 0.4889889657497406], "34": ["vertical_and_slash", 1000, 6096, 0.5186775326728821], "35": ["vertical_and_slash", 1000, 6096, 0.540533721446991], "36": ["vertical_and_slash", 1000, 6096, 0.524261474609375], "37": ["vertical_and_slash", 1000, 6096, 0.4721734821796417], "38": ["vertical_and_slash", 1000, 6096, 0.4477219879627228], "39": ["vertical_and_slash", 1000, 6096, 0.5071361660957336], "40": ["vertical_and_slash", 1000, 6096, 0.47985517978668213], "41": ["vertical_and_slash", 1000, 6096, 0.4668477773666382], "42": ["vertical_and_slash", 1000, 6096, 0.4370587170124054], "43": ["vertical_and_slash", 1000, 6096, 0.4554778039455414], "44": ["vertical_and_slash", 1000, 6096, 0.47508490085601807], "45": ["vertical_and_slash", 1000, 6096, 0.46971839666366577], "46": ["vertical_and_slash", 1000, 6096, 0.4878678321838379], "47": ["vertical_and_slash", 1000, 6096, 0.5569708943367004], "48": ["vertical_and_slash", 1000, 6096, 0.4150324761867523], "49": ["vertical_and_slash", 1000, 6096, 0.4905151128768921], "50": ["vertical_and_slash", 1000, 6096, 0.43700531125068665], "51": ["vertical_and_slash", 1000, 6096, 0.43443816900253296], "52": ["vertical_and_slash", 1000, 6096, 0.43187475204467773], "53": ["vertical_and_slash", 1000, 6096, 0.45011499524116516], "54": ["vertical_and_slash", 1000, 6096, 0.5036559700965881], "55": ["vertical_and_slash", 1000, 6096, 0.46208861470222473], "56": ["vertical_and_slash", 1000, 6096, 0.5059226155281067], "57": ["vertical_and_slash", 1000, 6096, 0.3998066782951355], "58": ["vertical_and_slash", 1000, 6096, 0.5521529316902161], "59": ["vertical_and_slash", 1000, 6096, 0.3958789110183716], "60": ["vertical_and_slash", 1000, 6096, 0.4112447202205658], "61": ["vertical_and_slash", 1000, 6096, 0.41214239597320557], "62": ["vertical_and_slash", 1000, 6096, 0.46674785017967224], "63": ["vertical_and_slash", 1000, 6096, 0.4109486937522888]}, {"0": ["vertical_and_slash", 1000, 6096, 0.44059738516807556], "1": ["vertical_and_slash", 1000, 6096, 0.43392717838287354], "2": ["vertical_and_slash", 1000, 6096, 0.46930402517318726], "3": ["vertical_and_slash", 1000, 6096, 0.5839933156967163], "4": ["vertical_and_slash", 1000, 6096, 0.4590402841567993], "5": ["vertical_and_slash", 1000, 6096, 0.45941856503486633], "6": ["vertical_and_slash", 1000, 6096, 0.4706774055957794], "7": ["vertical_and_slash", 1000, 6096, 0.450945645570755], "8": ["vertical_and_slash", 1000, 6096, 0.3782079219818115], "9": ["vertical_and_slash", 1000, 6096, 0.3728492856025696], "10": ["vertical_and_slash", 1000, 6096, 0.38549453020095825], "11": ["vertical_and_slash", 1000, 6096, 0.39694541692733765], "12": ["vertical_and_slash", 1000, 6096, 0.3991129696369171], "13": ["vertical_and_slash", 1000, 6096, 0.37146085500717163], "14": ["vertical_and_slash", 1000, 6096, 0.37608325481414795], "15": ["vertical_and_slash", 1000, 6096, 0.3979935944080353], "16": ["vertical_and_slash", 1000, 6096, 0.3903820812702179], "17": ["vertical_and_slash", 1000, 6096, 0.37855273485183716], "18": ["vertical_and_slash", 1000, 6096, 0.3896428048610687], "19": ["vertical_and_slash", 1000, 6096, 0.4237852394580841], "20": ["vertical_and_slash", 1000, 6096, 0.38398924469947815], "21": ["vertical_and_slash", 1000, 6096, 0.4070151150226593], "22": ["vertical_and_slash", 1000, 6096, 0.4061044454574585], "23": ["vertical_and_slash", 1000, 6096, 0.8632068634033203], "24": ["vertical_and_slash", 1000, 6096, 0.42937761545181274], "25": ["vertical_and_slash", 1000, 6096, 0.41877108812332153], "26": ["vertical_and_slash", 1000, 6096, 0.4391466975212097], "27": ["vertical_and_slash", 1000, 6096, 0.4374719560146332], "28": ["vertical_and_slash", 1000, 6096, 0.3985455632209778], "29": ["vertical_and_slash", 1000, 6096, 0.4640307128429413], "30": ["vertical_and_slash", 1000, 6096, 0.45760607719421387], "31": ["vertical_and_slash", 1000, 6096, 0.4295358955860138], "32": ["vertical_and_slash", 1000, 6096, 0.44049057364463806], "33": ["vertical_and_slash", 1000, 6096, 0.4621715545654297], "34": ["vertical_and_slash", 1000, 6096, 0.4912516176700592], "35": ["vertical_and_slash", 1000, 6096, 0.45964545011520386], "36": ["vertical_and_slash", 1000, 6096, 0.438959002494812], "37": ["vertical_and_slash", 1000, 6096, 0.4589058756828308], "38": ["vertical_and_slash", 1000, 6096, 0.45177242159843445], "39": ["vertical_and_slash", 1000, 6096, 0.42832842469215393], "40": ["vertical_and_slash", 1000, 6096, 0.4197799861431122], "41": ["vertical_and_slash", 1000, 6096, 0.46408891677856445], "42": ["vertical_and_slash", 1000, 6096, 0.4570249021053314], "43": ["vertical_and_slash", 1000, 6096, 0.4617656469345093], "44": ["vertical_and_slash", 1000, 6096, 0.4646436274051666], "45": ["vertical_and_slash", 1000, 6096, 0.44017234444618225], "46": ["vertical_and_slash", 1000, 6096, 0.45289623737335205], "47": ["vertical_and_slash", 1000, 6096, 0.4250416159629822], "48": ["vertical_and_slash", 1000, 6096, 0.52975994348526], "49": ["vertical_and_slash", 1000, 6096, 0.4815570116043091], "50": ["vertical_and_slash", 1000, 6096, 0.4452938735485077], "51": ["vertical_and_slash", 1000, 6096, 0.5241828560829163], "52": ["vertical_and_slash", 1000, 6096, 0.4219858944416046], "53": ["vertical_and_slash", 1000, 6096, 0.44048964977264404], "54": ["vertical_and_slash", 1000, 6096, 0.4164474904537201], "55": ["vertical_and_slash", 1000, 6096, 0.5230542421340942], "56": ["vertical_and_slash", 1000, 6096, 0.4202518165111542], "57": ["vertical_and_slash", 1000, 6096, 0.45435741543769836], "58": ["vertical_and_slash", 1000, 6096, 0.4135667383670807], "59": ["vertical_and_slash", 1000, 6096, 0.43343403935432434], "60": ["vertical_and_slash", 1000, 6096, 0.46540239453315735], "61": ["vertical_and_slash", 1000, 6096, 0.42419230937957764], "62": ["vertical_and_slash", 1000, 6096, 0.42752206325531006], "63": ["vertical_and_slash", 1000, 6096, 0.4722974896430969]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4491803050041199], "1": ["vertical_and_slash", 1000, 6096, 0.5550928115844727], "2": ["vertical_and_slash", 1000, 6096, 0.4515601396560669], "3": ["vertical_and_slash", 1000, 6096, 0.46383705735206604], "4": ["vertical_and_slash", 1000, 6096, 0.4622269570827484], "5": ["vertical_and_slash", 1000, 6096, 0.43974602222442627], "6": ["vertical_and_slash", 1000, 6096, 0.540428638458252], "7": ["vertical_and_slash", 1000, 6096, 0.4438239336013794], "8": ["vertical_and_slash", 1000, 6096, 0.37025290727615356], "9": ["vertical_and_slash", 1000, 6096, 0.37881314754486084], "10": ["vertical_and_slash", 1000, 6096, 0.39733871817588806], "11": ["vertical_and_slash", 1000, 6096, 0.38146084547042847], "12": ["vertical_and_slash", 1000, 6096, 0.39112624526023865], "13": ["vertical_and_slash", 1000, 6096, 0.3838358223438263], "14": ["vertical_and_slash", 1000, 6096, 0.3938346207141876], "15": ["vertical_and_slash", 1000, 6096, 0.3816402554512024], "16": ["vertical_and_slash", 1000, 6096, 0.4295480251312256], "17": ["vertical_and_slash", 1000, 6096, 0.43359655141830444], "18": ["vertical_and_slash", 1000, 6096, 0.44507545232772827], "19": ["vertical_and_slash", 1000, 6096, 0.4328117072582245], "20": ["vertical_and_slash", 1000, 6096, 0.46168890595436096], "21": ["vertical_and_slash", 1000, 6096, 0.4795302450656891], "22": ["vertical_and_slash", 1000, 6096, 0.423140287399292], "23": ["vertical_and_slash", 1000, 6096, 0.42151060700416565], "24": ["vertical_and_slash", 1000, 6096, 0.39807024598121643], "25": ["vertical_and_slash", 1000, 6096, 0.37432706356048584], "26": ["vertical_and_slash", 1000, 6096, 0.39261171221733093], "27": ["vertical_and_slash", 1000, 6096, 0.37564700841903687], "28": ["vertical_and_slash", 1000, 6096, 0.400588721036911], "29": ["vertical_and_slash", 1000, 6096, 0.39105749130249023], "30": ["vertical_and_slash", 1000, 6096, 0.3725428581237793], "31": ["vertical_and_slash", 1000, 6096, 0.3972933888435364], "32": ["vertical_and_slash", 1000, 6096, 0.4422115087509155], "33": ["vertical_and_slash", 1000, 6096, 0.43143555521965027], "34": ["vertical_and_slash", 1000, 6096, 0.4325573444366455], "35": ["vertical_and_slash", 1000, 6096, 0.43785175681114197], "36": ["vertical_and_slash", 1000, 6096, 0.44763830304145813], "37": ["vertical_and_slash", 1000, 6096, 0.45146289467811584], "38": ["vertical_and_slash", 1000, 6096, 0.40639352798461914], "39": ["vertical_and_slash", 1000, 6096, 0.46494582295417786], "40": ["vertical_and_slash", 1000, 6096, 0.511673629283905], "41": ["vertical_and_slash", 1000, 6096, 0.4421812891960144], "42": ["vertical_and_slash", 1000, 6096, 0.5228017568588257], "43": ["vertical_and_slash", 1000, 6096, 0.43951719999313354], "44": ["vertical_and_slash", 1000, 6096, 0.4728645980358124], "45": ["vertical_and_slash", 1000, 6096, 0.46488356590270996], "46": ["vertical_and_slash", 1000, 6096, 0.4896649718284607], "47": ["vertical_and_slash", 1000, 6096, 0.5028613805770874], "48": ["vertical_and_slash", 1000, 6096, 0.3910902738571167], "49": ["vertical_and_slash", 1000, 6096, 0.4265647828578949], "50": ["vertical_and_slash", 1000, 6096, 0.40619251132011414], "51": ["vertical_and_slash", 1000, 6096, 0.38427889347076416], "52": ["vertical_and_slash", 1000, 6096, 0.387980192899704], "53": ["vertical_and_slash", 1000, 6096, 0.4038885831832886], "54": ["vertical_and_slash", 1000, 6096, 0.38432350754737854], "55": ["vertical_and_slash", 1000, 6096, 0.40661248564720154], "56": ["vertical_and_slash", 1000, 6096, 0.5271414518356323], "57": ["vertical_and_slash", 1000, 6096, 0.47732970118522644], "58": ["vertical_and_slash", 1000, 6096, 0.4584406316280365], "59": ["vertical_and_slash", 1000, 6096, 0.4175199866294861], "60": ["vertical_and_slash", 1000, 6096, 0.43598857522010803], "61": ["vertical_and_slash", 1000, 6096, 0.4229041635990143], "62": ["vertical_and_slash", 1000, 6096, 0.41297516226768494], "63": ["vertical_and_slash", 1000, 6096, 0.44939979910850525]}, {"0": ["vertical_and_slash", 1000, 6096, 0.47387292981147766], "1": ["vertical_and_slash", 1000, 6096, 0.4753618836402893], "2": ["vertical_and_slash", 1000, 6096, 0.5118429660797119], "3": ["vertical_and_slash", 1000, 6096, 0.4929084777832031], "4": ["vertical_and_slash", 1000, 6096, 0.47649696469306946], "5": ["vertical_and_slash", 1000, 6096, 0.49600955843925476], "6": ["vertical_and_slash", 1000, 6096, 0.49038174748420715], "7": ["vertical_and_slash", 1000, 6096, 0.45804569125175476], "8": ["vertical_and_slash", 1000, 6096, 0.447040319442749], "9": ["vertical_and_slash", 1000, 6096, 0.49770376086235046], "10": ["vertical_and_slash", 1000, 6096, 0.45211902260780334], "11": ["vertical_and_slash", 1000, 6096, 0.4409184753894806], "12": ["vertical_and_slash", 1000, 6096, 0.46253517270088196], "13": ["vertical_and_slash", 1000, 6096, 0.4312135875225067], "14": ["vertical_and_slash", 1000, 6096, 0.45674318075180054], "15": ["vertical_and_slash", 1000, 6096, 0.46814796328544617], "16": ["vertical_and_slash", 1000, 6096, 0.4071671962738037], "17": ["vertical_and_slash", 1000, 6096, 0.3947398364543915], "18": ["vertical_and_slash", 1000, 6096, 0.45328158140182495], "19": ["vertical_and_slash", 1000, 6096, 0.3879552185535431], "20": ["vertical_and_slash", 1000, 6096, 0.40364035964012146], "21": ["vertical_and_slash", 1000, 6096, 0.4578745365142822], "22": ["vertical_and_slash", 1000, 6096, 0.41116172075271606], "23": ["vertical_and_slash", 1000, 6096, 0.45348817110061646], "24": ["vertical_and_slash", 1000, 6096, 0.47273576259613037], "25": ["vertical_and_slash", 1000, 6096, 0.5286158323287964], "26": ["vertical_and_slash", 1000, 6096, 0.47018516063690186], "27": ["vertical_and_slash", 1000, 6096, 0.5253229737281799], "28": ["vertical_and_slash", 1000, 6096, 0.4552764594554901], "29": ["vertical_and_slash", 1000, 6096, 0.48606762290000916], "30": ["vertical_and_slash", 1000, 6096, 0.478216290473938], "31": ["vertical_and_slash", 1000, 6096, 0.5026976466178894], "32": ["vertical_and_slash", 1000, 6096, 0.4540098011493683], "33": ["vertical_and_slash", 1000, 6096, 0.5384467244148254], "34": ["vertical_and_slash", 1000, 6096, 0.4502244293689728], "35": ["vertical_and_slash", 1000, 6096, 0.5228873491287231], "36": ["vertical_and_slash", 1000, 6096, 0.46251729130744934], "37": ["vertical_and_slash", 1000, 6096, 0.4385242164134979], "38": ["vertical_and_slash", 1000, 6096, 0.5055512189865112], "39": ["vertical_and_slash", 1000, 6096, 0.4755783975124359], "40": ["vertical_and_slash", 1000, 6096, 0.4777684807777405], "41": ["vertical_and_slash", 1000, 6096, 0.45129716396331787], "42": ["vertical_and_slash", 1000, 6096, 0.48987916111946106], "43": ["vertical_and_slash", 1000, 6096, 0.4808993339538574], "44": ["vertical_and_slash", 1000, 6096, 0.4350082278251648], "45": ["vertical_and_slash", 1000, 6096, 0.47876209020614624], "46": ["vertical_and_slash", 1000, 6096, 0.4958331882953644], "47": ["vertical_and_slash", 1000, 6096, 0.4725053310394287], "48": ["vertical_and_slash", 1000, 6096, 0.47200074791908264], "49": ["vertical_and_slash", 1000, 6096, 0.4188830554485321], "50": ["vertical_and_slash", 1000, 6096, 0.4303551912307739], "51": ["vertical_and_slash", 1000, 6096, 0.43282943964004517], "52": ["vertical_and_slash", 1000, 6096, 0.4296450912952423], "53": ["vertical_and_slash", 1000, 6096, 0.48929914832115173], "54": ["vertical_and_slash", 1000, 6096, 0.4310673773288727], "55": ["vertical_and_slash", 1000, 6096, 0.42790159583091736], "56": ["vertical_and_slash", 1000, 6096, 0.5130923390388489], "57": ["vertical_and_slash", 1000, 6096, 0.5312053561210632], "58": ["vertical_and_slash", 1000, 6096, 0.4942825138568878], "59": ["vertical_and_slash", 1000, 6096, 0.48329731822013855], "60": ["vertical_and_slash", 1000, 6096, 0.5531109571456909], "61": ["vertical_and_slash", 1000, 6096, 0.4988951086997986], "62": ["vertical_and_slash", 1000, 6096, 0.4679020047187805], "63": ["vertical_and_slash", 1000, 6096, 0.5723735690116882]}, {"0": ["vertical_and_slash", 1000, 6096, 0.5636101365089417], "1": ["vertical_and_slash", 1000, 6096, 0.5306821465492249], "2": ["vertical_and_slash", 1000, 6096, 0.5599849224090576], "3": ["vertical_and_slash", 1000, 6096, 0.5554834008216858], "4": ["vertical_and_slash", 1000, 6096, 0.6373623013496399], "5": ["vertical_and_slash", 1000, 6096, 0.5551227331161499], "6": ["vertical_and_slash", 1000, 6096, 0.5325362086296082], "7": ["vertical_and_slash", 1000, 6096, 0.7966234087944031], "8": ["vertical_and_slash", 1000, 6096, 0.49281081557273865], "9": ["vertical_and_slash", 1000, 6096, 0.504035472869873], "10": ["vertical_and_slash", 1000, 6096, 0.4639042019844055], "11": ["vertical_and_slash", 1000, 6096, 0.4632888734340668], "12": ["vertical_and_slash", 1000, 6096, 0.462544322013855], "13": ["vertical_and_slash", 1000, 6096, 0.4534291625022888], "14": ["vertical_and_slash", 1000, 6096, 0.4505586624145508], "15": ["vertical_and_slash", 1000, 6096, 0.45636382699012756], "16": ["vertical_and_slash", 1000, 6096, 0.473593145608902], "17": ["vertical_and_slash", 1000, 6096, 0.5129236578941345], "18": ["vertical_and_slash", 1000, 6096, 0.5581309199333191], "19": ["vertical_and_slash", 1000, 6096, 0.5805029273033142], "20": ["vertical_and_slash", 1000, 6096, 0.5017237663269043], "21": ["vertical_and_slash", 1000, 6096, 0.5130214095115662], "22": ["vertical_and_slash", 1000, 6096, 0.5000002980232239], "23": ["vertical_and_slash", 1000, 6096, 0.47976842522621155], "24": ["vertical_and_slash", 1000, 6096, 0.581958532333374], "25": ["vertical_and_slash", 1000, 6096, 0.5239659547805786], "26": ["vertical_and_slash", 1000, 6096, 0.5855763554573059], "27": ["vertical_and_slash", 1000, 6096, 0.660392701625824], "28": ["vertical_and_slash", 1000, 6096, 0.6228428483009338], "29": ["vertical_and_slash", 1000, 6096, 0.5769487619400024], "30": ["vertical_and_slash", 1000, 6096, 0.5207295417785645], "31": ["vertical_and_slash", 1000, 6096, 0.598313570022583], "32": ["vertical_and_slash", 1000, 6096, 0.5544818639755249], "33": ["vertical_and_slash", 1000, 6096, 0.5321142077445984], "34": ["vertical_and_slash", 1000, 6096, 0.473348468542099], "35": ["vertical_and_slash", 1000, 6096, 0.5306211709976196], "36": ["vertical_and_slash", 1000, 6096, 0.5441246032714844], "37": ["vertical_and_slash", 1000, 6096, 0.5346777439117432], "38": ["vertical_and_slash", 1000, 6096, 0.5144092440605164], "39": ["vertical_and_slash", 1000, 6096, 0.48490381240844727], "40": ["vertical_and_slash", 1000, 6096, 0.48038655519485474], "41": ["vertical_and_slash", 1000, 6096, 0.4326256811618805], "42": ["vertical_and_slash", 1000, 6096, 0.4173765778541565], "43": ["vertical_and_slash", 1000, 6096, 0.4368197023868561], "44": ["vertical_and_slash", 1000, 6096, 0.4842618703842163], "45": ["vertical_and_slash", 1000, 6096, 0.4223862886428833], "46": ["vertical_and_slash", 1000, 6096, 0.49422523379325867], "47": ["vertical_and_slash", 1000, 6096, 0.462192565202713], "48": ["vertical_and_slash", 1000, 6096, 0.5556728839874268], "49": ["vertical_and_slash", 1000, 6096, 0.4889889061450958], "50": ["vertical_and_slash", 1000, 6096, 0.4906291663646698], "51": ["vertical_and_slash", 1000, 6096, 0.4791358411312103], "52": ["vertical_and_slash", 1000, 6096, 0.5502273440361023], "53": ["vertical_and_slash", 1000, 6096, 0.5127010941505432], "54": ["vertical_and_slash", 1000, 6096, 0.5439939498901367], "55": ["vertical_and_slash", 1000, 6096, 0.4507691264152527], "56": ["vertical_and_slash", 1000, 6096, 0.5510099530220032], "57": ["vertical_and_slash", 1000, 6096, 0.5467963814735413], "58": ["vertical_and_slash", 1000, 6096, 0.554278552532196], "59": ["vertical_and_slash", 1000, 6096, 0.4962359666824341], "60": ["vertical_and_slash", 1000, 6096, 0.5385680794715881], "61": ["vertical_and_slash", 1000, 6096, 0.4654519855976105], "62": ["vertical_and_slash", 1000, 6096, 0.49327707290649414], "63": ["vertical_and_slash", 1000, 6096, 0.4645046591758728]}, {"0": ["vertical_and_slash", 1000, 6096, 0.4660588502883911], "1": ["vertical_and_slash", 1000, 6096, 0.4499935209751129], "2": ["vertical_and_slash", 1000, 6096, 0.556355893611908], "3": ["vertical_and_slash", 1000, 6096, 0.4940278232097626], "4": ["vertical_and_slash", 1000, 6096, 0.46935275197029114], "5": ["vertical_and_slash", 1000, 6096, 0.4845670461654663], "6": ["vertical_and_slash", 1000, 6096, 0.45483165979385376], "7": ["vertical_and_slash", 1000, 6096, 0.5473892092704773], "8": ["vertical_and_slash", 1000, 6096, 0.536383330821991], "9": ["vertical_and_slash", 1000, 6096, 0.45980268716812134], "10": ["vertical_and_slash", 1000, 6096, 0.5058341026306152], "11": ["vertical_and_slash", 1000, 6096, 0.6728416681289673], "12": ["vertical_and_slash", 1000, 6096, 0.539482593536377], "13": ["vertical_and_slash", 1000, 6096, 0.49266552925109863], "14": ["vertical_and_slash", 1000, 6096, 0.5015248656272888], "15": ["vertical_and_slash", 1000, 6096, 0.5347263216972351], "16": ["vertical_and_slash", 1000, 6096, 0.4574420154094696], "17": ["vertical_and_slash", 1000, 6096, 0.4682964086532593], "18": ["vertical_and_slash", 1000, 6096, 0.41336971521377563], "19": ["vertical_and_slash", 1000, 6096, 0.46123045682907104], "20": ["vertical_and_slash", 1000, 6096, 0.48761746287345886], "21": ["vertical_and_slash", 1000, 6096, 0.46481478214263916], "22": ["vertical_and_slash", 1000, 6096, 0.4297581613063812], "23": ["vertical_and_slash", 1000, 6096, 0.4376780092716217], "24": ["vertical_and_slash", 1000, 6096, 0.5530688166618347], "25": ["vertical_and_slash", 1000, 6096, 0.4590163230895996], "26": ["vertical_and_slash", 1000, 6096, 0.4526755213737488], "27": ["vertical_and_slash", 1000, 6096, 0.5249397158622742], "28": ["vertical_and_slash", 1000, 6096, 0.5311598777770996], "29": ["vertical_and_slash", 1000, 6096, 0.5106601119041443], "30": ["vertical_and_slash", 1000, 6096, 0.47056764364242554], "31": ["vertical_and_slash", 1000, 6096, 0.44593682885169983], "32": ["vertical_and_slash", 1000, 6096, 0.559239387512207], "33": ["vertical_and_slash", 1000, 6096, 0.5076427459716797], "34": ["vertical_and_slash", 1000, 6096, 0.6108396053314209], "35": ["vertical_and_slash", 1000, 6096, 0.5546644330024719], "36": ["vertical_and_slash", 1000, 6096, 0.5031907558441162], "37": ["vertical_and_slash", 1000, 6096, 0.524406909942627], "38": ["vertical_and_slash", 1000, 6096, 0.5165525674819946], "39": ["vertical_and_slash", 1000, 6096, 0.6001859903335571], "40": ["vertical_and_slash", 1000, 6096, 0.6040078401565552], "41": ["vertical_and_slash", 1000, 6096, 0.4807492792606354], "42": ["vertical_and_slash", 1000, 6096, 0.6196043491363525], "43": ["vertical_and_slash", 1000, 6096, 0.49187734723091125], "44": ["vertical_and_slash", 1000, 6096, 0.5518859028816223], "45": ["vertical_and_slash", 1000, 6096, 0.49562084674835205], "46": ["vertical_and_slash", 1000, 6096, 0.5680897831916809], "47": ["vertical_and_slash", 1000, 6096, 0.43709596991539], "48": ["vertical_and_slash", 1000, 6096, 0.5223337411880493], "49": ["vertical_and_slash", 1000, 6096, 0.5998830199241638], "50": ["vertical_and_slash", 1000, 6096, 0.5712031126022339], "51": ["vertical_and_slash", 1000, 6096, 0.5467070937156677], "52": ["vertical_and_slash", 1000, 6096, 0.4971073567867279], "53": ["vertical_and_slash", 1000, 6096, 0.5527136921882629], "54": ["vertical_and_slash", 1000, 6096, 0.5827455520629883], "55": ["vertical_and_slash", 1000, 6096, 0.5370614528656006], "56": ["vertical_and_slash", 1000, 6096, 0.45419761538505554], "57": ["vertical_and_slash", 1000, 6096, 0.7148821949958801], "58": ["vertical_and_slash", 1000, 6096, 0.5191951990127563], "59": ["vertical_and_slash", 1000, 6096, 0.4711379110813141], "60": ["vertical_and_slash", 1000, 6096, 0.5515417456626892], "61": ["vertical_and_slash", 1000, 6096, 0.5269593596458435], "62": ["vertical_and_slash", 1000, 6096, 0.5484921336174011], "63": ["vertical_and_slash", 1000, 6096, 0.6204590797424316]}]
diff --git a/minference/configs/Qwen2.5_7B_Instruct_128k_instruct_kv_out_v32_fit_o_best_pattern.json b/minference/configs/Qwen2.5_7B_Instruct_128k_instruct_kv_out_v32_fit_o_best_pattern.json
new file mode 100644
index 00000000..b04d65e1
--- /dev/null
+++ b/minference/configs/Qwen2.5_7B_Instruct_128k_instruct_kv_out_v32_fit_o_best_pattern.json
@@ -0,0 +1 @@
+[{"0": ["vertical_and_slash", 3500, 100, 0.6750829815864563], "1": ["vertical_and_slash", 1000, 6096, 0.9805771112442017], "2": ["vertical_and_slash", 1000, 6096, 0.8052300810813904], "3": ["vertical_and_slash", 500, 700, 0.918170154094696], "4": ["vertical_and_slash", 3500, 100, 0.7307244539260864], "5": ["vertical_and_slash", 1000, 6096, 0.9887239336967468], "6": ["vertical_and_slash", 30, 800, 0.9976220726966858], "7": ["vertical_and_slash", 1000, 6096, 0.9847933650016785], "8": ["vertical_and_slash", 1000, 6096, 0.9312621355056763], "9": ["vertical_and_slash", 1000, 6096, 0.8598348498344421], "10": ["vertical_and_slash", 1000, 6096, 0.7951297163963318], "11": ["vertical_and_slash", 1000, 6096, 0.904041051864624], "12": ["vertical_and_slash", 1000, 6096, 0.905684232711792], "13": ["vertical_and_slash", 1000, 6096, 0.8471306562423706], "14": ["vertical_and_slash", 3500, 100, 0.6442427635192871], "15": ["vertical_and_slash", 3500, 100, 0.7113548517227173], "16": ["vertical_and_slash", 1000, 6096, 0.9781556129455566], "17": ["vertical_and_slash", 1000, 6096, 0.7771812081336975], "18": ["vertical_and_slash", 1000, 6096, 0.5806995034217834], "19": ["vertical_and_slash", 1000, 6096, 0.8473479151725769], "20": ["vertical_and_slash", 1000, 6096, 0.9540221691131592], "21": ["vertical_and_slash", 1000, 6096, 0.9366317987442017], "22": ["vertical_and_slash", 1000, 6096, 0.750575065612793], "23": ["vertical_and_slash", 1000, 6096, 0.9571341276168823], "24": ["vertical_and_slash", 1000, 6096, 0.9624292850494385], "25": ["vertical_and_slash", 1000, 6096, 0.961791455745697], "26": ["vertical_and_slash", 1000, 6096, 0.9593926072120667], "27": ["vertical_and_slash", 1000, 6096, 0.9615839719772339]}, {"0": ["vertical_and_slash", 1000, 6096, 0.999679446220398], "1": ["vertical_and_slash", 3500, 100, 0.6681293845176697], "2": ["vertical_and_slash", 1000, 6096, 0.9987146258354187], "3": ["vertical_and_slash", 1000, 6096, 0.99982750415802], "4": ["vertical_and_slash", 1000, 6096, 0.9987889528274536], "5": ["vertical_and_slash", 1000, 6096, 0.9999328255653381], "6": ["vertical_and_slash", 1000, 6096, 0.6873452663421631], "7": ["vertical_and_slash", 1000, 6096, 0.8953464031219482], "8": ["vertical_and_slash", 1000, 6096, 0.825433611869812], "9": ["vertical_and_slash", 1000, 6096, 0.5478852987289429], "10": ["vertical_and_slash", 1000, 6096, 0.9211719632148743], "11": ["vertical_and_slash", 1000, 6096, 0.891532301902771], "12": ["vertical_and_slash", 1000, 6096, 0.9572229981422424], "13": ["vertical_and_slash", 1000, 6096, 0.5790613293647766], "14": ["vertical_and_slash", 1000, 6096, 0.9492497444152832], "15": ["vertical_and_slash", 1000, 6096, 0.9910810589790344], "16": ["vertical_and_slash", 1000, 6096, 0.7837988138198853], "17": ["vertical_and_slash", 1000, 6096, 0.9984248280525208], "18": ["vertical_and_slash", 1000, 6096, 0.86527419090271], "19": ["vertical_and_slash", 1000, 6096, 0.902651846408844], "20": ["vertical_and_slash", 1000, 6096, 0.9432386755943298], "21": ["vertical_and_slash", 3500, 100, 0.690281093120575], "22": ["vertical_and_slash", 1000, 6096, 0.6934974193572998], "23": ["vertical_and_slash", 30, 800, 0.9758475422859192], "24": ["vertical_and_slash", 30, 800, 0.9926055669784546], "25": ["vertical_and_slash", 1000, 6096, 0.8463826775550842], "26": ["vertical_and_slash", 3500, 100, 0.7129432559013367], "27": ["vertical_and_slash", 1000, 6096, 0.9563227295875549]}, {"0": ["vertical_and_slash", 3500, 100, 0.937682569026947], "1": ["vertical_and_slash", 3500, 100, 0.9309927225112915], "2": ["vertical_and_slash", 3500, 100, 0.9264195561408997], "3": ["vertical_and_slash", 3500, 100, 0.9423984289169312], "4": ["vertical_and_slash", 1000, 6096, 0.6362282633781433], "5": ["vertical_and_slash", 3500, 100, 0.9381661415100098], "6": ["vertical_and_slash", 3500, 100, 0.8798274993896484], "7": ["vertical_and_slash", 1000, 6096, 0.9354630708694458], "8": ["vertical_and_slash", 1000, 6096, 0.8675763607025146], "9": ["vertical_and_slash", 1000, 6096, 0.9294215440750122], "10": ["vertical_and_slash", 1000, 6096, 0.9030386805534363], "11": ["vertical_and_slash", 1000, 6096, 0.9264913201332092], "12": ["vertical_and_slash", 1000, 6096, 0.7738977670669556], "13": ["vertical_and_slash", 1000, 6096, 0.8057573437690735], "14": ["vertical_and_slash", 1000, 6096, 0.6162545680999756], "15": ["vertical_and_slash", 1000, 6096, 0.8160500526428223], "16": ["vertical_and_slash", 1000, 6096, 0.7907118797302246], "17": ["vertical_and_slash", 3500, 100, 0.7754966616630554], "18": ["vertical_and_slash", 3500, 100, 0.6583731770515442], "19": ["vertical_and_slash", 1000, 6096, 0.7325001358985901], "20": ["vertical_and_slash", 3500, 100, 0.8400217890739441], "21": ["vertical_and_slash", 1000, 6096, 0.9005410075187683], "22": ["vertical_and_slash", 1000, 6096, 0.6206564903259277], "23": ["vertical_and_slash", 1000, 6096, 0.8995620012283325], "24": ["vertical_and_slash", 1000, 6096, 0.7642338275909424], "25": ["vertical_and_slash", 1000, 6096, 0.9060980677604675], "26": ["vertical_and_slash", 1000, 6096, 0.6690731644630432], "27": ["vertical_and_slash", 1000, 6096, 0.9422726631164551]}, {"0": ["vertical_and_slash", 1000, 6096, 0.6443943381309509], "1": ["vertical_and_slash", 1000, 6096, 0.9848536849021912], "2": ["vertical_and_slash", 1000, 6096, 0.8956511616706848], "3": ["vertical_and_slash", 1000, 6096, 0.9640544056892395], "4": ["vertical_and_slash", 1000, 6096, 0.885956346988678], "5": ["vertical_and_slash", 1000, 6096, 0.9406574964523315], "6": ["vertical_and_slash", 1000, 6096, 0.94412761926651], "7": ["vertical_and_slash", 1000, 6096, 0.9971193075180054], "8": ["vertical_and_slash", 1000, 6096, 0.5039099454879761], "9": ["vertical_and_slash", 1000, 6096, 0.981869101524353], "10": ["vertical_and_slash", 1000, 6096, 0.9978828430175781], "11": ["vertical_and_slash", 1000, 6096, 0.9503500461578369], "12": ["vertical_and_slash", 1000, 6096, 0.9206511974334717], "13": ["vertical_and_slash", 1000, 6096, 0.8876844048500061], "14": ["vertical_and_slash", 1000, 6096, 0.7389187216758728], "15": ["vertical_and_slash", 1000, 6096, 0.934273362159729], "16": ["vertical_and_slash", 3500, 100, 0.6163461804389954], "17": ["vertical_and_slash", 1000, 6096, 0.7882677316665649], "18": ["vertical_and_slash", 1000, 6096, 0.6220505833625793], "19": ["vertical_and_slash", 1000, 6096, 0.8008646368980408], "20": ["vertical_and_slash", 1000, 6096, 0.8597842454910278], "21": ["vertical_and_slash", 1000, 6096, 0.8192393779754639], "22": ["vertical_and_slash", 1000, 6096, 0.9566972255706787], "23": ["vertical_and_slash", 1000, 6096, 0.90835040807724], "24": ["vertical_and_slash", 1000, 6096, 0.9798492789268494], "25": ["vertical_and_slash", 1000, 6096, 0.9638792276382446], "26": ["vertical_and_slash", 1000, 6096, 0.9422233700752258], "27": ["vertical_and_slash", 1000, 6096, 0.9642584323883057]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9416517019271851], "1": ["vertical_and_slash", 1000, 6096, 0.9115155935287476], "2": ["vertical_and_slash", 1000, 6096, 0.9919536709785461], "3": ["vertical_and_slash", 1000, 6096, 0.9658247828483582], "4": ["vertical_and_slash", 1000, 6096, 0.9270678162574768], "5": ["vertical_and_slash", 1000, 6096, 0.9699860215187073], "6": ["vertical_and_slash", 1000, 6096, 0.9457927942276001], "7": ["vertical_and_slash", 100, 800, 1.0], "8": ["vertical_and_slash", 1000, 6096, 0.8798156380653381], "9": ["vertical_and_slash", 1000, 6096, 0.9258736968040466], "10": ["vertical_and_slash", 1000, 6096, 0.9318532347679138], "11": ["vertical_and_slash", 100, 800, 0.984375], "12": ["vertical_and_slash", 100, 800, 1.0], "13": ["vertical_and_slash", 1000, 6096, 0.9952613115310669], "14": ["vertical_and_slash", 30, 800, 0.9801478385925293], "15": ["vertical_and_slash", 30, 800, 0.9578455090522766], "16": ["vertical_and_slash", 100, 800, 0.96484375], "17": ["vertical_and_slash", 1000, 6096, 0.5916932821273804], "18": ["vertical_and_slash", 30, 800, 0.9947859644889832], "19": ["vertical_and_slash", 30, 800, 0.9898229837417603], "20": ["vertical_and_slash", 30, 800, 0.9837746620178223], "21": ["vertical_and_slash", 30, 800, 0.9347994923591614], "22": ["vertical_and_slash", 30, 800, 0.9872448444366455], "23": ["vertical_and_slash", 30, 800, 0.997748851776123], "24": ["vertical_and_slash", 500, 700, 0.7037194967269897], "25": ["vertical_and_slash", 30, 800, 0.9900204539299011], "26": ["vertical_and_slash", 1000, 6096, 0.9468749761581421], "27": ["vertical_and_slash", 30, 800, 0.9923171401023865]}, {"0": ["vertical_and_slash", 30, 800, 0.8765897154808044], "1": ["vertical_and_slash", 500, 700, 0.9750522375106812], "2": ["vertical_and_slash", 500, 700, 0.9347320199012756], "3": ["vertical_and_slash", 500, 700, 0.9776737689971924], "4": ["vertical_and_slash", 500, 700, 0.927309513092041], "5": ["vertical_and_slash", 1000, 6096, 0.7631339430809021], "6": ["vertical_and_slash", 1000, 6096, 0.7299099564552307], "7": ["vertical_and_slash", 1000, 6096, 0.9909149408340454], "8": ["vertical_and_slash", 3500, 100, 0.9894052147865295], "9": ["vertical_and_slash", 3500, 100, 0.8932533264160156], "10": ["vertical_and_slash", 1000, 6096, 0.9002631306648254], "11": ["vertical_and_slash", 3500, 100, 0.6817510724067688], "12": ["vertical_and_slash", 1000, 6096, 0.938396155834198], "13": ["vertical_and_slash", 1000, 6096, 0.8958967924118042], "14": ["vertical_and_slash", 3500, 100, 0.9840639233589172], "15": ["vertical_and_slash", 1000, 6096, 0.9135857820510864], "16": ["vertical_and_slash", 3500, 100, 0.9779791831970215], "17": ["vertical_and_slash", 1000, 6096, 0.5101895332336426], "18": ["vertical_and_slash", 1000, 6096, 0.5636370778083801], "19": ["vertical_and_slash", 1000, 6096, 0.6438812613487244], "20": ["vertical_and_slash", 1000, 6096, 0.6584950685501099], "21": ["vertical_and_slash", 1000, 6096, 0.5612680315971375], "22": ["vertical_and_slash", 30, 800, 0.9790387749671936], "23": ["vertical_and_slash", 30, 800, 0.988126277923584], "24": ["vertical_and_slash", 100, 800, 0.94140625], "25": ["vertical_and_slash", 30, 800, 0.9846640229225159], "26": ["vertical_and_slash", 30, 800, 0.8738773465156555], "27": ["vertical_and_slash", 500, 700, 0.950529932975769]}, {"0": ["vertical_and_slash", 30, 800, 0.9940765500068665], "1": ["vertical_and_slash", 30, 800, 0.9962191581726074], "2": ["vertical_and_slash", 30, 800, 0.9980990886688232], "3": ["vertical_and_slash", 30, 800, 0.9929674863815308], "4": ["vertical_and_slash", 30, 800, 0.996401309967041], "5": ["vertical_and_slash", 30, 800, 0.981270968914032], "6": ["vertical_and_slash", 30, 800, 0.9985417127609253], "7": ["vertical_and_slash", 3500, 100, 0.8639557361602783], "8": ["vertical_and_slash", 1000, 6096, 0.8663111925125122], "9": ["vertical_and_slash", 1000, 6096, 0.7727228403091431], "10": ["vertical_and_slash", 1000, 6096, 0.7190098166465759], "11": ["vertical_and_slash", 3500, 100, 0.8344407081604004], "12": ["vertical_and_slash", 1000, 6096, 0.9553660154342651], "13": ["vertical_and_slash", 3500, 100, 0.9762938022613525], "14": ["vertical_and_slash", 1000, 6096, 0.9423325657844543], "15": ["vertical_and_slash", 30, 800, 0.9912431836128235], "16": ["vertical_and_slash", 30, 800, 0.9901708960533142], "17": ["vertical_and_slash", 3500, 100, 0.7286731600761414], "18": ["vertical_and_slash", 1000, 6096, 0.9008346796035767], "19": ["vertical_and_slash", 30, 800, 0.9865132570266724], "20": ["vertical_and_slash", 30, 800, 0.9902596473693848], "21": ["vertical_and_slash", 30, 800, 0.987083911895752], "22": ["vertical_and_slash", 30, 800, 0.9856812953948975], "23": ["vertical_and_slash", 1000, 6096, 0.640706479549408], "24": ["vertical_and_slash", 30, 800, 0.9729791283607483], "25": ["vertical_and_slash", 30, 800, 0.9913076162338257], "26": ["vertical_and_slash", 30, 800, 0.9903677105903625], "27": ["vertical_and_slash", 30, 800, 0.9784517884254456]}, {"0": ["vertical_and_slash", 30, 800, 0.9815305471420288], "1": ["vertical_and_slash", 100, 800, 0.87890625], "2": ["vertical_and_slash", 1000, 6096, 0.8910069465637207], "3": ["vertical_and_slash", 30, 800, 0.9630582928657532], "4": ["vertical_and_slash", 30, 800, 0.9894396066665649], "5": ["vertical_and_slash", 30, 800, 0.9901205897331238], "6": ["vertical_and_slash", 3500, 100, 0.5307775735855103], "7": ["vertical_and_slash", 1000, 6096, 0.8945309519767761], "8": ["vertical_and_slash", 30, 800, 0.9863417744636536], "9": ["vertical_and_slash", 30, 800, 0.9815919399261475], "10": ["vertical_and_slash", 1000, 6096, 0.8101935386657715], "11": ["vertical_and_slash", 30, 800, 0.9899166226387024], "12": ["vertical_and_slash", 500, 700, 0.9940841197967529], "13": ["vertical_and_slash", 30, 800, 0.9760785102844238], "14": ["vertical_and_slash", 1000, 6096, 0.6692083477973938], "15": ["vertical_and_slash", 3500, 100, 0.7875557541847229], "16": ["vertical_and_slash", 3500, 100, 0.9933667778968811], "17": ["vertical_and_slash", 1000, 6096, 0.9945434927940369], "18": ["vertical_and_slash", 3500, 100, 0.8973793387413025], "19": ["vertical_and_slash", 1000, 6096, 0.705904483795166], "20": ["vertical_and_slash", 30, 800, 0.958629310131073], "21": ["vertical_and_slash", 500, 700, 0.9835988879203796], "22": ["vertical_and_slash", 3500, 100, 0.9260191917419434], "23": ["vertical_and_slash", 3500, 100, 0.9437853097915649], "24": ["vertical_and_slash", 1000, 6096, 0.9394773840904236], "25": ["vertical_and_slash", 1000, 6096, 0.9844918251037598], "26": ["vertical_and_slash", 3500, 100, 0.8746518492698669], "27": ["vertical_and_slash", 1000, 6096, 0.9021381735801697]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9704672694206238], "1": ["vertical_and_slash", 30, 800, 0.9846382737159729], "2": ["vertical_and_slash", 1000, 6096, 0.9476844072341919], "3": ["vertical_and_slash", 3500, 100, 0.9300746917724609], "4": ["vertical_and_slash", 3500, 100, 0.8714134693145752], "5": ["vertical_and_slash", 1000, 6096, 0.9580811858177185], "6": ["vertical_and_slash", 1000, 6096, 0.9928625822067261], "7": ["vertical_and_slash", 1000, 6096, 0.9470779299736023], "8": ["vertical_and_slash", 30, 800, 0.9705231189727783], "9": ["vertical_and_slash", 1000, 6096, 0.9141854643821716], "10": ["vertical_and_slash", 30, 800, 0.978949785232544], "11": ["vertical_and_slash", 30, 800, 0.8739697337150574], "12": ["vertical_and_slash", 500, 700, 0.9466822147369385], "13": ["vertical_and_slash", 1000, 6096, 0.8348672389984131], "14": ["vertical_and_slash", 30, 800, 0.9781960844993591], "15": ["vertical_and_slash", 30, 800, 0.9973211288452148], "16": ["vertical_and_slash", 30, 800, 0.9987407326698303], "17": ["vertical_and_slash", 3500, 100, 0.9436909556388855], "18": ["vertical_and_slash", 30, 800, 0.9812127351760864], "19": ["vertical_and_slash", 30, 800, 0.9916983842849731], "20": ["vertical_and_slash", 30, 800, 0.9825637340545654], "21": ["vertical_and_slash", 1000, 6096, 0.9954416155815125], "22": ["vertical_and_slash", 1000, 6096, 0.6436072587966919], "23": ["vertical_and_slash", 1000, 6096, 0.8333381414413452], "24": ["vertical_and_slash", 1000, 6096, 0.9481961131095886], "25": ["vertical_and_slash", 1000, 6096, 0.9355611205101013], "26": ["vertical_and_slash", 1000, 6096, 0.5479784607887268], "27": ["vertical_and_slash", 1000, 6096, 0.8135812878608704]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9255619049072266], "1": ["vertical_and_slash", 3500, 100, 0.9221183061599731], "2": ["vertical_and_slash", 3500, 100, 0.7880069017410278], "3": ["vertical_and_slash", 1000, 6096, 0.957737147808075], "4": ["vertical_and_slash", 3500, 100, 0.902386486530304], "5": ["vertical_and_slash", 1000, 6096, 0.9594583511352539], "6": ["vertical_and_slash", 1000, 6096, 0.9658820033073425], "7": ["vertical_and_slash", 1000, 6096, 0.8921599984169006], "8": ["vertical_and_slash", 3500, 100, 0.587282121181488], "9": ["vertical_and_slash", 1000, 6096, 0.9727311134338379], "10": ["vertical_and_slash", 3500, 100, 0.8759869933128357], "11": ["vertical_and_slash", 1000, 6096, 0.9664274454116821], "12": ["vertical_and_slash", 1000, 6096, 0.9790003299713135], "13": ["vertical_and_slash", 1000, 6096, 0.8624147772789001], "14": ["vertical_and_slash", 1000, 6096, 0.9046944379806519], "15": ["vertical_and_slash", 3500, 100, 0.902532160282135], "16": ["vertical_and_slash", 1000, 6096, 0.9447299242019653], "17": ["vertical_and_slash", 1000, 6096, 0.7742043733596802], "18": ["vertical_and_slash", 3500, 100, 0.9317870736122131], "19": ["vertical_and_slash", 1000, 6096, 0.6389862298965454], "20": ["vertical_and_slash", 1000, 6096, 0.872114360332489], "21": ["vertical_and_slash", 3500, 100, 0.8341178297996521], "22": ["vertical_and_slash", 3500, 100, 0.7938392162322998], "23": ["vertical_and_slash", 1000, 6096, 0.8454285860061646], "24": ["vertical_and_slash", 1000, 6096, 0.9322804808616638], "25": ["vertical_and_slash", 3500, 100, 0.8644981980323792], "26": ["vertical_and_slash", 1000, 6096, 0.8821665644645691], "27": ["vertical_and_slash", 1000, 6096, 0.9186031222343445]}, {"0": ["vertical_and_slash", 3500, 100, 0.9386969804763794], "1": ["vertical_and_slash", 30, 800, 0.9781283140182495], "2": ["vertical_and_slash", 1000, 6096, 0.9235864877700806], "3": ["vertical_and_slash", 3500, 100, 0.940860390663147], "4": ["vertical_and_slash", 500, 700, 0.9786099791526794], "5": ["vertical_and_slash", 500, 700, 0.9615233540534973], "6": ["vertical_and_slash", 3500, 100, 0.780426025390625], "7": ["vertical_and_slash", 500, 700, 0.9750614762306213], "8": ["vertical_and_slash", 500, 700, 0.9821221828460693], "9": ["vertical_and_slash", 500, 700, 0.9754288196563721], "10": ["vertical_and_slash", 500, 700, 0.9245291352272034], "11": ["vertical_and_slash", 30, 800, 0.8216957449913025], "12": ["vertical_and_slash", 30, 800, 0.990125298500061], "13": ["vertical_and_slash", 500, 700, 0.9841389060020447], "14": ["vertical_and_slash", 30, 800, 0.9329437017440796], "15": ["vertical_and_slash", 30, 800, 0.9896363019943237], "16": ["vertical_and_slash", 500, 700, 0.9818762540817261], "17": ["vertical_and_slash", 3500, 100, 0.9923845529556274], "18": ["vertical_and_slash", 1000, 6096, 0.7936633825302124], "19": ["vertical_and_slash", 1000, 6096, 0.8555855751037598], "20": ["vertical_and_slash", 1000, 6096, 0.9654672741889954], "21": ["vertical_and_slash", 30, 800, 0.9842104911804199], "22": ["vertical_and_slash", 30, 800, 0.9923548102378845], "23": ["vertical_and_slash", 30, 800, 0.9805132746696472], "24": ["vertical_and_slash", 1000, 6096, 0.8944741487503052], "25": ["vertical_and_slash", 30, 800, 0.9880507588386536], "26": ["vertical_and_slash", 30, 800, 0.9848252534866333], "27": ["vertical_and_slash", 30, 800, 0.990683376789093]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7014197707176208], "1": ["vertical_and_slash", 30, 800, 0.9906507730484009], "2": ["vertical_and_slash", 1000, 6096, 0.8565399646759033], "3": ["vertical_and_slash", 30, 800, 0.9761000275611877], "4": ["vertical_and_slash", 500, 700, 0.9540215730667114], "5": ["vertical_and_slash", 500, 700, 0.9528164863586426], "6": ["vertical_and_slash", 30, 800, 0.9404529333114624], "7": ["vertical_and_slash", 100, 800, 0.98046875], "8": ["vertical_and_slash", 3500, 100, 0.5027956366539001], "9": ["vertical_and_slash", 100, 800, 0.88671875], "10": ["vertical_and_slash", 500, 700, 0.9342089891433716], "11": ["vertical_and_slash", 100, 800, 0.87890625], "12": ["vertical_and_slash", 3500, 100, 0.44585394859313965], "13": ["vertical_and_slash", 3500, 100, 0.9621792435646057], "14": ["vertical_and_slash", 30, 800, 0.9818242788314819], "15": ["vertical_and_slash", 30, 800, 0.9872978329658508], "16": ["vertical_and_slash", 30, 800, 0.9290117025375366], "17": ["vertical_and_slash", 500, 700, 0.9664343595504761], "18": ["vertical_and_slash", 30, 800, 0.9874595999717712], "19": ["vertical_and_slash", 30, 800, 0.9734411835670471], "20": ["vertical_and_slash", 30, 800, 0.987689197063446], "21": ["vertical_and_slash", 1000, 6096, 0.830070972442627], "22": ["vertical_and_slash", 1000, 6096, 0.8193408250808716], "23": ["vertical_and_slash", 1000, 6096, 0.6732114553451538], "24": ["vertical_and_slash", 3500, 100, 0.6378278732299805], "25": ["vertical_and_slash", 1000, 6096, 0.9034184813499451], "26": ["vertical_and_slash", 3500, 100, 0.9268975257873535], "27": ["vertical_and_slash", 1000, 6096, 0.899992823600769]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9845043420791626], "1": ["vertical_and_slash", 3500, 100, 0.9459904432296753], "2": ["vertical_and_slash", 3500, 100, 0.8677979707717896], "3": ["vertical_and_slash", 1000, 6096, 0.950700044631958], "4": ["vertical_and_slash", 3500, 100, 0.9338597059249878], "5": ["vertical_and_slash", 100, 800, 0.984375], "6": ["vertical_and_slash", 3500, 100, 0.723767876625061], "7": ["vertical_and_slash", 3500, 100, 0.8333364725112915], "8": ["vertical_and_slash", 1000, 6096, 0.9460318088531494], "9": ["vertical_and_slash", 1000, 6096, 0.958717405796051], "10": ["vertical_and_slash", 1000, 6096, 0.9514124989509583], "11": ["vertical_and_slash", 1000, 6096, 0.970104992389679], "12": ["vertical_and_slash", 1000, 6096, 0.8972886800765991], "13": ["vertical_and_slash", 1000, 6096, 0.9765167236328125], "14": ["vertical_and_slash", 1000, 6096, 0.9680219888687134], "15": ["vertical_and_slash", 1000, 6096, 0.8409132361412048], "16": ["vertical_and_slash", 3500, 100, 0.653377890586853], "17": ["vertical_and_slash", 1000, 6096, 0.5165567398071289], "18": ["vertical_and_slash", 1000, 6096, 0.9893617630004883], "19": ["vertical_and_slash", 1000, 6096, 0.9176812171936035], "20": ["vertical_and_slash", 3500, 100, 0.8301747441291809], "21": ["vertical_and_slash", 1000, 6096, 0.9036827087402344], "22": ["vertical_and_slash", 1000, 6096, 0.9301431775093079], "23": ["vertical_and_slash", 1000, 6096, 0.9032087922096252], "24": ["vertical_and_slash", 1000, 6096, 0.806071400642395], "25": ["vertical_and_slash", 500, 700, 0.9673863053321838], "26": ["vertical_and_slash", 1000, 6096, 0.9135075807571411], "27": ["vertical_and_slash", 1000, 6096, 0.9343668818473816]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8967662453651428], "1": ["vertical_and_slash", 1000, 6096, 0.9186690449714661], "2": ["vertical_and_slash", 1000, 6096, 0.9233105778694153], "3": ["vertical_and_slash", 1000, 6096, 0.9110817313194275], "4": ["vertical_and_slash", 1000, 6096, 0.8421552181243896], "5": ["vertical_and_slash", 1000, 6096, 0.8990441560745239], "6": ["vertical_and_slash", 3500, 100, 0.8867408037185669], "7": ["vertical_and_slash", 1000, 6096, 0.9886850714683533], "8": ["vertical_and_slash", 30, 800, 0.9791635870933533], "9": ["vertical_and_slash", 1000, 6096, 0.9414047002792358], "10": ["vertical_and_slash", 30, 800, 0.9734972715377808], "11": ["vertical_and_slash", 1000, 6096, 0.6580237150192261], "12": ["vertical_and_slash", 1000, 6096, 0.874021053314209], "13": ["vertical_and_slash", 30, 800, 0.9982300400733948], "14": ["vertical_and_slash", 1000, 6096, 0.8276069760322571], "15": ["vertical_and_slash", 1000, 6096, 0.9144404530525208], "16": ["vertical_and_slash", 1000, 6096, 0.7851533889770508], "17": ["vertical_and_slash", 1000, 6096, 0.8217888474464417], "18": ["vertical_and_slash", 1000, 6096, 0.7995612621307373], "19": ["vertical_and_slash", 1000, 6096, 0.7720385789871216], "20": ["vertical_and_slash", 1000, 6096, 0.8377131819725037], "21": ["vertical_and_slash", 1000, 6096, 0.8834578990936279], "22": ["vertical_and_slash", 1000, 6096, 0.934196412563324], "23": ["vertical_and_slash", 1000, 6096, 0.9010329842567444], "24": ["vertical_and_slash", 1000, 6096, 0.9080091714859009], "25": ["vertical_and_slash", 1000, 6096, 0.8898270726203918], "26": ["vertical_and_slash", 1000, 6096, 0.9437620043754578], "27": ["vertical_and_slash", 1000, 6096, 0.9125307202339172]}, {"0": ["vertical_and_slash", 1000, 6096, 0.6647223830223083], "1": ["vertical_and_slash", 1000, 6096, 0.6934041976928711], "2": ["vertical_and_slash", 1000, 6096, 0.8942349553108215], "3": ["vertical_and_slash", 1000, 6096, 0.4702666997909546], "4": ["vertical_and_slash", 1000, 6096, 0.9409206509590149], "5": ["vertical_and_slash", 3500, 100, 1.0001837015151978], "6": ["vertical_and_slash", 1000, 6096, 0.7396820187568665], "7": ["vertical_and_slash", 3500, 100, 0.9886468052864075], "8": ["vertical_and_slash", 3500, 100, 0.6820537447929382], "9": ["vertical_and_slash", 100, 800, 0.921875], "10": ["vertical_and_slash", 1000, 6096, 0.7334492802619934], "11": ["vertical_and_slash", 1000, 6096, 0.910801887512207], "12": ["vertical_and_slash", 3500, 100, 0.4942914545536041], "13": ["vertical_and_slash", 100, 800, 0.85546875], "14": ["vertical_and_slash", 1000, 6096, 0.8864116668701172], "15": ["vertical_and_slash", 3500, 100, 0.9293166995048523], "16": ["vertical_and_slash", 3500, 100, 0.8957590460777283], "17": ["vertical_and_slash", 1000, 6096, 0.9594305157661438], "18": ["vertical_and_slash", 1000, 6096, 0.7695325016975403], "19": ["vertical_and_slash", 1000, 6096, 0.8477590084075928], "20": ["vertical_and_slash", 1000, 6096, 0.7081705331802368], "21": ["vertical_and_slash", 3500, 100, 0.8224411606788635], "22": ["vertical_and_slash", 1000, 6096, 0.703486442565918], "23": ["vertical_and_slash", 3500, 100, 0.7805135250091553], "24": ["vertical_and_slash", 1000, 6096, 0.9295625686645508], "25": ["vertical_and_slash", 1000, 6096, 0.9823433756828308], "26": ["vertical_and_slash", 1000, 6096, 0.875329852104187], "27": ["vertical_and_slash", 1000, 6096, 0.6856881976127625]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9829742908477783], "1": ["vertical_and_slash", 1000, 6096, 0.7207006216049194], "2": ["vertical_and_slash", 1000, 6096, 0.9685685038566589], "3": ["vertical_and_slash", 1000, 6096, 0.9824396967887878], "4": ["vertical_and_slash", 1000, 6096, 0.7485611438751221], "5": ["vertical_and_slash", 1000, 6096, 0.9940901398658752], "6": ["vertical_and_slash", 1000, 6096, 0.9946538209915161], "7": ["vertical_and_slash", 30, 800, 0.8307965397834778], "8": ["vertical_and_slash", 500, 700, 0.9881348609924316], "9": ["vertical_and_slash", 100, 750, 0.992007315158844], "10": ["vertical_and_slash", 500, 700, 0.943379282951355], "11": ["vertical_and_slash", 100, 750, 0.9991031885147095], "12": ["vertical_and_slash", 500, 700, 0.9775263071060181], "13": ["vertical_and_slash", 500, 700, 0.9813495874404907], "14": ["vertical_and_slash", 3500, 100, 0.4861743748188019], "15": ["vertical_and_slash", 1000, 6096, 0.920661449432373], "16": ["vertical_and_slash", 500, 700, 0.9851991534233093], "17": ["vertical_and_slash", 3500, 100, 0.8947359323501587], "18": ["vertical_and_slash", 1000, 6096, 0.7658404111862183], "19": ["vertical_and_slash", 100, 800, 0.83984375], "20": ["vertical_and_slash", 1000, 6096, 0.6112598776817322], "21": ["vertical_and_slash", 500, 700, 0.944545328617096], "22": ["vertical_and_slash", 1000, 6096, 0.7944725155830383], "23": ["vertical_and_slash", 1000, 6096, 0.7917705178260803], "24": ["vertical_and_slash", 1000, 6096, 0.5064239501953125], "25": ["vertical_and_slash", 1000, 6096, 0.6259453296661377], "26": ["vertical_and_slash", 3500, 100, 0.8557173609733582], "27": ["vertical_and_slash", 500, 700, 0.917644739151001]}, {"0": ["vertical_and_slash", 3500, 100, 0.6924666166305542], "1": ["vertical_and_slash", 3500, 100, 0.8591611981391907], "2": ["vertical_and_slash", 3500, 100, 0.6443263292312622], "3": ["vertical_and_slash", 3500, 100, 0.7151119112968445], "4": ["vertical_and_slash", 1000, 6096, 0.8455731868743896], "5": ["vertical_and_slash", 3500, 100, 0.5850971937179565], "6": ["vertical_and_slash", 3500, 100, 0.9161069989204407], "7": ["vertical_and_slash", 3500, 100, 0.6830418109893799], "8": ["vertical_and_slash", 100, 800, 0.98046875], "9": ["vertical_and_slash", 1000, 6096, 0.6874740123748779], "10": ["vertical_and_slash", 1000, 6096, 0.9198310971260071], "11": ["vertical_and_slash", 3500, 100, 0.5894737243652344], "12": ["vertical_and_slash", 1000, 6096, 0.5572332739830017], "13": ["vertical_and_slash", 1000, 6096, 0.5696163177490234], "14": ["vertical_and_slash", 3500, 100, 0.67469322681427], "15": ["vertical_and_slash", 1000, 6096, 0.7836475372314453], "16": ["vertical_and_slash", 3500, 100, 0.9059060215950012], "17": ["vertical_and_slash", 3500, 100, 0.9301673173904419], "18": ["vertical_and_slash", 3500, 100, 0.9061617851257324], "19": ["vertical_and_slash", 1000, 6096, 0.6480774879455566], "20": ["vertical_and_slash", 3500, 100, 0.7150899171829224], "21": ["vertical_and_slash", 500, 700, 0.9598405957221985], "22": ["vertical_and_slash", 1000, 6096, 0.892973780632019], "23": ["vertical_and_slash", 500, 700, 0.9304614663124084], "24": ["vertical_and_slash", 1000, 6096, 0.9003311395645142], "25": ["vertical_and_slash", 1000, 6096, 0.9597203731536865], "26": ["vertical_and_slash", 1000, 6096, 0.9169187545776367], "27": ["vertical_and_slash", 1000, 6096, 0.8284030556678772]}, {"0": ["vertical_and_slash", 1000, 6096, 0.7870258092880249], "1": ["vertical_and_slash", 3500, 100, 0.9561753273010254], "2": ["vertical_and_slash", 1000, 6096, 0.9502847194671631], "3": ["vertical_and_slash", 1000, 6096, 0.9458599090576172], "4": ["vertical_and_slash", 1000, 6096, 0.983038604259491], "5": ["vertical_and_slash", 500, 700, 0.9306663274765015], "6": ["vertical_and_slash", 1000, 6096, 0.9474708437919617], "7": ["vertical_and_slash", 1000, 6096, 0.6626478433609009], "8": ["vertical_and_slash", 1000, 6096, 0.6083192229270935], "9": ["vertical_and_slash", 1000, 6096, 0.8787986636161804], "10": ["vertical_and_slash", 1000, 6096, 0.6313983201980591], "11": ["vertical_and_slash", 100, 800, 0.9765625], "12": ["vertical_and_slash", 1000, 6096, 0.8851125240325928], "13": ["vertical_and_slash", 1000, 6096, 0.7471399903297424], "14": ["vertical_and_slash", 3500, 100, 0.823978841304779], "15": ["vertical_and_slash", 3500, 100, 0.6992096304893494], "16": ["vertical_and_slash", 100, 800, 0.8671875], "17": ["vertical_and_slash", 100, 800, 0.828125], "18": ["vertical_and_slash", 3500, 100, 0.8418475389480591], "19": ["vertical_and_slash", 1000, 6096, 0.5056701302528381], "20": ["vertical_and_slash", 3500, 100, 0.7014374732971191], "21": ["vertical_and_slash", 1000, 6096, 0.9509202837944031], "22": ["vertical_and_slash", 1000, 6096, 0.8527569770812988], "23": ["vertical_and_slash", 1000, 6096, 0.9129858613014221], "24": ["vertical_and_slash", 3500, 100, 0.663795530796051], "25": ["vertical_and_slash", 1000, 6096, 0.9446316361427307], "26": ["vertical_and_slash", 3500, 100, 0.6898106336593628], "27": ["vertical_and_slash", 1000, 6096, 0.9013346433639526]}, {"0": ["vertical_and_slash", 100, 750, 0.9516309499740601], "1": ["vertical_and_slash", 3500, 100, 0.9530265927314758], "2": ["vertical_and_slash", 500, 700, 0.9479495286941528], "3": ["vertical_and_slash", 500, 700, 0.9605967402458191], "4": ["vertical_and_slash", 500, 700, 0.9592612385749817], "5": ["vertical_and_slash", 500, 700, 0.9429551959037781], "6": ["vertical_and_slash", 500, 700, 0.8644149899482727], "7": ["vertical_and_slash", 30, 800, 1.0000064373016357], "8": ["vertical_and_slash", 1000, 6096, 0.9511156678199768], "9": ["vertical_and_slash", 500, 700, 0.931096613407135], "10": ["vertical_and_slash", 3500, 100, 0.5491603016853333], "11": ["vertical_and_slash", 1000, 6096, 0.7933111190795898], "12": ["vertical_and_slash", 100, 800, 0.8515625], "13": ["vertical_and_slash", 3500, 100, 0.9444908499717712], "14": ["vertical_and_slash", 3500, 100, 0.8150294423103333], "15": ["vertical_and_slash", 3500, 100, 0.92420494556427], "16": ["vertical_and_slash", 3500, 100, 0.5426702499389648], "17": ["vertical_and_slash", 500, 700, 0.967065691947937], "18": ["vertical_and_slash", 3500, 100, 0.7650114893913269], "19": ["vertical_and_slash", 1000, 6096, 0.7211912274360657], "20": ["vertical_and_slash", 1000, 6096, 0.8799037337303162], "21": ["vertical_and_slash", 500, 700, 0.9614995121955872], "22": ["vertical_and_slash", 1000, 6096, 0.9363625645637512], "23": ["vertical_and_slash", 3500, 100, 0.6777249574661255], "24": ["vertical_and_slash", 1000, 6096, 0.905726969242096], "25": ["vertical_and_slash", 1000, 6096, 0.9575398564338684], "26": ["vertical_and_slash", 1000, 6096, 0.8252630829811096], "27": ["vertical_and_slash", 1000, 6096, 0.9448449015617371]}, {"0": ["vertical_and_slash", 30, 800, 1.0000189542770386], "1": ["vertical_and_slash", 3500, 100, 0.7812989354133606], "2": ["vertical_and_slash", 3500, 100, 0.95023512840271], "3": ["vertical_and_slash", 3500, 100, 0.9918785691261292], "4": ["vertical_and_slash", 1000, 6096, 0.6573578119277954], "5": ["vertical_and_slash", 3500, 100, 0.8104897737503052], "6": ["vertical_and_slash", 3500, 100, 0.736077070236206], "7": ["vertical_and_slash", 100, 750, 0.9734572768211365], "8": ["vertical_and_slash", 100, 750, 0.9851570725440979], "9": ["vertical_and_slash", 500, 700, 0.9506796598434448], "10": ["vertical_and_slash", 3500, 100, 0.8702376484870911], "11": ["vertical_and_slash", 1000, 6096, 0.7537837028503418], "12": ["vertical_and_slash", 500, 700, 0.9368585348129272], "13": ["vertical_and_slash", 100, 750, 0.9949247241020203], "14": ["vertical_and_slash", 100, 800, 0.6328125], "15": ["vertical_and_slash", 3500, 100, 0.877793550491333], "16": ["vertical_and_slash", 3500, 100, 0.9036691188812256], "17": ["vertical_and_slash", 3500, 100, 0.9620419144630432], "18": ["vertical_and_slash", 3500, 100, 0.7708083987236023], "19": ["vertical_and_slash", 3500, 100, 0.9611032605171204], "20": ["vertical_and_slash", 3500, 100, 0.9523343443870544], "21": ["vertical_and_slash", 3500, 100, 0.9313843250274658], "22": ["vertical_and_slash", 3500, 100, 0.8399397730827332], "23": ["vertical_and_slash", 3500, 100, 0.76487135887146], "24": ["vertical_and_slash", 3500, 100, 0.9502568244934082], "25": ["vertical_and_slash", 3500, 100, 0.9089641571044922], "26": ["vertical_and_slash", 3500, 100, 0.690402626991272], "27": ["vertical_and_slash", 1000, 6096, 0.7801981568336487]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9159413576126099], "1": ["vertical_and_slash", 500, 700, 0.9605610966682434], "2": ["vertical_and_slash", 1000, 6096, 0.8440356850624084], "3": ["vertical_and_slash", 1000, 6096, 0.905092179775238], "4": ["vertical_and_slash", 500, 700, 0.912187933921814], "5": ["vertical_and_slash", 1000, 6096, 0.7955954670906067], "6": ["vertical_and_slash", 3500, 100, 0.8192487955093384], "7": ["vertical_and_slash", 3500, 100, 0.8681324124336243], "8": ["vertical_and_slash", 1000, 6096, 0.5407619476318359], "9": ["vertical_and_slash", 1000, 6096, 0.5471318364143372], "10": ["vertical_and_slash", 100, 800, 0.97265625], "11": ["vertical_and_slash", 1000, 6096, 0.7762816548347473], "12": ["vertical_and_slash", 3500, 100, 0.8634917140007019], "13": ["vertical_and_slash", 100, 800, 0.9765625], "14": ["vertical_and_slash", 3500, 100, 0.8586200475692749], "15": ["vertical_and_slash", 1000, 6096, 0.908331036567688], "16": ["vertical_and_slash", 3500, 100, 0.9884516000747681], "17": ["vertical_and_slash", 3500, 100, 0.703482449054718], "18": ["vertical_and_slash", 1000, 6096, 0.7293429970741272], "19": ["vertical_and_slash", 3500, 100, 0.9010433554649353], "20": ["vertical_and_slash", 3500, 100, 0.9552940130233765], "21": ["vertical_and_slash", 1000, 6096, 0.7608126997947693], "22": ["vertical_and_slash", 1000, 6096, 0.9385209679603577], "23": ["vertical_and_slash", 1000, 6096, 0.956599771976471], "24": ["vertical_and_slash", 1000, 6096, 0.8710024952888489], "25": ["vertical_and_slash", 3500, 100, 0.7606588006019592], "26": ["vertical_and_slash", 1000, 6096, 0.7958735823631287], "27": ["vertical_and_slash", 1000, 6096, 0.8293774724006653]}, {"0": ["vertical_and_slash", 1000, 6096, 0.8493539094924927], "1": ["vertical_and_slash", 3500, 100, 0.9704030156135559], "2": ["vertical_and_slash", 1000, 6096, 0.892375111579895], "3": ["vertical_and_slash", 1000, 6096, 0.9775978922843933], "4": ["vertical_and_slash", 3500, 100, 0.990434467792511], "5": ["vertical_and_slash", 3500, 100, 0.8386834859848022], "6": ["vertical_and_slash", 1000, 6096, 0.9319282174110413], "7": ["vertical_and_slash", 1000, 6096, 0.9702367186546326], "8": ["vertical_and_slash", 100, 800, 0.984375], "9": ["vertical_and_slash", 3500, 100, 0.9866511225700378], "10": ["vertical_and_slash", 3500, 100, 0.9985076189041138], "11": ["vertical_and_slash", 3500, 100, 0.9933387041091919], "12": ["vertical_and_slash", 3500, 100, 0.9702840447425842], "13": ["vertical_and_slash", 3500, 100, 0.8703793883323669], "14": ["vertical_and_slash", 1000, 6096, 0.8367708921432495], "15": ["vertical_and_slash", 30, 800, 0.9986119270324707], "16": ["vertical_and_slash", 1000, 6096, 0.9723682999610901], "17": ["vertical_and_slash", 1000, 6096, 0.6487687826156616], "18": ["vertical_and_slash", 1000, 6096, 0.9587192535400391], "19": ["vertical_and_slash", 3500, 100, 0.8507857322692871], "20": ["vertical_and_slash", 30, 800, 0.9730740189552307], "21": ["vertical_and_slash", 3500, 100, 0.9618991613388062], "22": ["vertical_and_slash", 1000, 6096, 0.9648565649986267], "23": ["vertical_and_slash", 1000, 6096, 0.9644651412963867], "24": ["vertical_and_slash", 1000, 6096, 0.9925320148468018], "25": ["vertical_and_slash", 3500, 100, 0.8604639172554016], "26": ["vertical_and_slash", 1000, 6096, 0.9907099604606628], "27": ["vertical_and_slash", 1000, 6096, 0.9783667922019958]}, {"0": ["vertical_and_slash", 3500, 100, 0.8745191693305969], "1": ["vertical_and_slash", 3500, 100, 0.9138997197151184], "2": ["vertical_and_slash", 100, 800, 0.91796875], "3": ["vertical_and_slash", 1000, 6096, 0.6723800301551819], "4": ["vertical_and_slash", 1000, 6096, 0.8992129564285278], "5": ["vertical_and_slash", 1000, 6096, 0.6409047842025757], "6": ["vertical_and_slash", 1000, 6096, 0.7251356244087219], "7": ["vertical_and_slash", 1000, 6096, 0.8066868782043457], "8": ["vertical_and_slash", 1000, 6096, 0.8305136561393738], "9": ["vertical_and_slash", 1000, 6096, 0.9651612043380737], "10": ["vertical_and_slash", 1000, 6096, 0.9071229696273804], "11": ["vertical_and_slash", 1000, 6096, 0.987766683101654], "12": ["vertical_and_slash", 1000, 6096, 0.9971622824668884], "13": ["vertical_and_slash", 1000, 6096, 0.9850742816925049], "14": ["vertical_and_slash", 3500, 100, 0.9865512251853943], "15": ["vertical_and_slash", 1000, 6096, 0.9994776844978333], "16": ["vertical_and_slash", 1000, 6096, 0.9906349778175354], "17": ["vertical_and_slash", 3500, 100, 0.9941802024841309], "18": ["vertical_and_slash", 3500, 100, 0.9773752689361572], "19": ["vertical_and_slash", 1000, 6096, 0.970836877822876], "20": ["vertical_and_slash", 3500, 100, 0.9667485356330872], "21": ["vertical_and_slash", 3500, 100, 0.687179684638977], "22": ["vertical_and_slash", 1000, 6096, 0.9673103094100952], "23": ["vertical_and_slash", 1000, 6096, 0.9230855107307434], "24": ["vertical_and_slash", 1000, 6096, 0.9796652793884277], "25": ["vertical_and_slash", 1000, 6096, 0.9788067936897278], "26": ["vertical_and_slash", 1000, 6096, 0.9844804406166077], "27": ["vertical_and_slash", 1000, 6096, 0.9884434938430786]}, {"0": ["vertical_and_slash", 1000, 6096, 1.000038504600525], "1": ["vertical_and_slash", 3500, 100, 0.9941023588180542], "2": ["vertical_and_slash", 3500, 100, 0.9975664019584656], "3": ["vertical_and_slash", 1000, 6096, 0.98569256067276], "4": ["vertical_and_slash", 1000, 6096, 0.9957911968231201], "5": ["vertical_and_slash", 1000, 6096, 0.998773455619812], "6": ["vertical_and_slash", 1000, 6096, 0.9927323460578918], "7": ["vertical_and_slash", 3500, 100, 0.9634103178977966], "8": ["vertical_and_slash", 1000, 6096, 0.8786033987998962], "9": ["vertical_and_slash", 1000, 6096, 0.855072557926178], "10": ["vertical_and_slash", 100, 800, 0.98046875], "11": ["vertical_and_slash", 1000, 6096, 0.8532230854034424], "12": ["vertical_and_slash", 1000, 6096, 0.8525854349136353], "13": ["vertical_and_slash", 1000, 6096, 0.765890896320343], "14": ["vertical_and_slash", 1000, 6096, 0.9484980702400208], "15": ["vertical_and_slash", 1000, 6096, 0.9614062309265137], "16": ["vertical_and_slash", 1000, 6096, 0.959348738193512], "17": ["vertical_and_slash", 1000, 6096, 0.9856812953948975], "18": ["vertical_and_slash", 3500, 100, 0.8322045207023621], "19": ["vertical_and_slash", 1000, 6096, 0.8192600011825562], "20": ["vertical_and_slash", 1000, 6096, 0.8806852698326111], "21": ["vertical_and_slash", 3500, 100, 0.9675425887107849], "22": ["vertical_and_slash", 1000, 6096, 0.8807272911071777], "23": ["vertical_and_slash", 3500, 100, 0.9965254664421082], "24": ["vertical_and_slash", 3500, 100, 0.9755669832229614], "25": ["vertical_and_slash", 1000, 6096, 0.9037275314331055], "26": ["vertical_and_slash", 3500, 100, 0.9864920377731323], "27": ["vertical_and_slash", 500, 700, 0.9479431509971619]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9386948943138123], "1": ["vertical_and_slash", 30, 800, 0.9946699142456055], "2": ["vertical_and_slash", 500, 700, 0.9499735832214355], "3": ["vertical_and_slash", 1000, 6096, 0.9934840202331543], "4": ["vertical_and_slash", 3500, 100, 0.9760491251945496], "5": ["vertical_and_slash", 30, 800, 0.9524878263473511], "6": ["vertical_and_slash", 500, 700, 0.9868755340576172], "7": ["vertical_and_slash", 3500, 100, 0.9672828316688538], "8": ["vertical_and_slash", 3500, 100, 0.9808163642883301], "9": ["vertical_and_slash", 3500, 100, 0.9952189326286316], "10": ["vertical_and_slash", 3500, 100, 0.9910376667976379], "11": ["vertical_and_slash", 3500, 100, 0.9631356596946716], "12": ["vertical_and_slash", 3500, 100, 0.9869235754013062], "13": ["vertical_and_slash", 3500, 100, 0.6277638673782349], "14": ["vertical_and_slash", 3500, 100, 0.9969587922096252], "15": ["vertical_and_slash", 1000, 6096, 0.990169882774353], "16": ["vertical_and_slash", 3500, 100, 0.9979966282844543], "17": ["vertical_and_slash", 3500, 100, 0.9865444302558899], "18": ["vertical_and_slash", 1000, 6096, 0.9588218331336975], "19": ["vertical_and_slash", 3500, 100, 0.999292254447937], "20": ["vertical_and_slash", 3500, 100, 0.9950262904167175], "21": ["vertical_and_slash", 1000, 6096, 0.9836653470993042], "22": ["vertical_and_slash", 1000, 6096, 0.9830690622329712], "23": ["vertical_and_slash", 1000, 6096, 0.9948968291282654], "24": ["vertical_and_slash", 1000, 6096, 0.9425172209739685], "25": ["vertical_and_slash", 1000, 6096, 0.98743736743927], "26": ["vertical_and_slash", 1000, 6096, 0.9738022685050964], "27": ["vertical_and_slash", 3500, 100, 0.9369613528251648]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9886358380317688], "1": ["vertical_and_slash", 3500, 100, 0.8968799710273743], "2": ["vertical_and_slash", 3500, 100, 0.9865190386772156], "3": ["vertical_and_slash", 3500, 100, 0.9866040945053101], "4": ["vertical_and_slash", 1000, 6096, 0.9527764916419983], "5": ["vertical_and_slash", 3500, 100, 0.9903413653373718], "6": ["vertical_and_slash", 3500, 100, 0.9875758290290833], "7": ["vertical_and_slash", 3500, 100, 0.9743981957435608], "8": ["vertical_and_slash", 3500, 100, 0.9398495554924011], "9": ["vertical_and_slash", 3500, 100, 0.9608177542686462], "10": ["vertical_and_slash", 3500, 100, 0.9327642321586609], "11": ["vertical_and_slash", 3500, 100, 0.9610418081283569], "12": ["vertical_and_slash", 3500, 100, 0.7950856685638428], "13": ["vertical_and_slash", 1000, 6096, 0.8581352233886719], "14": ["vertical_and_slash", 500, 700, 0.9941531419754028], "15": ["vertical_and_slash", 1000, 6096, 0.6474716067314148], "16": ["vertical_and_slash", 1000, 6096, 0.8489192128181458], "17": ["vertical_and_slash", 3500, 100, 0.9771746397018433], "18": ["vertical_and_slash", 30, 800, 0.9288356304168701], "19": ["vertical_and_slash", 30, 800, 0.9712263941764832], "20": ["vertical_and_slash", 3500, 100, 0.7129408717155457], "21": ["vertical_and_slash", 1000, 6096, 0.9798778891563416], "22": ["vertical_and_slash", 1000, 6096, 0.9738019108772278], "23": ["vertical_and_slash", 1000, 6096, 0.9911205172538757], "24": ["vertical_and_slash", 1000, 6096, 0.8216277956962585], "25": ["vertical_and_slash", 1000, 6096, 0.950930118560791], "26": ["vertical_and_slash", 3500, 100, 0.9760453104972839], "27": ["vertical_and_slash", 3500, 100, 0.9889680743217468]}, {"0": ["vertical_and_slash", 3500, 100, 0.9715936779975891], "1": ["vertical_and_slash", 3500, 100, 0.9720566272735596], "2": ["vertical_and_slash", 3500, 100, 0.9753875136375427], "3": ["vertical_and_slash", 3500, 100, 0.9864369630813599], "4": ["vertical_and_slash", 3500, 100, 0.9902608394622803], "5": ["vertical_and_slash", 3500, 100, 0.9871652722358704], "6": ["vertical_and_slash", 3500, 100, 0.9877868294715881], "7": ["vertical_and_slash", 1000, 6096, 0.7928203344345093], "8": ["vertical_and_slash", 30, 800, 0.9940297603607178], "9": ["vertical_and_slash", 1000, 6096, 0.8952949047088623], "10": ["vertical_and_slash", 1000, 6096, 0.8851021528244019], "11": ["vertical_and_slash", 30, 800, 0.9964791536331177], "12": ["vertical_and_slash", 3500, 100, 0.9338324666023254], "13": ["vertical_and_slash", 1000, 6096, 0.8905411958694458], "14": ["vertical_and_slash", 1000, 6096, 0.8696432113647461], "15": ["vertical_and_slash", 1000, 6096, 0.9554032683372498], "16": ["vertical_and_slash", 1000, 6096, 0.9713241457939148], "17": ["vertical_and_slash", 1000, 6096, 0.9806569814682007], "18": ["vertical_and_slash", 3500, 100, 0.9807780981063843], "19": ["vertical_and_slash", 3500, 100, 0.9720619320869446], "20": ["vertical_and_slash", 1000, 6096, 0.9717702269554138], "21": ["vertical_and_slash", 1000, 6096, 0.7935751080513], "22": ["vertical_and_slash", 1000, 6096, 0.9250394105911255], "23": ["vertical_and_slash", 3500, 100, 0.9081441164016724], "24": ["vertical_and_slash", 3500, 100, 0.8211441040039062], "25": ["vertical_and_slash", 3500, 100, 0.8812087178230286], "26": ["vertical_and_slash", 500, 700, 0.9544926881790161], "27": ["vertical_and_slash", 3500, 100, 0.9467933177947998]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9007910490036011], "1": ["vertical_and_slash", 1000, 6096, 0.9436451196670532], "2": ["vertical_and_slash", 1000, 6096, 0.8619889616966248], "3": ["vertical_and_slash", 3500, 100, 0.9261324405670166], "4": ["vertical_and_slash", 1000, 6096, 0.9136027097702026], "5": ["vertical_and_slash", 3500, 100, 0.9574146270751953], "6": ["vertical_and_slash", 1000, 6096, 0.7774612307548523], "7": ["vertical_and_slash", 1000, 6096, 0.8998187184333801], "8": ["vertical_and_slash", 1000, 6096, 0.9201644062995911], "9": ["vertical_and_slash", 1000, 6096, 0.7296996116638184], "10": ["vertical_and_slash", 30, 800, 0.9583849906921387], "11": ["vertical_and_slash", 1000, 6096, 0.9813587069511414], "12": ["vertical_and_slash", 30, 800, 0.9112449884414673], "13": ["vertical_and_slash", 30, 800, 0.9529325366020203], "14": ["vertical_and_slash", 1000, 6096, 0.8890124559402466], "15": ["vertical_and_slash", 1000, 6096, 0.9021613597869873], "16": ["vertical_and_slash", 1000, 6096, 0.9225684404373169], "17": ["vertical_and_slash", 1000, 6096, 0.8930850625038147], "18": ["vertical_and_slash", 1000, 6096, 0.9344975352287292], "19": ["vertical_and_slash", 3500, 100, 0.8462615013122559], "20": ["vertical_and_slash", 1000, 6096, 0.9107160568237305], "21": ["vertical_and_slash", 3500, 100, 0.8705074191093445], "22": ["vertical_and_slash", 1000, 6096, 0.767874538898468], "23": ["vertical_and_slash", 1000, 6096, 0.9286450147628784], "24": ["vertical_and_slash", 1000, 6096, 0.7197266817092896], "25": ["vertical_and_slash", 1000, 6096, 0.9577316641807556], "26": ["vertical_and_slash", 1000, 6096, 0.9439008831977844], "27": ["vertical_and_slash", 1000, 6096, 0.7233526706695557]}]
diff --git a/minference/configs/Qwen2.5_7B_Instruct_1M.json b/minference/configs/Qwen2.5_7B_Instruct_1M.json
new file mode 100644
index 00000000..ba575a8d
--- /dev/null
+++ b/minference/configs/Qwen2.5_7B_Instruct_1M.json
@@ -0,0 +1 @@
+[{"0": ["vertical_and_slash", 9192, 14288, 0.6920984983444214], "1": ["vertical_and_slash", 1000, 6096, 0.9862403869628906], "2": ["vertical_and_slash", 3048, 8144, 0.8198256492614746], "3": ["vertical_and_slash", 11340, 16336, 0.9518619179725647], "4": ["vertical_and_slash", 5096, 10192, 0.739506721496582], "5": ["vertical_and_slash", 1000, 6096, 0.9902973771095276], "6": ["vertical_and_slash", 30, 800, 0.9972315430641174], "7": ["vertical_and_slash", 1000, 6096, 0.9914392828941345], "8": ["vertical_and_slash", 1000, 6096, 0.950901210308075], "9": ["vertical_and_slash", 3048, 8144, 0.8771697878837585], "10": ["vertical_and_slash", 13388, 18384, 0.8314550518989563], "11": ["vertical_and_slash", 1000, 6096, 0.9330742359161377], "12": ["vertical_and_slash", 1000, 6096, 0.9362020492553711], "13": ["vertical_and_slash", 3048, 8144, 0.8856230974197388], "14": ["vertical_and_slash", 9192, 14288, 0.6533831357955933], "15": ["vertical_and_slash", 13388, 18384, 0.8002538084983826], "16": ["vertical_and_slash", 1000, 6096, 0.9781253933906555], "17": ["vertical_and_slash", 3048, 8144, 0.8541072010993958], "18": ["vertical_and_slash", 9192, 14288, 0.6799413561820984], "19": ["vertical_and_slash", 3048, 8144, 0.8869160413742065], "20": ["vertical_and_slash", 1000, 6096, 0.9579253196716309], "21": ["vertical_and_slash", 1000, 6096, 0.9464394450187683], "22": ["vertical_and_slash", 3048, 8144, 0.8140186071395874], "23": ["vertical_and_slash", 1000, 6096, 0.9577401876449585], "24": ["vertical_and_slash", 1000, 6096, 0.9639880061149597], "25": ["vertical_and_slash", 1000, 6096, 0.9603379368782043], "26": ["vertical_and_slash", 1000, 6096, 0.9592313170433044], "27": ["vertical_and_slash", 1000, 6096, 0.9610494375228882]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9999175667762756], "1": ["vertical_and_slash", 11692, 8292, 0.5583158731460571], "2": ["vertical_and_slash", 11340, 16336, 0.9962033033370972], "3": ["vertical_and_slash", 1000, 6096, 0.9998870491981506], "4": ["vertical_and_slash", 1000, 6096, 0.9992745518684387], "5": ["vertical_and_slash", 1000, 6096, 0.9997513890266418], "6": ["vertical_and_slash", 13388, 18384, 0.8439157009124756], "7": ["vertical_and_slash", 1000, 6096, 0.9102388024330139], "8": ["vertical_and_slash", 3048, 8144, 0.8252120018005371], "9": ["vertical_and_slash", 9192, 14288, 0.5839092135429382], "10": ["vertical_and_slash", 1000, 6096, 0.9527733325958252], "11": ["vertical_and_slash", 1000, 6096, 0.9262151122093201], "12": ["vertical_and_slash", 1000, 6096, 0.9747101664543152], "13": ["vertical_and_slash", 9192, 14288, 0.693157434463501], "14": ["vertical_and_slash", 1000, 6096, 0.9503929615020752], "15": ["vertical_and_slash", 1000, 6096, 0.9907119870185852], "16": ["vertical_and_slash", 5096, 10192, 0.7641042470932007], "17": ["vertical_and_slash", 1000, 6096, 0.9974478483200073], "18": ["vertical_and_slash", 3048, 8144, 0.864757776260376], "19": ["vertical_and_slash", 1000, 6096, 0.9005423188209534], "20": ["vertical_and_slash", 1000, 6096, 0.934573769569397], "21": ["vertical_and_slash", 9192, 14288, 0.5810629725456238], "22": ["vertical_and_slash", 5096, 10192, 0.7144602537155151], "23": ["vertical_and_slash", 1000, 6096, 0.9564751386642456], "24": ["vertical_and_slash", 1000, 6096, 0.9916792511940002], "25": ["vertical_and_slash", 3048, 8144, 0.8495540618896484], "26": ["vertical_and_slash", 9192, 14288, 0.6452760696411133], "27": ["vertical_and_slash", 1000, 6096, 0.9232876300811768]}, {"0": ["vertical_and_slash", 9744, 6244, 0.9077025055885315], "1": ["vertical_and_slash", 5548, 2148, 0.8882887959480286], "2": ["vertical_and_slash", 3500, 100, 0.9153436422348022], "3": ["vertical_and_slash", 3500, 100, 0.9312531352043152], "4": ["vertical_and_slash", 9192, 14288, 0.6427125334739685], "5": ["vertical_and_slash", 3500, 100, 0.9152134656906128], "6": ["vertical_and_slash", 5548, 2148, 0.8578056693077087], "7": ["vertical_and_slash", 1000, 6096, 0.932381808757782], "8": ["vertical_and_slash", 3048, 8144, 0.8542991280555725], "9": ["vertical_and_slash", 1000, 6096, 0.9245713949203491], "10": ["vertical_and_slash", 3048, 8144, 0.8989690542221069], "11": ["vertical_and_slash", 1000, 6096, 0.9269236326217651], "12": ["vertical_and_slash", 5096, 10192, 0.7965206503868103], "13": ["vertical_and_slash", 5096, 10192, 0.7843445539474487], "14": ["vertical_and_slash", 9192, 14288, 0.5634094476699829], "15": ["vertical_and_slash", 5096, 10192, 0.7882068157196045], "16": ["vertical_and_slash", 5096, 10192, 0.7812356352806091], "17": ["vertical_and_slash", 9192, 14288, 0.6630747318267822], "18": ["vertical_and_slash", 11692, 8292, 0.695554792881012], "19": ["vertical_and_slash", 9192, 14288, 0.6496447920799255], "20": ["vertical_and_slash", 9192, 14288, 0.6239688992500305], "21": ["vertical_and_slash", 3048, 8144, 0.8381555676460266], "22": ["vertical_and_slash", 9192, 14288, 0.5935300588607788], "23": ["vertical_and_slash", 1000, 6096, 0.9077279567718506], "24": ["vertical_and_slash", 5096, 10192, 0.7705077528953552], "25": ["vertical_and_slash", 3048, 8144, 0.8881292343139648], "26": ["vertical_and_slash", 15436, 20432, 0.654614269733429], "27": ["vertical_and_slash", 1000, 6096, 0.9183178544044495]}, {"0": ["vertical_and_slash", 9192, 14288, 0.6299600005149841], "1": ["vertical_and_slash", 1000, 6096, 0.9939385652542114], "2": ["vertical_and_slash", 1000, 6096, 0.9255406856536865], "3": ["vertical_and_slash", 1000, 6096, 0.9815036058425903], "4": ["vertical_and_slash", 1000, 6096, 0.9068899154663086], "5": ["vertical_and_slash", 1000, 6096, 0.9562771916389465], "6": ["vertical_and_slash", 1000, 6096, 0.9499258399009705], "7": ["vertical_and_slash", 1000, 6096, 0.9981131553649902], "8": ["vertical_and_slash", 9192, 14288, 0.47260501980781555], "9": ["vertical_and_slash", 1000, 6096, 0.9922735691070557], "10": ["vertical_and_slash", 1000, 6096, 0.9986152052879333], "11": ["vertical_and_slash", 1000, 6096, 0.9634198546409607], "12": ["vertical_and_slash", 1000, 6096, 0.9510660171508789], "13": ["vertical_and_slash", 1000, 6096, 0.9214046597480774], "14": ["vertical_and_slash", 5096, 10192, 0.7424683570861816], "15": ["vertical_and_slash", 1000, 6096, 0.9193776845932007], "16": ["vertical_and_slash", 11692, 8292, 0.6805881261825562], "17": ["vertical_and_slash", 5096, 10192, 0.7827474474906921], "18": ["vertical_and_slash", 9192, 14288, 0.6499208211898804], "19": ["vertical_and_slash", 3048, 8144, 0.8205469846725464], "20": ["vertical_and_slash", 1000, 6096, 0.9113396406173706], "21": ["vertical_and_slash", 3048, 8144, 0.8239019513130188], "22": ["vertical_and_slash", 1000, 6096, 0.953626275062561], "23": ["vertical_and_slash", 3048, 8144, 0.8758710622787476], "24": ["vertical_and_slash", 1000, 6096, 0.9802020788192749], "25": ["vertical_and_slash", 1000, 6096, 0.9784566760063171], "26": ["vertical_and_slash", 1000, 6096, 0.9459996819496155], "27": ["vertical_and_slash", 1000, 6096, 0.9493811726570129]}, {"0": ["vertical_and_slash", 3048, 8144, 0.8282588720321655], "1": ["vertical_and_slash", 3048, 8144, 0.8945374488830566], "2": ["vertical_and_slash", 1000, 6096, 0.9859223961830139], "3": ["vertical_and_slash", 1000, 6096, 0.9584572911262512], "4": ["vertical_and_slash", 1000, 6096, 0.9155289530754089], "5": ["vertical_and_slash", 1000, 6096, 0.9548629522323608], "6": ["vertical_and_slash", 3048, 8144, 0.8657681345939636], "7": ["vertical_and_slash", 1000, 6096, 0.9909697771072388], "8": ["vertical_and_slash", 3048, 8144, 0.8176682591438293], "9": ["vertical_and_slash", 1000, 6096, 0.9185464978218079], "10": ["vertical_and_slash", 3048, 8144, 0.8876745104789734], "11": ["vertical_and_slash", 1000, 6096, 0.9989773631095886], "12": ["vertical_and_slash", 100, 800, 1.0], "13": ["vertical_and_slash", 13840, 10340, 0.9865253567695618], "14": ["vertical_and_slash", 30, 800, 0.9833630919456482], "15": ["vertical_and_slash", 100, 800, 0.9375], "16": ["vertical_and_slash", 1000, 6096, 0.9603279829025269], "17": ["vertical_and_slash", 9192, 14288, 0.5568011403083801], "18": ["vertical_and_slash", 30, 800, 0.9944095611572266], "19": ["vertical_and_slash", 100, 800, 0.98828125], "20": ["vertical_and_slash", 30, 800, 0.9845749139785767], "21": ["vertical_and_slash", 1000, 6096, 0.9226506948471069], "22": ["vertical_and_slash", 30, 800, 0.9778012633323669], "23": ["vertical_and_slash", 30, 800, 0.9957334399223328], "24": ["vertical_and_slash", 5096, 10192, 0.7061529159545898], "25": ["vertical_and_slash", 30, 800, 0.9853928685188293], "26": ["vertical_and_slash", 3048, 8144, 0.8172721862792969], "27": ["vertical_and_slash", 30, 800, 0.9829623103141785]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9093767404556274], "1": ["vertical_and_slash", 30, 800, 0.9735046625137329], "2": ["vertical_and_slash", 1000, 6096, 0.9702109098434448], "3": ["vertical_and_slash", 30, 800, 0.9810881614685059], "4": ["vertical_and_slash", 1000, 6096, 0.956884503364563], "5": ["vertical_and_slash", 15436, 20432, 0.5826653242111206], "6": ["vertical_and_slash", 9192, 14288, 0.6588539481163025], "7": ["vertical_and_slash", 1000, 6096, 0.9873548150062561], "8": ["vertical_and_slash", 1000, 6096, 0.9430211186408997], "9": ["vertical_and_slash", 5096, 10192, 0.7686501145362854], "10": ["vertical_and_slash", 1000, 6096, 0.9063356518745422], "11": ["vertical_and_slash", 9192, 14288, 0.6081218123435974], "12": ["vertical_and_slash", 1000, 6096, 0.9467446208000183], "13": ["vertical_and_slash", 3048, 8144, 0.8317232728004456], "14": ["vertical_and_slash", 1000, 6096, 0.9639446139335632], "15": ["vertical_and_slash", 3048, 8144, 0.8432199358940125], "16": ["vertical_and_slash", 1000, 6096, 0.9603729844093323], "17": ["vertical_and_slash", 9192, 14288, 0.5282297134399414], "18": ["vertical_and_slash", 9192, 14288, 0.5313483476638794], "19": ["vertical_and_slash", 19532, 24528, 0.6716174483299255], "20": ["vertical_and_slash", 9192, 14288, 0.5943314433097839], "21": ["vertical_and_slash", 9192, 14288, 0.5765253901481628], "22": ["vertical_and_slash", 30, 800, 0.9723267555236816], "23": ["vertical_and_slash", 30, 800, 0.9864580631256104], "24": ["vertical_and_slash", 100, 800, 0.9375], "25": ["vertical_and_slash", 30, 800, 0.9841305613517761], "26": ["vertical_and_slash", 100, 800, 0.9375], "27": ["vertical_and_slash", 100, 800, 0.94140625]}, {"0": ["vertical_and_slash", 30, 800, 0.9887828826904297], "1": ["vertical_and_slash", 30, 800, 0.991061270236969], "2": ["vertical_and_slash", 30, 800, 0.9967905282974243], "3": ["vertical_and_slash", 30, 800, 0.9886236786842346], "4": ["vertical_and_slash", 30, 800, 0.995130717754364], "5": ["vertical_and_slash", 30, 800, 0.980113685131073], "6": ["vertical_and_slash", 30, 800, 0.9969208836555481], "7": ["vertical_and_slash", 1000, 6096, 0.960637092590332], "8": ["vertical_and_slash", 1000, 6096, 0.9405831694602966], "9": ["vertical_and_slash", 1000, 6096, 0.9176963567733765], "10": ["vertical_and_slash", 3048, 8144, 0.819543182849884], "11": ["vertical_and_slash", 3048, 8144, 0.849609911441803], "12": ["vertical_and_slash", 1000, 6096, 0.9364283084869385], "13": ["vertical_and_slash", 3500, 100, 0.9509801864624023], "14": ["vertical_and_slash", 30, 800, 0.9800994396209717], "15": ["vertical_and_slash", 30, 800, 0.9964476227760315], "16": ["vertical_and_slash", 30, 800, 0.9962098598480225], "17": ["vertical_and_slash", 3048, 8144, 0.8155508637428284], "18": ["vertical_and_slash", 5096, 10192, 0.969977855682373], "19": ["vertical_and_slash", 30, 800, 0.9929144978523254], "20": ["vertical_and_slash", 30, 800, 0.9956931471824646], "21": ["vertical_and_slash", 30, 800, 0.9900340437889099], "22": ["vertical_and_slash", 30, 800, 0.9817578196525574], "23": ["vertical_and_slash", 5096, 10192, 0.7279469966888428], "24": ["vertical_and_slash", 30, 800, 0.977938711643219], "25": ["vertical_and_slash", 30, 800, 0.9940697550773621], "26": ["vertical_and_slash", 30, 800, 0.9888582825660706], "27": ["vertical_and_slash", 30, 800, 0.9776446223258972]}, {"0": ["vertical_and_slash", 6274, 6944, 0.9849215745925903], "1": ["vertical_and_slash", 100, 800, 0.91796875], "2": ["vertical_and_slash", 3048, 8144, 0.8702616691589355], "3": ["vertical_and_slash", 1000, 6096, 0.9742972254753113], "4": ["vertical_and_slash", 30, 800, 0.9930776953697205], "5": ["vertical_and_slash", 30, 800, 0.9923169016838074], "6": ["vertical_and_slash", 11340, 16336, 0.7224187850952148], "7": ["vertical_and_slash", 1000, 6096, 0.9252712726593018], "8": ["vertical_and_slash", 1000, 6096, 0.987740695476532], "9": ["vertical_and_slash", 1000, 6096, 0.9918652176856995], "10": ["vertical_and_slash", 3048, 8144, 0.8553299307823181], "11": ["vertical_and_slash", 30, 800, 0.9839231967926025], "12": ["vertical_and_slash", 1000, 6096, 0.9922063946723938], "13": ["vertical_and_slash", 30, 800, 0.9844004511833191], "14": ["vertical_and_slash", 7244, 12240, 0.9108995199203491], "15": ["vertical_and_slash", 5096, 10192, 0.7733249068260193], "16": ["vertical_and_slash", 100, 800, 0.9921875], "17": ["vertical_and_slash", 1000, 6096, 0.9894708395004272], "18": ["vertical_and_slash", 6344, 6944, 0.96484375], "19": ["vertical_and_slash", 5096, 10192, 0.7390482425689697], "20": ["vertical_and_slash", 30, 800, 0.9494436979293823], "21": ["vertical_and_slash", 1000, 6096, 0.9951373934745789], "22": ["vertical_and_slash", 1000, 6096, 0.9152458906173706], "23": ["vertical_and_slash", 1000, 6096, 0.9692571759223938], "24": ["vertical_and_slash", 1000, 6096, 0.9701493978500366], "25": ["vertical_and_slash", 1000, 6096, 0.983892023563385], "26": ["vertical_and_slash", 3048, 8144, 0.8999871611595154], "27": ["vertical_and_slash", 1000, 6096, 0.9546129703521729]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9800829291343689], "1": ["vertical_and_slash", 1000, 6096, 0.9934086799621582], "2": ["vertical_and_slash", 1000, 6096, 0.9579113721847534], "3": ["vertical_and_slash", 1000, 6096, 0.910209059715271], "4": ["vertical_and_slash", 3048, 8144, 0.8276802897453308], "5": ["vertical_and_slash", 1000, 6096, 0.9891079068183899], "6": ["vertical_and_slash", 1000, 6096, 0.9916367530822754], "7": ["vertical_and_slash", 1000, 6096, 0.9807681441307068], "8": ["vertical_and_slash", 1000, 6096, 0.996056318283081], "9": ["vertical_and_slash", 1000, 6096, 0.9689919948577881], "10": ["vertical_and_slash", 1000, 6096, 0.9973182678222656], "11": ["vertical_and_slash", 1000, 6096, 0.9486907720565796], "12": ["vertical_and_slash", 1000, 6096, 0.9872133135795593], "13": ["vertical_and_slash", 1000, 6096, 0.9248751997947693], "14": ["vertical_and_slash", 500, 700, 0.9737105965614319], "15": ["vertical_and_slash", 30, 800, 0.9953463673591614], "16": ["vertical_and_slash", 500, 700, 0.996398389339447], "17": ["vertical_and_slash", 7596, 4196, 0.9452576041221619], "18": ["vertical_and_slash", 500, 700, 0.9803798794746399], "19": ["vertical_and_slash", 30, 800, 0.9829242825508118], "20": ["vertical_and_slash", 30, 800, 0.9800650477409363], "21": ["vertical_and_slash", 1000, 6096, 0.9978428483009338], "22": ["vertical_and_slash", 15436, 20432, 0.6166174411773682], "23": ["vertical_and_slash", 13388, 18384, 0.8101190328598022], "24": ["vertical_and_slash", 1000, 6096, 0.9721275568008423], "25": ["vertical_and_slash", 7244, 12240, 0.9621387720108032], "26": ["vertical_and_slash", 5096, 10192, 0.761796236038208], "27": ["vertical_and_slash", 15436, 20432, 0.735977292060852]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9662788510322571], "1": ["vertical_and_slash", 1000, 6096, 0.933112621307373], "2": ["vertical_and_slash", 3048, 8144, 0.845474123954773], "3": ["vertical_and_slash", 11340, 16336, 0.9396570324897766], "4": ["vertical_and_slash", 5548, 2148, 0.8758905529975891], "5": ["vertical_and_slash", 1000, 6096, 0.9595933556556702], "6": ["vertical_and_slash", 1000, 6096, 0.9791407585144043], "7": ["vertical_and_slash", 1000, 6096, 0.9448844194412231], "8": ["vertical_and_slash", 15436, 20432, 0.7103061079978943], "9": ["vertical_and_slash", 1000, 6096, 0.985871434211731], "10": ["vertical_and_slash", 7596, 4196, 0.7926585078239441], "11": ["vertical_and_slash", 5096, 10192, 0.9791630506515503], "12": ["vertical_and_slash", 1000, 6096, 0.9933391809463501], "13": ["vertical_and_slash", 11340, 16336, 0.9010775089263916], "14": ["vertical_and_slash", 1000, 6096, 0.9843372702598572], "15": ["vertical_and_slash", 3048, 8144, 0.8298346996307373], "16": ["vertical_and_slash", 1000, 6096, 0.9661977887153625], "17": ["vertical_and_slash", 1000, 6096, 0.90715092420578], "18": ["vertical_and_slash", 1000, 6096, 0.9170965552330017], "19": ["vertical_and_slash", 9192, 14288, 0.6205919981002808], "20": ["vertical_and_slash", 1000, 6096, 0.9696729779243469], "21": ["vertical_and_slash", 1000, 6096, 0.9067583084106445], "22": ["vertical_and_slash", 7596, 4196, 0.7575717568397522], "23": ["vertical_and_slash", 7244, 12240, 0.9475517868995667], "24": ["vertical_and_slash", 1000, 6096, 0.966336727142334], "25": ["vertical_and_slash", 5548, 2148, 0.8904582262039185], "26": ["vertical_and_slash", 1000, 6096, 0.9011759757995605], "27": ["vertical_and_slash", 1000, 6096, 0.9367696046829224]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9443923234939575], "1": ["vertical_and_slash", 7244, 12240, 0.9806238412857056], "2": ["vertical_and_slash", 13840, 10340, 0.957985520362854], "3": ["vertical_and_slash", 11340, 16336, 0.9818342924118042], "4": ["vertical_and_slash", 500, 700, 0.9885478019714355], "5": ["vertical_and_slash", 1000, 6096, 0.9833213090896606], "6": ["vertical_and_slash", 7596, 4196, 0.7089235782623291], "7": ["vertical_and_slash", 1000, 6096, 0.9972561597824097], "8": ["vertical_and_slash", 1000, 6096, 0.995736837387085], "9": ["vertical_and_slash", 1000, 6096, 0.9955088496208191], "10": ["vertical_and_slash", 1000, 6096, 0.9935648441314697], "11": ["vertical_and_slash", 1000, 6096, 0.9156711101531982], "12": ["vertical_and_slash", 1000, 6096, 0.9951316118240356], "13": ["vertical_and_slash", 1000, 6096, 0.9924508929252625], "14": ["vertical_and_slash", 1000, 6096, 0.9403036832809448], "15": ["vertical_and_slash", 30, 800, 0.9910191297531128], "16": ["vertical_and_slash", 500, 700, 0.9839996099472046], "17": ["vertical_and_slash", 3500, 100, 0.9900570511817932], "18": ["vertical_and_slash", 1000, 6096, 0.901885449886322], "19": ["vertical_and_slash", 1000, 6096, 0.9504903554916382], "20": ["vertical_and_slash", 1000, 6096, 0.9650601744651794], "21": ["vertical_and_slash", 10840, 10940, 0.990660548210144], "22": ["vertical_and_slash", 30, 800, 0.9931259155273438], "23": ["vertical_and_slash", 6274, 6944, 0.9908908605575562], "24": ["vertical_and_slash", 1000, 6096, 0.9198976755142212], "25": ["vertical_and_slash", 30, 800, 0.9933606386184692], "26": ["vertical_and_slash", 6274, 6944, 0.9890289306640625], "27": ["vertical_and_slash", 30, 800, 0.9929149746894836]}, {"0": ["vertical_and_slash", 3048, 8144, 0.8870794177055359], "1": ["vertical_and_slash", 30, 800, 0.9871358275413513], "2": ["vertical_and_slash", 1000, 6096, 0.9092150330543518], "3": ["vertical_and_slash", 1000, 6096, 0.9849765300750732], "4": ["vertical_and_slash", 500, 700, 0.9693071246147156], "5": ["vertical_and_slash", 30, 800, 0.9693368673324585], "6": ["vertical_and_slash", 1000, 6096, 0.9419503211975098], "7": ["vertical_and_slash", 30, 800, 0.9812013506889343], "8": ["vertical_and_slash", 15436, 20432, 0.6496729254722595], "9": ["vertical_and_slash", 100, 800, 0.94140625], "10": ["vertical_and_slash", 1000, 6096, 0.9741862416267395], "11": ["vertical_and_slash", 4196, 4896, 0.95703125], "12": ["vertical_and_slash", 15788, 12388, 0.46300187706947327], "13": ["vertical_and_slash", 10440, 11040, 0.96875], "14": ["vertical_and_slash", 30, 800, 0.9830393195152283], "15": ["vertical_and_slash", 30, 800, 0.9927415251731873], "16": ["vertical_and_slash", 1000, 6096, 0.9534035325050354], "17": ["vertical_and_slash", 100, 800, 0.984375], "18": ["vertical_and_slash", 30, 800, 0.9904463291168213], "19": ["vertical_and_slash", 6274, 6944, 0.9873848557472229], "20": ["vertical_and_slash", 6274, 6944, 0.991168737411499], "21": ["vertical_and_slash", 7244, 12240, 0.9426537752151489], "22": ["vertical_and_slash", 3048, 8144, 0.8975484371185303], "23": ["vertical_and_slash", 5096, 10192, 0.7350636124610901], "24": ["vertical_and_slash", 11340, 16336, 0.9229165315628052], "25": ["vertical_and_slash", 1000, 6096, 0.9412928223609924], "26": ["vertical_and_slash", 5096, 10192, 0.9706910252571106], "27": ["vertical_and_slash", 5096, 10192, 0.9476442933082581]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9863660931587219], "1": ["vertical_and_slash", 4196, 4896, 0.95703125], "2": ["vertical_and_slash", 5548, 2148, 0.8480055332183838], "3": ["vertical_and_slash", 500, 700, 0.9538192749023438], "4": ["vertical_and_slash", 4196, 4896, 0.953125], "5": ["vertical_and_slash", 6344, 6944, 0.98046875], "6": ["vertical_and_slash", 5096, 10192, 0.7229321599006653], "7": ["vertical_and_slash", 17936, 14436, 0.7735021114349365], "8": ["vertical_and_slash", 1000, 6096, 0.9491978883743286], "9": ["vertical_and_slash", 1000, 6096, 0.9595749974250793], "10": ["vertical_and_slash", 1000, 6096, 0.942698061466217], "11": ["vertical_and_slash", 1000, 6096, 0.9691566228866577], "12": ["vertical_and_slash", 3048, 8144, 0.8794195055961609], "13": ["vertical_and_slash", 1000, 6096, 0.9725112915039062], "14": ["vertical_and_slash", 1000, 6096, 0.9814248085021973], "15": ["vertical_and_slash", 3048, 8144, 0.8134639859199524], "16": ["vertical_and_slash", 5096, 10192, 0.7067453265190125], "17": ["vertical_and_slash", 9192, 14288, 0.5412349700927734], "18": ["vertical_and_slash", 1000, 6096, 0.9959503412246704], "19": ["vertical_and_slash", 1000, 6096, 0.9272978901863098], "20": ["vertical_and_slash", 11340, 16336, 0.934908926486969], "21": ["vertical_and_slash", 1000, 6096, 0.9556578397750854], "22": ["vertical_and_slash", 1000, 6096, 0.9684078097343445], "23": ["vertical_and_slash", 1000, 6096, 0.9632892608642578], "24": ["vertical_and_slash", 13388, 18384, 0.8790978789329529], "25": ["vertical_and_slash", 1000, 6096, 0.9878668785095215], "26": ["vertical_and_slash", 1000, 6096, 0.9301976561546326], "27": ["vertical_and_slash", 1000, 6096, 0.9775260090827942]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9639821648597717], "1": ["vertical_and_slash", 1000, 6096, 0.9714324474334717], "2": ["vertical_and_slash", 1000, 6096, 0.9500700235366821], "3": ["vertical_and_slash", 1000, 6096, 0.9642037153244019], "4": ["vertical_and_slash", 1000, 6096, 0.94118332862854], "5": ["vertical_and_slash", 1000, 6096, 0.9665160179138184], "6": ["vertical_and_slash", 5548, 2148, 0.8518802523612976], "7": ["vertical_and_slash", 1000, 6096, 0.9963601231575012], "8": ["vertical_and_slash", 30, 800, 0.9910691380500793], "9": ["vertical_and_slash", 100, 800, 0.9765625], "10": ["vertical_and_slash", 30, 800, 0.9925505518913269], "11": ["vertical_and_slash", 9192, 14288, 0.6851534247398376], "12": ["vertical_and_slash", 1000, 6096, 0.9318822622299194], "13": ["vertical_and_slash", 30, 800, 0.9992240071296692], "14": ["vertical_and_slash", 7244, 12240, 0.9547796845436096], "15": ["vertical_and_slash", 1000, 6096, 0.9696962237358093], "16": ["vertical_and_slash", 1000, 6096, 0.9316620826721191], "17": ["vertical_and_slash", 1000, 6096, 0.9258195161819458], "18": ["vertical_and_slash", 3048, 8144, 0.8255325555801392], "19": ["vertical_and_slash", 3048, 8144, 0.8840956687927246], "20": ["vertical_and_slash", 1000, 6096, 0.9662702083587646], "21": ["vertical_and_slash", 1000, 6096, 0.9504241943359375], "22": ["vertical_and_slash", 1000, 6096, 0.9732783436775208], "23": ["vertical_and_slash", 1000, 6096, 0.9553895592689514], "24": ["vertical_and_slash", 1000, 6096, 0.9737565517425537], "25": ["vertical_and_slash", 1000, 6096, 0.9609088897705078], "26": ["vertical_and_slash", 1000, 6096, 0.9755902886390686], "27": ["vertical_and_slash", 1000, 6096, 0.9681479930877686]}, {"0": ["vertical_and_slash", 19532, 24528, 0.6649974584579468], "1": ["vertical_and_slash", 5096, 10192, 0.7836548686027527], "2": ["vertical_and_slash", 11340, 16336, 0.937842607498169], "3": ["vertical_and_slash", 9192, 14288, 0.47755563259124756], "4": ["vertical_and_slash", 11340, 16336, 0.9645791053771973], "5": ["vertical_and_slash", 11340, 16336, 1.0000194311141968], "6": ["vertical_and_slash", 13388, 18384, 0.8536562919616699], "7": ["vertical_and_slash", 13840, 10340, 0.9791404604911804], "8": ["vertical_and_slash", 22032, 18532, 0.680722177028656], "9": ["vertical_and_slash", 3048, 8144, 0.8969497084617615], "10": ["vertical_and_slash", 3048, 8144, 0.8135265707969666], "11": ["vertical_and_slash", 11340, 16336, 0.9405107498168945], "12": ["vertical_and_slash", 9192, 14288, 0.6148377656936646], "13": ["vertical_and_slash", 8392, 8992, 0.87890625], "14": ["vertical_and_slash", 7244, 12240, 0.9699380993843079], "15": ["vertical_and_slash", 7244, 12240, 0.9827472567558289], "16": ["vertical_and_slash", 1000, 6096, 0.9853057861328125], "17": ["vertical_and_slash", 1000, 6096, 0.9891751408576965], "18": ["vertical_and_slash", 1000, 6096, 0.9121230244636536], "19": ["vertical_and_slash", 7244, 12240, 0.935899555683136], "20": ["vertical_and_slash", 3048, 8144, 0.8554543852806091], "21": ["vertical_and_slash", 3048, 8144, 0.8143891096115112], "22": ["vertical_and_slash", 5096, 10192, 0.7405089139938354], "23": ["vertical_and_slash", 9292, 14288, 0.8643857836723328], "24": ["vertical_and_slash", 1000, 6096, 0.916479229927063], "25": ["vertical_and_slash", 1000, 6096, 0.9841439127922058], "26": ["vertical_and_slash", 1000, 6096, 0.9474544525146484], "27": ["vertical_and_slash", 7144, 12240, 0.8183215260505676]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9937507510185242], "1": ["vertical_and_slash", 9292, 14288, 0.8320319652557373], "2": ["vertical_and_slash", 1000, 6096, 0.9843978881835938], "3": ["vertical_and_slash", 1000, 6096, 0.99434894323349], "4": ["vertical_and_slash", 3048, 8144, 0.8445190787315369], "5": ["vertical_and_slash", 1000, 6096, 0.9977291226387024], "6": ["vertical_and_slash", 1000, 6096, 0.9971616864204407], "7": ["vertical_and_slash", 6344, 6944, 0.92578125], "8": ["vertical_and_slash", 30, 800, 0.9918319582939148], "9": ["vertical_and_slash", 30, 800, 0.9931505918502808], "10": ["vertical_and_slash", 500, 700, 0.9629374146461487], "11": ["vertical_and_slash", 30, 800, 0.9984971880912781], "12": ["vertical_and_slash", 500, 700, 0.9874173402786255], "13": ["vertical_and_slash", 30, 800, 0.9872177839279175], "14": ["vertical_and_slash", 8292, 8992, 0.6796875], "15": ["vertical_and_slash", 7244, 12240, 0.9822348356246948], "16": ["vertical_and_slash", 4196, 4896, 0.984375], "17": ["vertical_and_slash", 1000, 6096, 0.9580352902412415], "18": ["vertical_and_slash", 13388, 18384, 0.8853479027748108], "19": ["vertical_and_slash", 6344, 6944, 0.953125], "20": ["vertical_and_slash", 13388, 18384, 0.8276364803314209], "21": ["vertical_and_slash", 500, 700, 0.9783065915107727], "22": ["vertical_and_slash", 1000, 6096, 0.9267724752426147], "23": ["vertical_and_slash", 3048, 8144, 0.8524566888809204], "24": ["vertical_and_slash", 9192, 14288, 0.663494348526001], "25": ["vertical_and_slash", 3048, 8144, 0.8196530342102051], "26": ["vertical_and_slash", 3048, 8144, 0.8980101346969604], "27": ["vertical_and_slash", 4596, 4796, 0.9499697089195251]}, {"0": ["vertical_and_slash", 11340, 16336, 0.9402349591255188], "1": ["vertical_and_slash", 10440, 11040, 0.9453125], "2": ["vertical_and_slash", 3048, 8144, 0.8996480703353882], "3": ["vertical_and_slash", 1000, 6096, 0.9078497886657715], "4": ["vertical_and_slash", 11340, 16336, 0.958014965057373], "5": ["vertical_and_slash", 5096, 10192, 0.796527087688446], "6": ["vertical_and_slash", 7244, 12240, 0.9736052751541138], "7": ["vertical_and_slash", 11340, 16336, 0.9348691701889038], "8": ["vertical_and_slash", 1000, 6096, 0.9979962706565857], "9": ["vertical_and_slash", 15436, 20432, 0.7974388003349304], "10": ["vertical_and_slash", 1000, 6096, 0.9191967248916626], "11": ["vertical_and_slash", 5096, 10192, 0.7944039702415466], "12": ["vertical_and_slash", 11340, 16336, 0.7239288091659546], "13": ["vertical_and_slash", 15436, 20432, 0.6403535008430481], "14": ["vertical_and_slash", 13388, 18384, 0.8503284454345703], "15": ["vertical_and_slash", 9292, 14288, 0.8853875994682312], "16": ["vertical_and_slash", 11340, 16336, 0.9519768953323364], "17": ["vertical_and_slash", 13840, 10340, 0.9313420653343201], "18": ["vertical_and_slash", 7244, 12240, 0.9588450789451599], "19": ["vertical_and_slash", 5096, 10192, 0.7538775205612183], "20": ["vertical_and_slash", 17936, 14436, 0.7871273756027222], "21": ["vertical_and_slash", 4596, 4796, 0.983285665512085], "22": ["vertical_and_slash", 7244, 12240, 0.9609666466712952], "23": ["vertical_and_slash", 1000, 6096, 0.974759042263031], "24": ["vertical_and_slash", 1000, 6096, 0.9698718190193176], "25": ["vertical_and_slash", 1000, 6096, 0.9898294806480408], "26": ["vertical_and_slash", 1000, 6096, 0.9256659150123596], "27": ["vertical_and_slash", 7244, 12240, 0.9485111832618713]}, {"0": ["vertical_and_slash", 9292, 14288, 0.8835593461990356], "1": ["vertical_and_slash", 100, 800, 0.98046875], "2": ["vertical_and_slash", 1000, 6096, 0.9635430574417114], "3": ["vertical_and_slash", 1000, 6096, 0.9747232794761658], "4": ["vertical_and_slash", 1000, 6096, 0.9661608934402466], "5": ["vertical_and_slash", 6344, 6944, 0.953125], "6": ["vertical_and_slash", 1000, 6096, 0.9636247754096985], "7": ["vertical_and_slash", 13388, 18384, 0.8315137028694153], "8": ["vertical_and_slash", 13388, 18384, 0.8079983592033386], "9": ["vertical_and_slash", 3048, 8144, 0.8652046322822571], "10": ["vertical_and_slash", 5096, 10192, 0.7678346633911133], "11": ["vertical_and_slash", 4196, 4896, 0.9921875], "12": ["vertical_and_slash", 1000, 6096, 0.9621175527572632], "13": ["vertical_and_slash", 1000, 6096, 0.9095380902290344], "14": ["vertical_and_slash", 15888, 12388, 0.8682130575180054], "15": ["vertical_and_slash", 5096, 10192, 0.7981836199760437], "16": ["vertical_and_slash", 4196, 4896, 0.94921875], "17": ["vertical_and_slash", 3048, 8144, 0.8423125743865967], "18": ["vertical_and_slash", 11340, 16336, 0.9536053538322449], "19": ["vertical_and_slash", 19532, 24528, 0.6965113878250122], "20": ["vertical_and_slash", 11340, 16336, 0.765586256980896], "21": ["vertical_and_slash", 1000, 6096, 0.9789958000183105], "22": ["vertical_and_slash", 1000, 6096, 0.941737174987793], "23": ["vertical_and_slash", 1000, 6096, 0.9770992398262024], "24": ["vertical_and_slash", 13388, 18384, 0.8724013566970825], "25": ["vertical_and_slash", 1000, 6096, 0.9776409268379211], "26": ["vertical_and_slash", 3048, 8144, 0.8984430432319641], "27": ["vertical_and_slash", 1000, 6096, 0.9618942141532898]}, {"0": ["vertical_and_slash", 30, 800, 0.9397658705711365], "1": ["vertical_and_slash", 1000, 6096, 0.9678259491920471], "2": ["vertical_and_slash", 500, 700, 0.9607874751091003], "3": ["vertical_and_slash", 30, 800, 0.9745917320251465], "4": ["vertical_and_slash", 30, 800, 0.9659082889556885], "5": ["vertical_and_slash", 6274, 6944, 0.947303056716919], "6": ["vertical_and_slash", 30, 800, 0.9196206331253052], "7": ["vertical_and_slash", 30, 800, 1.000028133392334], "8": ["vertical_and_slash", 100, 800, 0.96484375], "9": ["vertical_and_slash", 500, 700, 0.9743742346763611], "10": ["vertical_and_slash", 11340, 16336, 0.7517896294593811], "11": ["vertical_and_slash", 1000, 6096, 0.9228399991989136], "12": ["vertical_and_slash", 1000, 6096, 0.9122115969657898], "13": ["vertical_and_slash", 4196, 4896, 0.96875], "14": ["vertical_and_slash", 1000, 6096, 0.9384160041809082], "15": ["vertical_and_slash", 1000, 6096, 0.955692708492279], "16": ["vertical_and_slash", 15436, 20432, 0.7190804481506348], "17": ["vertical_and_slash", 500, 700, 0.9635783433914185], "18": ["vertical_and_slash", 1000, 6096, 0.9114483594894409], "19": ["vertical_and_slash", 3048, 8144, 0.8773453831672668], "20": ["vertical_and_slash", 1000, 6096, 0.9607734084129333], "21": ["vertical_and_slash", 500, 700, 0.9821328520774841], "22": ["vertical_and_slash", 7244, 12240, 0.9674672484397888], "23": ["vertical_and_slash", 5096, 10192, 0.7276496887207031], "24": ["vertical_and_slash", 1000, 6096, 0.97364741563797], "25": ["vertical_and_slash", 1000, 6096, 0.9671147465705872], "26": ["vertical_and_slash", 1000, 6096, 0.9272445440292358], "27": ["vertical_and_slash", 4196, 4896, 0.96875]}, {"0": ["vertical_and_slash", 4126, 4896, 1.0000450611114502], "1": ["vertical_and_slash", 3048, 8144, 0.8740337491035461], "2": ["vertical_and_slash", 9744, 6244, 0.9127500057220459], "3": ["vertical_and_slash", 1000, 6096, 0.9917009472846985], "4": ["vertical_and_slash", 3048, 8144, 0.8467518091201782], "5": ["vertical_and_slash", 5096, 10192, 0.9295762181282043], "6": ["vertical_and_slash", 13388, 18384, 0.8826475143432617], "7": ["vertical_and_slash", 30, 800, 0.9837709069252014], "8": ["vertical_and_slash", 30, 800, 0.9920428991317749], "9": ["vertical_and_slash", 30, 800, 0.9717660546302795], "10": ["vertical_and_slash", 6274, 6944, 0.9047616124153137], "11": ["vertical_and_slash", 5096, 10192, 0.7814681529998779], "12": ["vertical_and_slash", 500, 700, 0.9585937857627869], "13": ["vertical_and_slash", 30, 800, 0.9965728521347046], "14": ["vertical_and_slash", 3048, 8144, 0.831355631351471], "15": ["vertical_and_slash", 11340, 16336, 0.9474310874938965], "16": ["vertical_and_slash", 11340, 16336, 0.9529578685760498], "17": ["vertical_and_slash", 11340, 16336, 0.9589705467224121], "18": ["vertical_and_slash", 7244, 12240, 0.9103161692619324], "19": ["vertical_and_slash", 11340, 16336, 0.9817567467689514], "20": ["vertical_and_slash", 11340, 16336, 0.9756876230239868], "21": ["vertical_and_slash", 7244, 12240, 0.9710824489593506], "22": ["vertical_and_slash", 11340, 16336, 0.9776445627212524], "23": ["vertical_and_slash", 11340, 16336, 0.9071230888366699], "24": ["vertical_and_slash", 1000, 6096, 0.9814715385437012], "25": ["vertical_and_slash", 1000, 6096, 0.9802733659744263], "26": ["vertical_and_slash", 1000, 6096, 0.9048901200294495], "27": ["vertical_and_slash", 7244, 12240, 0.948653519153595]}, {"0": ["vertical_and_slash", 30, 800, 0.9610792398452759], "1": ["vertical_and_slash", 30, 800, 0.9808367490768433], "2": ["vertical_and_slash", 3048, 8144, 0.8854068517684937], "3": ["vertical_and_slash", 6344, 6944, 0.95703125], "4": ["vertical_and_slash", 100, 800, 0.95703125], "5": ["vertical_and_slash", 3048, 8144, 0.8755441904067993], "6": ["vertical_and_slash", 100, 800, 0.92578125], "7": ["vertical_and_slash", 100, 800, 0.921875], "8": ["vertical_and_slash", 9192, 14288, 0.6833025813102722], "9": ["vertical_and_slash", 5096, 10192, 0.7133266925811768], "10": ["vertical_and_slash", 1000, 6096, 0.9820693731307983], "11": ["vertical_and_slash", 1000, 6096, 0.9388710856437683], "12": ["vertical_and_slash", 3048, 8144, 0.8800644278526306], "13": ["vertical_and_slash", 4196, 4896, 0.98828125], "14": ["vertical_and_slash", 1000, 6096, 0.9447485208511353], "15": ["vertical_and_slash", 1000, 6096, 0.9788766503334045], "16": ["vertical_and_slash", 1000, 6096, 0.9929926991462708], "17": ["vertical_and_slash", 7144, 12240, 0.8362777233123779], "18": ["vertical_and_slash", 3048, 8144, 0.8786005973815918], "19": ["vertical_and_slash", 7244, 12240, 0.9762356281280518], "20": ["vertical_and_slash", 1000, 6096, 0.9894784688949585], "21": ["vertical_and_slash", 1000, 6096, 0.9231916666030884], "22": ["vertical_and_slash", 1000, 6096, 0.9846755266189575], "23": ["vertical_and_slash", 1000, 6096, 0.9899857640266418], "24": ["vertical_and_slash", 1000, 6096, 0.9508718848228455], "25": ["vertical_and_slash", 1000, 6096, 0.9378311038017273], "26": ["vertical_and_slash", 13388, 18384, 0.895405650138855], "27": ["vertical_and_slash", 1000, 6096, 0.969482421875]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9123888611793518], "1": ["vertical_and_slash", 7244, 12240, 0.9673317074775696], "2": ["vertical_and_slash", 7244, 12240, 0.9695746302604675], "3": ["vertical_and_slash", 1000, 6096, 0.9790319204330444], "4": ["vertical_and_slash", 11340, 16336, 0.9740945100784302], "5": ["vertical_and_slash", 11340, 16336, 0.9472271800041199], "6": ["vertical_and_slash", 1000, 6096, 0.9376558661460876], "7": ["vertical_and_slash", 1000, 6096, 0.9759783148765564], "8": ["vertical_and_slash", 100, 800, 0.984375], "9": ["vertical_and_slash", 100, 800, 0.9921875], "10": ["vertical_and_slash", 3500, 100, 0.9970564246177673], "11": ["vertical_and_slash", 9744, 6244, 0.9889124035835266], "12": ["vertical_and_slash", 100, 800, 0.9765625], "13": ["vertical_and_slash", 11340, 16336, 0.9012805819511414], "14": ["vertical_and_slash", 1000, 6096, 0.939403235912323], "15": ["vertical_and_slash", 30, 800, 0.9993904829025269], "16": ["vertical_and_slash", 7244, 12240, 0.9905298352241516], "17": ["vertical_and_slash", 5096, 10192, 0.7282559275627136], "18": ["vertical_and_slash", 1000, 6096, 0.9881381988525391], "19": ["vertical_and_slash", 1000, 6096, 0.9709164500236511], "20": ["vertical_and_slash", 1000, 6096, 0.9852944016456604], "21": ["vertical_and_slash", 1000, 6096, 0.9899225234985352], "22": ["vertical_and_slash", 11340, 16336, 0.9815132021903992], "23": ["vertical_and_slash", 1000, 6096, 0.9717292785644531], "24": ["vertical_and_slash", 7244, 12240, 0.9921572208404541], "25": ["vertical_and_slash", 11340, 16336, 0.9181587100028992], "26": ["vertical_and_slash", 1000, 6096, 0.9939910769462585], "27": ["vertical_and_slash", 7244, 12240, 0.9894195199012756]}, {"0": ["vertical_and_slash", 5096, 10192, 0.9363827705383301], "1": ["vertical_and_slash", 11340, 16336, 0.9809366464614868], "2": ["vertical_and_slash", 1000, 6096, 0.9730969071388245], "3": ["vertical_and_slash", 13388, 18384, 0.8695287704467773], "4": ["vertical_and_slash", 11340, 16336, 0.9124827980995178], "5": ["vertical_and_slash", 11340, 16336, 0.7494962215423584], "6": ["vertical_and_slash", 13388, 18384, 0.8308557868003845], "7": ["vertical_and_slash", 11340, 16336, 0.957757830619812], "8": ["vertical_and_slash", 1000, 6096, 0.9518009424209595], "9": ["vertical_and_slash", 1000, 6096, 0.9622867703437805], "10": ["vertical_and_slash", 1000, 6096, 0.979856550693512], "11": ["vertical_and_slash", 1000, 6096, 0.9948192834854126], "12": ["vertical_and_slash", 7244, 12240, 0.9975912570953369], "13": ["vertical_and_slash", 7244, 12240, 0.9956138730049133], "14": ["vertical_and_slash", 1000, 6096, 0.9788697957992554], "15": ["vertical_and_slash", 6344, 6944, 1.0], "16": ["vertical_and_slash", 7244, 12240, 0.9795857667922974], "17": ["vertical_and_slash", 11340, 16336, 0.9933436512947083], "18": ["vertical_and_slash", 1000, 6096, 0.9908619523048401], "19": ["vertical_and_slash", 7244, 12240, 0.9774454832077026], "20": ["vertical_and_slash", 1000, 6096, 0.9626350402832031], "21": ["vertical_and_slash", 9292, 14288, 0.823958158493042], "22": ["vertical_and_slash", 11340, 16336, 0.9598273634910583], "23": ["vertical_and_slash", 1000, 6096, 0.9597938656806946], "24": ["vertical_and_slash", 11340, 16336, 0.9751371145248413], "25": ["vertical_and_slash", 7244, 12240, 0.9830336570739746], "26": ["vertical_and_slash", 7244, 12240, 0.9900059103965759], "27": ["vertical_and_slash", 11340, 16336, 0.9930396676063538]}, {"0": ["vertical_and_slash", 4196, 4896, 1.0], "1": ["vertical_and_slash", 1000, 6096, 0.9948315024375916], "2": ["vertical_and_slash", 11340, 16336, 0.9978001117706299], "3": ["vertical_and_slash", 1000, 6096, 0.9911418557167053], "4": ["vertical_and_slash", 1000, 6096, 0.9911423921585083], "5": ["vertical_and_slash", 1000, 6096, 0.9994310140609741], "6": ["vertical_and_slash", 1000, 6096, 0.9933038949966431], "7": ["vertical_and_slash", 1000, 6096, 0.9815270304679871], "8": ["vertical_and_slash", 1000, 6096, 0.9188183546066284], "9": ["vertical_and_slash", 7244, 12240, 0.949191689491272], "10": ["vertical_and_slash", 10440, 11040, 0.99609375], "11": ["vertical_and_slash", 11340, 16336, 0.9600533246994019], "12": ["vertical_and_slash", 11340, 16336, 0.9197850823402405], "13": ["vertical_and_slash", 11340, 16336, 0.9567902684211731], "14": ["vertical_and_slash", 7244, 12240, 0.9746363759040833], "15": ["vertical_and_slash", 7244, 12240, 0.9832977652549744], "16": ["vertical_and_slash", 1000, 6096, 0.9724404811859131], "17": ["vertical_and_slash", 1000, 6096, 0.9983720183372498], "18": ["vertical_and_slash", 7244, 12240, 0.9574311375617981], "19": ["vertical_and_slash", 11340, 16336, 0.9586833715438843], "20": ["vertical_and_slash", 7244, 12240, 0.9386707544326782], "21": ["vertical_and_slash", 7596, 4196, 0.9758472442626953], "22": ["vertical_and_slash", 1000, 6096, 0.9404810070991516], "23": ["vertical_and_slash", 9744, 6244, 0.9920862913131714], "24": ["vertical_and_slash", 1000, 6096, 0.9855319261550903], "25": ["vertical_and_slash", 1000, 6096, 0.9087734818458557], "26": ["vertical_and_slash", 1000, 6096, 0.9804069995880127], "27": ["vertical_and_slash", 1000, 6096, 0.9705759882926941]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9644368290901184], "1": ["vertical_and_slash", 30, 800, 0.9986089468002319], "2": ["vertical_and_slash", 1000, 6096, 0.9745039343833923], "3": ["vertical_and_slash", 1000, 6096, 0.9963008761405945], "4": ["vertical_and_slash", 7596, 4196, 0.9790559411048889], "5": ["vertical_and_slash", 1000, 6096, 0.9741643071174622], "6": ["vertical_and_slash", 1000, 6096, 0.9873979687690735], "7": ["vertical_and_slash", 7596, 4196, 0.9851264357566833], "8": ["vertical_and_slash", 9744, 6244, 0.9803036451339722], "9": ["vertical_and_slash", 1000, 6096, 0.9777622818946838], "10": ["vertical_and_slash", 1000, 6096, 0.9912298321723938], "11": ["vertical_and_slash", 1000, 6096, 0.962704598903656], "12": ["vertical_and_slash", 7596, 4196, 0.9825693368911743], "13": ["vertical_and_slash", 5096, 10192, 0.7750788331031799], "14": ["vertical_and_slash", 1000, 6096, 0.9972183704376221], "15": ["vertical_and_slash", 1000, 6096, 0.9945282340049744], "16": ["vertical_and_slash", 9744, 6244, 0.9972888231277466], "17": ["vertical_and_slash", 1000, 6096, 0.9963790774345398], "18": ["vertical_and_slash", 1000, 6096, 0.9650697112083435], "19": ["vertical_and_slash", 7596, 4196, 0.9987449645996094], "20": ["vertical_and_slash", 5096, 10192, 0.9960426092147827], "21": ["vertical_and_slash", 7244, 12240, 0.9878199100494385], "22": ["vertical_and_slash", 1000, 6096, 0.9935839176177979], "23": ["vertical_and_slash", 7244, 12240, 0.9975365400314331], "24": ["vertical_and_slash", 1000, 6096, 0.968522310256958], "25": ["vertical_and_slash", 7244, 12240, 0.9933074116706848], "26": ["vertical_and_slash", 7244, 12240, 0.9941068887710571], "27": ["vertical_and_slash", 1000, 6096, 0.9910241365432739]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9869405627250671], "1": ["vertical_and_slash", 9744, 6244, 0.9560899138450623], "2": ["vertical_and_slash", 9744, 6244, 0.9869239330291748], "3": ["vertical_and_slash", 3500, 100, 0.9870764017105103], "4": ["vertical_and_slash", 1000, 6096, 0.9273780584335327], "5": ["vertical_and_slash", 7596, 4196, 0.9860992431640625], "6": ["vertical_and_slash", 9744, 6244, 0.9839791059494019], "7": ["vertical_and_slash", 1000, 6096, 0.9863566756248474], "8": ["vertical_and_slash", 7244, 12240, 0.9857996106147766], "9": ["vertical_and_slash", 1000, 6096, 0.9804321527481079], "10": ["vertical_and_slash", 7244, 12240, 0.9676904082298279], "11": ["vertical_and_slash", 7596, 4196, 0.9908879399299622], "12": ["vertical_and_slash", 7244, 12240, 0.9486064314842224], "13": ["vertical_and_slash", 7244, 12240, 0.9652743339538574], "14": ["vertical_and_slash", 30, 800, 0.9901286363601685], "15": ["vertical_and_slash", 5096, 10192, 0.7799577116966248], "16": ["vertical_and_slash", 3048, 8144, 0.8891290426254272], "17": ["vertical_and_slash", 7596, 4196, 0.9794012308120728], "18": ["vertical_and_slash", 30, 800, 0.951410710811615], "19": ["vertical_and_slash", 30, 800, 0.9806299209594727], "20": ["vertical_and_slash", 3048, 8144, 0.8039274215698242], "21": ["vertical_and_slash", 1000, 6096, 0.9898089170455933], "22": ["vertical_and_slash", 1000, 6096, 0.986038863658905], "23": ["vertical_and_slash", 7244, 12240, 0.9948249459266663], "24": ["vertical_and_slash", 1000, 6096, 0.9429303407669067], "25": ["vertical_and_slash", 11340, 16336, 0.978610098361969], "26": ["vertical_and_slash", 1000, 6096, 0.9908936023712158], "27": ["vertical_and_slash", 1000, 6096, 0.9936961531639099]}, {"0": ["vertical_and_slash", 7244, 12240, 0.9856012463569641], "1": ["vertical_and_slash", 1000, 6096, 0.9747336506843567], "2": ["vertical_and_slash", 7596, 4196, 0.9834318161010742], "3": ["vertical_and_slash", 1000, 6096, 0.9862018823623657], "4": ["vertical_and_slash", 1000, 6096, 0.9937860369682312], "5": ["vertical_and_slash", 1000, 6096, 0.9886960387229919], "6": ["vertical_and_slash", 7596, 4196, 0.9887931942939758], "7": ["vertical_and_slash", 3048, 8144, 0.8064029216766357], "8": ["vertical_and_slash", 500, 700, 0.9941562414169312], "9": ["vertical_and_slash", 1000, 6096, 0.9482210278511047], "10": ["vertical_and_slash", 1000, 6096, 0.933103084564209], "11": ["vertical_and_slash", 500, 700, 0.9964345097541809], "12": ["vertical_and_slash", 1000, 6096, 0.9534383416175842], "13": ["vertical_and_slash", 1000, 6096, 0.9467671513557434], "14": ["vertical_and_slash", 7244, 12240, 0.9333358407020569], "15": ["vertical_and_slash", 11340, 16336, 0.9803599715232849], "16": ["vertical_and_slash", 1000, 6096, 0.9709724187850952], "17": ["vertical_and_slash", 1000, 6096, 0.9853048324584961], "18": ["vertical_and_slash", 1000, 6096, 0.9850086569786072], "19": ["vertical_and_slash", 7244, 12240, 0.9774631261825562], "20": ["vertical_and_slash", 1000, 6096, 0.9743291139602661], "21": ["vertical_and_slash", 3048, 8144, 0.8560544848442078], "22": ["vertical_and_slash", 3048, 8144, 0.8773953914642334], "23": ["vertical_and_slash", 1000, 6096, 0.9476152062416077], "24": ["vertical_and_slash", 1000, 6096, 0.9344698786735535], "25": ["vertical_and_slash", 11340, 16336, 0.9541553854942322], "26": ["vertical_and_slash", 1000, 6096, 0.9854223728179932], "27": ["vertical_and_slash", 1000, 6096, 0.9799894690513611]}, {"0": ["vertical_and_slash", 1000, 6096, 0.9208858609199524], "1": ["vertical_and_slash", 1000, 6096, 0.9677489995956421], "2": ["vertical_and_slash", 1000, 6096, 0.9397966861724854], "3": ["vertical_and_slash", 1000, 6096, 0.9308995008468628], "4": ["vertical_and_slash", 1000, 6096, 0.9643043279647827], "5": ["vertical_and_slash", 1000, 6096, 0.9657536149024963], "6": ["vertical_and_slash", 3048, 8144, 0.8598212599754333], "7": ["vertical_and_slash", 10370, 11040, 0.9467752575874329], "8": ["vertical_and_slash", 1000, 6096, 0.9844503998756409], "9": ["vertical_and_slash", 1000, 6096, 0.9382181763648987], "10": ["vertical_and_slash", 10370, 11040, 0.972872257232666], "11": ["vertical_and_slash", 10370, 11040, 0.9927771687507629], "12": ["vertical_and_slash", 10370, 11040, 0.9328325390815735], "13": ["vertical_and_slash", 10370, 11040, 0.9946746826171875], "14": ["vertical_and_slash", 1000, 6096, 0.9508710503578186], "15": ["vertical_and_slash", 1000, 6096, 0.9318876266479492], "16": ["vertical_and_slash", 1000, 6096, 0.9340384602546692], "17": ["vertical_and_slash", 7244, 12240, 0.9182395339012146], "18": ["vertical_and_slash", 1000, 6096, 0.9607776403427124], "19": ["vertical_and_slash", 1000, 6096, 0.9068328738212585], "20": ["vertical_and_slash", 1000, 6096, 0.9481955170631409], "21": ["vertical_and_slash", 1000, 6096, 0.9357800483703613], "22": ["vertical_and_slash", 3048, 8144, 0.8631651997566223], "23": ["vertical_and_slash", 7244, 12240, 0.9462882876396179], "24": ["vertical_and_slash", 3048, 8144, 0.808625340461731], "25": ["vertical_and_slash", 1000, 6096, 0.9675469398498535], "26": ["vertical_and_slash", 1000, 6096, 0.9622009992599487], "27": ["vertical_and_slash", 3048, 8144, 0.8711419701576233]}]
diff --git a/minference/configs/leank/llama3.1-8b-instruct.pth b/minference/configs/leank/llama3.1-8b-instruct.pth
new file mode 100644
index 00000000..cb058f4f
Binary files /dev/null and b/minference/configs/leank/llama3.1-8b-instruct.pth differ
diff --git a/minference/configs/leank/qwen2.5-7b-insturct.pth b/minference/configs/leank/qwen2.5-7b-insturct.pth
new file mode 100644
index 00000000..c47d3c55
Binary files /dev/null and b/minference/configs/leank/qwen2.5-7b-insturct.pth differ
diff --git a/minference/configs/model2path.py b/minference/configs/model2path.py
index d71500c7..a8a209b4 100644
--- a/minference/configs/model2path.py
+++ b/minference/configs/model2path.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import os
@@ -23,11 +23,38 @@
"Qwen/Qwen2-7B-Instruct": os.path.join(
BASE_DIR, "Qwen2_7B_Instruct_128k_instruct_kv_out_v32_fit_o_best_pattern.json"
),
+ "Qwen/Qwen2.5-7B-Instruct": os.path.join(
+ BASE_DIR,
+ "Qwen2.5_7B_Instruct_128k_instruct_kv_out_v32_fit_o_best_pattern.json",
+ ),
+ "Qwen/Qwen2.5-32B-Instruct": os.path.join(
+ BASE_DIR,
+ "Qwen2.5_32B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json",
+ ),
+ "Qwen/Qwen2.5-72B-Instruct": os.path.join(
+ BASE_DIR,
+ "Qwen2.5_72B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json",
+ ),
+ "Qwen/Qwen2.5-7B-Instruct-1M": os.path.join(
+ BASE_DIR, "Qwen2.5_7B_Instruct_1M.json"
+ ),
+ "Qwen/Qwen2.5-14B-Instruct-1M": os.path.join(
+ BASE_DIR, "Qwen2.5_14B_Instruct_1M.json"
+ ),
"THUDM/glm-4-9b-chat-1m": os.path.join(
BASE_DIR, "GLM_4_9B_1M_instruct_kv_out_v32_fit_o_best_pattern.json"
),
"meta-llama/Meta-Llama-3.1-8B-Instruct": os.path.join(
- BASE_DIR, "Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern.json"
+ BASE_DIR, "Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json"
+ ),
+ "meta-llama/Meta-Llama-3.1-70B-Instruct": os.path.join(
+ BASE_DIR, "Llama_3.1_70B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json"
+ ),
+ "meta-llama/Llama-3.1-8B-Instruct": os.path.join(
+ BASE_DIR, "Llama_3.1_8B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json"
+ ),
+ "meta-llama/Llama-3.1-70B-Instruct": os.path.join(
+ BASE_DIR, "Llama_3.1_70B_Instruct_128k_kv_out_v32_fit_o_best_pattern_v2.json"
),
"gradientai/Llama-3-70B-Instruct-Gradient-262k": os.path.join(
BASE_DIR, "Llama_3_70B_Instruct_262k_kv_out_v32_fit_o_best_pattern.json"
@@ -37,6 +64,22 @@
),
}
+LEANKPATNS = {
+ "Meta-Llama-3.1-8B-Instruct": os.path.join(
+ BASE_DIR, "leank/llama3.1-8b-instruct.pth"
+ ),
+ "Qwen2.5-7B-Instruct": os.path.join(BASE_DIR, "leank/qwen2.5-7b-insturct.pth"),
+}
+
def get_support_models():
return list(MODEL2PATH.keys())
+
+
+def check_path():
+ for name, path in MODEL2PATH.items():
+ assert os.path.exists(path), f"{name} Config does not exist! Please check it."
+
+
+if __name__ == "__main__":
+ check_path()
diff --git a/minference/dist_ops/__init__.py b/minference/dist_ops/__init__.py
new file mode 100644
index 00000000..cf6d46ee
--- /dev/null
+++ b/minference/dist_ops/__init__.py
@@ -0,0 +1,8 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from .minfer_dr_striped import minfer_dr_stripe_func
+from .minfer_striped import minfer_stripe_func
+from .minfer_zigzag import minfer_zigzag_func
+from .moba_zigzag import moba_zigzag_func
+from .xattn_zigzag import xattn_zigzag_func
diff --git a/minference/dist_ops/dr_striped_attention.py b/minference/dist_ops/dr_striped_attention.py
new file mode 100644
index 00000000..03f29318
--- /dev/null
+++ b/minference/dist_ops/dr_striped_attention.py
@@ -0,0 +1,691 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+from typing import List
+
+import torch
+import torch.distributed as dist
+from flash_attn.flash_attn_interface import _flash_attn_backward, _flash_attn_forward
+
+from .utils import (
+ RingComm,
+ get_default_args,
+ recover_striped_output,
+ shuffle_striped_input,
+ update_out_and_lse,
+)
+
+
+def get_inner_group():
+ rank = dist.get_rank()
+ local_world_size = int(os.environ.get("LOCAL_WORLD_SIZE"))
+ inner_group = [i for i in range(local_world_size)]
+
+ rank_offset = (rank // local_world_size) * local_world_size
+ inner_group = [rank_offset + i for i in inner_group]
+
+ return inner_group
+
+
+def get_outer_group():
+ local_rank = int(os.environ["LOCAL_RANK"])
+ local_world_size = int(os.environ.get("LOCAL_WORLD_SIZE"))
+ world_size = dist.get_world_size()
+
+ outer_group = []
+ i = local_rank
+ while i < world_size:
+ outer_group.append(i)
+ i += local_world_size
+
+ return outer_group
+
+
+def stripe_fwd_inner(
+ process_group,
+ outer_step: int,
+ outer_rank: int,
+ inner_ring_list: List[int],
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ softmax_scale,
+ granularity=1,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ inner_comm = RingComm(process_group, False, inner_ring_list)
+ inner_rank = int(os.environ["LOCAL_RANK"])
+ num_inner_steps = len(inner_ring_list)
+
+ out, lse = None, None
+ next_k, next_v = None, None
+
+ for inner_step in range(num_inner_steps):
+ if inner_step + 1 != num_inner_steps:
+ next_k, next_v = inner_comm.send_recv_kv(k, v)
+
+ def forward(q_, k_, v_, dropout_p_, softmax_scale_, causal_, alibi_slopes_):
+ params = get_default_args(_flash_attn_forward).copy()
+ params.update(
+ {
+ "q": q_,
+ "k": k_,
+ "v": v_,
+ "dropout_p": dropout_p_,
+ "softmax_scale": softmax_scale_,
+ "causal": causal_,
+ "alibi_slopes": alibi_slopes_,
+ "return_softmax": True and dropout_p_ > 0,
+ }
+ )
+
+ if "window_size" in params:
+ params.update({"window_size": window_size})
+ else:
+ params.update(
+ {
+ "window_size_left": window_size[0],
+ "window_size_right": window_size[1],
+ }
+ )
+
+ outputs = _flash_attn_forward(**params)
+ if len(outputs) == 8:
+ block_out, _, _, _, _, block_lse, _, _ = outputs
+ else:
+ assert len(outputs) == 4
+ block_out, block_lse, _, _ = outputs
+ return block_out, block_lse
+
+ if outer_step == 0 and inner_step > inner_rank:
+ block_out, block_lse = forward(
+ q[:, granularity:],
+ k[:, :-granularity],
+ v[:, :-granularity],
+ dropout_p,
+ softmax_scale,
+ causal,
+ alibi_slopes,
+ )
+ out, lse = update_out_and_lse(
+ out,
+ lse,
+ block_out,
+ block_lse,
+ slice_=(slice(None), slice(granularity, None)),
+ )
+ else:
+ block_out, block_lse = forward(
+ q,
+ k,
+ v,
+ dropout_p,
+ softmax_scale,
+ causal,
+ alibi_slopes,
+ )
+ out, lse = update_out_and_lse(out, lse, block_out, block_lse)
+
+ if inner_step + 1 != num_inner_steps:
+ inner_comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ lse = lse.squeeze(dim=-1).transpose(1, 2)
+ return out, lse
+
+
+def stripe_fwd_outer(
+ process_group,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ softmax_scale,
+ granularity=1,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ inner_ring_list: List[int] = None,
+ outer_ring_list: List[int] = None,
+):
+ assert (
+ causal
+ ), "stripe flash attn only supports causal attention, if not causal, use ring flash attn instead"
+ outer_comm = RingComm(process_group, False, outer_ring_list)
+
+ global_rank = dist.get_rank()
+ outer_rank = outer_ring_list.index(global_rank)
+ num_outer_steps = len(outer_ring_list)
+
+ out = None
+ lse = None
+
+ next_k, next_v = None, None
+ for outer_step in range(num_outer_steps):
+ if outer_step + 1 != num_outer_steps:
+ next_k, next_v = outer_comm.send_recv_kv(k, v)
+
+ if outer_step <= outer_rank:
+ block_out, block_lse = stripe_fwd_inner(
+ process_group,
+ outer_step,
+ outer_rank,
+ inner_ring_list,
+ q,
+ k,
+ v,
+ softmax_scale,
+ granularity,
+ dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=deterministic,
+ )
+ out, lse = update_out_and_lse(out, lse, block_out, block_lse)
+ else:
+ # Before the step index goes beyond the current rank, the received KV indices are not greater than those of the Q in the current rank
+ # After the step index goes beyond the current rank, only the KV indices before the last granularity are no greater than those of the Q after the first granularity
+ # this conclusion holds after the step index goes beyond the current rank (not just step index == current rank)
+ block_out, block_lse = stripe_fwd_inner(
+ process_group,
+ outer_step,
+ outer_rank,
+ inner_ring_list,
+ q[:, granularity:],
+ k[:, :-granularity],
+ v[:, :-granularity],
+ softmax_scale,
+ granularity,
+ dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=deterministic,
+ )
+ out, lse = update_out_and_lse(
+ out,
+ lse,
+ block_out,
+ block_lse,
+ slice_=(slice(None), slice(granularity, None)),
+ )
+
+ if outer_step + 1 != num_outer_steps:
+ outer_comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ lse = lse.squeeze(dim=-1).transpose(1, 2)
+ return out, lse
+
+
+def stripe_backward_inner(
+ process_group,
+ outer_step,
+ inner_ring_list,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ granularity=1,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert (
+ causal
+ ), "stripe flash attn only supports causal attention, if not causal, ring flash attn instead"
+ kv_comm = RingComm(process_group, False, inner_ring_list)
+ d_kv_comm = RingComm(process_group, False, inner_ring_list)
+
+ inner_rank = int(os.environ["LOCAL_RANK"])
+ num_inner_step = len(inner_ring_list)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ block_dq_buffer = torch.empty(q.shape, dtype=q.dtype, device=q.device)
+ block_dk_buffer = torch.empty(k.shape, dtype=k.dtype, device=k.device)
+ block_dv_buffer = torch.empty(v.shape, dtype=v.dtype, device=v.device)
+ for inner_step in range(num_inner_step):
+ if inner_step + 1 != num_inner_step:
+ next_k, next_v = kv_comm.send_recv_kv(k, v)
+
+ shift_causal = outer_step == 0 and inner_step > inner_rank
+ softmax_lse_1 = None
+
+ def backward(
+ dout_,
+ q_,
+ k_,
+ v_,
+ out_,
+ softmax_lse_,
+ block_dq_buffer_,
+ block_dk_buffer_,
+ block_dv_buffer_,
+ ):
+ params = get_default_args(_flash_attn_backward).copy()
+ params.update(
+ {
+ "dout": dout_,
+ "q": q_,
+ "k": k_,
+ "v": v_,
+ "out": out_,
+ "softmax_lse": softmax_lse_,
+ "dq": block_dq_buffer_,
+ "dk": block_dk_buffer_,
+ "dv": block_dv_buffer_,
+ "dropout_p": dropout_p,
+ "softmax_scale": softmax_scale,
+ "causal": causal,
+ "alibi_slopes": alibi_slopes,
+ "deterministic": deterministic,
+ }
+ )
+ if "window_size" in params:
+ params.update({"window_size": window_size})
+ else:
+ params.update(
+ {
+ "window_size_left": window_size[0],
+ "window_size_right": window_size[1],
+ }
+ )
+ _flash_attn_backward(**params)
+
+ if not shift_causal:
+ backward(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_dq_buffer,
+ block_dk_buffer,
+ block_dv_buffer,
+ )
+ else:
+ if softmax_lse_1 is None:
+ # lazy init, since the last rank does not need softmax_lse_1
+ softmax_lse_1 = softmax_lse[:, :, granularity:].contiguous()
+ backward(
+ dout[:, granularity:],
+ q[:, granularity:],
+ k[:, :-granularity],
+ v[:, :-granularity],
+ out[:, granularity:],
+ softmax_lse_1,
+ block_dq_buffer[:, granularity:],
+ block_dk_buffer[:, :-granularity],
+ block_dv_buffer[:, :-granularity],
+ )
+
+ if dq is None:
+ dq = block_dq_buffer.to(torch.float32)
+ dk = block_dk_buffer.to(torch.float32)
+ dv = block_dv_buffer.to(torch.float32)
+ else:
+ if not shift_causal:
+ dq += block_dq_buffer
+ else:
+ dq[:, granularity:] += block_dq_buffer[:, granularity:]
+
+ d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ if not shift_causal:
+ dk = block_dk_buffer + dk
+ dv = block_dv_buffer + dv
+ else:
+ dk[:, :-granularity] += block_dk_buffer[:, :-granularity]
+ dv[:, :-granularity] += block_dv_buffer[:, :-granularity]
+
+ if inner_step + 1 != num_inner_step:
+ kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+def stripe_backward_outer(
+ process_group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ inner_ring_list: List[int],
+ outer_ring_list: List[int],
+ granularity=1,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert (
+ causal
+ ), "stripe flash attn only supports causal attention, if not causal, ring flash attn instead"
+
+ outer_kv_comm = RingComm(process_group, False, outer_ring_list)
+ outer_dkv_comm = RingComm(process_group, False, outer_ring_list)
+
+ global_rank = dist.get_rank()
+ outer_rank = outer_ring_list.index(global_rank)
+ num_outer_steps = len(outer_ring_list)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ block_dq_buffer = torch.empty(q.shape, dtype=q.dtype, device=q.device)
+ block_dk_buffer = torch.empty(k.shape, dtype=k.dtype, device=k.device)
+ block_dv_buffer = torch.empty(v.shape, dtype=v.dtype, device=v.device)
+ for outer_step in range(num_outer_steps):
+ if outer_step + 1 != num_outer_steps:
+ next_k, next_v = outer_kv_comm.send_recv_kv(k, v)
+
+ softmax_lse_1 = None
+ outer_shift = outer_step > outer_rank
+
+ if not outer_shift:
+ block_dq_buffer, block_dk_buffer, block_dv_buffer = stripe_backward_inner(
+ process_group,
+ outer_step,
+ inner_ring_list,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ granularity,
+ dropout_p,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ )
+ else:
+ if softmax_lse_1 is None:
+ # lazy init, since the last rank does not need softmax_lse_1
+ softmax_lse_1 = softmax_lse[:, :, granularity:].contiguous()
+ block_dq_buffer, block_dk_buffer, block_dv_buffer = stripe_backward_inner(
+ process_group,
+ outer_step,
+ inner_ring_list,
+ dout[:, granularity:],
+ q[:, granularity:],
+ k[:, :-granularity],
+ v[:, :-granularity],
+ out[:, granularity:],
+ softmax_lse_1,
+ softmax_scale,
+ granularity,
+ dropout_p,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ )
+
+ if dq is None:
+ dq = block_dq_buffer.to(torch.float32)
+ dk = block_dk_buffer.to(torch.float32)
+ dv = block_dv_buffer.to(torch.float32)
+ else:
+ if not outer_shift:
+ dq += block_dq_buffer
+ else:
+ dq[:, granularity:] += block_dq_buffer
+
+ outer_dkv_comm.wait()
+
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ if not outer_shift:
+ dk = block_dk_buffer + dk
+ dv = block_dv_buffer + dv
+ else:
+ dk[:, :-granularity] += block_dk_buffer
+ dv[:, :-granularity] += block_dv_buffer
+
+ if outer_step + 1 != num_outer_steps:
+ outer_kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = outer_dkv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ outer_dkv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+class DRStripeFlashAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_softmax,
+ group,
+ ):
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+
+ assert alibi_slopes is None
+ inner_ring_list = (
+ get_inner_group()
+ ) # ranks in the current node, length = num of cards within this node
+ outer_ring_list = (
+ get_outer_group()
+ ) # corresponding ranks in other nodes, length = num of nodes
+
+ q = shuffle_striped_input(
+ to_send=q, dim=1, granularity=granularity, process_group=group
+ )
+ k = shuffle_striped_input(
+ to_send=k, dim=1, granularity=granularity, process_group=group
+ )
+ v = shuffle_striped_input(
+ to_send=v, dim=1, granularity=granularity, process_group=group
+ )
+
+ out, softmax_lse = stripe_fwd_outer(
+ group,
+ q,
+ k,
+ v,
+ softmax_scale=softmax_scale,
+ granularity=granularity,
+ dropout_p=dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=False,
+ inner_ring_list=inner_ring_list,
+ outer_ring_list=outer_ring_list,
+ )
+ # this should be out_padded
+ ctx.save_for_backward(q, k, v, out, softmax_lse)
+ ctx.inner_ring_list = inner_ring_list
+ ctx.outer_ring_list = outer_ring_list
+ ctx.dropout_p = dropout_p
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.causal = causal
+ ctx.window_size = window_size
+ ctx.alibi_slopes = alibi_slopes
+ ctx.deterministic = deterministic
+ ctx.group = group
+ out = recover_striped_output(
+ out, dim=1, granularity=granularity, process_group=group
+ )
+ if return_softmax:
+ softmax_lse = recover_striped_output(
+ softmax_lse, dim=2, granularity=granularity, process_group=group
+ )
+ return (out, softmax_lse, None)
+ return out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ dout = shuffle_striped_input(
+ to_send=dout, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ q, k, v, out, softmax_lse = ctx.saved_tensors
+ inner_ring_list, outer_ring_list = ctx.inner_ring_list, ctx.outer_ring_list
+ dq, dk, dv = stripe_backward_outer(
+ ctx.group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ ctx.softmax_scale,
+ inner_ring_list=inner_ring_list,
+ outer_ring_list=outer_ring_list,
+ granularity=ctx.granularity,
+ dropout_p=ctx.dropout_p,
+ causal=ctx.causal,
+ window_size=ctx.window_size,
+ alibi_slopes=ctx.alibi_slopes,
+ deterministic=ctx.deterministic,
+ )
+
+ dq = recover_striped_output(
+ dq, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dk = recover_striped_output(
+ dk, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dv = recover_striped_output(
+ dv, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ return dq, dk, dv, None, None, None, None, None, None, None, None, None
+
+
+def dr_stripe_flash_attn_qkvpacked_func(
+ qkv, # [B, N, 3, H, D]
+ dropout_p=0.0,
+ softmax_scale=None,
+ granularity=1,
+ causal=False,
+ window_size=(-1, -1), # -1 means infinite context window
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return DRStripeFlashAttnFunc.apply(
+ qkv[:, :, 0],
+ qkv[:, :, 1],
+ qkv[:, :, 2],
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def dr_stripe_flash_attn_kvpacked_func(
+ q,
+ kv,
+ dropout_p=0.0,
+ softmax_scale=None,
+ granularity=1,
+ causal=False,
+ window_size=(-1, -1), # -1 means infinite context window
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return DRStripeFlashAttnFunc.apply(
+ q,
+ kv[:, :, 0],
+ kv[:, :, 1],
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def dr_stripe_flash_attn_func(
+ q,
+ k,
+ v,
+ dropout_p=0.0,
+ softmax_scale=None,
+ granularity=1,
+ causal=False,
+ window_size=(-1, -1), # -1 means infinite context window
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return DRStripeFlashAttnFunc.apply(
+ q,
+ k,
+ v,
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
diff --git a/minference/dist_ops/minfer_dr_striped.py b/minference/dist_ops/minfer_dr_striped.py
new file mode 100644
index 00000000..7d827443
--- /dev/null
+++ b/minference/dist_ops/minfer_dr_striped.py
@@ -0,0 +1,974 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+from typing import Dict, List, Tuple
+
+import torch
+import torch.distributed as dist
+
+from minference.ops.op_utils.vertical_slash_utils import (
+ build_index,
+ convert_blockmask,
+ extract_kv,
+ merge_kv,
+)
+from minference.ops.pit_sparse_flash_attention_v3 import (
+ bar_attn_bwd,
+ block_attn_bwd,
+ block_bar_attn_bwd,
+ block_bar_attn_fwd,
+)
+from minference.ops.utils import use_triton
+
+from .utils import (
+ RingComm,
+ get_inner_ring,
+ get_outer_ring,
+ recover_striped_output,
+ shuffle_striped_input,
+)
+
+
+# ------------------------------------------------------------------------
+# CUDA-based Implementation (Block-Sparse-Attention version)
+def minfer_dr_stripe_forward_inner(
+ process_group: dist.ProcessGroup,
+ outer_step: int,
+ outer_offset: int,
+ outer_rank: int,
+ inner_ring: List[int],
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ bar_k: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ bar_v: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ granularity: int = 128,
+):
+ inner_comm = RingComm(process_group, False, inner_ring)
+ inner_rank = inner_ring.index(inner_comm.rank)
+ num_inner_steps = len(inner_ring)
+
+ next_k, next_v = None, None
+ block_idx, block_cnt = convert_blockmask(
+ block_mask, block_size_M=granularity, block_size_N=64
+ )
+
+ for inner_step in range(num_inner_steps):
+ if inner_step + 1 != num_inner_steps:
+ next_k, next_v = inner_comm.send_recv_kv(k, v)
+
+ block_causal = (outer_step == 0) and (inner_step == 0)
+ inner_offset = (inner_rank - inner_step) % num_inner_steps
+ offset = outer_offset * num_inner_steps + inner_offset
+
+ out, lse = block_bar_attn_fwd(
+ q,
+ k,
+ v,
+ out,
+ lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ block_idx[inner_offset],
+ block_cnt[inner_offset],
+ granularity=granularity,
+ step=offset,
+ causal=block_causal,
+ )
+ extract_kv(k, v, bar_k, bar_v, v_idx, v_cnt, step=offset)
+
+ if inner_step + 1 != num_inner_steps:
+ inner_comm.wait()
+ k, v = next_k, next_v
+
+ return out, lse
+
+
+def minfer_dr_stripe_forward_outer(
+ process_group: dist.ProcessGroup,
+ outer_ring: List[int],
+ inner_ring: List[int],
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ granularity: int = 128,
+):
+ outer_comm = RingComm(process_group, False, outer_ring)
+ outer_rank = outer_ring.index(outer_comm.rank)
+ num_outer_steps = len(outer_ring)
+
+ out, lse = None, None
+ next_k, next_v = None, None
+ inner_block_masks = block_mask.chunk(num_outer_steps, dim=0)
+
+ batch_size, _, num_qo_heads, head_dim = q.shape
+ max_v_size = v_idx.shape[-1]
+ bar_k = torch.empty(
+ (batch_size, max_v_size, num_qo_heads, head_dim), dtype=q.dtype, device=q.device
+ )
+ bar_v = torch.empty(
+ (batch_size, max_v_size, num_qo_heads, head_dim), dtype=q.dtype, device=q.device
+ )
+
+ for outer_step in range(num_outer_steps):
+ if outer_step + 1 != num_outer_steps:
+ next_k, next_v = outer_comm.send_recv_kv(k, v)
+ outer_offset = (outer_rank - outer_step) % num_outer_steps
+
+ out, lse = minfer_dr_stripe_forward_inner(
+ process_group,
+ outer_step,
+ outer_offset,
+ outer_rank,
+ inner_ring,
+ q,
+ k,
+ v,
+ out,
+ lse,
+ layer_idx,
+ softmax_scale,
+ inner_block_masks[outer_offset],
+ bar_idx,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ bar_k,
+ bar_v,
+ granularity,
+ )
+
+ if outer_step + 1 != num_outer_steps:
+ outer_comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ return out, lse, bar_k, bar_v
+
+
+def minfer_dr_stripe_backward_inner(
+ process_group: dist.ProcessGroup,
+ outer_step: int,
+ outer_offset: int,
+ outer_rank: int,
+ inner_ring: List[int],
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ dq: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ bar_dk: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ bar_dv: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ granularity: int = 128,
+):
+ inner_kv_comm = RingComm(process_group, False, inner_ring)
+ inner_d_kv_comm = RingComm(process_group, False, inner_ring)
+ inner_rank = inner_ring.index(inner_kv_comm.rank)
+ num_inner_steps = len(inner_ring)
+
+ dk, dv = None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ for inner_step in range(num_inner_steps):
+ if inner_step + 1 != num_inner_steps:
+ next_k, next_v = inner_kv_comm.send_recv_kv(k, v)
+ block_causal = (outer_step == 0) and (inner_step == 0)
+ offset = (
+ outer_offset * num_inner_steps + (inner_rank - inner_step) % num_inner_steps
+ )
+
+ # Block Mask
+ step_dq, step_dk, step_dv = block_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ block_mask[offset],
+ granularity=granularity,
+ deterministic=False,
+ causal=block_causal,
+ )
+
+ # Update dQ, dK, dV
+ if inner_step == 0:
+ # TODO: check if float32 is necessary
+ dk = step_dk.to(torch.float32)
+ dv = step_dv.to(torch.float32)
+ else:
+ inner_d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ dk += step_dk
+ dv += step_dv
+ dq += step_dq
+ merge_kv(dk, dv, bar_dk, bar_dv, v_idx, v_cnt, step=offset)
+ if inner_step + 1 != num_inner_steps:
+ inner_kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = inner_d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ inner_d_kv_comm.wait()
+ return dq, next_dk, next_dv
+
+
+def minfer_dr_stripe_backward_outer(
+ process_group: dist.ProcessGroup,
+ outer_ring: List[int],
+ inner_ring: List[int],
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ bar_pos: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ bar_k: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ bar_v: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ granularity: int = 128,
+):
+ outer_kv_comm = RingComm(process_group, False, outer_ring)
+ outer_d_kv_comm = RingComm(process_group, False, outer_ring)
+ outer_rank = outer_ring.index(outer_kv_comm.rank)
+ num_outer_steps = len(outer_ring)
+
+ dq, dk, dv = None, None, None
+ next_k, next_v = None, None
+ next_dk, next_dv = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ # Bar Mask
+ full_bar_cnt = torch.stack([bar_cnt[..., 0], bar_cnt[..., -1]], dim=-1)
+ dq, bar_dk, bar_dv = bar_attn_bwd(
+ dout,
+ q,
+ bar_k,
+ bar_v,
+ out,
+ None,
+ None,
+ None,
+ softmax_lse,
+ softmax_scale,
+ bar_pos,
+ full_bar_cnt,
+ granularity=granularity,
+ deterministic=False,
+ step=0,
+ )
+
+ for outer_step in range(num_outer_steps):
+ if outer_step + 1 != num_outer_steps:
+ next_k, next_v = outer_kv_comm.send_recv_kv(k, v)
+ outer_offset = (outer_rank - outer_step) % num_outer_steps
+
+ dq, step_dk, step_dv = minfer_dr_stripe_backward_inner(
+ process_group,
+ outer_step,
+ outer_offset,
+ outer_rank,
+ inner_ring,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx,
+ softmax_scale,
+ block_mask,
+ v_idx,
+ v_cnt,
+ dq,
+ bar_dk,
+ bar_dv,
+ granularity,
+ )
+
+ if outer_step == 0:
+ # TODO: check if float32 is necessary
+ dk, dv = step_dk, step_dv
+ else:
+ outer_d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ dk += step_dk
+ dv += step_dv
+
+ if outer_step + 1 != num_outer_steps:
+ outer_kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = outer_d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ outer_d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+# ---------------------------------------------------------------------
+# Purely Triton-based Implementation
+def minfer_dr_stripe_triton_forward_inner(
+ process_group: dist.ProcessGroup,
+ outer_step: int,
+ outer_offset: int,
+ inner_ring: List[int],
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ inner_comm = RingComm(process_group, False, inner_ring)
+ inner_rank = inner_ring.index(inner_comm.rank)
+ num_inner_steps = len(inner_ring)
+
+ next_k, next_v = None, None
+
+ for inner_step in range(num_inner_steps):
+ if inner_step + 1 != num_inner_steps:
+ next_k, next_v = inner_comm.send_recv_kv(k, v)
+
+ block_causal = (outer_step == 0) and (inner_step == 0)
+ offset = (
+ outer_offset * num_inner_steps + (inner_rank - inner_step) % num_inner_steps
+ )
+
+ out, lse = block_bar_attn_fwd(
+ q,
+ k,
+ v,
+ out,
+ lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ block_idx[offset],
+ block_cnt[offset],
+ granularity=granularity,
+ step=offset,
+ causal=block_causal,
+ )
+
+ if inner_step + 1 != num_inner_steps:
+ inner_comm.wait()
+ k, v = next_k, next_v
+
+ return out, lse
+
+
+def minfer_dr_stripe_triton_forward_outer(
+ process_group: dist.ProcessGroup,
+ outer_ring: List[int],
+ inner_ring: List[int],
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ outer_comm = RingComm(process_group, False, outer_ring)
+ outer_rank = outer_ring.index(outer_comm.rank)
+ num_outer_steps = len(outer_ring)
+
+ out = None
+ lse = None
+
+ next_k, next_v = None, None
+ for outer_step in range(num_outer_steps):
+ if outer_step + 1 != num_outer_steps:
+ next_k, next_v = outer_comm.send_recv_kv(k, v)
+
+ outer_offset = (outer_rank - outer_step) % num_outer_steps
+ out, lse = minfer_dr_stripe_triton_forward_inner(
+ process_group,
+ outer_step,
+ outer_offset,
+ inner_ring,
+ q,
+ k,
+ v,
+ out,
+ lse,
+ softmax_scale,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ granularity,
+ )
+
+ if outer_step + 1 != num_outer_steps:
+ outer_comm.wait()
+ k, v = next_k, next_v
+
+ # out = out.to(q.dtype)
+ # lse = lse.squeeze(dim=-1).transpose(1, 2)
+ return out, lse
+
+
+def minfer_dr_stripe_triton_backward_inner(
+ process_group: dist.ProcessGroup,
+ outer_step: int,
+ outer_offset: int,
+ inner_ring: List[int],
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ inner_kv_comm = RingComm(process_group, False, inner_ring)
+ inner_d_kv_comm = RingComm(process_group, False, inner_ring)
+ inner_rank = inner_ring.index(inner_kv_comm.rank)
+ num_inner_steps = len(inner_ring)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ for inner_step in range(num_inner_steps):
+ if inner_step + 1 != num_inner_steps:
+ next_k, next_v = inner_kv_comm.send_recv_kv(k, v)
+
+ block_causal = (outer_step == 0) and (inner_step == 0)
+ offset = (
+ outer_offset * num_inner_steps + (inner_rank - inner_step) % num_inner_steps
+ )
+
+ dq, step_dk, step_dv = block_bar_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ dq,
+ None,
+ None,
+ softmax_lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ block_idx[offset],
+ block_cnt[offset],
+ granularity=granularity,
+ deterministic=False,
+ step=offset,
+ causal=block_causal,
+ )
+
+ # Update dQ, dK, dV
+ if inner_step == 0:
+ # TODO: check if float32 is necessary
+ dk = step_dk.to(torch.float32)
+ dv = step_dv.to(torch.float32)
+ else:
+ inner_d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+ dk += step_dk
+ dv += step_dv
+
+ if inner_step + 1 != num_inner_steps:
+ inner_kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = inner_d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ inner_d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+def minfer_dr_stripe_triton_backward_outer(
+ process_group: dist.ProcessGroup,
+ outer_ring: List[int],
+ inner_ring: List[int],
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ outer_kv_comm = RingComm(process_group, False, outer_ring)
+ outer_d_kv_comm = RingComm(process_group, False, outer_ring)
+ outer_rank = outer_ring.index(outer_kv_comm.rank)
+ num_outer_steps = len(outer_ring)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ for outer_step in range(num_outer_steps):
+ if outer_step + 1 != num_outer_steps:
+ next_k, next_v = outer_kv_comm.send_recv_kv(k, v)
+
+ outer_offset = (outer_rank - outer_step) % num_outer_steps
+ step_dq, step_dk, step_dv = minfer_dr_stripe_triton_backward_inner(
+ process_group,
+ outer_step,
+ outer_offset,
+ inner_ring,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ granularity,
+ )
+
+ if outer_step == 0:
+ # TODO: check if float32 is necessary
+ dq = step_dq.to(torch.float32)
+ dk = step_dk.to(torch.float32)
+ dv = step_dv.to(torch.float32)
+ else:
+ dq += step_dq
+ outer_d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+ dk += step_dk
+ dv += step_dv
+
+ if outer_step + 1 != num_outer_steps:
+ outer_kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = outer_d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ outer_d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+# -----------------------------------------------------------
+# Attention Classes
+class MInferDRStripeFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_softmax,
+ group,
+ ):
+ batch_size, num_tokens_local, num_qo_heads, head_dim = q.shape
+ if softmax_scale is None:
+ softmax_scale = head_dim ** (-0.5)
+ inner_ring = get_inner_ring(group)
+ outer_ring = get_outer_ring(group)
+
+ # ----------------------------------------------
+ # Index Build
+ block_mask, bar_idx, bar_cnt, bar_pos, v_idx, v_cnt = build_index(
+ q, k, v_size, s_size, num_tokens_local, granularity=granularity, group=group
+ )
+
+ # ----------------------------------------------
+ # Shuffle
+ q = shuffle_striped_input(
+ to_send=q, dim=1, granularity=granularity, process_group=group
+ )
+ k = shuffle_striped_input(
+ to_send=k, dim=1, granularity=granularity, process_group=group
+ )
+ v = shuffle_striped_input(
+ to_send=v, dim=1, granularity=granularity, process_group=group
+ )
+
+ # ----------------------------------------------
+ # Compute
+ out, softmax_lse, bar_k, bar_v = minfer_dr_stripe_forward_outer(
+ group,
+ outer_ring,
+ inner_ring,
+ q,
+ k,
+ v,
+ layer_idx,
+ softmax_scale,
+ block_mask,
+ bar_idx,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ granularity,
+ )
+
+ # ----------------------------------------------
+ # Recover
+ recovered_out = recover_striped_output(
+ out, dim=1, granularity=granularity, process_group=group
+ )
+ if return_softmax:
+ recovered_softmax_lse = recover_striped_output(
+ softmax_lse, dim=2, granularity=granularity, process_group=group
+ )
+
+ # ----------------------------------------------
+ # Saving tensors for backward
+ ctx.save_for_backward(
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_mask,
+ bar_pos,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ bar_k,
+ bar_v,
+ )
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.group = group
+ ctx.inner_ring = inner_ring
+ ctx.outer_ring = outer_ring
+ ctx.layer_idx = layer_idx
+
+ # ----------------------------------------------
+ # Output and Return
+ if return_softmax:
+ return (recovered_out, recovered_softmax_lse, None)
+ return recovered_out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ (
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_mask,
+ bar_pos,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ bar_k,
+ bar_v,
+ ) = ctx.saved_tensors
+ inner_ring = ctx.inner_ring
+ layer_idx = ctx.layer_idx
+ group = ctx.group
+
+ # ----------------------------------------------
+ # Shuffle
+ dout = shuffle_striped_input(
+ to_send=dout, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+
+ # ----------------------------------------------
+ # Compute
+ dq, dk, dv = minfer_dr_stripe_backward_outer(
+ ctx.group,
+ ctx.outer_ring,
+ ctx.inner_ring,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx,
+ ctx.softmax_scale,
+ block_mask,
+ bar_pos,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ bar_k,
+ bar_v,
+ granularity=ctx.granularity,
+ )
+
+ # ----------------------------------------------
+ # Recover
+ dq = recover_striped_output(
+ dq, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dk = recover_striped_output(
+ dk, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dv = recover_striped_output(
+ dv, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+
+ return dq, dk, dv, None, None, None, None, None, None, None
+
+
+class MInferDRStripeTritonFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_softmax,
+ group,
+ ):
+ batch_size, num_tokens_local, num_qo_heads, head_dim = q.shape
+ if softmax_scale is None:
+ softmax_scale = head_dim ** (-0.5)
+
+ # build index TODO: move convert_indices() into the first step
+ block_mask, bar_idx, bar_cnt, _, _, _ = build_index(
+ q, k, v_size, s_size, num_tokens_local, granularity=granularity, group=group
+ )
+ block_idx, block_cnt = convert_blockmask(
+ block_mask, block_size_M=granularity, block_size_N=64
+ )
+
+ # TODO: remove shuffle
+ q = shuffle_striped_input(
+ to_send=q, dim=1, granularity=granularity, process_group=group
+ )
+ k = shuffle_striped_input(
+ to_send=k, dim=1, granularity=granularity, process_group=group
+ )
+ v = shuffle_striped_input(
+ to_send=v, dim=1, granularity=granularity, process_group=group
+ )
+
+ inner_ring = get_inner_ring(group)
+ outer_ring = get_outer_ring(group)
+ out, softmax_lse = minfer_dr_stripe_triton_forward_outer(
+ group,
+ outer_ring,
+ inner_ring,
+ q,
+ k,
+ v,
+ softmax_scale,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ granularity,
+ )
+
+ # this should be out_padded
+ ctx.save_for_backward(
+ q, k, v, out, softmax_lse, block_idx, block_cnt, bar_idx, bar_cnt
+ )
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.group = group
+ ctx.inner_ring = inner_ring
+ ctx.outer_ring = outer_ring
+ ctx.layer_idx = layer_idx
+
+ out = recover_striped_output(
+ out, dim=1, granularity=granularity, process_group=group
+ )
+ if return_softmax:
+ softmax_lse = recover_striped_output(
+ softmax_lse, dim=2, granularity=granularity, process_group=group
+ )
+ return (out, softmax_lse, None)
+ return out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ dout = shuffle_striped_input(
+ to_send=dout, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ (
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ ) = ctx.saved_tensors
+
+ dq, dk, dv = minfer_dr_stripe_triton_backward_outer(
+ ctx.group,
+ ctx.outer_ring,
+ ctx.inner_ring,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ ctx.softmax_scale,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ ctx.granularity,
+ )
+ dq = recover_striped_output(
+ dq, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dk = recover_striped_output(
+ dk, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dv = recover_striped_output(
+ dv, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ return dq, dk, dv, None, None, None, None, None, None, None
+
+
+# ---------------------------------------------------------------------
+# Wrapped Attention Functions
+def minfer_dr_stripe_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ v_size: List[int], # [num_heads]
+ s_size: List[int], # [num_heads]
+ layer_idx: int = 0,
+ dropout_p: float = 0.0,
+ softmax_scale: float = None,
+ granularity: int = 128,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[int, int] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+):
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+
+ if not use_triton():
+ return MInferDRStripeFunc.apply(
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ group,
+ )
+ else:
+ return MInferDRStripeTritonFunc.apply(
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ group,
+ )
+
+
+def minfer_dr_stripe_qkvpacked_func(
+ qkv: torch.Tensor, # [batch_size, num_tokens, 3, num_heads, head_dim]
+ *args,
+ **kwargs,
+):
+ return minfer_dr_stripe_func(
+ qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2], *args, **kwargs
+ )
+
+
+def minfer_dr_stripe_kvpacked_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ kv: torch.Tensor, # [batch_size, num_tokens, 2, num_heads, head_dim]
+ *args,
+ **kwargs,
+):
+ return minfer_dr_stripe_func(q, kv[:, :, 0], kv[:, :, 1], *args, **kwargs)
diff --git a/minference/dist_ops/minfer_striped.py b/minference/dist_ops/minfer_striped.py
new file mode 100644
index 00000000..8a32d2db
--- /dev/null
+++ b/minference/dist_ops/minfer_striped.py
@@ -0,0 +1,666 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+import sys
+from typing import Dict, List, Tuple
+
+import torch
+import torch.distributed as dist
+import triton
+
+from minference.dist_ops.utils import (
+ RingComm,
+ recover_striped_output,
+ shuffle_striped_input,
+)
+from minference.ops.op_utils.vertical_slash_utils import (
+ build_index,
+ convert_blockmask,
+ extract_kv,
+ merge_kv,
+)
+from minference.ops.pit_sparse_flash_attention_v3 import (
+ bar_attn_bwd,
+ block_attn_bwd,
+ block_bar_attn_bwd,
+ block_bar_attn_fwd,
+)
+from minference.ops.utils import use_triton
+
+if torch.version.hip is None:
+ original_flags = sys.getdlopenflags()
+ try:
+ sys.setdlopenflags(os.RTLD_LAZY | os.RTLD_GLOBAL)
+ import block_sparse_attn_cuda # type: ignore
+ from block_sparse_attn.block_sparse_attn_interface import ( # type: ignore
+ convert_blockmask_col_reverse,
+ convert_blockmask_row_reverse,
+ )
+
+ # NOTE: Block-Sparse-Attention/csrc/block_sparse_attn/src/flash_blockmask.h: add head_idx to blockmask_ptr
+ except ModuleNotFoundError as e:
+ print(f"[Warning] Failed to import block_sparse_attn_cuda: {e}")
+ finally:
+ # Restore original flags for future imports
+ sys.setdlopenflags(original_flags)
+ # NOTE: Block-Sparse-Attention/csrc/block_sparse_attn/src/flash_blockmask.h: add head_idx to blockmask_ptr
+
+
+# ------------------------------------------------------------------
+# CUDA-based Implementation
+def minfer_stripe_forward(
+ process_group: dist.ProcessGroup,
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ granularity: int = 128,
+):
+ comm = RingComm(process_group, zigzag=False)
+
+ out, lse = None, None
+ block_idx, block_cnt = convert_blockmask(
+ block_mask, block_size_M=granularity, block_size_N=64
+ )
+
+ batch_size, _, num_qo_heads, head_dim = q.shape
+ max_v_size = v_idx.shape[-1]
+ bar_k = torch.empty(
+ (batch_size, max_v_size, num_qo_heads, head_dim), dtype=q.dtype, device=q.device
+ )
+ bar_v = torch.empty(
+ (batch_size, max_v_size, num_qo_heads, head_dim), dtype=q.dtype, device=q.device
+ )
+
+ for step in range(comm.world_size):
+ if step + 1 != comm.world_size:
+ next_k, next_v = comm.send_recv_kv(k, v)
+ block_causal = step == 0
+ offset = (comm.rank - step) % comm.world_size
+
+ out, lse = block_bar_attn_fwd(
+ q,
+ k,
+ v,
+ out,
+ lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ block_idx[offset],
+ block_cnt[offset],
+ granularity=granularity,
+ step=offset,
+ causal=block_causal,
+ )
+ extract_kv(k, v, bar_k, bar_v, v_idx, v_cnt, step=offset)
+
+ if step + 1 != comm.world_size:
+ comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ # lse = lse.squeeze(dim=-1).transpose(1, 2)
+ return out, lse, bar_k, bar_v
+
+
+def minfer_stripe_backward(
+ process_group: dist.ProcessGroup,
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ bar_pos: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ bar_k: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ bar_v: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ granularity: int = 128,
+):
+ kv_comm = RingComm(process_group, zigzag=False)
+ d_kv_comm = RingComm(process_group, zigzag=False)
+
+ dq, dk, dv = None, None, None
+ next_k, next_v = None, None
+ next_dk, next_dv = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+ block_mask = convert_blockmask_col_reverse(block_mask, causal=True)
+
+ # Bar Mask
+ full_bar_cnt = torch.stack([bar_cnt[..., 0], bar_cnt[..., -1]], dim=-1)
+ dq, bar_dk, bar_dv = bar_attn_bwd(
+ dout,
+ q,
+ bar_k,
+ bar_v,
+ out,
+ None,
+ None,
+ None,
+ softmax_lse,
+ softmax_scale,
+ bar_pos,
+ full_bar_cnt,
+ granularity=granularity,
+ deterministic=False,
+ step=0,
+ )
+
+ for step in range(kv_comm.world_size):
+ if step + 1 != kv_comm.world_size:
+ next_k, next_v = kv_comm.send_recv_kv(k, v)
+ block_causal = step == 0
+ offset = (kv_comm.rank - step) % kv_comm.world_size
+
+ # Block Mask
+ step_dq, step_dk, step_dv = block_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ block_mask[offset],
+ granularity=granularity,
+ deterministic=False,
+ causal=block_causal,
+ converted=True,
+ )
+
+ # Update dQ, dK, dV
+ if step == 0:
+ # TODO: check if float32 is necessary
+ dk = step_dk.to(torch.float32)
+ dv = step_dv.to(torch.float32)
+ else:
+ d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+ dk += step_dk
+ dv += step_dv
+ dq += step_dq
+ merge_kv(dk, dv, bar_dk, bar_dv, v_idx, v_cnt, step=offset)
+
+ if step + 1 != kv_comm.world_size:
+ kv_comm.wait()
+ k, v = next_k, next_v
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+# ------------------------------------------------------------------
+# Triton-based Implementation
+def minfer_stripe_triton_forward(
+ process_group: dist.ProcessGroup,
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ layer_idx: int,
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ comm = RingComm(process_group)
+ out, lse = None, None
+ next_k, next_v = None, None
+
+ for step in range(comm.world_size):
+ if step + 1 != comm.world_size:
+ next_k, next_v = comm.send_recv_kv(k, v)
+ block_causal = step == 0
+ offset = (comm.rank - step) % comm.world_size
+
+ out, lse = block_bar_attn_fwd(
+ q,
+ k,
+ v,
+ out,
+ lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ block_idx[offset],
+ block_cnt[offset],
+ granularity=granularity,
+ step=offset,
+ causal=block_causal,
+ )
+
+ if step + 1 != comm.world_size:
+ comm.wait()
+ k, v = next_k, next_v
+
+ return out, lse
+
+
+def minfer_stripe_triton_backward(
+ process_group: dist.ProcessGroup,
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ layer_idx: int,
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ kv_comm = RingComm(process_group)
+ d_kv_comm = RingComm(process_group)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ for step in range(kv_comm.world_size):
+ if step + 1 != kv_comm.world_size:
+ next_k, next_v = kv_comm.send_recv_kv(k, v)
+ block_causal = step == 0
+ offset = (kv_comm.rank - step) % kv_comm.world_size
+
+ dq, step_dk, step_dv = block_bar_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ dq,
+ None,
+ None,
+ softmax_lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ block_idx[offset],
+ block_cnt[offset],
+ granularity=granularity,
+ deterministic=False,
+ step=offset,
+ causal=block_causal,
+ )
+
+ # Update dQ, dK, dV
+ if step == 0:
+ dk = step_dk
+ dv = step_dv
+ else:
+ d_kv_comm.wait()
+
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+ dk += step_dk
+ dv += step_dv
+
+ if step + 1 != kv_comm.world_size:
+ kv_comm.wait()
+ k, v = next_k, next_v
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+# ------------------------------------------------------------------
+# Attention Classes
+class MInferStripeFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_softmax,
+ group,
+ ):
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+ batch_size, num_tokens_local, num_qo_heads, head_dim = q.shape
+
+ # Indexing
+ block_mask, bar_idx, bar_cnt, bar_pos, v_idx, v_cnt = build_index(
+ q, k, v_size, s_size, num_tokens_local, granularity=granularity, group=group
+ )
+
+ # Shuffle
+ q = shuffle_striped_input(
+ to_send=q, dim=1, granularity=granularity, process_group=group
+ )
+ k = shuffle_striped_input(
+ to_send=k, dim=1, granularity=granularity, process_group=group
+ )
+ v = shuffle_striped_input(
+ to_send=v, dim=1, granularity=granularity, process_group=group
+ )
+
+ # Compute
+ out, softmax_lse, bar_k, bar_v = minfer_stripe_forward(
+ group,
+ q,
+ k,
+ v,
+ layer_idx,
+ softmax_scale,
+ block_mask,
+ bar_idx,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ granularity=granularity,
+ )
+
+ # Saving tensors for backward
+ ctx.save_for_backward(
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_mask,
+ bar_pos,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ bar_k,
+ bar_v,
+ )
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.group = group
+ ctx.layer_idx = layer_idx
+
+ # Recover outputs
+ out = recover_striped_output(
+ out, dim=1, granularity=granularity, process_group=group
+ )
+ if return_softmax:
+ softmax_lse = recover_striped_output(
+ softmax_lse, dim=2, granularity=granularity, process_group=group
+ )
+
+ # Output and Return
+ if return_softmax:
+ return (out, softmax_lse, None)
+ return out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ (
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_mask,
+ bar_pos,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ bar_k,
+ bar_v,
+ ) = ctx.saved_tensors
+ softmax_scale = ctx.softmax_scale
+ granularity = ctx.granularity
+ layer_idx = ctx.layer_idx
+ group = ctx.group
+
+ # Shuffle
+ dout = shuffle_striped_input(
+ to_send=dout, dim=1, granularity=granularity, process_group=group
+ )
+
+ # Compute
+ dq, dk, dv = minfer_stripe_backward(
+ group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx,
+ softmax_scale,
+ block_mask,
+ bar_pos,
+ bar_cnt,
+ v_idx,
+ v_cnt,
+ bar_k,
+ bar_v,
+ granularity=granularity,
+ )
+
+ # Recover
+ dq = recover_striped_output(
+ dq, dim=1, granularity=granularity, process_group=group
+ )
+ dk = recover_striped_output(
+ dk, dim=1, granularity=granularity, process_group=group
+ )
+ dv = recover_striped_output(
+ dv, dim=1, granularity=granularity, process_group=group
+ )
+ return dq, dk, dv, None, None, None, None, None, None, None
+
+
+class MInferStripeTritonFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_softmax,
+ group,
+ ):
+ batch_size, num_tokens_local, num_qo_heads, head_dim = q.shape
+ if softmax_scale is None:
+ softmax_scale = head_dim ** (-0.5)
+
+ # built block_idx: [world_size, batch_size, num_qo_heads, num_blocks_local, num_blocks_local]
+ block_mask, bar_idx, bar_cnt, _, _, _ = build_index(
+ q, k, v_size, s_size, num_tokens_local, granularity=granularity, group=group
+ )
+ block_idx, block_cnt = convert_blockmask(
+ block_mask, block_size_M=granularity, block_size_N=64
+ )
+
+ q = shuffle_striped_input(
+ to_send=q, dim=1, granularity=granularity, process_group=group
+ )
+ k = shuffle_striped_input(
+ to_send=k, dim=1, granularity=granularity, process_group=group
+ )
+ v = shuffle_striped_input(
+ to_send=v, dim=1, granularity=granularity, process_group=group
+ )
+
+ # slash attn
+ out, softmax_lse = minfer_stripe_triton_forward(
+ group,
+ q,
+ k,
+ v,
+ layer_idx,
+ softmax_scale,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ granularity=granularity,
+ )
+
+ # this should be out_padded
+ ctx.save_for_backward(
+ q, k, v, out, softmax_lse, block_idx, block_cnt, bar_idx, bar_cnt
+ )
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.group = group
+ ctx.layer_idx = layer_idx
+
+ out = recover_striped_output(
+ out, dim=1, granularity=granularity, process_group=group
+ )
+ if return_softmax:
+ softmax_lse = recover_striped_output(
+ softmax_lse, dim=2, granularity=granularity, process_group=group
+ )
+ return (out, softmax_lse, None)
+ return out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ layer_idx = ctx.layer_idx
+ dout = shuffle_striped_input(
+ to_send=dout, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ (
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ ) = ctx.saved_tensors
+
+ dq, dk, dv = minfer_stripe_triton_backward(
+ ctx.group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx,
+ ctx.softmax_scale,
+ block_idx,
+ block_cnt,
+ bar_idx,
+ bar_cnt,
+ granularity=ctx.granularity,
+ )
+
+ dq = recover_striped_output(
+ dq, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dk = recover_striped_output(
+ dk, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dv = recover_striped_output(
+ dv, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+
+ return dq, dk, dv, None, None, None, None, None, None, None
+
+
+# ------------------------------------------------------------------
+# Wrapped Attention Functions
+# ------------------
+# CUDA-based
+def minfer_stripe_func( # the one used for nnscaler training
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ v_size: List[int], # [num_heads]
+ s_size: List[int], # [num_heads]
+ layer_idx: int = 0,
+ dropout_p: float = 0.0,
+ softmax_scale: float = None,
+ granularity: int = 128,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[int, int] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+) -> torch.Tensor:
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+
+ if not use_triton():
+ return MInferStripeFunc.apply(
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ group,
+ )
+ else:
+ return MInferStripeTritonFunc.apply(
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ group,
+ )
+
+
+def minfer_stripe_qkvpacked_func(
+ qkv: torch.Tensor, # [batch_size, num_tokens, 3, num_heads, head_dim]
+ *args,
+ **kwargs,
+):
+ return minfer_stripe_func(qkv[:, :, 0], qkv[:, :, 1], qkv[:, :, 2], *args, **kwargs)
+
+
+def minfer_stripe_kvpacked_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ kv: torch.Tensor, # [batch_size, num_tokens, 2, num_heads, head_dim]
+ *args,
+ **kwargs,
+):
+ return minfer_stripe_func(q, kv[:, :, 0], kv[:, :, 1], *args, **kwargs)
diff --git a/minference/dist_ops/minfer_zigzag.py b/minference/dist_ops/minfer_zigzag.py
new file mode 100644
index 00000000..1fe5e819
--- /dev/null
+++ b/minference/dist_ops/minfer_zigzag.py
@@ -0,0 +1,384 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+from typing import Dict, List, Tuple
+
+import torch
+import torch.distributed as dist
+import triton
+
+from minference.ops.op_utils.vertical_slash_utils import build_index, convert_blockmask
+from minference.ops.pit_sparse_flash_attention_v3 import (
+ bar_attn_bwd,
+ block_attn_bwd,
+ block_bar_attn_fwd,
+)
+
+from .utils import RingComm, recover_zigzag_output, shuffle_zigzag_input
+
+
+def minfer_zigzag_forward(
+ process_group: dist.ProcessGroup,
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ comm = RingComm(process_group, zigzag=True)
+ ring_list = comm.ring_list
+ ring_index = ring_list.index(comm.rank)
+
+ out, lse = None, None
+ block_idx, block_cnt = convert_blockmask(
+ block_mask, block_size_M=granularity, block_size_N=64
+ )
+
+ for step in range(comm.world_size):
+ if step + 1 != comm.world_size:
+ next_k, next_v = comm.send_recv_kv(k, v)
+ block_causal = step == 0
+ offset = (ring_index - step) % comm.world_size
+
+ # ----------------------------------------------
+ out, lse = block_bar_attn_fwd(
+ q,
+ k,
+ v,
+ out,
+ lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ block_idx[offset],
+ block_cnt[offset],
+ granularity=granularity,
+ step=offset,
+ causal=block_causal,
+ )
+
+ if step + 1 != comm.world_size:
+ comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ return out, lse
+
+
+def minfer_zigzag_backward(
+ process_group: dist.ProcessGroup,
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int = 128,
+):
+ kv_comm = RingComm(process_group, zigzag=True)
+ d_kv_comm = RingComm(process_group, zigzag=True)
+ ring_list = kv_comm.ring_list
+ ring_index = ring_list.index(kv_comm.rank)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ for step in range(kv_comm.world_size):
+ if step + 1 != kv_comm.world_size:
+ next_k, next_v = kv_comm.send_recv_kv(k, v)
+ block_causal = step == 0
+ offset = (ring_index - step) % kv_comm.world_size
+
+ # ----------------------------------------------
+ # Block Mask
+ step_dq, step_dk, step_dv = block_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ block_mask[offset],
+ granularity=granularity,
+ deterministic=False,
+ causal=block_causal,
+ )
+
+ # ----------------------------------------------
+ # Bar Mask
+ step_dq, step_dk, step_dv = bar_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ step_dq,
+ step_dk,
+ step_dv,
+ softmax_lse,
+ softmax_scale,
+ bar_idx,
+ bar_cnt,
+ granularity=granularity,
+ deterministic=False,
+ step=offset,
+ )
+
+ # ----------------------------------------------
+ # Update dQ, dK, dV
+ if step == 0:
+ # TODO: check if float32 is necessary
+ dq = step_dq.to(torch.float32)
+ dk = step_dk.to(torch.float32)
+ dv = step_dv.to(torch.float32)
+ else:
+ d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ dq += step_dq
+ dk += step_dk
+ dv += step_dv
+
+ if step + 1 != kv_comm.world_size:
+ kv_comm.wait()
+ k, v = next_k, next_v
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+class MInferZigzagAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_softmax,
+ group,
+ ):
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+ batch_size, num_tokens_local, num_qo_heads, head_dim = q.shape
+
+ # ------------------------------------------------------------------
+ # Index Build
+ block_mask, bar_idx, bar_cnt, bar_pos, v_idx, v_cnt = build_index(
+ q,
+ k,
+ v_size,
+ s_size,
+ num_tokens_local,
+ stripe_transform=False,
+ zigzag_transform=True,
+ granularity=granularity,
+ group=group,
+ )
+
+ # ----------------------------------------------
+ # Shuffle
+ q = shuffle_zigzag_input(to_send=q, dim=1, process_group=group)
+ k = shuffle_zigzag_input(to_send=k, dim=1, process_group=group)
+ v = shuffle_zigzag_input(to_send=v, dim=1, process_group=group)
+
+ # ----------------------------------------------
+ # Compute
+ out, softmax_lse = minfer_zigzag_forward(
+ group,
+ q,
+ k,
+ v,
+ layer_idx,
+ softmax_scale,
+ block_mask,
+ bar_idx,
+ bar_cnt,
+ granularity=granularity,
+ )
+
+ # ----------------------------------------------
+ # Recover outputs
+ recovered_out = recover_zigzag_output(out, dim=1, process_group=group)
+ if return_softmax:
+ recovered_softmax_lse = recover_zigzag_output(
+ softmax_lse, dim=2, process_group=group
+ )
+
+ # ----------------------------------------------
+ # Saving tensors for backward
+ ctx.save_for_backward(q, k, v, out, softmax_lse, block_mask, bar_idx, bar_cnt)
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.group = group
+ ctx.layer_idx = layer_idx
+
+ # Output and Return
+ if return_softmax:
+ return (recovered_out, recovered_softmax_lse, None)
+ return recovered_out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ q, k, v, out, softmax_lse, block_mask, bar_idx, bar_cnt = ctx.saved_tensors
+ softmax_scale = ctx.softmax_scale
+ granularity = ctx.granularity
+ layer_idx = ctx.layer_idx
+ group = ctx.group
+
+ # ----------------------------------------------
+ # Shuffle
+ dout = shuffle_zigzag_input(to_send=dout, dim=1, process_group=group)
+
+ # ----------------------------------------------
+ # Compute
+ dq, dk, dv = minfer_zigzag_backward(
+ group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx,
+ softmax_scale,
+ block_mask,
+ bar_idx,
+ bar_cnt,
+ granularity=granularity,
+ )
+
+ # ----------------------------------------------
+ # Recover
+ dq = recover_zigzag_output(dq, dim=1, process_group=group)
+ dk = recover_zigzag_output(dk, dim=1, process_group=group)
+ dv = recover_zigzag_output(dv, dim=1, process_group=group)
+
+ return dq, dk, dv, None, None, None, None, None, None, None
+
+
+def minfer_zigzag_qkvpacked_func(
+ qkv: torch.Tensor, # [batch_size, num_tokens, 3, num_heads, head_dim]
+ v_size: List[int], # [num_heads]
+ s_size: List[int], # [num_heads]
+ layer_idx: int = 0,
+ dropout_p: float = 0.0,
+ softmax_scale: float = None,
+ granularity: int = 128,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[int, int] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+):
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+ return MInferZigzagAttnFunc.apply(
+ qkv[:, :, 0],
+ qkv[:, :, 1],
+ qkv[:, :, 2],
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ group,
+ )
+
+
+def minfer_zigzag_kvpacked_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ kv: torch.Tensor, # [batch_size, num_tokens, 2, num_heads, head_dim]
+ v_size: List[int], # [num_heads]
+ s_size: List[int], # [num_heads]
+ layer_idx: int = 0,
+ dropout_p: float = 0.0,
+ softmax_scale: float = None,
+ granularity: int = 128,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[int, int] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+):
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+ return MInferZigzagAttnFunc.apply(
+ q,
+ kv[:, :, 0],
+ kv[:, :, 1],
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ group,
+ )
+
+
+def minfer_zigzag_func( # the one used for nnscaler training
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ v_size: List[int], # [num_heads]
+ s_size: List[int], # [num_heads]
+ layer_idx: int = 0,
+ dropout_p: float = 0.0,
+ softmax_scale: float = None,
+ granularity: int = 128,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[int, int] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+) -> torch.Tensor:
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+
+ return MInferZigzagAttnFunc.apply(
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ layer_idx,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ group,
+ )
diff --git a/minference/dist_ops/moba_zigzag.py b/minference/dist_ops/moba_zigzag.py
new file mode 100644
index 00000000..963d652d
--- /dev/null
+++ b/minference/dist_ops/moba_zigzag.py
@@ -0,0 +1,1301 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Credits: This logger implementation is inspired by project https://github.com/zhuzilin/ring-flash-attention
+import os
+from typing import Dict, List, Tuple
+
+import torch
+import torch.distributed as dist
+from einops import rearrange
+from flash_attn.flash_attn_interface import (
+ _flash_attn_varlen_backward,
+ _flash_attn_varlen_forward,
+)
+
+from minference.ops.op_utils.moba_utils import (
+ compute_moba_gate,
+ shuffle_input_all,
+ shuffle_input_only,
+ tensor_4d_to_3d,
+)
+
+from .utils import RingComm, get_default_args, recover_zigzag_output, update_out_and_lse
+
+
+def moba_zigzag_attn_fwd_step(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor, # [S, H, D]
+ step: int,
+ causal: bool,
+ # q_seq_offsets: torch.Tensor,
+ num_q_blocks: int,
+ k_seq_offsets: torch.Tensor,
+ gate_mask: torch.Tensor, # [num_filtered_chunk, num_head, seq_len]
+ cu_chunk: torch.Tensor,
+ filtered_chunk_indices: torch.Tensor,
+ num_filtered_chunk: int,
+ chunk_to_batch: torch.Tensor,
+ moba_chunk_size: int,
+ moba_topk: int,
+ softmax_scale,
+ dropout_p=0,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+) -> Tuple[torch.Tensor, torch.Tensor, int]:
+ _, _, seq_len = gate_mask.shape
+ q_block_seq_len, num_head, head_dim = q.shape
+ k_block_seq_len, k_num_head, _ = k.shape
+ if num_head > k_num_head:
+ k = torch.repeat_interleave(k, num_head // k_num_head, dim=1)
+ v = torch.repeat_interleave(v, num_head // k_num_head, dim=1)
+
+ block_seq_len = q_block_seq_len // num_q_blocks
+
+ # assumption: block_seq_len is divisible by moba_chunk_size
+ assert (
+ block_seq_len % moba_chunk_size == 0
+ ), "block_seq_len should be divisible by moba_chunk_size"
+
+ kv = torch.stack((k, v), dim=1)
+ k_seq_offset_list = [
+ k_seq_offsets[i].detach().cpu().item() for i in range(len(k_seq_offsets))
+ ]
+ filtered_kv_indices = torch.arange(
+ 0,
+ min(k_seq_offset_list[0] + block_seq_len, num_filtered_chunk * moba_chunk_size)
+ - k_seq_offset_list[0],
+ device=k.device,
+ dtype=torch.int32,
+ )
+ kv_chunk_indices = torch.arange(
+ k_seq_offset_list[0],
+ min(k_seq_offset_list[0] + block_seq_len, num_filtered_chunk * moba_chunk_size),
+ moba_chunk_size,
+ device=k.device,
+ dtype=torch.int32,
+ )
+ if len(k_seq_offset_list) > 1:
+ filtered_kv_indices = torch.cat(
+ [
+ filtered_kv_indices,
+ torch.arange(
+ block_seq_len,
+ min(
+ k_seq_offset_list[1] + block_seq_len,
+ num_filtered_chunk * moba_chunk_size,
+ )
+ - k_seq_offset_list[1]
+ + block_seq_len,
+ device=k.device,
+ dtype=torch.int32,
+ ),
+ ]
+ )
+ kv_chunk_indices = torch.cat(
+ [
+ kv_chunk_indices,
+ torch.arange(
+ k_seq_offset_list[1],
+ min(
+ k_seq_offset_list[1] + block_seq_len,
+ num_filtered_chunk * moba_chunk_size,
+ ),
+ moba_chunk_size,
+ device=k.device,
+ dtype=torch.int32,
+ ),
+ ]
+ )
+ filtered_kv = kv.index_select(0, filtered_kv_indices)
+ kv_chunk_indices = kv_chunk_indices // moba_chunk_size
+ num_filtered_kv_chunks = len(kv_chunk_indices)
+
+ q_indices = torch.arange(
+ 0 if num_q_blocks == 2 else block_seq_len,
+ 2 * block_seq_len,
+ device=q.device,
+ dtype=torch.int32,
+ )
+
+ # varlen trick: combining all q index that needs moba attn
+ # the result will be like [ C0H0 ][ C0H1 ][ C0H2 ][ ... ][ CnHm ]
+ gate_mask_q = gate_mask.index_select(0, kv_chunk_indices)
+ gate_mask_q = gate_mask_q.index_select(
+ 2, q_indices
+ ) # we need to know which part(s) of the two query blocks should be activated
+
+ moba_q_indices = gate_mask_q.reshape(gate_mask_q.shape[0], -1).nonzero(
+ as_tuple=True
+ )[-1]
+ moba_seqlen_q = gate_mask_q.sum(dim=-1).flatten()
+
+ # -----------------------------------------------------------
+ # select all q that needs moba attn based on the moba_q_indices
+ moba_q = rearrange(q, "s h d -> ( h s ) d")
+ moba_q = moba_q.index_select(0, moba_q_indices) # [ selected_HS, D ]
+ moba_q = moba_q.unsqueeze(1)
+
+ # moba_q_sh_indices represents the position in the origin q tensor of each q token inside moba_q
+ moba_q_sh_indices = (
+ moba_q_indices % q_block_seq_len * num_head + moba_q_indices // q_block_seq_len
+ )
+
+ """ prepare moba kv """
+ # Since moba_q is organized as HS * N, we need to reorganize kv to adapt to q
+
+ # cut off zero experts
+ q_zero_mask = moba_seqlen_q == 0
+ valid_expert_mask = ~q_zero_mask
+ zero_expert_count = q_zero_mask.sum()
+ # only keep the kv that has q select > 0
+ if zero_expert_count > 0:
+ moba_seqlen_q = moba_seqlen_q[valid_expert_mask]
+
+ # moba cu_seqlen for flash attn
+ moba_cu_seqlen_q = torch.cat(
+ (
+ torch.tensor([0], device=q.device, dtype=moba_seqlen_q.dtype),
+ moba_seqlen_q.cumsum(dim=0),
+ ),
+ dim=0,
+ ).to(torch.int32)
+
+ # -----------------------------------------------------------------------------------
+ # here `x` only stands for a dimension (stack dimension for KV)
+ moba_kv = rearrange(filtered_kv, "s x h d -> h s x d") # [H, K_S, 2, D ]
+
+ moba_kv = moba_kv.split(
+ moba_chunk_size, dim=1
+ ) # tuple of (num_selected_chunks) elements with shape [H, chunk_size, 2, D]
+ moba_kv = torch.cat(
+ moba_kv, dim=0
+ ) # [H x num_selected_chunks, chunk_size, 2, D ] after split
+
+ # The transformation is aimed for masking out by valid_expert_mask where the mask selects elements along (H x num_selected_chunks) dimension
+ if zero_expert_count > 0:
+ assert valid_expert_mask.sum() == moba_kv.shape[0] - zero_expert_count
+ moba_kv = moba_kv[
+ valid_expert_mask
+ ] # cut off zero Q expert from kv , or the grad may be nan
+
+ moba_kv = moba_kv.flatten(start_dim=0, end_dim=1).unsqueeze(
+ 2
+ ) # [H x num_selected_chunks x chunk_size, 2, 1, D]
+ moba_cu_seqlen_kv = (
+ torch.arange(
+ 0,
+ num_filtered_kv_chunks * num_head + 1 - zero_expert_count,
+ dtype=torch.int32,
+ device=q.device,
+ )
+ * moba_chunk_size
+ )
+
+ # Shape check
+ assert (
+ moba_cu_seqlen_kv.shape == moba_cu_seqlen_q.shape
+ ), f"moba_cu_seqlen_kv.shape != moba_cu_seqlen_q.shape {moba_cu_seqlen_kv.shape} != {moba_cu_seqlen_q.shape}"
+
+ softmax_scale = softmax_scale = head_dim ** (-0.5)
+
+ self_attn_cu_seqlen = [0] + [moba_chunk_size] * (q_block_seq_len // moba_chunk_size)
+ if q_block_seq_len % moba_chunk_size != 0:
+ self_attn_cu_seqlen.append(q_block_seq_len % moba_chunk_size)
+ self_attn_cu_seqlen = torch.tensor(
+ self_attn_cu_seqlen, device=q.device, dtype=torch.int32
+ )
+ self_attn_cu_seqlen = self_attn_cu_seqlen.cumsum(dim=0, dtype=torch.int32)
+
+ # -----------------------------------------------------------------------------------
+ # self attn
+ if causal:
+ # out, softmax_lse, S_dmask, rng_state
+ self_attn_out_sh, self_attn_lse_hs, _, _ = _flash_attn_varlen_forward(
+ q=q,
+ k=k,
+ v=v,
+ cu_seqlens_q=self_attn_cu_seqlen,
+ cu_seqlens_k=self_attn_cu_seqlen,
+ max_seqlen_q=q_block_seq_len,
+ max_seqlen_k=k_block_seq_len,
+ softmax_scale=softmax_scale,
+ causal=True,
+ dropout_p=0.0,
+ )
+ else:
+ # self_attn_out_sh, self_attn_lse_hs = None, None
+ self_attn_out_sh = torch.zeros(
+ (q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32
+ )
+ self_attn_lse_hs = torch.zeros(
+ (num_head, q_block_seq_len), device=q.device, dtype=torch.float32
+ ) + (-float("inf"))
+
+ # moba attn
+ # moba_attn_lse_hs - [1, num_nonzero_elems]
+ if moba_q.shape[0] > 0:
+ # out, softmax_lse, S_dmask, rng_state
+ moba_attn_out, moba_attn_lse_hs, _, _ = _flash_attn_varlen_forward(
+ q=moba_q,
+ k=moba_kv[:, 0],
+ v=moba_kv[:, 1],
+ cu_seqlens_q=moba_cu_seqlen_q,
+ cu_seqlens_k=moba_cu_seqlen_kv,
+ max_seqlen_q=q_block_seq_len,
+ max_seqlen_k=moba_chunk_size,
+ softmax_scale=softmax_scale,
+ causal=False,
+ dropout_p=0.0,
+ )
+ else:
+ moba_attn_lse_hs = torch.zeros(
+ (1, moba_q.shape[0]), device=q.device, dtype=torch.float32
+ ) + (-float("inf"))
+
+ # -----------------------------------------------------------------------------------
+ # If no queries need to be computed with the current KV chunk and no causal attention is needed, return None to skip the output update
+ if not causal and moba_q.shape[0] == 0:
+ return None, None
+
+ # -----------------------------------------------------------------------------------
+ # Processing output and lse
+ # output buffer [S, H, D], same shape as q
+ output = torch.zeros(
+ (q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32
+ )
+ # flatten vS & H for index ops
+ output_2d = output.view(-1, q.shape[2])
+
+ # --------------------------------------------------
+ moba_attn_lse: torch.Tensor = (
+ moba_attn_lse_hs.t().contiguous()
+ ) # [ num_nonzero_elems, 1 ]
+ self_attn_lse_sh = self_attn_lse_hs.t().contiguous() # [q_S, H]
+
+ # calc mixed_lse
+ # minus max lse to avoid exp explosion
+ max_lse_1d = self_attn_lse_sh.view(-1) # [ vS ]
+ max_lse_1d = max_lse_1d.index_reduce(
+ 0, moba_q_sh_indices, moba_attn_lse.view(-1), "amax"
+ )
+ self_attn_lse_sh = self_attn_lse_sh - max_lse_1d.view_as(self_attn_lse_sh)
+ moba_attn_lse = (
+ moba_attn_lse.view(-1)
+ .sub(max_lse_1d.index_select(0, moba_q_sh_indices))
+ .reshape_as(moba_attn_lse)
+ )
+
+ # --------------------------------------------------
+ # Build mixed attn lse
+ mixed_attn_se_sh = (
+ self_attn_lse_sh.exp() if causal else torch.zeros_like(self_attn_lse_sh)
+ )
+ moba_attn_se = (
+ moba_attn_lse.exp() if moba_q.shape[0] > 0 else torch.zeros_like(moba_attn_lse)
+ )
+
+ # index_add_: converting elements from 1D tensor (num_nonzero_elems) to matrices (HS)
+ # Now, mixed_attn_se_sh is the sum of LSE of self attn and LSE of moba attn (including multiple LSEs corresponding to the same q token but in different HS positions)
+ mixed_attn_se_sh.view(-1).index_add_(0, moba_q_sh_indices, moba_attn_se.view(-1))
+ mixed_attn_lse_sh = mixed_attn_se_sh.log()
+
+ # ----------------------------------------------------
+ # Compute factor of self-attention and add to output
+ if causal:
+ factor = (self_attn_lse_sh - mixed_attn_lse_sh).exp() # [ vS, H ]
+ self_attn_out_sh = self_attn_out_sh * factor.unsqueeze(-1)
+ output_2d += self_attn_out_sh.reshape_as(output_2d)
+
+ # add moba output
+ # ----------------------------------------------------
+ # Compute factor of moba-attention and add to output
+ if moba_q.shape[0] > 0:
+ mixed_attn_lse = (
+ mixed_attn_lse_sh.view(-1)
+ .index_select(0, moba_q_sh_indices)
+ .view_as(moba_attn_lse)
+ )
+ factor = (moba_attn_lse - mixed_attn_lse).exp() # [ vS, H ]
+ moba_attn_out = moba_attn_out * factor.unsqueeze(-1)
+
+ raw_attn_out = moba_attn_out.view(-1, moba_attn_out.shape[-1])
+ output_2d.index_add_(0, moba_q_sh_indices, raw_attn_out.to(output_2d.dtype))
+
+ output = output.to(q.dtype)
+
+ # add back max lse
+ mixed_attn_lse_sh = mixed_attn_lse_sh + max_lse_1d.view_as(mixed_attn_se_sh)
+ return output, mixed_attn_lse_sh.t()
+
+
+def moba_zigzag_attn_fwd(
+ process_group,
+ q: torch.Tensor, # [S, H, D]
+ k: torch.Tensor,
+ v: torch.Tensor,
+ seq_offsets: torch.Tensor, # sequence offsets for Q
+ layer_idx: int,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ moba_chunk_size,
+ moba_topk,
+ softmax_scale,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert causal == True, "zigzag ring is meaningless for causal=False"
+ comm = RingComm(process_group, zigzag=True)
+
+ block_seq_len = q.shape[0] // 2
+ seq_len, num_q_heads, head_dim = q.shape
+
+ out, lse = None, None
+ next_k, next_v = None, None
+
+ kv_seq_offsets = torch.clone(seq_offsets)
+ next_kv_seq_offsets = None
+
+ def fwd_step(
+ q_,
+ k_,
+ v_,
+ step_,
+ causal_,
+ # q_seq_offsets,
+ num_q_blocks,
+ k_seq_offsets,
+ ):
+ return moba_zigzag_attn_fwd_step(
+ q_,
+ k_,
+ v_,
+ step_,
+ causal_,
+ num_q_blocks,
+ k_seq_offsets,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ moba_chunk_size,
+ moba_topk,
+ softmax_scale,
+ dropout_p,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ )
+
+ for step in range(comm.world_size):
+ if step + 1 != comm.world_size:
+ # when step < N-1, do the ring-communication to get KV to be used in the next round
+ next_k, next_v, next_kv_seq_offsets = comm.send_recv_kv_offsets(
+ k, v, kv_seq_offsets
+ )
+
+ if step == 0:
+ # Do softmax(QK^T / sqrt(d_k))V on the currently hold K and V
+ # and record the output and the LSE
+ block_out, block_lse = fwd_step(
+ q,
+ k,
+ v,
+ step,
+ causal_=True,
+ num_q_blocks=2,
+ k_seq_offsets=kv_seq_offsets,
+ )
+
+ out, lse = update_out_and_lse(
+ out,
+ lse,
+ block_out,
+ block_lse,
+ use_triton_kernel=False,
+ )
+ elif step <= comm.revert_rank:
+ k0 = k[:block_seq_len]
+ v0 = v[:block_seq_len]
+ block_out, block_lse = fwd_step(
+ q,
+ k0,
+ v0,
+ step,
+ causal_=False,
+ num_q_blocks=2,
+ k_seq_offsets=kv_seq_offsets[0:1],
+ )
+
+ if block_out is not None:
+ out, lse = update_out_and_lse(
+ out,
+ lse,
+ block_out,
+ block_lse,
+ use_triton_kernel=False,
+ )
+ else:
+ q1 = q[block_seq_len:]
+ block_out, block_lse = fwd_step(
+ q1,
+ k,
+ v,
+ step,
+ causal_=False,
+ num_q_blocks=1,
+ k_seq_offsets=kv_seq_offsets,
+ )
+
+ if block_out is not None:
+ out, lse = update_out_and_lse(
+ out,
+ lse,
+ block_out,
+ block_lse,
+ slice_=(slice(block_seq_len, None)),
+ use_triton_kernel=False,
+ )
+
+ if step + 1 != comm.world_size:
+ comm.wait()
+ k, v, kv_seq_offsets = next_k, next_v, next_kv_seq_offsets
+
+ out = out.to(q.dtype) # [S, H, D]
+ lse = lse.squeeze(dim=-1).transpose(0, 1) # [H, S]
+ return out, lse
+
+
+def moba_zigzag_attn_bwd_step(
+ step: int,
+ dout, # [blk_S, H, D]
+ out, # [blk_S, H, D]
+ causal: bool,
+ q: torch.Tensor, # [blk_S, H, D]
+ k: torch.Tensor, # [blk_S, H, D]
+ v: torch.Tensor, # [blk_S, H, D]
+ softmax_lse: torch.Tensor, # [H, blk_S]
+ num_q_blocks: int,
+ k_seq_offsets: torch.Tensor,
+ layer_idx: int,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch: torch.Tensor,
+ moba_chunk_size: int,
+ moba_topk: int,
+ softmax_scale,
+ dropout_p=0,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+) -> Tuple[torch.Tensor, torch.Tensor, int]:
+ _, _, seq_len = gate_mask.shape
+ q_block_seq_len, num_head, head_dim = q.shape
+ k_block_seq_len, k_num_head, _ = k.shape
+ if num_head > k_num_head:
+ k = torch.repeat_interleave(k, num_head // k_num_head, dim=1)
+ v = torch.repeat_interleave(v, num_head // k_num_head, dim=1)
+
+ block_seq_len = q_block_seq_len // num_q_blocks
+
+ # assumption: block_seq_len is divisible by moba_chunk_size
+ assert (
+ block_seq_len % moba_chunk_size == 0
+ ), "block_seq_len should be divisible by moba_chunk_size"
+
+ # -----------------------------------------------------------------------------------
+ dq = torch.zeros_like(q, dtype=q.dtype)
+ dk = torch.zeros_like(k, dtype=k.dtype)
+ dv = torch.zeros_like(v, dtype=v.dtype)
+
+ kv = torch.stack((k, v), dim=1)
+ dkv = torch.stack((dk, dv), dim=1)
+ # -----------------------------------------------------------------------------------
+
+ k_seq_offset_list = [
+ k_seq_offsets[i].detach().cpu().item() for i in range(len(k_seq_offsets))
+ ]
+ filtered_kv_indices = torch.arange(
+ 0,
+ min(k_seq_offset_list[0] + block_seq_len, num_filtered_chunk * moba_chunk_size)
+ - k_seq_offset_list[0],
+ device=k.device,
+ dtype=torch.int32,
+ )
+ kv_chunk_indices = torch.arange(
+ k_seq_offset_list[0],
+ min(k_seq_offset_list[0] + block_seq_len, num_filtered_chunk * moba_chunk_size),
+ moba_chunk_size,
+ device=k.device,
+ dtype=torch.int32,
+ )
+ if len(k_seq_offset_list) > 1:
+ filtered_kv_indices = torch.cat(
+ [
+ filtered_kv_indices,
+ torch.arange(
+ block_seq_len,
+ min(
+ k_seq_offset_list[1] + block_seq_len,
+ num_filtered_chunk * moba_chunk_size,
+ )
+ - k_seq_offset_list[1]
+ + block_seq_len,
+ device=k.device,
+ dtype=torch.int32,
+ ),
+ ]
+ )
+ kv_chunk_indices = torch.cat(
+ [
+ kv_chunk_indices,
+ torch.arange(
+ k_seq_offset_list[1],
+ min(
+ k_seq_offset_list[1] + block_seq_len,
+ num_filtered_chunk * moba_chunk_size,
+ ),
+ moba_chunk_size,
+ device=k.device,
+ dtype=torch.int32,
+ ),
+ ]
+ )
+ filtered_kv = kv.index_select(0, filtered_kv_indices)
+ filtered_dkv = dkv.index_select(0, filtered_kv_indices)
+
+ kv_chunk_indices = kv_chunk_indices // moba_chunk_size
+ num_filtered_kv_chunks = len(kv_chunk_indices)
+
+ q_indices = torch.arange(
+ 0 if num_q_blocks == 2 else block_seq_len,
+ 2 * block_seq_len,
+ device=q.device,
+ dtype=torch.int32,
+ )
+
+ # varlen trick: combining all q index that needs moba attn
+ # the result will be like [ C0H0 ][ C0H1 ][ C0H2 ][ ... ][ CnHm ]
+ gate_mask_q = gate_mask.index_select(0, kv_chunk_indices)
+ gate_mask_q = gate_mask_q.index_select(
+ 2, q_indices
+ ) # we need to know which part(s) of the two query blocks should be activated
+
+ # equivalent to einops.rearrange(q, "n h s -> n (h s)"). ([s] [s] ... [s] for h times)
+ # [HS indices] * N (total size: all non-zero elements in HS dimension, potentially repeat)
+ # [num_selected_chunks, HS indices of non-zero elements]
+ # gate_mask has been filtered by q_indices. If we still need use gate_mask_q for indexing, it should be offset by block_seq_len if num_q_blocks == 1
+ # + (0 if num_q_blocks == 2 else block_seq_len)
+ moba_q_indices = gate_mask_q.reshape(gate_mask_q.shape[0], -1).nonzero(
+ as_tuple=True
+ )[-1]
+
+ # moba_seqlen_q indicates that how many q chunks are selected for each kv chunk - head
+ # moba_seqlen_q has shape (num_selecte_chunks * num_heads, ) => varlen_forward computes attention by (num_selecte_chunks * num_heads) times
+ moba_seqlen_q = gate_mask_q.sum(dim=-1).flatten()
+
+ # -----------------------------------------------------------
+ # select all q that needs moba attn based on the moba_q_indices
+ moba_q = rearrange(q, "s h d -> ( h s ) d")
+ moba_q = moba_q.index_select(0, moba_q_indices) # [ selected_HS, D ]
+
+ # [ selected_S, 1, D ] (pseudo head dim for flash attn)
+ moba_q = moba_q.unsqueeze(1)
+
+ # moba_q_sh_indices represents the position in the origin q tensor of each q token inside moba_q
+ # note that original q has shape (S, H, D) while moba_q_indices is based on (H S)
+ # Ignoring D, q has the flattend form like [H] [H] ... [H] for S times
+ # => moba_q_sh_indices is the index of each token in the original q tensor
+ moba_q_sh_indices = (
+ moba_q_indices % q_block_seq_len * num_head + moba_q_indices // q_block_seq_len
+ )
+
+ """ prepare moba kv """
+ # Since moba_q is organized as HS * N, we need to reorganize kv to adapt to q
+ # cut off zero experts
+ q_zero_mask = moba_seqlen_q == 0
+ valid_expert_mask = ~q_zero_mask
+ zero_expert_count = q_zero_mask.sum()
+ # only keep the kv that has q select > 0
+ if zero_expert_count > 0:
+ moba_seqlen_q = moba_seqlen_q[valid_expert_mask]
+
+ # moba cu_seqlen for flash attn
+ moba_cu_seqlen_q = torch.cat(
+ (
+ torch.tensor([0], device=q.device, dtype=moba_seqlen_q.dtype),
+ moba_seqlen_q.cumsum(dim=0),
+ ),
+ dim=0,
+ ).to(torch.int32)
+
+ # ------------------------------
+ # Select dout and output
+ d_moba_out = (
+ # [num_non-zero_elements, D]
+ dout.view(-1, head_dim)
+ .index_select(0, moba_q_sh_indices)
+ .unsqueeze(1)
+ )
+ moba_out = (
+ # [num_non-zero_elements, D]
+ out.view(-1, head_dim)
+ .index_select(0, moba_q_sh_indices)
+ .unsqueeze(1)
+ )
+
+ # -----------------------------------------------------------------------------------
+ # here `x` only stands for a dimension (stack dimension for KV)
+ moba_kv = rearrange(filtered_kv, "s x h d -> h s x d") # [H, K_S, 2, D ]
+ moba_dkv = rearrange(filtered_dkv, "s x h d -> h s x d") # [H, K_S, 2, D ]
+
+ moba_kv = moba_kv.split(
+ moba_chunk_size, dim=1
+ ) # tuple of (num_selected_chunks) elements with shape [H, chunk_size, 2, D]
+ moba_kv = torch.cat(
+ moba_kv, dim=0
+ ) # [H x num_selected_chunks, chunk_size, 2, D ] after split
+ moba_dkv = torch.cat(
+ moba_dkv.split(moba_chunk_size, dim=1), dim=0
+ ) # [H x num_selected_chunks, chunk_size, 2, D ] after split
+
+ # The transformation is aimed for masking out by valid_expert_mask where the mask selects elements along (H x num_selected_chunks) dimension
+ if zero_expert_count > 0:
+ assert valid_expert_mask.sum() == moba_kv.shape[0] - zero_expert_count
+
+ # cut off zero Q expert from kv , or the grad may be nan
+ moba_kv = moba_kv[valid_expert_mask]
+ moba_dkv = moba_dkv[valid_expert_mask]
+
+ moba_kv = moba_kv.flatten(start_dim=0, end_dim=1).unsqueeze(
+ 2
+ ) # [H x num_selected_chunks x chunk_size, 2, 1, D]
+ moba_dkv = moba_dkv.flatten(start_dim=0, end_dim=1).unsqueeze(
+ 2
+ ) # [H x num_selected_chunks x chunk_size, 2, 1, D]
+
+ moba_cu_seqlen_kv = (
+ torch.arange(
+ 0,
+ num_filtered_kv_chunks * num_head + 1 - zero_expert_count,
+ dtype=torch.int32,
+ device=q.device,
+ )
+ * moba_chunk_size
+ )
+
+ # Shape check
+ assert (
+ moba_cu_seqlen_kv.shape == moba_cu_seqlen_q.shape
+ ), f"moba_cu_seqlen_kv.shape != moba_cu_seqlen_q.shape {moba_cu_seqlen_kv.shape} != {moba_cu_seqlen_q.shape}"
+
+ self_attn_cu_seqlen = [0] + [moba_chunk_size] * (q_block_seq_len // moba_chunk_size)
+ if q_block_seq_len % moba_chunk_size != 0:
+ self_attn_cu_seqlen.append(q_block_seq_len % moba_chunk_size)
+ self_attn_cu_seqlen = torch.tensor(
+ self_attn_cu_seqlen, device=q.device, dtype=torch.int32
+ )
+ self_attn_cu_seqlen = self_attn_cu_seqlen.cumsum(dim=0, dtype=torch.int32)
+
+ # -----------------------------------------------------------------------------------
+ # self attn
+ if causal:
+ dq_, dk_, dv_ = (
+ torch.empty_like(dq),
+ torch.empty_like(dkv[:, 0]),
+ torch.empty_like(dkv[:, 1]),
+ )
+ _flash_attn_varlen_backward(
+ dout=dout,
+ out=out,
+ q=q,
+ k=k,
+ v=v,
+ dq=dq_,
+ dk=dk_,
+ dv=dv_,
+ softmax_lse=softmax_lse.contiguous(),
+ cu_seqlens_q=self_attn_cu_seqlen,
+ cu_seqlens_k=self_attn_cu_seqlen,
+ max_seqlen_q=q_block_seq_len,
+ max_seqlen_k=k_block_seq_len,
+ softmax_scale=softmax_scale,
+ causal=True,
+ dropout_p=0.0,
+ window_size_left=window_size[0],
+ window_size_right=window_size[1],
+ alibi_slopes=alibi_slopes,
+ deterministic=deterministic,
+ softcap=0.0,
+ )
+ dq, dkv[:, 0], dkv[:, 1] = dq + dq_, dk_ + dkv[:, 0], dv_ + dkv[:, 1]
+
+ if moba_q.shape[0] > 0:
+ softmax_lse_sh = rearrange(softmax_lse.contiguous(), "h s -> (s h)")
+ moba_attn_lse = (
+ # [1, num_non-zero_elements]
+ softmax_lse_sh.index_select(0, moba_q_sh_indices).view(1, -1)
+ )
+
+ moba_dq_, moba_dk_, moba_dv_ = (
+ torch.empty_like(moba_q),
+ torch.empty_like(moba_kv[:, 0]),
+ torch.empty_like(moba_kv[:, 1]),
+ )
+ _flash_attn_varlen_backward(
+ dout=d_moba_out,
+ out=moba_out,
+ q=moba_q,
+ k=moba_kv[:, 0],
+ v=moba_kv[:, 1],
+ dq=moba_dq_,
+ dk=moba_dk_,
+ dv=moba_dv_,
+ softmax_lse=moba_attn_lse,
+ cu_seqlens_q=moba_cu_seqlen_q,
+ cu_seqlens_k=moba_cu_seqlen_kv,
+ max_seqlen_q=q_block_seq_len,
+ max_seqlen_k=moba_chunk_size,
+ softmax_scale=softmax_scale,
+ causal=False,
+ dropout_p=0.0,
+ window_size_left=window_size[0],
+ window_size_right=window_size[1],
+ alibi_slopes=alibi_slopes,
+ deterministic=deterministic,
+ softcap=0.0,
+ )
+
+ dq.view(-1, q.shape[-1]).index_add_(
+ 0, moba_q_sh_indices, moba_dq_.view(-1, head_dim).to(dq.dtype)
+ )
+ moba_dkv[:, 0] = moba_dkv[:, 0] + moba_dk_
+ moba_dkv[:, 1] = moba_dkv[:, 1] + moba_dv_
+
+ # ------------------------------------------------------------------------------------
+ # Backpropagate moba_dkv to dk and dv
+ moba_dkv = moba_dkv.squeeze(2) # [H x num_selected_chunks x chunk_size, 2, D]
+ moba_dkv = moba_dkv.unflatten(
+ 0, (-1, moba_chunk_size)
+ ) # [H x num_selected_chunks, chunk_size, 2, D]
+
+ if zero_expert_count > 0:
+ full_moba_dkv = torch.zeros(
+ (moba_dkv.shape[0] + zero_expert_count, moba_chunk_size, 2, head_dim),
+ dtype=moba_dkv.dtype,
+ device=moba_dkv.device,
+ )
+ full_moba_dkv[valid_expert_mask] = moba_dkv
+ moba_dkv = full_moba_dkv # [H x num_selected_chunks, chunk_size, 2, D]
+ moba_dkv = moba_dkv.split(
+ num_head, dim=0
+ ) # [H, num_selected_chunks, chunk_size, 2, D]
+ moba_dkv = torch.cat(
+ moba_dkv, dim=1
+ ) # [H, num_selected_chunks x chunk_size, 2, D]
+
+ filtered_dkv = rearrange(moba_dkv, "h s x d -> s x h d")
+ dkv.index_add_(0, filtered_kv_indices, filtered_dkv) # [K_S, 2, H, D]
+
+ if num_head > k_num_head:
+ num_kv_replicas = num_head // k_num_head
+ dkv_reshaped = dkv.view(-1, 2, k_num_head, num_kv_replicas, head_dim)
+ dkv = dkv_reshaped.sum(dim=3)
+
+ return dq, dkv[:, 0], dkv[:, 1]
+
+
+def moba_zigzag_attn_bwd(
+ process_group,
+ dout, # [blk_S, H, D]
+ q: torch.Tensor, # [blk_S, H, D]
+ k: torch.Tensor, # [blk_S, H, D]
+ v: torch.Tensor, # [blk_S, H, D]
+ out, # [blk_S, H, D]
+ softmax_lse, # [H, blk_S]
+ seq_offsets: torch.Tensor, # sequence offsets for Q
+ layer_idx: int,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ moba_chunk_size,
+ moba_topk,
+ softmax_scale,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert causal == True, "zigzag ring is meaningless for causal=False"
+
+ kv_comm = RingComm(process_group, zigzag=True)
+ d_kv_comm = RingComm(process_group, zigzag=True)
+
+ kv_seq_offsets = torch.clone(seq_offsets)
+ seq_len, num_q_heads, head_dim = q.shape
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ dout1 = dout.chunk(2, dim=0)[1]
+ q1 = q.chunk(2, dim=0)[1]
+ out1 = out.chunk(2, dim=0)[1]
+ softmax_lse1 = softmax_lse.chunk(2, dim=1)[1].contiguous()
+ block_seq_len = q.shape[0] // 2
+
+ def backward(step, dout_, q_, k_, v_, out_, k_seq_offsets, softmax_lse_, causal):
+ seqlen_q = q_.shape[0]
+ seqlen_kv = k_.shape[0]
+
+ params = get_default_args(moba_zigzag_attn_bwd_step).copy()
+ params.update(
+ {
+ "step": step,
+ "causal": causal,
+ "dout": dout_,
+ "out": out_,
+ "q": q_,
+ "k": k_,
+ "v": v_,
+ "softmax_lse": softmax_lse_,
+ "num_q_blocks": 1 if seqlen_q == block_seq_len else 2,
+ "k_seq_offsets": k_seq_offsets,
+ "layer_idx": layer_idx,
+ "gate_mask": gate_mask,
+ "cu_chunk": cu_chunk,
+ "filtered_chunk_indices": filtered_chunk_indices,
+ "num_filtered_chunk": num_filtered_chunk,
+ "chunk_to_batch": chunk_to_batch,
+ "moba_chunk_size": moba_chunk_size,
+ "moba_topk": moba_topk,
+ "dropout_p": dropout_p,
+ "softmax_scale": softmax_scale,
+ "alibi_slopes": alibi_slopes,
+ "deterministic": deterministic,
+ }
+ )
+ if "window_size" in params:
+ params.update({"window_size": window_size})
+ else:
+ params.update(
+ {
+ "window_size_left": window_size[0],
+ "window_size_right": window_size[1],
+ }
+ )
+ return moba_zigzag_attn_bwd_step(**params)
+
+ for step in range(kv_comm.world_size):
+ if step + 1 != kv_comm.world_size:
+ # next_k, next_v = kv_comm.send_recv_kv(k, v)
+ next_k, next_v, next_kv_seq_offsets = kv_comm.send_recv_kv_offsets(
+ k, v, kv_seq_offsets
+ )
+
+ if step == 0:
+ dq_buffer, dk_buffer, dv_buffer = backward(
+ step, dout, q, k, v, out, kv_seq_offsets, softmax_lse, causal=True
+ )
+ dq = dq_buffer.to(torch.float32)
+ dk = dk_buffer.to(torch.float32)
+ dv = dv_buffer.to(torch.float32)
+ else:
+ if step <= kv_comm.revert_rank:
+ k0 = k[:block_seq_len]
+ v0 = v[:block_seq_len]
+ dq_buffer, dk_buffer, dv_buffer = backward(
+ step,
+ dout,
+ q,
+ k0,
+ v0,
+ out,
+ kv_seq_offsets[0:1],
+ softmax_lse,
+ causal=False,
+ )
+ dq += dq_buffer
+ else:
+ dq_buffer, dk_buffer, dv_buffer = backward(
+ step,
+ dout1,
+ q1,
+ k,
+ v,
+ out1,
+ kv_seq_offsets,
+ softmax_lse1,
+ causal=False,
+ )
+
+ # use the first half in dq_buffer.
+ dq[block_seq_len:] += dq_buffer
+
+ d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ if step <= kv_comm.revert_rank:
+ dk[:block_seq_len] += dk_buffer
+ dv[:block_seq_len] += dv_buffer
+ else:
+ dk += dk_buffer
+ dv += dv_buffer
+
+ if step + 1 != kv_comm.world_size:
+ kv_comm.wait()
+ k, v, kv_seq_offsets = next_k, next_v, next_kv_seq_offsets
+
+ # the finally received dk and dv will be the same as the first dk and dv (corresponding to local Q)
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+"""
+In nnscaler, sequence are stored in the initial order, e.g., [0 1 2 3 4 5 6 7].
+However, zigzag ring flash attention requires the sequence to be in the order of [0 7 2 5 3 4 1 6].
+As a result:
+- in forward, we need to shuffle q, k, v and recover the out
+- in backward, we need to shuffle dout and recover the dq, dk, dv
+"""
+
+
+class MoBAZigzagRingFlashAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q: torch.Tensor, # [batch * seq_block_len, n_heads, head_dim]
+ k: torch.Tensor,
+ v: torch.Tensor,
+ seq_offset: torch.Tensor,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ cu_seqlens,
+ moba_chunk_size,
+ moba_topk,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_softmax,
+ group,
+ ):
+ # Note seq_len here refers to the total sequence length
+ seq_lens = cu_seqlens[1:] - cu_seqlens[:-1]
+ assert (
+ seq_lens.min() == seq_lens.max()
+ ), "Current implementation of MoBA Zigzag Ring Attention does not support variable sequence lengths within a batch"
+ seq_len = (
+ seq_lens.detach().cpu()[0].item()
+ ) # all sequences in the batch have the same length
+
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+ assert alibi_slopes is None
+
+ # ---------------------------
+ # Compute gate values before shuffling
+ (
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ ) = compute_moba_gate(
+ q,
+ k,
+ v,
+ seq_offset,
+ cu_seqlens,
+ moba_chunk_size,
+ moba_topk,
+ )
+
+ q, seq_offsets, gate_mask = shuffle_input_all(
+ to_send=q, gate_mask=gate_mask, seq_offset=seq_offset, process_group=group
+ )
+ k = shuffle_input_only(to_send=k, process_group=group)
+ v = shuffle_input_only(to_send=v, process_group=group)
+ k = k.contiguous()
+ v = v.contiguous()
+
+ q_3d, k_3d, v_3d = tensor_4d_to_3d(q), tensor_4d_to_3d(k), tensor_4d_to_3d(v)
+
+ out_3d, softmax_lse = moba_zigzag_attn_fwd(
+ group,
+ q_3d,
+ k_3d,
+ v_3d,
+ seq_offsets, # sequence offsets for Q
+ layer_idx,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ moba_chunk_size,
+ moba_topk,
+ softmax_scale=softmax_scale,
+ dropout_p=dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=False,
+ )
+
+ out = out_3d.reshape(*q.shape)
+ ctx.save_for_backward(
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ seq_offsets,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ chunk_to_batch,
+ )
+ ctx.num_filtered_chunk = num_filtered_chunk
+ ctx.moba_chunk_size = moba_chunk_size
+ ctx.moba_topk = moba_topk
+
+ ctx.dropout_p = dropout_p
+ ctx.softmax_scale = softmax_scale
+ ctx.causal = causal
+ ctx.window_size = window_size
+ ctx.alibi_slopes = alibi_slopes
+ ctx.deterministic = deterministic
+ ctx.group = group
+ ctx.layer_idx = layer_idx
+ ctx.seq_len = seq_len
+
+ out = recover_zigzag_output(out, dim=1, process_group=group)
+ return out if not return_softmax else (out, softmax_lse, None)
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ (
+ q,
+ k,
+ v,
+ out,
+ softmax_lse, # [n_heads, seq_block_len]
+ seq_offsets,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ chunk_to_batch,
+ ) = ctx.saved_tensors
+
+ q_3d, k_3d, v_3d, out_3d = (
+ tensor_4d_to_3d(q),
+ tensor_4d_to_3d(k),
+ tensor_4d_to_3d(v),
+ tensor_4d_to_3d(out),
+ )
+
+ dout = shuffle_input_only(to_send=dout, process_group=ctx.group)
+ dout_3d = tensor_4d_to_3d(dout)
+
+ num_filtered_chunk = ctx.num_filtered_chunk
+ moba_chunk_size = ctx.moba_chunk_size
+ moba_topk = ctx.moba_topk
+
+ dq_3d, dk_3d, dv_3d = moba_zigzag_attn_bwd(
+ ctx.group,
+ dout_3d,
+ q_3d,
+ k_3d,
+ v_3d,
+ out_3d,
+ softmax_lse,
+ seq_offsets,
+ ctx.layer_idx,
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ moba_chunk_size,
+ moba_topk,
+ softmax_scale=ctx.softmax_scale,
+ dropout_p=ctx.dropout_p,
+ causal=ctx.causal,
+ window_size=ctx.window_size,
+ alibi_slopes=ctx.alibi_slopes,
+ deterministic=ctx.deterministic,
+ )
+
+ dq, dk, dv = (
+ dq_3d.reshape(*q.shape),
+ dk_3d.reshape(*k.shape),
+ dv_3d.reshape(*v.shape),
+ )
+
+ dq = recover_zigzag_output(dq, dim=1, process_group=ctx.group)
+ dk = recover_zigzag_output(dk, dim=1, process_group=ctx.group)
+ dv = recover_zigzag_output(dv, dim=1, process_group=ctx.group)
+ return (
+ dq,
+ dk,
+ dv,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ )
+
+
+def moba_zigzag_qkvpacked_func(
+ qkv,
+ seq_offset: torch.Tensor,
+ layer_idx: int,
+ cu_seqlens,
+ moba_chunk_size,
+ moba_topk,
+ dropout_p=0.0,
+ softmax_scale=None,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return MoBAZigzagRingFlashAttnFunc.apply(
+ qkv[:, :, 0],
+ qkv[:, :, 1],
+ qkv[:, :, 2],
+ seq_offset,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ cu_seqlens,
+ moba_chunk_size,
+ moba_topk,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def moba_zigzag_kvpacked_func(
+ q,
+ kv,
+ seq_offset: torch.Tensor,
+ layer_idx: int,
+ cu_seqlens,
+ moba_chunk_size,
+ moba_topk,
+ dropout_p=0.0,
+ softmax_scale=None,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return MoBAZigzagRingFlashAttnFunc.apply(
+ q,
+ kv[:, :, 0],
+ kv[:, :, 1],
+ seq_offset,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ cu_seqlens,
+ moba_chunk_size,
+ moba_topk,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def moba_zigzag_func(
+ q,
+ k,
+ v, # [batch_size, seq_block_len, n_heads, head_dim]
+ layer_idx: int,
+ global_seq_len: int,
+ moba_chunk_size,
+ moba_topk,
+ dropout_p=0.0,
+ softmax_scale=None,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ batch_size = q.shape[0]
+ cu_seqlens = torch.cumsum(
+ torch.tensor([0] + [global_seq_len] * batch_size, device=q.device),
+ dim=0,
+ dtype=torch.int32,
+ )
+
+ rank = dist.get_rank()
+ world_size = dist.get_world_size()
+ seq_offset = torch.arange(0, global_seq_len, global_seq_len // world_size)[
+ rank : rank + 1
+ ]
+
+ return MoBAZigzagRingFlashAttnFunc.apply(
+ q,
+ k,
+ v,
+ seq_offset,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ cu_seqlens,
+ moba_chunk_size,
+ moba_topk,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
diff --git a/minference/dist_ops/ring_attention.py b/minference/dist_ops/ring_attention.py
new file mode 100644
index 00000000..f0b06322
--- /dev/null
+++ b/minference/dist_ops/ring_attention.py
@@ -0,0 +1,272 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# TODO: replace with zhuzilin's implementation
+
+import torch
+import torch.distributed as dist
+from flash_attn.flash_attn_interface import _flash_attn_backward, _flash_attn_forward
+
+from .utils import GlobalMemoryBuffer, recover_zigzag_output, shuffle_zigzag_input
+
+_GLOBAL_MEMORY_BUFFER = GlobalMemoryBuffer()
+
+
+def ring_flash_attn_forward(
+ process_group,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ softmax_scale,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ block_len = q.size(1) // 2
+ curr_rank = dist.get_rank(process_group)
+ world_size = dist.get_world_size(process_group)
+ keep_idx = 2 * curr_rank
+ dual_rank = world_size - curr_rank - 1
+ dual_send_idx = 2 * dual_rank + 1
+ up_rank = min(keep_idx, dual_send_idx)
+ down_rank = max(keep_idx, dual_send_idx)
+
+ up_q = q[:, :block_len]
+ if causal:
+ up_k = k[:, : (up_rank + 1) * block_len]
+ up_v = v[:, : (up_rank + 1) * block_len]
+ else:
+ up_k, up_v = k, v
+ up_out, _, _, _, _, up_lse, _, _ = _flash_attn_forward(
+ up_q,
+ up_k,
+ up_v,
+ dropout_p,
+ softmax_scale,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ return_softmax=True and dropout_p > 0,
+ )
+
+ down_q = q[:, block_len:]
+ if causal:
+ down_k = k[:, : (down_rank + 1) * block_len]
+ down_v = v[:, : (down_rank + 1) * block_len]
+ else:
+ down_k, down_v = k, v
+ down_out, _, _, _, _, down_lse, _, _ = _flash_attn_forward(
+ down_q,
+ down_k,
+ down_v,
+ dropout_p,
+ softmax_scale,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ return_softmax=True and dropout_p > 0,
+ )
+
+ out = torch.cat([up_out, down_out], dim=1)
+ return out, up_lse, down_lse
+
+
+def ring_flash_attn_backward(
+ process_group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ up_lse,
+ down_lse,
+ softmax_scale,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ block_len = q.size(1) // 2
+ curr_rank = dist.get_rank(process_group)
+ world_size = dist.get_world_size(process_group)
+ keep_idx = 2 * curr_rank
+ dual_rank = world_size - curr_rank - 1
+ dual_send_idx = 2 * dual_rank + 1
+ up_rank = min(keep_idx, dual_send_idx)
+ down_rank = max(keep_idx, dual_send_idx)
+
+ dq = torch.zeros_like(q)
+ dk_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(k.size(), k.dtype, "bwd_dk")
+ dk_buffer.zero_()
+ dv_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(v.size(), v.dtype, "bwd_dv")
+ dv_buffer.zero_()
+
+ up_q = q[:, :block_len]
+ up_out = out[:, :block_len]
+ up_dout = dout[:, :block_len]
+ if causal:
+ up_k = k[:, : (up_rank + 1) * block_len]
+ up_v = v[:, : (up_rank + 1) * block_len]
+ else:
+ up_k, up_v = k, v
+ _flash_attn_backward(
+ up_dout,
+ up_q,
+ up_k,
+ up_v,
+ up_out,
+ up_lse,
+ dq[:, :block_len],
+ dk_buffer[:, : (up_rank + 1) * block_len],
+ dv_buffer[:, : (up_rank + 1) * block_len],
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ rng_state=None,
+ )
+
+ down_q = q[:, block_len:]
+ down_out = out[:, block_len:]
+ down_dout = dout[:, block_len:]
+ # TODO: optimize the buffer allocation
+ down_dk_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(k.size(), k.dtype, "bwd_down_dk")
+ down_dk_buffer.zero_()
+ down_dv_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(v.size(), v.dtype, "bwd_down_dv")
+ down_dv_buffer.zero_()
+ if causal:
+ down_k = k[:, : (down_rank + 1) * block_len]
+ down_v = v[:, : (down_rank + 1) * block_len]
+ else:
+ down_k, down_v = k, v
+ _flash_attn_backward(
+ down_dout,
+ down_q,
+ down_k,
+ down_v,
+ down_out,
+ down_lse,
+ dq[:, block_len:],
+ down_dk_buffer[:, : (down_rank + 1) * block_len],
+ down_dv_buffer[:, : (down_rank + 1) * block_len],
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ rng_state=None,
+ )
+ dk_buffer.add_(down_dk_buffer)
+ dv_buffer.add_(down_dv_buffer)
+
+ dim_size = list(k.size())
+ dim_size[1] = dim_size[1] // world_size
+ dk = torch.empty(dim_size, dtype=k.dtype, device=k.device)
+ dv = torch.empty(dim_size, dtype=v.dtype, device=v.device)
+ dist._reduce_scatter_base(dk, dk_buffer, group=process_group)
+ dist._reduce_scatter_base(dv, dv_buffer, group=process_group)
+
+ return dq, dk, dv
+
+
+"""
+In nnscaler, sequence are stored in the initial order, e.g., [0 1 2 3 4 5 6 7].
+However, ring flash attention requires the sequence to be in the order of [0 7 2 5 3 4 1 6].
+As a result:
+- in forward, we need to shuffle q, all gather k, v and recover the out
+- in backward, we need to shuffle dout and recover the dq, reduce scatter dk, dv
+"""
+
+
+class RingFlashAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_softmax,
+ group,
+ ):
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+ assert alibi_slopes is None
+
+ q = shuffle_zigzag_input(to_send=q, process_group=group)
+ world_size = dist.get_world_size(group)
+ dim_size = list(k.size())
+ dim_size[1] = dim_size[1] * world_size
+ k_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(dim_size, k.dtype, "fwd_k")
+ v_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(dim_size, v.dtype, "fwd_v")
+ torch.distributed._all_gather_base(k_buffer, k, group=group)
+ torch.distributed._all_gather_base(v_buffer, v, group=group)
+
+ out, up_lse, down_lse = ring_flash_attn_forward(
+ group,
+ q,
+ k_buffer,
+ v_buffer,
+ softmax_scale=softmax_scale,
+ dropout_p=dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=False,
+ )
+ # this should be out_padded
+ ctx.save_for_backward(q, k, v, out, up_lse, down_lse)
+ ctx.dropout_p = dropout_p
+ ctx.softmax_scale = softmax_scale
+ ctx.causal = causal
+ ctx.window_size = window_size
+ ctx.alibi_slopes = alibi_slopes
+ ctx.deterministic = deterministic
+ ctx.group = group
+ out = recover_zigzag_output(out, process_group=group)
+ return out if not return_softmax else (out, softmax_lse, None)
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ dout = shuffle_zigzag_input(to_send=dout, process_group=ctx.group)
+ q, k, v, out, up_lse, down_lse = ctx.saved_tensors
+ world_size = dist.get_world_size(ctx.group)
+ dim_size = list(k.size())
+ dim_size[1] = dim_size[1] * world_size
+ k_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(dim_size, k.dtype, "fwd_k")
+ v_buffer = _GLOBAL_MEMORY_BUFFER.get_tensor(dim_size, v.dtype, "fwd_v")
+ torch.distributed._all_gather_base(k_buffer, k, group=ctx.group)
+ torch.distributed._all_gather_base(v_buffer, v, group=ctx.group)
+
+ dq, dk, dv = ring_flash_attn_backward(
+ ctx.group,
+ dout,
+ q,
+ k_buffer,
+ v_buffer,
+ out,
+ up_lse,
+ down_lse,
+ softmax_scale=ctx.softmax_scale,
+ dropout_p=ctx.dropout_p,
+ causal=ctx.causal,
+ window_size=ctx.window_size,
+ alibi_slopes=ctx.alibi_slopes,
+ deterministic=ctx.deterministic,
+ )
+ dq = recover_zigzag_output(dq, ctx.group)
+ return dq, dk, dv, None, None, None, None, None, None, None, None
diff --git a/minference/dist_ops/striped_attention.py b/minference/dist_ops/striped_attention.py
new file mode 100644
index 00000000..fd8cf805
--- /dev/null
+++ b/minference/dist_ops/striped_attention.py
@@ -0,0 +1,469 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from typing import Dict, List, Tuple
+
+import torch
+import torch.distributed as dist
+from flash_attn.flash_attn_interface import _flash_attn_backward, _flash_attn_forward
+
+from .utils import (
+ RingComm,
+ get_default_args,
+ recover_striped_output,
+ shuffle_striped_input,
+ update_out_and_lse,
+)
+
+
+def stripe_flash_attn_forward(
+ process_group,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer_idx: int,
+ softmax_scale,
+ granularity=1,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert (
+ causal
+ ), "stripe flash attn only supports causal attention, if not causal, use ring flash attn instead"
+ comm = RingComm(process_group)
+ bsz, seq_len, num_heads, head_dim = q.shape
+
+ out, lse = None, None
+ next_k, next_v = None, None
+
+ def forward(q_, k_, v_, causal_):
+ params = get_default_args(_flash_attn_forward).copy()
+ params.update(
+ {
+ "q": q_,
+ "k": k_,
+ "v": v_,
+ "dropout_p": dropout_p,
+ "softmax_scale": softmax_scale,
+ "causal": causal_,
+ "alibi_slopes": alibi_slopes,
+ "return_softmax": True and dropout_p > 0,
+ }
+ )
+ if "window_size" in params:
+ params.update({"window_size": window_size})
+ else:
+ params.update(
+ {
+ "window_size_left": window_size[0],
+ "window_size_right": window_size[1],
+ }
+ )
+ outputs = _flash_attn_forward(**params)
+ if len(outputs) == 8:
+ block_out, _, _, _, _, block_lse, _, _ = outputs
+ else:
+ assert len(outputs) == 4
+ block_out, block_lse, _, _ = outputs
+ return block_out, block_lse
+
+ for step in range(comm.world_size):
+ if step + 1 != comm.world_size:
+ next_k, next_v = comm.send_recv_kv(k, v)
+
+ shift = 1 if step > comm.rank else 0
+ if shift == 0:
+ step_out, step_lse = forward(q, k, v, causal)
+ out, lse = update_out_and_lse(out, lse, step_out, step_lse)
+ else:
+ # Before the step index goes beyond the current rank, the received KV indices are not greater than those of the Q in the current rank
+ # After the step index goes beyond the current rank, only the KV indices before the last granularity are no greater than those of the Q after the first granularity
+ # this conclusion holds after the step index goes beyond the current rank (not just step index == current rank)
+ step_out, step_lse = forward(
+ q[:, granularity:], k[:, :-granularity], v[:, :-granularity], causal
+ )
+ out, lse = update_out_and_lse(
+ out,
+ lse,
+ step_out,
+ step_lse,
+ slice_=(slice(None), slice(granularity, None)),
+ )
+
+ if step + 1 != comm.world_size:
+ comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ lse = lse.squeeze(dim=-1).transpose(1, 2)
+ return out, lse
+
+
+def stripe_flash_attn_backward(
+ process_group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx: int,
+ softmax_scale,
+ granularity=1,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert (
+ causal
+ ), "stripe flash attn only supports causal attention, if not causal, ring flash attn instead"
+
+ kv_comm = RingComm(process_group)
+ d_kv_comm = RingComm(process_group)
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ block_dq_buffer = torch.empty(q.shape, dtype=q.dtype, device=q.device)
+ block_dk_buffer = torch.empty(k.shape, dtype=k.dtype, device=k.device)
+ block_dv_buffer = torch.empty(v.shape, dtype=v.dtype, device=v.device)
+
+ def backward(
+ granularity_,
+ ):
+ if granularity_ == 0:
+ k_, v_ = k, v
+ dk_, dv_ = block_dk_buffer, block_dv_buffer
+ else:
+ k_, v_ = k[:, :-granularity_], v[:, :-granularity_]
+ dk_, dv_ = (
+ block_dk_buffer[:, :-granularity_],
+ block_dv_buffer[:, :-granularity_],
+ )
+ params = get_default_args(_flash_attn_backward).copy()
+ params.update(
+ {
+ "dout": dout[:, granularity_:],
+ "q": q[:, granularity_:],
+ "k": k_,
+ "v": v_,
+ "out": out[:, granularity_:],
+ "softmax_lse": softmax_lse[:, :, granularity_:].contiguous(),
+ "dq": block_dq_buffer[:, granularity_:],
+ "dk": dk_,
+ "dv": dv_,
+ "dropout_p": dropout_p,
+ "softmax_scale": softmax_scale,
+ "causal": causal,
+ "alibi_slopes": alibi_slopes,
+ "deterministic": deterministic,
+ }
+ )
+ if "window_size" in params:
+ params.update({"window_size": window_size})
+ else:
+ params.update(
+ {
+ "window_size_left": window_size[0],
+ "window_size_right": window_size[1],
+ }
+ )
+ params.update(
+ {"rng_state": torch.zeros((2,), dtype=torch.int64, device=q.device)}
+ )
+ _flash_attn_backward(**params)
+
+ for step in range(kv_comm.world_size):
+ if step + 1 != kv_comm.world_size:
+ next_k, next_v = kv_comm.send_recv_kv(k, v)
+
+ shift_causal = 1 if step > kv_comm.rank else 0
+ if shift_causal == 0:
+ backward(granularity_=0)
+ else:
+ backward(granularity_=granularity)
+
+ if dq is None:
+ dq = block_dq_buffer.to(torch.float32)
+ dk = block_dk_buffer.to(torch.float32)
+ dv = block_dv_buffer.to(torch.float32)
+ else:
+ d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ if shift_causal == 0:
+ dq += block_dq_buffer
+ dk = block_dk_buffer + dk
+ dv = block_dv_buffer + dv
+ else:
+ dq[:, granularity:] += block_dq_buffer[:, granularity:]
+ dk[:, :-granularity] += block_dk_buffer[:, :-granularity]
+ dv[:, :-granularity] += block_dv_buffer[:, :-granularity]
+
+ if step + 1 != kv_comm.world_size:
+ kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+class StripeFlashAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_softmax,
+ group,
+ ):
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+ assert alibi_slopes is None
+
+ # -----------------------------------------
+ # Shuffle
+ q = shuffle_striped_input(
+ to_send=q, dim=1, granularity=granularity, process_group=group
+ )
+ k = shuffle_striped_input(
+ to_send=k, dim=1, granularity=granularity, process_group=group
+ )
+ v = shuffle_striped_input(
+ to_send=v, dim=1, granularity=granularity, process_group=group
+ )
+ k, v = k.contiguous(), v.contiguous()
+
+ # ----------------------------------------------
+ # Compute
+ out, softmax_lse = stripe_flash_attn_forward(
+ group,
+ q,
+ k,
+ v,
+ layer_idx,
+ softmax_scale=softmax_scale,
+ granularity=granularity,
+ dropout_p=dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=False,
+ )
+
+ # ----------------------------------------------
+ # Recover outputs
+ recovered_out = recover_striped_output(
+ out, dim=1, granularity=granularity, process_group=group
+ )
+ if return_softmax:
+ recovered_softmax_lse = recover_striped_output(
+ softmax_lse, dim=2, granularity=granularity, process_group=group
+ )
+
+ # ----------------------------------------------
+ # this should be out_padded
+ ctx.save_for_backward(q, k, v, out, softmax_lse)
+ ctx.dropout_p = dropout_p
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.causal = causal
+ ctx.window_size = window_size
+ ctx.alibi_slopes = alibi_slopes
+ ctx.deterministic = deterministic
+ ctx.layer_idx = layer_idx
+ ctx.return_softmax = return_softmax
+ ctx.group = group
+
+ # ----------------------------------------------
+ # Output and return
+ if return_softmax:
+ return (recovered_out, recovered_softmax_lse, None)
+ return recovered_out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ q, k, v, out, softmax_lse = ctx.saved_tensors
+ layer_idx = ctx.layer_idx
+ (
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_softmax,
+ group,
+ ) = (
+ ctx.dropout_p,
+ ctx.softmax_scale,
+ ctx.causal,
+ ctx.window_size,
+ ctx.alibi_slopes,
+ ctx.deterministic,
+ ctx.return_softmax,
+ ctx.group,
+ )
+
+ # ----------------------------------------------
+ # Shuffle
+ dout = shuffle_striped_input(
+ to_send=dout, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+
+ # ----------------------------------------------
+ # Compute
+ dq, dk, dv = stripe_flash_attn_backward(
+ ctx.group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx=layer_idx,
+ softmax_scale=ctx.softmax_scale,
+ granularity=ctx.granularity,
+ dropout_p=ctx.dropout_p,
+ causal=ctx.causal,
+ window_size=ctx.window_size,
+ alibi_slopes=ctx.alibi_slopes,
+ deterministic=ctx.deterministic,
+ )
+
+ # ----------------------------------------------
+ # Recover
+ dq = recover_striped_output(
+ dq, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dk = recover_striped_output(
+ dk, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ dv = recover_striped_output(
+ dv, dim=1, granularity=ctx.granularity, process_group=ctx.group
+ )
+ return (
+ dq,
+ dk,
+ dv,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ )
+
+
+def stripe_flash_attn_qkvpacked_func(
+ qkv, # [B, N, 3, H, D]
+ dropout_p=0.0,
+ softmax_scale=None,
+ granularity=1,
+ causal=False,
+ window_size=(-1, -1), # -1 means infinite context window
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return StripeFlashAttnFunc.apply(
+ qkv[:, :, 0],
+ qkv[:, :, 1],
+ qkv[:, :, 2],
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def stripe_flash_attn_kvpacked_func(
+ q,
+ kv,
+ dropout_p=0.0,
+ softmax_scale=None,
+ granularity=1,
+ causal=False,
+ window_size=(-1, -1), # -1 means infinite context window
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return StripeFlashAttnFunc.apply(
+ q,
+ kv[:, :, 0],
+ kv[:, :, 1],
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def stripe_flash_attn_func(
+ q,
+ k,
+ v,
+ layer_idx: int,
+ dropout_p=0.0,
+ softmax_scale=None,
+ granularity=1,
+ causal=False,
+ window_size=(-1, -1), # -1 means infinite context window
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return StripeFlashAttnFunc.apply(
+ q,
+ k,
+ v,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
diff --git a/minference/dist_ops/test/minfer_ring_test.py b/minference/dist_ops/test/minfer_ring_test.py
new file mode 100644
index 00000000..a59e58be
--- /dev/null
+++ b/minference/dist_ops/test/minfer_ring_test.py
@@ -0,0 +1,173 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from __future__ import annotations
+
+import os
+import random
+from types import SimpleNamespace
+from typing import Callable
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+from minference.dist_ops.minfer_dr_striped import minfer_dr_stripe_func
+from minference.dist_ops.minfer_striped import minfer_stripe_func
+from minference.dist_ops.minfer_zigzag import minfer_zigzag_func
+from minference.dist_ops.test.raw_test_utils import (
+ SEED_BASE,
+ check_forward_and_qkv_grads,
+ create_full_inputs,
+ gather_sequence_shards,
+ init_process_group,
+ slice_local_inputs,
+)
+from minference.ops.pit_sparse_flash_attention_v3 import minference_flash_attn_func
+from minference.ops.utils import set_seed
+
+# ------------- constants ------------------------------------------------------
+_ATOL = 1e-2
+_RTOL = 1e-2
+_WORLD_SIZE = 4
+
+_ATTENTION_IMPLS: dict[str, Callable] = {
+ "minfer_zigzag": minfer_zigzag_func,
+ "minfer_stripe": minfer_stripe_func,
+ "minfer_dr_stripe": minfer_dr_stripe_func,
+}
+
+
+def _run_worker(
+ rank: int,
+ world_size: int,
+ port: str,
+ cfg: SimpleNamespace,
+ attn_op_name: str,
+) -> None:
+ """Worker function executed in every spawned GPU process."""
+ init_process_group(rank, world_size, port)
+
+ device = torch.device(f"cuda:{rank}")
+ torch.cuda.set_device(device)
+ dtype = torch.bfloat16
+ set_seed(SEED_BASE + rank)
+
+ attn_op: Callable = _ATTENTION_IMPLS[attn_op_name]
+ q, k, v, dout = create_full_inputs(rank, cfg, device, dtype)
+ q_local, k_local, v_local, dout_local = slice_local_inputs(
+ rank, world_size, q, k, v, dout
+ )
+
+ # ----------------- forward / backward on the candidate kernel ------------
+ out_local = attn_op(
+ q_local,
+ k_local,
+ v_local,
+ cfg.v_size,
+ cfg.s_size,
+ layer_idx=0,
+ )
+ torch.autograd.backward(out_local, dout_local)
+
+ final_out = gather_sequence_shards(out_local, world_size)
+ grads = tuple(
+ gather_sequence_shards(grad, world_size)
+ for grad in (q_local.grad, k_local.grad, v_local.grad)
+ )
+
+ torch.distributed.barrier()
+ torch.cuda.synchronize()
+ # ----------------- reference: dense Flash-Attention ----------------------
+ if rank == 0:
+ q_ref = q.detach().clone().requires_grad_()
+ k_ref = k.detach().clone().requires_grad_()
+ v_ref = v.detach().clone().requires_grad_()
+
+ out_ref = minference_flash_attn_func(
+ q_ref,
+ k_ref,
+ v_ref,
+ cfg.v_size,
+ cfg.s_size,
+ causal=True,
+ )
+ torch.autograd.backward(out_ref, dout)
+ ref_grads = (q_ref.grad, k_ref.grad, v_ref.grad)
+
+ check_forward_and_qkv_grads(
+ cfg.seq_len,
+ final_out,
+ out_ref,
+ grads,
+ ref_grads,
+ atol=_ATOL,
+ rtol=_RTOL,
+ raise_on_fail=True,
+ )
+ dist.destroy_process_group()
+
+
+# ------------- pytest entry-point --------------------------------------------
+@pytest.mark.skipif(torch.cuda.device_count() < _WORLD_SIZE, reason="Not enough GPUs")
+@pytest.mark.parametrize("seq_len", [131072, 262144, 524288])
+@pytest.mark.parametrize("batch_sz", [1])
+@pytest.mark.parametrize("head_dim", [64, 128])
+@pytest.mark.parametrize("sparsity", [0.9, 0.95])
+@pytest.mark.parametrize("num_qkv_head_pair", [(4, 1), (4, 4)])
+@pytest.mark.parametrize("use_triton", [True, False])
+@pytest.mark.parametrize(
+ "attn_op_name", ["minfer_zigzag", "minfer_stripe", "minfer_dr_stripe"]
+)
+def test_sparse_attention_kernels(
+ seq_len: int,
+ batch_sz: int,
+ head_dim: int,
+ sparsity: float,
+ num_qkv_head_pair: tuple[int, int],
+ use_triton: bool,
+ attn_op_name: str,
+):
+ """
+ Compare every sparse kernel against the dense Flash-Attention reference on
+ both forward pass and input-gradient w.r.t Q/K/V.
+ """
+ port = str(random.randint(12000, 20000))
+ if attn_op_name == "minfer_zigzag" and use_triton:
+ pytest.skip("minfer_zigzag is not implemented with the Triton path")
+
+ cfg = SimpleNamespace(
+ batch_size=batch_sz,
+ seq_len=seq_len,
+ head_dim=head_dim,
+ sparsity=sparsity,
+ ones=False,
+ num_qo_heads=num_qkv_head_pair[0],
+ num_kv_heads=num_qkv_head_pair[1],
+ )
+ # derived sizes used by both candidate and reference kernels
+ cfg.v_size = [int((1 - cfg.sparsity) * 0.1 * cfg.seq_len)] * cfg.num_qo_heads
+ cfg.s_size = [int((1 - cfg.sparsity) * 0.2 * cfg.seq_len)] * cfg.num_qo_heads
+
+ print(f"=" * 80)
+ print(f"Testing {attn_op_name} with configuration:\n{cfg}")
+ print(f"=" * 80)
+ prev_force_triton = os.environ.get("FORCE_TRITON")
+ try:
+ if use_triton:
+ os.environ["FORCE_TRITON"] = "1"
+ else:
+ os.environ.pop("FORCE_TRITON", None)
+
+ mp.spawn(
+ _run_worker,
+ args=(_WORLD_SIZE, port, cfg, attn_op_name),
+ nprocs=_WORLD_SIZE,
+ join=True,
+ )
+ finally:
+ if prev_force_triton is None:
+ os.environ.pop("FORCE_TRITON", None)
+ else:
+ os.environ["FORCE_TRITON"] = prev_force_triton
diff --git a/minference/dist_ops/test/minfer_ring_test_raw.py b/minference/dist_ops/test/minfer_ring_test_raw.py
new file mode 100644
index 00000000..a274d6c1
--- /dev/null
+++ b/minference/dist_ops/test/minfer_ring_test_raw.py
@@ -0,0 +1,165 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""Standalone distributed correctness checks for Minference raw kernels."""
+from __future__ import annotations
+
+import random
+from types import SimpleNamespace
+from typing import Callable
+
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+from minference.dist_ops.minfer_dr_striped import minfer_dr_stripe_func
+from minference.dist_ops.minfer_striped import minfer_stripe_func
+from minference.dist_ops.minfer_zigzag import minfer_zigzag_func
+from minference.dist_ops.test.raw_test_utils import (
+ SEED_BASE,
+ check_forward_and_qkv_grads,
+ create_full_inputs,
+ gather_sequence_shards,
+ init_process_group,
+ slice_local_inputs,
+)
+from minference.ops.pit_sparse_flash_attention_v3 import minference_flash_attn_func
+from minference.ops.utils import set_seed
+
+# ------------- constants ------------------------------------------------------
+_ATOL = 1e-2
+_RTOL = 1e-2
+_WORLD_SIZE = 4
+
+_ATTENTION_IMPLS: dict[str, Callable] = {
+ "minfer_zigzag": minfer_zigzag_func,
+ "minfer_stripe": minfer_stripe_func,
+ "minfer_dr_stripe": minfer_dr_stripe_func,
+}
+
+
+def _run_worker(
+ rank: int,
+ world_size: int,
+ port: str,
+ cfg: SimpleNamespace,
+ attn_op_name: str,
+) -> None:
+ """Worker function executed in every spawned GPU process."""
+ init_process_group(rank, world_size, port)
+
+ device = torch.device(f"cuda:{rank}")
+ torch.cuda.set_device(device)
+ dtype = torch.bfloat16
+ set_seed(SEED_BASE + rank)
+
+ attn_op: Callable = _ATTENTION_IMPLS[attn_op_name]
+
+ q, k, v, dout = create_full_inputs(rank, cfg, device, dtype)
+ q_local, k_local, v_local, dout_local = slice_local_inputs(
+ rank, world_size, q, k, v, dout
+ )
+
+ # ----------------- forward / backward on the candidate kernel ------------
+ out_local = attn_op(
+ q_local,
+ k_local,
+ v_local,
+ cfg.v_size,
+ cfg.s_size,
+ layer_idx=0,
+ )
+ torch.autograd.backward(out_local, dout_local)
+
+ final_out = gather_sequence_shards(out_local, world_size)
+ grads = tuple(
+ gather_sequence_shards(grad, world_size)
+ for grad in (q_local.grad, k_local.grad, v_local.grad)
+ )
+ torch.distributed.barrier()
+ torch.cuda.synchronize()
+
+ # ----------------- reference: single machine MInference Forward/Backward ----------------------
+ if rank == 0:
+ print(f"Rank {rank} | Running reference forward/backward", flush=True)
+ q_ref = q.detach().clone().requires_grad_()
+ k_ref = k.detach().clone().requires_grad_()
+ v_ref = v.detach().clone().requires_grad_()
+
+ print(
+ f"Rank {rank} | Running reference forward with q_ref.shape={q_ref.shape}, k_ref.shape={k_ref.shape}, v_ref.shape={v_ref.shape}",
+ flush=True,
+ )
+ out_ref = minference_flash_attn_func(
+ q_ref,
+ k_ref,
+ v_ref,
+ cfg.v_size,
+ cfg.s_size,
+ causal=True,
+ )
+
+ torch.autograd.backward(out_ref, dout)
+ ref_grads = (q_ref.grad, k_ref.grad, v_ref.grad)
+
+ check_forward_and_qkv_grads(
+ cfg.seq_len,
+ final_out,
+ out_ref,
+ grads,
+ ref_grads,
+ atol=_ATOL,
+ rtol=_RTOL,
+ )
+
+ dist.destroy_process_group()
+
+
+def run_minfer_kernel_test(
+ seq_len: int,
+ batch_sz: int,
+ head_dim: int,
+ sparsity: float,
+ ones: bool,
+ num_qo_heads: int,
+ num_kv_heads: int,
+ attn_op_name: str,
+):
+ """Compare a distributed Minference kernel against dense reference outputs."""
+ port = str(random.randint(12000, 20000))
+
+ cfg = SimpleNamespace(
+ batch_size=batch_sz,
+ seq_len=seq_len,
+ head_dim=head_dim,
+ sparsity=sparsity,
+ ones=ones,
+ num_qo_heads=num_qo_heads,
+ num_kv_heads=num_kv_heads,
+ )
+ # derived sizes used by both candidate and reference kernels
+ cfg.v_size = [int((1 - cfg.sparsity) * 0.1 * cfg.seq_len)] * cfg.num_qo_heads
+ cfg.s_size = [int((1 - cfg.sparsity) * 0.2 * cfg.seq_len)] * cfg.num_qo_heads
+
+ print("=" * 80)
+ print(f"Testing {attn_op_name} with configuration:\n{cfg}")
+ print("=" * 80)
+ mp.spawn(
+ _run_worker,
+ args=(_WORLD_SIZE, port, cfg, attn_op_name),
+ nprocs=_WORLD_SIZE,
+ join=True,
+ )
+
+
+if __name__ == "__main__":
+ run_minfer_kernel_test(
+ seq_len=512 * 1024,
+ batch_sz=1,
+ head_dim=128,
+ sparsity=0.9,
+ ones=False,
+ num_qo_heads=4,
+ num_kv_heads=1,
+ attn_op_name="minfer_stripe",
+ )
diff --git a/minference/dist_ops/test/moba_ring_test.py b/minference/dist_ops/test/moba_ring_test.py
new file mode 100644
index 00000000..957b2dfe
--- /dev/null
+++ b/minference/dist_ops/test/moba_ring_test.py
@@ -0,0 +1,160 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from __future__ import annotations
+
+import functools
+import os
+import random
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+from torch.multiprocessing.spawn import ProcessRaisedException
+
+from minference.dist_ops.moba_zigzag import moba_zigzag_func
+from minference.dist_ops.test.raw_test_utils import (
+ SEED_BASE,
+ check_forward_and_qkv_grads,
+ create_full_inputs,
+ gather_sequence_shards,
+ init_process_group,
+ slice_local_inputs,
+)
+from minference.ops.moba import moba_attn_func
+from minference.ops.utils import set_seed
+
+# ------------- constants ------------------------------------------------------
+_ATOL = 1e-2
+_RTOL = 1e-2
+_WORLD_SIZE = 4
+
+
+# ------------- helpers --------------------------------------------------------
+def skip_if_cuda_oom(test_func):
+ """Decorator: convert worker OOM raised by spawn into a pytest skip."""
+
+ @functools.wraps(test_func)
+ def _wrapper(*args, **kwargs):
+ try:
+ return test_func(*args, **kwargs)
+ except (torch.OutOfMemoryError, ProcessRaisedException) as exc:
+ if "OutOfMemoryError" not in str(exc):
+ raise
+ torch.cuda.empty_cache()
+ pytest.skip("skipped because the GPU ran out of memory")
+
+ return _wrapper
+
+
+def _run_worker(
+ rank: int,
+ world_size: int,
+ port: str,
+ cfg: SimpleNamespace,
+) -> None:
+ """Worker function executed in every spawned GPU process."""
+ init_process_group(rank, world_size, port)
+
+ device = torch.device(f"cuda:{rank}")
+ torch.cuda.set_device(device)
+ dtype = torch.bfloat16
+ set_seed(SEED_BASE + rank)
+ q, k, v, dout = create_full_inputs(rank, cfg, device, dtype)
+ q_local, k_local, v_local, dout_local = slice_local_inputs(
+ rank, world_size, q, k, v, dout
+ )
+
+ # ----------------- forward / backward on the candidate kernel ------------
+ out_local = moba_zigzag_func(
+ q_local,
+ k_local,
+ v_local,
+ layer_idx=0,
+ global_seq_len=cfg.seq_len,
+ moba_chunk_size=cfg.moba_chunk_size,
+ moba_topk=cfg.moba_topk,
+ )
+ torch.autograd.backward(out_local, dout_local)
+
+ final_out = gather_sequence_shards(out_local, world_size)
+ grads = tuple(
+ gather_sequence_shards(grad, world_size)
+ for grad in (q_local.grad, k_local.grad, v_local.grad)
+ )
+ torch.distributed.barrier()
+ torch.cuda.synchronize()
+ # ---------------------------------------
+ if rank == 0:
+ q_ref = q.detach().clone().requires_grad_()
+ k_ref = k.detach().clone().requires_grad_()
+ v_ref = v.detach().clone().requires_grad_()
+
+ out_ref = moba_attn_func(
+ q_ref,
+ k_ref,
+ v_ref,
+ global_seq_len=cfg.seq_len,
+ moba_chunk_size=cfg.moba_chunk_size,
+ moba_topk=cfg.moba_topk,
+ )
+ torch.autograd.backward(out_ref, dout)
+ ref_grads = (q_ref.grad, k_ref.grad, v_ref.grad)
+
+ check_forward_and_qkv_grads(
+ cfg.seq_len,
+ final_out,
+ out_ref,
+ grads,
+ ref_grads,
+ atol=_ATOL,
+ rtol=_RTOL,
+ raise_on_fail=True,
+ )
+
+ dist.destroy_process_group()
+
+
+# ------------- pytest entry-point --------------------------------------------
+@skip_if_cuda_oom
+@pytest.mark.skipif(torch.cuda.device_count() < _WORLD_SIZE, reason="Not enough GPUs")
+@pytest.mark.parametrize("seq_len", [16384, 32768])
+@pytest.mark.parametrize("head_dim", [64, 128])
+@pytest.mark.parametrize("num_qkv_head_pair", [(4, 1), (4, 4)])
+@pytest.mark.parametrize("moba_chunk_size", [128, 256])
+@pytest.mark.parametrize("moba_topk", [8, 16])
+def test_moba_kernels(
+ seq_len: int,
+ head_dim: int,
+ num_qkv_head_pair: tuple[int, int],
+ moba_chunk_size: int,
+ moba_topk: int,
+):
+ """
+ Compare every sparse kernel against the dense Flash-Attention reference on
+ both forward pass and input-gradient w.r.t Q/K/V.
+ """
+
+ port = str(random.randint(12000, 20000))
+ cfg = SimpleNamespace(
+ batch_size=1,
+ seq_len=seq_len,
+ head_dim=head_dim,
+ ones=False,
+ num_qo_heads=num_qkv_head_pair[0],
+ num_kv_heads=num_qkv_head_pair[1],
+ moba_chunk_size=moba_chunk_size,
+ moba_topk=moba_topk,
+ )
+
+ print(f"=" * 80)
+ print(f"Testing MoBA (w. Zigzag) with configuration:\n{cfg}")
+ print(f"=" * 80)
+ mp.spawn(
+ _run_worker,
+ args=(_WORLD_SIZE, port, cfg),
+ nprocs=_WORLD_SIZE,
+ join=True,
+ )
diff --git a/minference/dist_ops/test/moba_ring_test_raw.py b/minference/dist_ops/test/moba_ring_test_raw.py
new file mode 100644
index 00000000..90d5db74
--- /dev/null
+++ b/minference/dist_ops/test/moba_ring_test_raw.py
@@ -0,0 +1,138 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""Standalone distributed correctness checks for MoBA kernels."""
+from __future__ import annotations
+
+import random
+from types import SimpleNamespace
+
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+from minference.dist_ops.moba_zigzag import moba_zigzag_func
+from minference.dist_ops.test.raw_test_utils import (
+ SEED_BASE,
+ check_forward_and_qkv_grads,
+ create_full_inputs,
+ gather_sequence_shards,
+ init_process_group,
+ slice_local_inputs,
+)
+from minference.ops.moba import moba_attn_func
+from minference.ops.utils import set_seed
+
+# ------------- constants ------------------------------------------------------
+_ATOL = 1e-2
+_RTOL = 1e-2
+_WORLD_SIZE = 4
+
+
+def _run_worker(
+ rank: int,
+ world_size: int,
+ port: str,
+ cfg: SimpleNamespace,
+) -> None:
+ """Worker function executed in every spawned GPU process."""
+ init_process_group(rank, world_size, port)
+
+ device = torch.device(f"cuda:{rank}")
+ torch.cuda.set_device(device)
+ dtype = torch.bfloat16
+ set_seed(SEED_BASE + rank)
+ q, k, v, dout = create_full_inputs(rank, cfg, device, dtype)
+ q_local, k_local, v_local, dout_local = slice_local_inputs(
+ rank, world_size, q, k, v, dout
+ )
+
+ # ----------------- forward / backward on the candidate kernel ------------
+ out_local = moba_zigzag_func(
+ q_local,
+ k_local,
+ v_local,
+ layer_idx=0,
+ global_seq_len=cfg.seq_len,
+ moba_chunk_size=cfg.moba_chunk_size,
+ moba_topk=cfg.moba_topk,
+ )
+ torch.autograd.backward(out_local, dout_local)
+
+ final_out = gather_sequence_shards(out_local, world_size)
+ grads = tuple(
+ gather_sequence_shards(grad, world_size)
+ for grad in (q_local.grad, k_local.grad, v_local.grad)
+ )
+
+ if rank == 0:
+ q_ref = q.detach().clone().requires_grad_()
+ k_ref = k.detach().clone().requires_grad_()
+ v_ref = v.detach().clone().requires_grad_()
+
+ out_ref = moba_attn_func(
+ q_ref,
+ k_ref,
+ v_ref,
+ global_seq_len=cfg.seq_len,
+ moba_chunk_size=cfg.moba_chunk_size,
+ moba_topk=cfg.moba_topk,
+ )
+ torch.autograd.backward(out_ref, dout)
+ ref_grads = (q_ref.grad, k_ref.grad, v_ref.grad)
+
+ check_forward_and_qkv_grads(
+ cfg.seq_len,
+ final_out,
+ out_ref,
+ grads,
+ ref_grads,
+ atol=_ATOL,
+ rtol=_RTOL,
+ )
+ dist.destroy_process_group()
+
+
+def run_moba_kernel_test(
+ seq_len: int = 4096,
+ batch_size: int = 1,
+ head_dim: int = 64,
+ ones: bool = True,
+ num_qkv_head_pair: tuple[int, int] = (2, 2),
+ moba_chunk_size: int = 512,
+ moba_topk: int = 8,
+):
+ """Compare distributed MoBA-Zigzag outputs with dense MoBA reference."""
+ port = str(random.randint(12000, 20000))
+ cfg = SimpleNamespace(
+ batch_size=batch_size,
+ seq_len=seq_len,
+ head_dim=head_dim,
+ ones=ones,
+ num_qo_heads=num_qkv_head_pair[0],
+ num_kv_heads=num_qkv_head_pair[1],
+ moba_chunk_size=moba_chunk_size,
+ moba_topk=moba_topk,
+ )
+
+ print("=" * 80)
+ print(f"Testing MoBA (w. Zigzag) with configuration:\n{cfg}")
+ print("=" * 80)
+ mp.spawn(
+ _run_worker,
+ args=(_WORLD_SIZE, port, cfg),
+ nprocs=_WORLD_SIZE,
+ join=True,
+ )
+
+
+if __name__ == "__main__":
+ run_moba_kernel_test(
+ seq_len=16384,
+ batch_size=1,
+ head_dim=128,
+ ones=False,
+ num_qkv_head_pair=(4, 1),
+ moba_chunk_size=128,
+ moba_topk=8,
+ )
diff --git a/minference/dist_ops/test/raw_test_utils.py b/minference/dist_ops/test/raw_test_utils.py
new file mode 100644
index 00000000..81c6981b
--- /dev/null
+++ b/minference/dist_ops/test/raw_test_utils.py
@@ -0,0 +1,174 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""Shared helpers for standalone distributed raw-kernel tests."""
+from __future__ import annotations
+
+import os
+from types import SimpleNamespace
+
+import torch
+import torch.distributed as dist
+
+from minference.ops.utils import check_by_correct_rate
+
+SEED_BASE = 2025
+
+
+def init_process_group(rank: int, world_size: int, port: str) -> None:
+ """Initialize NCCL backend for the current worker."""
+ os.environ.update(
+ {
+ "MASTER_ADDR": "127.0.0.1",
+ "MASTER_PORT": port,
+ "RANK": str(rank),
+ "WORLD_SIZE": str(world_size),
+ "LOCAL_RANK": str(rank % min(world_size, torch.cuda.device_count())),
+ "LOCAL_WORLD_SIZE": str(min(world_size, torch.cuda.device_count())),
+ }
+ )
+ dist.init_process_group("nccl", rank=rank, world_size=world_size)
+
+
+def create_full_inputs(
+ rank: int,
+ cfg: SimpleNamespace,
+ device: torch.device,
+ dtype: torch.dtype,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Create full-sequence inputs on rank 0 and broadcast to all ranks."""
+ if rank == 0:
+ rand_or_one = (
+ torch.randn if not cfg.ones else lambda s, **k: torch.ones(*s, **k)
+ )
+ q = rand_or_one(
+ (cfg.batch_size, cfg.seq_len, cfg.num_qo_heads, cfg.head_dim),
+ dtype=dtype,
+ device=device,
+ )
+ k = rand_or_one(
+ (cfg.batch_size, cfg.seq_len, cfg.num_kv_heads, cfg.head_dim),
+ dtype=dtype,
+ device=device,
+ )
+ v = rand_or_one(
+ (cfg.batch_size, cfg.seq_len, cfg.num_kv_heads, cfg.head_dim),
+ dtype=dtype,
+ device=device,
+ )
+ dout = rand_or_one(
+ (cfg.batch_size, cfg.seq_len, cfg.num_qo_heads, cfg.head_dim),
+ dtype=dtype,
+ device=device,
+ )
+ else:
+ shape_q = (cfg.batch_size, cfg.seq_len, cfg.num_qo_heads, cfg.head_dim)
+ shape_kv = (cfg.batch_size, cfg.seq_len, cfg.num_kv_heads, cfg.head_dim)
+ q = torch.empty(shape_q, device=device, dtype=dtype)
+ k = torch.empty(shape_kv, device=device, dtype=dtype)
+ v = torch.empty(shape_kv, device=device, dtype=dtype)
+ dout = torch.empty(shape_q, device=device, dtype=dtype)
+
+ for tensor in (q, k, v, dout):
+ dist.broadcast(tensor, src=0)
+
+ return q, k, v, dout
+
+
+def slice_local_inputs(
+ rank: int,
+ world_size: int,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ dout: torch.Tensor,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Slice local sequence shard and set gradients."""
+ local_ctx = q.size(1) // world_size
+ sl = slice(rank * local_ctx, (rank + 1) * local_ctx)
+ q_local = q[:, sl].clone().detach().requires_grad_()
+ k_local = k[:, sl].clone().detach().requires_grad_()
+ v_local = v[:, sl].clone().detach().requires_grad_()
+ dout_local = dout[:, sl].clone()
+ return q_local, k_local, v_local, dout_local
+
+
+def gather_sequence_shards(local_tensor: torch.Tensor, world_size: int) -> torch.Tensor:
+ """All-gather sequence shards and concatenate along sequence dim."""
+ gathered = [torch.empty_like(local_tensor) for _ in range(world_size)]
+ dist.all_gather(gathered, local_tensor)
+ return torch.cat(gathered, dim=1)
+
+
+def check_forward_and_qkv_grads(
+ seq_len: int,
+ final_out: torch.Tensor,
+ out_ref: torch.Tensor,
+ grads: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
+ ref_grads: tuple[torch.Tensor, torch.Tensor, torch.Tensor],
+ atol: float,
+ rtol: float,
+ raise_on_fail: bool = False,
+) -> bool:
+ """Run a unified forward/QKV check flow with detailed error analysis."""
+
+ def _format_index(flat_index: int, shape: torch.Size) -> tuple[int, ...]:
+ coords = []
+ stride = 1
+ for size in reversed(shape):
+ coords.append((flat_index // stride) % size)
+ stride *= size
+ return tuple(reversed(coords))
+
+ def _analyze_tensor(name: str, got: torch.Tensor, ref: torch.Tensor) -> bool:
+ # got_fp = got.float()
+ # ref_fp = ref.float()
+ diff = (got - ref).abs()
+ max_diff = diff.max()
+ mean_diff = diff.mean()
+ min_diff = diff.min()
+
+ ok = check_by_correct_rate(got, ref, ATOL=atol, RTOL=rtol)
+ status = "PASS" if ok else "FAIL"
+ print(
+ (
+ f"[{status}] {name}: max_diff={max_diff.item():.6e}, "
+ f"mean_diff={mean_diff.item():.6e}, "
+ f" min_diff={min_diff.item():.6e}"
+ ),
+ flush=True,
+ )
+
+ if not ok:
+ flat_idx = int(diff.argmax().item())
+ idx = _format_index(flat_idx, diff.shape)
+ got_val = got[idx].item()
+ ref_val = ref[idx].item()
+ print(
+ (
+ f" worst_idx={idx}, got={got_val:.6e}, ref={ref_val:.6e}, "
+ f"abs_diff={max_diff.item():.6e}"
+ ),
+ flush=True,
+ )
+ return ok
+
+ _ = seq_len # reserved for optional sequence-wise diagnostics
+ overall_ok = True
+ forward_ok = _analyze_tensor("forward output", final_out, out_ref)
+ if not forward_ok:
+ overall_ok = False
+ if raise_on_fail:
+ raise AssertionError(
+ "forward output mismatch; see printed diff analysis for details"
+ )
+ return overall_ok
+
+ for name, grad, ref_grad in zip(("Q-grad", "K-grad", "V-grad"), grads, ref_grads):
+ grad_ok = _analyze_tensor(name, grad, ref_grad)
+ overall_ok = overall_ok and grad_ok
+
+ if raise_on_fail and not overall_ok:
+ raise AssertionError("gradient mismatch; see printed diff analysis for details")
+
+ return overall_ok
diff --git a/minference/dist_ops/test/run_ring_pytests.sh b/minference/dist_ops/test/run_ring_pytests.sh
new file mode 100755
index 00000000..07743a65
--- /dev/null
+++ b/minference/dist_ops/test/run_ring_pytests.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+set -u
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+
+if command -v conda >/dev/null 2>&1; then
+ eval "$(conda shell.bash hook)" >/dev/null 2>&1
+ if [[ "${CONDA_DEFAULT_ENV:-}" != "mtrain" ]]; then
+ conda activate mtrain
+ fi
+fi
+
+cd "${REPO_ROOT}" || exit 1
+
+declare -a TEST_FILES=(
+ "minference/dist_ops/test/minfer_ring_test.py"
+ "minference/dist_ops/test/moba_ring_test.py"
+ "minference/dist_ops/test/xattn_ring_test.py"
+)
+
+declare -a LOG_FILES=(
+ "${SCRIPT_DIR}/minfer_ring_test.log"
+ "${SCRIPT_DIR}/moba_ring_test.log"
+ "${SCRIPT_DIR}/xattn_ring_test.log"
+)
+
+overall_rc=0
+for i in "${!TEST_FILES[@]}"; do
+ test_file="${TEST_FILES[$i]}"
+ log_file="${LOG_FILES[$i]}"
+
+ echo "================================================================" | tee "${log_file}"
+ echo "Running: ${test_file}" | tee -a "${log_file}"
+ echo "Started: $(date -Iseconds)" | tee -a "${log_file}"
+ echo "================================================================" | tee -a "${log_file}"
+
+ pytest -s "${test_file}" 2>&1 | tee -a "${log_file}"
+ rc=${PIPESTATUS[0]}
+
+ echo "" | tee -a "${log_file}"
+ echo "Finished: $(date -Iseconds) (exit_code=${rc})" | tee -a "${log_file}"
+
+ if [[ ${rc} -ne 0 ]]; then
+ overall_rc=1
+ fi
+done
+exit "${overall_rc}"
diff --git a/minference/dist_ops/test/xattn_ring_test.py b/minference/dist_ops/test/xattn_ring_test.py
new file mode 100644
index 00000000..0bec4c78
--- /dev/null
+++ b/minference/dist_ops/test/xattn_ring_test.py
@@ -0,0 +1,152 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from __future__ import annotations
+
+import os
+import random
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+from minference.dist_ops.test.raw_test_utils import (
+ SEED_BASE,
+ check_forward_and_qkv_grads,
+ create_full_inputs,
+ gather_sequence_shards,
+ init_process_group,
+ slice_local_inputs,
+)
+from minference.dist_ops.xattn_zigzag import xattn_zigzag_func
+from minference.ops.utils import set_seed
+from minference.ops.xattention_fa import xattn_flash_attn_func
+
+# ------------- constants ------------------------------------------------------
+_ATOL = 1e-1
+_RTOL = 1e-1
+_WORLD_SIZE = 4
+
+
+def _run_worker(
+ rank: int,
+ world_size: int,
+ port: str,
+ cfg: SimpleNamespace,
+) -> None:
+ """Worker function executed in every spawned GPU process."""
+ init_process_group(rank, world_size, port)
+
+ device = torch.device(f"cuda:{rank}")
+ torch.cuda.set_device(device)
+ dtype = torch.bfloat16
+ set_seed(SEED_BASE + rank)
+ q, k, v, dout = create_full_inputs(rank, cfg, device, dtype)
+ q_local, k_local, v_local, dout_local = slice_local_inputs(
+ rank, world_size, q, k, v, dout
+ )
+
+ # ----------------- forward / backward on the candidate kernel ------------
+ out_local = xattn_zigzag_func(
+ q_local,
+ k_local,
+ v_local,
+ layer_idx=0,
+ xattn_params=cfg.xattn_params,
+ granularity=128,
+ )
+ torch.autograd.backward(out_local, dout_local)
+
+ final_out = gather_sequence_shards(out_local, world_size)
+ grads = tuple(
+ gather_sequence_shards(grad, world_size)
+ for grad in (q_local.grad, k_local.grad, v_local.grad)
+ )
+ torch.distributed.barrier()
+ torch.cuda.synchronize()
+ # ---------------------------------------
+ if rank == 0:
+ q_ref = q.detach().clone().requires_grad_()
+ k_ref = k.detach().clone().requires_grad_()
+ v_ref = v.detach().clone().requires_grad_()
+
+ single_machine_params = cfg.xattn_params.copy()
+ single_machine_params["chunk_size"] = cfg.seq_len // _WORLD_SIZE
+ out_ref = xattn_flash_attn_func(
+ q_ref,
+ k_ref,
+ v_ref,
+ head_indices=list(range(cfg.num_qo_heads)),
+ xattn_params=single_machine_params,
+ granularity=128,
+ )
+ torch.autograd.backward(out_ref, dout)
+ ref_grads = (q_ref.grad, k_ref.grad, v_ref.grad)
+
+ check_forward_and_qkv_grads(
+ cfg.seq_len,
+ final_out,
+ out_ref,
+ grads,
+ ref_grads,
+ atol=_ATOL,
+ rtol=_RTOL,
+ raise_on_fail=True,
+ )
+
+ dist.destroy_process_group()
+
+
+# ------------- pytest entry-point --------------------------------------------
+@pytest.mark.skipif(torch.cuda.device_count() < _WORLD_SIZE, reason="Not enough GPUs")
+@pytest.mark.parametrize("seq_len", [131072, 262144, 524288])
+@pytest.mark.parametrize("head_dim", [64, 128])
+@pytest.mark.parametrize("num_qkv_head_pair", [(4, 1), (4, 4)])
+@pytest.mark.parametrize("stride", [16, 32])
+@pytest.mark.parametrize("threshold", [0.9, 0.95])
+def test_xattention_kernels(
+ seq_len: int,
+ head_dim: int,
+ num_qkv_head_pair: tuple[int, int],
+ stride: int,
+ threshold: float,
+):
+ """
+ Compare every sparse kernel against the dense Flash-Attention reference on
+ both forward pass and input-gradient w.r.t Q/K/V.
+ """
+
+ port = str(random.randint(12000, 20000))
+ xattn_params = {
+ "stride": stride,
+ "norm": 1,
+ "softmax": True,
+ "threshold": threshold,
+ "select_mode": "inverse",
+ "use_triton": True,
+ "causal": True,
+ "kdb": 1,
+ "keep_sink": False,
+ "keep_recent": False,
+ }
+ cfg = SimpleNamespace(
+ batch_size=1,
+ seq_len=seq_len,
+ head_dim=head_dim,
+ ones=False,
+ num_qo_heads=num_qkv_head_pair[0],
+ num_kv_heads=num_qkv_head_pair[1],
+ xattn_params=xattn_params,
+ )
+
+ print(f"=" * 80)
+ print(f"Testing XAttention (w. Zigzag) with configuration:\n{cfg}")
+ print(f"=" * 80)
+ mp.spawn(
+ _run_worker,
+ args=(_WORLD_SIZE, port, cfg),
+ nprocs=_WORLD_SIZE,
+ join=True,
+ )
diff --git a/minference/dist_ops/test/xattn_ring_test_raw.py b/minference/dist_ops/test/xattn_ring_test_raw.py
new file mode 100644
index 00000000..85d4af14
--- /dev/null
+++ b/minference/dist_ops/test/xattn_ring_test_raw.py
@@ -0,0 +1,155 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""Standalone distributed correctness checks for XAttention raw kernels."""
+from __future__ import annotations
+
+import random
+from types import SimpleNamespace
+
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+from minference.dist_ops.test.raw_test_utils import (
+ SEED_BASE,
+ check_forward_and_qkv_grads,
+ create_full_inputs,
+ gather_sequence_shards,
+ init_process_group,
+ slice_local_inputs,
+)
+from minference.dist_ops.xattn_zigzag import xattn_zigzag_func
+from minference.ops.utils import set_seed
+from minference.ops.xattention_fa import xattn_flash_attn_func
+
+# ------------- constants ------------------------------------------------------
+_ATOL = 1e-1
+_RTOL = 1e-1
+_WORLD_SIZE = 4
+
+
+def _run_worker(
+ rank: int,
+ world_size: int,
+ port: str,
+ cfg: SimpleNamespace,
+) -> None:
+ """Worker function executed in every spawned GPU process."""
+ init_process_group(rank, world_size, port)
+
+ device = torch.device(f"cuda:{rank}")
+ torch.cuda.set_device(device)
+ dtype = torch.bfloat16
+ set_seed(SEED_BASE + rank)
+ q, k, v, dout = create_full_inputs(rank, cfg, device, dtype)
+ q_local, k_local, v_local, dout_local = slice_local_inputs(
+ rank, world_size, q, k, v, dout
+ )
+
+ # ----------------- forward / backward on the candidate kernel ------------
+ out_local = xattn_zigzag_func(
+ q_local,
+ k_local,
+ v_local,
+ layer_idx=0,
+ xattn_params=cfg.xattn_params,
+ granularity=128,
+ )
+ torch.autograd.backward(out_local, dout_local)
+
+ final_out = gather_sequence_shards(out_local, world_size)
+ grads = tuple(
+ gather_sequence_shards(grad, world_size)
+ for grad in (q_local.grad, k_local.grad, v_local.grad)
+ )
+ dist.barrier()
+ torch.cuda.synchronize()
+
+ if rank == 0:
+ q_ref = q.detach().clone().requires_grad_()
+ k_ref = k.detach().clone().requires_grad_()
+ v_ref = v.detach().clone().requires_grad_()
+
+ single_machine_params = cfg.xattn_params.copy()
+ single_machine_params["chunk_size"] = cfg.seq_len // _WORLD_SIZE
+ out_ref = xattn_flash_attn_func(
+ q_ref,
+ k_ref,
+ v_ref,
+ head_indices=list(range(cfg.num_qo_heads)),
+ xattn_params=single_machine_params,
+ granularity=128,
+ )
+ torch.autograd.backward(out_ref, dout)
+ ref_grads = (q_ref.grad, k_ref.grad, v_ref.grad)
+
+ check_forward_and_qkv_grads(
+ cfg.seq_len,
+ final_out,
+ out_ref,
+ grads,
+ ref_grads,
+ atol=_ATOL,
+ rtol=_RTOL,
+ )
+
+ dist.destroy_process_group()
+
+
+def run_xattention_kernel_test(
+ seq_len: int = 4096,
+ batch_sz: int = 1,
+ head_dim: int = 64,
+ ones: bool = True,
+ num_qo_heads: int = 2,
+ num_kv_heads: int = 2,
+ stride: int = 16,
+ threshold: float = 0.9,
+):
+ """Compare distributed XAttention-Zigzag outputs with dense reference."""
+ port = str(random.randint(12000, 20000))
+ xattn_params = {
+ "stride": stride,
+ "norm": 1,
+ "softmax": True,
+ "threshold": threshold,
+ "select_mode": "inverse",
+ "use_triton": True,
+ "causal": True,
+ "kdb": 1,
+ "keep_sink": False,
+ "keep_recent": False,
+ }
+ cfg = SimpleNamespace(
+ batch_size=batch_sz,
+ seq_len=seq_len,
+ head_dim=head_dim,
+ ones=ones,
+ num_qo_heads=num_qo_heads,
+ num_kv_heads=num_kv_heads,
+ xattn_params=xattn_params,
+ )
+
+ print("=" * 80)
+ print(f"Testing XAttention (w. Zigzag) with configuration:\n{cfg}")
+ print("=" * 80)
+ mp.spawn(
+ _run_worker,
+ args=(_WORLD_SIZE, port, cfg),
+ nprocs=_WORLD_SIZE,
+ join=True,
+ )
+
+
+if __name__ == "__main__":
+ run_xattention_kernel_test(
+ seq_len=512 * 1024,
+ batch_sz=1,
+ head_dim=64,
+ ones=False,
+ num_qo_heads=4,
+ num_kv_heads=1,
+ stride=16,
+ threshold=0.95,
+ )
diff --git a/minference/dist_ops/utils.py b/minference/dist_ops/utils.py
new file mode 100644
index 00000000..9647bd6a
--- /dev/null
+++ b/minference/dist_ops/utils.py
@@ -0,0 +1,580 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import inspect
+import math
+import operator
+
+# Credits: This logger implementation is inspired by project https://github.com/zhuzilin/ring-flash-attention
+import os
+from functools import cache, reduce
+from typing import Dict, List, Optional, Tuple
+
+import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+from torch.distributed.distributed_c10d import P2POp
+
+PROCESS_GROUPS: Dict[str, dist.ProcessGroup] = {}
+
+
+@cache
+def _get_default_args(func):
+ spec = inspect.getfullargspec(func)
+ defaults = spec.defaults if spec.defaults is not None else ()
+ padded_defaults = (None,) * (len(spec.args) - len(defaults)) + defaults
+ args = dict(zip(spec.args, padded_defaults))
+ if "softcap" in args:
+ args["softcap"] = 0.0
+ return args
+
+
+def get_default_args(func):
+ if inspect.isfunction(func):
+ return _get_default_args(func)
+ else:
+ # Use the origin _init_fn in CustomOpDef
+ return _get_default_args(func._init_fn)
+
+
+# copy from megatron/core/utils.py
+class GlobalMemoryBuffer:
+ """Global buffer to avoid dynamic memory allocations.
+ Caller should ensure that buffers of the same name
+ are not used concurrently."""
+
+ def __init__(self):
+ self.buffer = {}
+
+ def get_tensor(self, tensor_shape, dtype, name):
+ required_len = reduce(operator.mul, tensor_shape, 1)
+ if (
+ self.buffer.get((name, dtype), None) is None
+ or self.buffer[(name, dtype)].numel() < required_len
+ ):
+ self.buffer[(name, dtype)] = torch.empty(
+ required_len,
+ dtype=dtype,
+ device=torch.cuda.current_device(),
+ requires_grad=False,
+ )
+
+ return self.buffer[(name, dtype)][0:required_len].view(*tensor_shape)
+
+
+@triton.jit
+def _update_out_and_lse_kernel(
+ Out0,
+ Lse0,
+ Out1,
+ Lse1,
+ stride_oz0,
+ stride_om0,
+ stride_oh0,
+ stride_od0,
+ stride_lz0,
+ stride_lm0,
+ stride_lh0,
+ stride_oz1,
+ stride_om1,
+ stride_oh1,
+ stride_od1,
+ stride_lz1,
+ stride_lm1,
+ stride_lh1,
+ num_tokens,
+ BLOCK_M: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+):
+ start_m = tl.program_id(0)
+ head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+
+ if start_m * BLOCK_M >= num_tokens:
+ return
+
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_d = tl.arange(0, BLOCK_D)
+ m_mask = offs_m < num_tokens
+
+ o0_ptrs = (
+ Out0
+ + batch_idx * stride_oz0
+ + head_idx * stride_oh0
+ + offs_m[:, None] * stride_om0
+ + offs_d[None, :] * stride_od0
+ )
+ o1_ptrs = (
+ Out1
+ + batch_idx * stride_oz1
+ + head_idx * stride_oh1
+ + offs_m[:, None] * stride_om1
+ + offs_d[None, :] * stride_od1
+ )
+ lse0_ptrs = (
+ Lse0 + batch_idx * stride_lz0 + head_idx * stride_lh0 + offs_m * stride_lm0
+ )
+ lse1_ptrs = (
+ Lse1 + batch_idx * stride_lz1 + head_idx * stride_lh1 + offs_m * stride_lm1
+ )
+
+ lse0 = tl.load(lse0_ptrs, mask=m_mask, other=float("-inf"))
+ lse1 = tl.load(lse1_ptrs, mask=m_mask, other=float("-inf"))
+ o0 = tl.load(o0_ptrs, mask=m_mask[:, None], other=0.0).to(tl.float32)
+ o1 = tl.load(o1_ptrs, mask=m_mask[:, None], other=0.0).to(tl.float32)
+
+ m_mask &= (lse0 - lse1) < 88.0
+
+ theta = tl.math.exp(lse0 - lse1)
+ alpha0 = 1 / (1 + 1 / theta)
+ alpha1 = 1 / (1 + theta)
+ o = alpha0[:, None] * o0 + alpha1[:, None] * o1
+ lse = lse1 - tl.math.log(alpha1)
+
+ tl.store(o0_ptrs, o.to(Out0.type.element_ty), mask=m_mask[:, None])
+ tl.store(lse0_ptrs, lse, mask=m_mask)
+
+
+def _update_out_and_lse_triton(
+ out: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ lse: torch.Tensor, # [batch_size, num_tokens, num_heads, 1]
+ block_out: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ block_lse: torch.Tensor, # [batch_size, num_heads, num_tokens] => [batch_size, num_tokens, num_heads, 1]
+ step_idx: Optional[int] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ block_lse = block_lse.transpose(-2, -1).unsqueeze(dim=-1)
+ batch_size, num_tokens, num_heads, head_dim = out.shape
+ block_M = 128
+ block_D = head_dim
+ _update_out_and_lse_kernel[
+ (triton.cdiv(num_tokens, block_M), num_heads, batch_size)
+ ](
+ out,
+ lse,
+ block_out,
+ block_lse,
+ out.stride(0),
+ out.stride(1),
+ out.stride(2),
+ out.stride(3),
+ lse.stride(0),
+ lse.stride(1),
+ lse.stride(2),
+ block_out.stride(0),
+ block_out.stride(1),
+ block_out.stride(2),
+ block_out.stride(3),
+ block_lse.stride(0),
+ block_lse.stride(1),
+ block_lse.stride(2),
+ num_tokens,
+ BLOCK_M=block_M,
+ BLOCK_D=block_D,
+ num_warps=4,
+ num_stages=1,
+ )
+ return out, lse
+
+
+@torch.jit.script
+def _update_out_and_lse_torch(
+ out: torch.Tensor,
+ lse: torch.Tensor,
+ block_out: torch.Tensor,
+ block_lse: torch.Tensor,
+ step_idx: Optional[int] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ block_out = block_out.to(torch.float32)
+ block_lse = block_lse.transpose(-2, -1).unsqueeze(dim=-1)
+
+ # new_lse = lse + torch.log(1 + torch.exp(block_lse - lse))
+ # torch.exp(lse - new_lse) * out + torch.exp(block_lse - new_lse) * block_out
+ # For additional context and discussion, please refer to:
+ # https://github.com/zhuzilin/ring-flash-attention/pull/34#issuecomment-2076126795
+ out = out - F.sigmoid(block_lse - lse) * (out - block_out)
+ lse = lse - F.logsigmoid(lse - block_lse)
+
+ return out, lse
+
+
+def update_out_and_lse(
+ out: Optional[torch.Tensor],
+ lse: Optional[torch.Tensor],
+ block_out: torch.Tensor,
+ block_lse: torch.Tensor,
+ slice_=None,
+ step_idx: Optional[int] = None,
+ use_triton_kernel: bool = True,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ if use_triton_kernel:
+ _update_out_and_lse = _update_out_and_lse_triton
+ else:
+ _update_out_and_lse = _update_out_and_lse_torch
+ if out is None:
+ if slice_ is not None:
+ raise RuntimeError("first update_out_and_lse should not pass slice_ args")
+ out = block_out.to(torch.float32)
+ lse = block_lse.transpose(-2, -1).unsqueeze(dim=-1)
+ elif slice_ is not None:
+ slice_out, slice_lse = out[slice_], lse[slice_]
+ slice_out, slice_lse = _update_out_and_lse(
+ slice_out, slice_lse, block_out, block_lse, step_idx=step_idx
+ )
+ out[slice_], lse[slice_] = slice_out, slice_lse
+ else:
+ out, lse = _update_out_and_lse(
+ out, lse, block_out, block_lse, step_idx=step_idx
+ )
+
+ return out, lse
+
+
+class RingComm:
+ def __init__(
+ self,
+ process_group: dist.ProcessGroup,
+ zigzag: bool = False,
+ ring_list: Optional[list] = None,
+ ):
+ self._process_group = process_group
+ self._ops: List[P2POp] = []
+ self.rank = dist.get_rank(self._process_group)
+ self.world_size = dist.get_world_size(self._process_group)
+ self._reqs = None
+
+ if ring_list is not None:
+ curr_idx = ring_list.index(self.rank)
+ self.send_rank = ring_list[(curr_idx + 1) % len(ring_list)]
+ self.recv_rank = ring_list[(curr_idx - 1 + len(ring_list)) % len(ring_list)]
+ elif zigzag:
+ parts = self.world_size // 2
+ self.ring_list = []
+ for i in range(parts):
+ self.ring_list.extend([i, self.world_size - i - 1])
+ self.revert_rank = self.ring_list.index(self.rank)
+ offset = (dist.get_rank() // self.world_size) * self.world_size
+ self.send_rank = (
+ self.ring_list[(self.revert_rank + 1) % self.world_size] + offset
+ )
+ self.recv_rank = (
+ self.ring_list[(self.revert_rank - 1) % self.world_size] + offset
+ )
+ else:
+ self.send_rank = (self.rank + 1) % self.world_size
+ self.recv_rank = (self.rank - 1) % self.world_size
+
+ def send_recv(
+ self,
+ to_send: torch.Tensor,
+ recv_tensor: Optional[torch.Tensor] = None,
+ step_idx: int = 0,
+ fwd: int = 1,
+ ) -> torch.Tensor:
+ if recv_tensor is None:
+ res = torch.empty_like(to_send)
+ else:
+ res = recv_tensor
+
+ send_op = dist.P2POp(
+ dist.isend,
+ to_send,
+ self.send_rank,
+ group=self._process_group,
+ tag=2 * (step_idx * (self.rank + 1)) + fwd,
+ )
+ recv_op = dist.P2POp(
+ dist.irecv,
+ res,
+ self.recv_rank,
+ group=self._process_group,
+ tag=2 * (step_idx * (self.rank + 1)) + fwd,
+ )
+
+ self._ops.append(send_op)
+ self._ops.append(recv_op)
+ return res
+
+ def commit(self):
+ if self._reqs is not None:
+ raise RuntimeError("commit called twice")
+ self._reqs = dist.batch_isend_irecv(self._ops)
+
+ def wait(self):
+ if self._reqs is None:
+ raise RuntimeError("wait called before commit")
+ for req in self._reqs:
+ req.wait()
+ self._reqs = None
+ self._ops = []
+
+ def send_recv_kv(
+ self,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ k_buffer: Optional[torch.Tensor] = None,
+ v_buffer: Optional[torch.Tensor] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ next_k, next_v = self.send_recv(k, k_buffer), self.send_recv(v, v_buffer)
+ self.commit()
+ return next_k, next_v
+
+ def send_recv_kv_offsets(
+ self,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ kv_seq_offsets: torch.Tensor,
+ k_buffer: Optional[torch.Tensor] = None,
+ v_buffer: Optional[torch.Tensor] = None,
+ kv_seq_offsets_buffer: Optional[torch.Tensor] = None,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ next_k, next_v = self.send_recv(k, k_buffer), self.send_recv(v, v_buffer)
+ next_kv_seq_offsets = self.send_recv(kv_seq_offsets, kv_seq_offsets_buffer)
+
+ self.commit()
+ return next_k, next_v, next_kv_seq_offsets
+
+
+def shuffle_zigzag_input(
+ to_send: torch.Tensor, dim: int = 1, process_group: dist.ProcessGroup = None
+):
+ dim %= len(to_send.shape)
+
+ if not to_send.is_contiguous():
+ to_send = to_send.contiguous()
+
+ # We must use outplace, otherwise it will raise error at backward due to inplace operations.
+ # We can not change to_send directly and create a new tensor to store the result.
+ to_send_f = torch.zeros_like(to_send)
+
+ # assume the input sequence length is 8, and computation runs on 4 GPUs
+ # the seq is represented as [0 1 2 3 4 5 6 7], world size is 4
+ # the input status before `shuffle_zigzag_input` is
+ # - gpu A: [0 1]
+ # - gpu B: [2 3]
+ # - gpu C: [4 5]
+ # - gpu D: [6 7]
+ # the value of `to_send_slice` is
+ # - gpu A: [1]
+ # - gpu B: [3]
+ # - gpu C: [5]
+ # - gpu D: [7]
+ block_seq_len = to_send.shape[dim] // 2
+ left_slicer = [slice(None)] * dim + [slice(None, block_seq_len)]
+ right_slicer = [slice(None)] * dim + [slice(block_seq_len, None)]
+ to_send_slice = to_send[right_slicer].contiguous()
+
+ rank = dist.get_rank(process_group)
+ world_size = dist.get_world_size(process_group)
+
+ res = torch.zeros_like(to_send_slice)
+
+ _ops = []
+ offset = (dist.get_rank() // world_size) * world_size
+ # rank src_rank
+ # 0 3
+ # 1 2
+ # 2 1
+ # 3 0
+ src_rank = (world_size - rank - 1) % world_size + offset
+ send_op = dist.P2POp(dist.isend, to_send_slice, src_rank, group=process_group)
+ recv_op = dist.P2POp(dist.irecv, res, src_rank, group=process_group)
+
+ _ops.append(send_op)
+ _ops.append(recv_op)
+
+ response = dist.batch_isend_irecv(_ops)
+ for resp in response:
+ resp.wait()
+
+ if rank >= world_size // 2: # D: 6 7, -> 1 6
+ to_send_f[right_slicer] = to_send[left_slicer]
+ to_send_f[left_slicer] = res
+ else: # A: 0 1, -> 0 7
+ to_send_f[left_slicer] = to_send[left_slicer]
+ to_send_f[right_slicer] = res
+ # after shuffle, the status of `to_send_f`
+ # GPU A: [0 7]
+ # GPU B: [2 5]
+ # GPU C: [3 4]
+ # GPU D: [1 6]
+
+ return to_send_f
+
+
+def recover_zigzag_output(
+ to_send: torch.Tensor, dim: int = 1, process_group: dist.ProcessGroup = None
+):
+ dim %= len(to_send.shape)
+
+ if not to_send.is_contiguous():
+ to_send = to_send.contiguous()
+
+ to_send_f = torch.zeros_like(to_send)
+
+ block_seq_len = to_send.shape[dim] // 2
+ left_slicer = [slice(None)] * dim + [slice(None, block_seq_len)]
+ right_slicer = [slice(None)] * dim + [slice(block_seq_len, None)]
+
+ rank = dist.get_rank(process_group)
+ world_size = dist.get_world_size(process_group)
+
+ if rank >= world_size // 2:
+ to_send_slice = to_send[left_slicer].contiguous()
+ else:
+ to_send_slice = to_send[right_slicer].contiguous()
+ res = torch.zeros_like(to_send_slice)
+
+ assert to_send_slice.is_contiguous()
+ assert res.is_contiguous()
+
+ _ops = []
+ offset = (dist.get_rank() // world_size) * world_size
+ src_rank = (world_size - rank - 1) % world_size + offset
+ send_op = dist.P2POp(dist.isend, to_send_slice, src_rank, group=process_group)
+ recv_op = dist.P2POp(dist.irecv, res, src_rank, group=process_group)
+
+ _ops.append(send_op)
+ _ops.append(recv_op)
+
+ response = dist.batch_isend_irecv(_ops)
+ for resp in response:
+ resp.wait()
+
+ if rank >= world_size // 2:
+ to_send_f[left_slicer] = to_send[right_slicer]
+ to_send_f[right_slicer] = res
+ else:
+ to_send_f[left_slicer] = to_send[left_slicer]
+ to_send_f[right_slicer] = res
+
+ return to_send_f.contiguous()
+
+
+def shuffle_block_mask_zigzag(
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ num_blocks_per_chunk: int,
+ group: dist.ProcessGroup,
+):
+ rank = dist.get_rank(group)
+ world_size = dist.get_world_size(group)
+
+ # ---------------------------------------
+ # Shuffle Query chunks
+ block_mask = shuffle_zigzag_input(
+ to_send=block_mask, dim=-2, process_group=group
+ ) # [batch_size, num_qo_heads, num_blocks_local, num_blocks]
+
+ # ---------------------------------------
+ # Shuffle Key chunks
+ ring_list = RingComm(group, zigzag=True).ring_list
+ ring_index = ring_list.index(rank)
+
+ shuffled_block_mask_list = []
+ for i in range(world_size):
+ rank_src = ring_list[(ring_index - i) % world_size]
+
+ curr_chunk_index = 2 * rank_src
+ rev_chunk_index = 2 * world_size - 1 - curr_chunk_index
+ if curr_chunk_index > rev_chunk_index:
+ curr_chunk_index, rev_chunk_index = rev_chunk_index, curr_chunk_index
+
+ shuffled_block_mask_list.append(
+ torch.cat(
+ [
+ block_mask[
+ ...,
+ curr_chunk_index
+ * num_blocks_per_chunk : (curr_chunk_index + 1)
+ * num_blocks_per_chunk,
+ ],
+ block_mask[
+ ...,
+ rev_chunk_index
+ * num_blocks_per_chunk : (rev_chunk_index + 1)
+ * num_blocks_per_chunk,
+ ],
+ ],
+ dim=-1,
+ )
+ )
+ block_mask = torch.stack(
+ shuffled_block_mask_list, dim=0
+ ).contiguous() # [world_size, batch_size, num_qo_heads, num_blocks_local, num_blocks_local]
+ return block_mask
+
+
+def shuffle_striped_input(
+ to_send: torch.Tensor, # [B, N / W, H, D]
+ granularity: int = 1,
+ dim: int = 1,
+ process_group: dist.ProcessGroup = None,
+):
+ # 00, 01, 02, 03, 04, 05, 06, 07 => 00, 04, 08, 12, 16, 20, 24, 28
+ # 08, 09, 10, 11, 12, 13, 14, 15 => 01, 05, 09, 13, 17, 21, 25, 29
+ # 16, 17, 18, 19, 20, 21, 22, 23 => 02, 06, 10, 14, 18, 22, 26, 30
+ # 24, 25, 26, 27, 28, 39, 30, 31 => 03, 07, 11, 15, 19, 23, 27, 31
+ shape = to_send.shape
+ dim %= len(shape)
+ world_size = dist.get_world_size(process_group)
+ input_reshape = to_send.reshape(
+ (*shape[:dim], -1, world_size * granularity, *shape[dim + 1 :])
+ )
+ input_list = [
+ x.contiguous() for x in input_reshape.split(granularity, dim=dim + 1)
+ ] # [N / W / (W * G), W*, G]
+ output_list = [torch.empty_like(x) for x in input_list] # [W*, N / W / (W * G), G]
+
+ dist.all_to_all(output_list, input_list, group=process_group)
+ return torch.stack(output_list, dim=dim).reshape(shape).contiguous()
+
+
+def recover_striped_output(
+ to_send: torch.Tensor, # [B, N / W, H, D]
+ granularity: int = 1,
+ dim: int = 1,
+ process_group: dist.ProcessGroup = None,
+):
+ # 00, 04, 08, 12, 16, 20, 24, 28 => 00, 01, 02, 03, 04, 05, 06, 07
+ # 01, 05, 09, 13, 17, 21, 25, 29 => 08, 09, 10, 11, 12, 13, 14, 15
+ # 02, 06, 10, 14, 18, 22, 26, 30 => 16, 17, 18, 19, 20, 21, 22, 23
+ # 03, 07, 11, 15, 19, 23, 27, 31 => 24, 25, 26, 27, 28, 39, 30, 31
+ shape = to_send.shape
+ dim %= len(shape)
+ world_size = dist.get_world_size(process_group)
+
+ input_reshape = to_send.reshape(
+ (*shape[:dim], world_size, -1, granularity, *shape[dim + 1 :])
+ )
+ input_list = [
+ x.squeeze(dim).contiguous() for x in input_reshape.split(1, dim=dim)
+ ] # [W*, N / W / (W * G), G]
+ output_list = [torch.empty_like(x) for x in input_list] # [N / W / (W * G), W*, G]
+
+ dist.all_to_all(output_list, input_list, group=process_group)
+ return torch.stack(output_list, dim=dim + 1).reshape(shape).contiguous()
+
+
+# --------------------------------------------------------------------
+# Double-Ring Related
+def get_inner_ring(group: dist.ProcessGroup):
+ rank = dist.get_rank(group)
+ local_rank = int(os.environ["LOCAL_RANK"])
+ local_world_size = int(os.environ.get("LOCAL_WORLD_SIZE"))
+ assert rank % local_world_size == local_rank
+ return [i + (rank - local_rank) for i in range(local_world_size)]
+
+
+def get_outer_ring(group: dist.ProcessGroup):
+ rank = dist.get_rank(group)
+ world_size = dist.get_world_size(group)
+ local_rank = int(os.environ["LOCAL_RANK"])
+ local_world_size = int(os.environ.get("LOCAL_WORLD_SIZE"))
+ assert rank % local_world_size == local_rank
+ return [
+ i * local_world_size + local_rank for i in range(world_size // local_world_size)
+ ]
diff --git a/minference/dist_ops/xattn_zigzag.py b/minference/dist_ops/xattn_zigzag.py
new file mode 100644
index 00000000..ca67dffe
--- /dev/null
+++ b/minference/dist_ops/xattn_zigzag.py
@@ -0,0 +1,588 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import math
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import torch
+import torch.distributed as dist
+import triton
+
+from minference.ops.op_utils.vertical_slash_utils import convert_blockmask
+from minference.ops.op_utils.xattn_utils import LN2, find_blocks_chunked
+from minference.ops.pit_sparse_flash_attention_v3 import (
+ block_attn_bwd,
+ block_attn_fwd,
+ triton_block_attn_bwd,
+ triton_block_attn_fwd,
+)
+from minference.ops.utils import use_triton
+from minference.ops.xattention_fa import (
+ flat_group_gemm_fuse_reshape,
+ softmax_fuse_block_sum,
+)
+
+from .utils import (
+ RingComm,
+ recover_zigzag_output,
+ shuffle_block_mask_zigzag,
+ shuffle_zigzag_input,
+ update_out_and_lse,
+)
+
+
+def xattn_zigzag_estimate(
+ query_states: torch.Tensor, # (batch_size, num_q_head, q_len, head_dim)
+ key_states: torch.Tensor, # (batch_size, num_kv_head, k_len, head_dim)
+ block_size,
+ stride,
+ norm=1,
+ softmax=True,
+ threshold=0.9,
+ select_mode="inverse",
+ use_triton=True,
+ causal=True,
+ kdb: int = 1,
+ keep_sink=False,
+ keep_recent=False,
+ group: dist.group = None,
+) -> torch.Tensor:
+ batch_size, num_kv_head, k_len_local, head_dim = key_states.shape
+ batch_size, num_q_head, q_len_local, head_dim = query_states.shape
+
+ rank = dist.get_rank(group)
+ world_size = dist.get_world_size(group)
+
+ k_gather_list = [torch.empty_like(key_states) for _ in range(world_size)]
+ dist.all_gather(k_gather_list, key_states.contiguous(), group=group)
+ k_gathered = torch.cat(k_gather_list, dim=2)
+ k_len = k_gathered.shape[2]
+
+ if num_q_head > num_kv_head:
+ k_gathered = torch.repeat_interleave(
+ k_gathered.contiguous(), num_q_head // num_kv_head, dim=1
+ )
+
+ chunk_size = q_len_local // 2
+ q_chunk_num = 2
+ q_block_num = q_len_local // block_size
+ q_block_num_per_chunk = chunk_size // block_size
+
+ # assert num_kv_head == num_q_head
+ attn_sum_list = []
+ simple_mask_list = []
+
+ num_strides_in_k = k_len // stride
+ num_strides_per_chunk = chunk_size // stride
+ num_strides_per_block = block_size // stride
+ num_blocks_per_chunk = num_strides_per_chunk // num_strides_per_block
+
+ attn_weight_slices = [None, None]
+ for chunk_idx in range(q_chunk_num):
+ global_chunk_idx = rank * 2 + chunk_idx
+
+ # Local start index
+ q_chunk_start = chunk_idx * chunk_size
+ q_chunk_end = (chunk_idx + 1) * chunk_size
+
+ # Global start index (stride-level)
+ q_chunk_start_stride_global = global_chunk_idx * num_strides_per_chunk
+ q_chunk_end_stride_global = (global_chunk_idx + 1) * num_strides_per_chunk
+
+ # attn_weights_slice: (batch_size, num_heads, chunk_size // stride, kv_len // stride)
+ # (i.e. the attention sum of each SxS stride block)
+ # This step is agnostic to block size and just computes the attention sum in each stride block
+ attn_weight_slice = flat_group_gemm_fuse_reshape(
+ # query_states, key_states, stride, chunk_start, chunk_end, is_causal=True
+ query_states[
+ :,
+ :,
+ q_chunk_start:q_chunk_end,
+ :,
+ ],
+ k_gathered,
+ stride,
+ q_chunk_start_stride_global,
+ q_chunk_end_stride_global,
+ is_causal=causal,
+ )
+ attn_weight_slices[chunk_idx] = attn_weight_slice
+ del k_gathered, k_gather_list
+
+ for chunk_idx in range(q_chunk_num):
+ global_chunk_idx = rank * 2 + chunk_idx
+
+ # Local start index
+ q_chunk_start = chunk_idx * chunk_size
+ q_chunk_end = (chunk_idx + 1) * chunk_size
+
+ # Global start index (block-level)
+ q_block_start = global_chunk_idx * q_block_num_per_chunk
+ q_block_end = (global_chunk_idx + 1) * q_block_num_per_chunk
+
+ # Global start index (stride-level)
+ q_chunk_start_stride_global = global_chunk_idx * num_strides_per_chunk
+ q_chunk_end_stride_global = (global_chunk_idx + 1) * num_strides_per_chunk
+
+ attn_weight_slice = attn_weight_slices[chunk_idx]
+
+ # (batch_size, num_heads, q_block_num, k_block_num),
+ attn_sum = softmax_fuse_block_sum(
+ attn_weight_slice, # (batch_size, num_heads, chunk_size // stride, kv_len // stride)
+ num_strides_per_block,
+ min(4096, num_strides_per_block),
+ q_chunk_start_stride_global,
+ q_chunk_end_stride_global,
+ num_strides_in_k,
+ 1 / LN2 / math.sqrt(head_dim) / stride / norm,
+ is_causal=causal,
+ )
+
+ # (batch_size, head_num, num_blocks_per_chunk, block_num)
+ simple_mask = find_blocks_chunked(
+ attn_sum,
+ global_chunk_idx * num_blocks_per_chunk,
+ threshold,
+ None,
+ decoding=False,
+ mode="prefill",
+ causal=causal,
+ )
+
+ del attn_weight_slice
+ if causal:
+ simple_mask[:, :, :, q_block_start:q_block_end] = torch.where(
+ torch.tril(
+ torch.ones(
+ q_block_num_per_chunk,
+ q_block_num_per_chunk,
+ dtype=bool,
+ device=key_states.device,
+ ),
+ diagonal=0,
+ ),
+ simple_mask[:, :, :, q_block_start:q_block_end],
+ False,
+ )
+ simple_mask[:, :, :, q_block_end:] = 0
+ if keep_sink:
+ simple_mask[:, :, 0, :] = True
+ if keep_recent:
+ eye_matrix = torch.eye(
+ q_block_num_per_chunk, device=simple_mask.device, dtype=bool
+ )
+ eye_matrix_expanded = (
+ eye_matrix.unsqueeze(0)
+ .unsqueeze(0)
+ .expand(1, num_kv_head, q_block_num_per_chunk, q_block_num_per_chunk)
+ )
+ simple_mask[:, :, :, q_block_start:q_block_end] = torch.where(
+ eye_matrix_expanded,
+ True,
+ simple_mask[:, :, :, q_block_start:q_block_end],
+ )
+
+ attn_sum_list.append(attn_sum)
+ simple_mask_list.append(simple_mask)
+
+ attn_sums = torch.cat(attn_sum_list, dim=-2)
+ simple_masks = torch.cat(
+ simple_mask_list, dim=-2
+ ) # (batch_size, head_num, q_local_block_num, k_global_block_num)
+ return attn_sums, simple_masks
+
+
+def xattn_zigzag_forward(
+ process_group: dist.ProcessGroup,
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ layer_idx: int,
+ softmax_scale: float,
+ granularity: int = 128,
+ block_idx: Optional[torch.Tensor] = None,
+ block_cnt: Optional[torch.Tensor] = None,
+):
+ comm = RingComm(process_group, zigzag=True)
+ out, lse = None, None
+ next_k, next_v = None, None
+
+ for step in range(comm.world_size):
+ if step + 1 != comm.world_size:
+ next_k, next_v = comm.send_recv_kv(k, v)
+
+ # [batch_size, num_qo_heads, num_blocks_local, num_blocks_local]
+ block_mask_step = block_mask[step]
+ block_causal = step == 0
+
+ if use_triton():
+ # TODO: block_mask here needs to be converted to block_idx before passing to triton
+ block_out, block_lse = triton_block_attn_fwd(
+ q,
+ k,
+ v,
+ block_idx=block_idx[step],
+ block_cnt=block_cnt[step],
+ softmax_scale=softmax_scale,
+ granularity=granularity,
+ causal=block_causal,
+ step=step,
+ )
+ else:
+ block_out, block_lse = block_attn_fwd(
+ q,
+ k,
+ v,
+ block_mask=block_mask_step,
+ softmax_scale=softmax_scale,
+ granularity=granularity,
+ causal=block_causal,
+ step_idx=step,
+ )
+
+ out, lse = update_out_and_lse(
+ out, lse, block_out, block_lse, use_triton_kernel=False
+ )
+ if step + 1 != comm.world_size:
+ comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ lse = lse.squeeze(dim=-1).transpose(1, 2)
+ return out, lse
+
+
+def xattn_zigzag_backward(
+ process_group: dist.ProcessGroup,
+ dout: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ out: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ layer_idx: int,
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [world_size, batch_size, num_qo_heads, num_blocks, num_blocks]
+ granularity: int = 128,
+ block_idx: Optional[
+ torch.Tensor
+ ] = None, # [world_size, batch_size, num_qo_heads, num_blocks_local, num_blocks]
+ block_cnt: Optional[
+ torch.Tensor
+ ] = None, # [world_size, batch_size, num_qo_heads, num_blocks_local]
+):
+ kv_comm = RingComm(process_group, zigzag=True)
+ d_kv_comm = RingComm(process_group, zigzag=True)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ for step in range(kv_comm.world_size):
+ if step + 1 != kv_comm.world_size:
+ next_k, next_v = kv_comm.send_recv_kv(k, v)
+
+ block_causal = step == 0
+ block_mask_step = block_mask[step]
+
+ # --------------------------------
+ # Block Mask
+ if use_triton():
+ step_dq, step_dk, step_dv = triton_block_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ block_idx[step],
+ block_cnt[step],
+ granularity=granularity,
+ deterministic=False,
+ causal=block_causal,
+ step=step,
+ )
+ else:
+ step_dq, step_dk, step_dv = block_attn_bwd(
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ softmax_scale,
+ block_mask_step,
+ granularity=granularity,
+ deterministic=False,
+ causal=block_causal,
+ )
+
+ # Update dQ, dK, dV
+ if step == 0:
+ # TODO: check if float32 is necessary
+ dq = step_dq.to(torch.float32)
+ dk = step_dk.to(torch.float32)
+ dv = step_dv.to(torch.float32)
+ else:
+ d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ dq += step_dq
+ dk += step_dk
+ dv += step_dv
+
+ if step + 1 != kv_comm.world_size:
+ kv_comm.wait()
+ k, v = next_k, next_v
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk, dv, dk_comm_buffer, dv_comm_buffer
+ )
+
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+class XAttnZigzagFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer_idx,
+ xattn_params, # Dict[str, Any]
+ granularity,
+ causal,
+ softmax_scale,
+ return_softmax,
+ deterministic,
+ group,
+ ):
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+
+ # ----------------------------------------------
+ # Index Building
+ # block_mask [batch_size, num_qo_heads, num_blocks_local, num_blocks]
+ _, block_mask = xattn_zigzag_estimate(
+ q.transpose(1, 2), k.transpose(1, 2), block_size=granularity, **xattn_params
+ )
+
+ # ------------------------------------------------------------------
+ # QKV Shuffling
+ q = shuffle_zigzag_input(to_send=q, dim=1, process_group=group)
+ k = shuffle_zigzag_input(to_send=k, dim=1, process_group=group)
+ v = shuffle_zigzag_input(to_send=v, dim=1, process_group=group)
+
+ # ------------------------------------------------------------------
+ # Index Shuffling
+ block_mask = shuffle_block_mask_zigzag(
+ block_mask, num_blocks_per_chunk=q.shape[1] // 2 // granularity, group=group
+ ).to(q.device)
+ if use_triton():
+ block_idx, block_cnt = convert_blockmask(
+ block_mask, block_size_M=granularity, block_size_N=64
+ )
+ else:
+ block_idx, block_cnt = None, None
+ block_mask = block_mask.contiguous()
+
+ # ----------------------------------------------
+ # Compute
+ out, softmax_lse = xattn_zigzag_forward(
+ group,
+ q,
+ k,
+ v,
+ block_mask,
+ layer_idx,
+ softmax_scale,
+ granularity=granularity,
+ block_idx=block_idx,
+ block_cnt=block_cnt,
+ )
+
+ # ----------------------------------------------
+ # Recover outputs
+ recovered_out = recover_zigzag_output(out, dim=1, process_group=group)
+ if return_softmax:
+ recovered_softmax_lse = recover_zigzag_output(
+ softmax_lse, dim=2, process_group=group
+ )
+
+ # -------------------------------
+ # Variale Saving
+ if use_triton():
+ ctx.save_for_backward(
+ q, k, v, out, softmax_lse, block_mask, block_idx, block_cnt
+ )
+ else:
+ ctx.save_for_backward(q, k, v, out, softmax_lse, block_mask)
+ ctx.softmax_scale = softmax_scale
+ ctx.granularity = granularity
+ ctx.group = group
+ ctx.layer_idx = layer_idx
+
+ # -------------------------------
+ # Recover outputs
+ if return_softmax:
+ return (recovered_out, recovered_softmax_lse, None)
+ return recovered_out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ if use_triton():
+ (
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ block_mask,
+ block_idx,
+ block_cnt,
+ ) = ctx.saved_tensors
+ else:
+ q, k, v, out, softmax_lse, block_mask = ctx.saved_tensors
+ block_idx, block_cnt = None, None
+ softmax_scale = ctx.softmax_scale
+ granularity = ctx.granularity
+ layer_idx = ctx.layer_idx
+ group = ctx.group
+
+ dout = shuffle_zigzag_input(to_send=dout, dim=1, process_group=group)
+
+ # ----------------------------------------------
+ # Compute
+ dq, dk, dv = xattn_zigzag_backward(
+ group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ softmax_lse,
+ layer_idx,
+ softmax_scale,
+ block_mask,
+ granularity,
+ block_idx=block_idx,
+ block_cnt=block_cnt,
+ )
+
+ dq = recover_zigzag_output(dq, dim=1, process_group=group)
+ dk = recover_zigzag_output(dk, dim=1, process_group=group)
+ dv = recover_zigzag_output(dv, dim=1, process_group=group)
+ return dq, dk, dv, None, None, None, None, None, None, None, None, None
+
+
+def xattn_zigzag_qkvpacked_func(
+ qkv: torch.Tensor, # [batch_size, num_tokens, 3, num_heads, head_dim]
+ layer_idx: int,
+ xattn_params: Dict[str, Any],
+ granularity: int = 128,
+ dropout_p: int = 0.0,
+ softmax_scale: float = None,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[float, float] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+):
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+ return XAttnZigzagFunc.apply(
+ qkv[:, :, 0],
+ qkv[:, :, 1],
+ qkv[:, :, 2],
+ layer_idx,
+ xattn_params,
+ granularity,
+ causal,
+ softmax_scale,
+ return_attn_probs,
+ deterministic,
+ group,
+ )
+
+
+def xattn_zigzag_kvpacked_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ kv: torch.Tensor, # [batch_size, num_tokens, 2, num_heads, head_dim]
+ layer_idx: int,
+ xattn_params: Dict[str, Any],
+ granularity: int = 128,
+ dropout_p: int = 0.0,
+ softmax_scale: float = None,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[float, float] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+):
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+
+ return XAttnZigzagFunc.apply(
+ q,
+ kv[:, :, 0],
+ kv[:, :, 1],
+ layer_idx,
+ xattn_params,
+ granularity,
+ causal,
+ softmax_scale,
+ return_attn_probs,
+ deterministic,
+ group,
+ )
+
+
+def xattn_zigzag_func( # the one used for nnscaler training
+ q: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_heads, head_dim]
+ layer_idx: int,
+ xattn_params: Dict[str, Any],
+ granularity: int = 128,
+ dropout_p: int = 0.0,
+ softmax_scale: float = None,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[float, float] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ group: dist.ProcessGroup = None,
+) -> torch.Tensor:
+ assert causal
+ assert dropout_p == 0
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ assert not deterministic
+
+ return XAttnZigzagFunc.apply(
+ q,
+ k,
+ v,
+ layer_idx,
+ xattn_params,
+ granularity,
+ causal,
+ softmax_scale,
+ return_attn_probs,
+ deterministic,
+ group,
+ )
diff --git a/minference/dist_ops/zigzag_attention.py b/minference/dist_ops/zigzag_attention.py
new file mode 100644
index 00000000..bdae51cf
--- /dev/null
+++ b/minference/dist_ops/zigzag_attention.py
@@ -0,0 +1,457 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import copy
+
+# Credits: This logger implementation is inspired by project https://github.com/zhuzilin/ring-flash-attention
+import os
+from time import perf_counter
+from typing import Dict, List, Tuple
+
+import torch
+import torch.distributed as dist
+from flash_attn.flash_attn_interface import _flash_attn_backward, _flash_attn_forward
+
+from .utils import (
+ RingComm,
+ get_default_args,
+ recover_zigzag_output,
+ shuffle_zigzag_input,
+ update_out_and_lse,
+)
+
+
+def zigzag_ring_flash_attn_forward(
+ process_group,
+ q: torch.Tensor, # [B, S, H, D]
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer_idx: int,
+ softmax_scale,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert causal == True, "zigzag ring is meaningless for causal=False"
+ comm = RingComm(process_group, zigzag=True)
+
+ bsz, seq_len, num_heads, head_dim = q.shape
+ block_seq_len = q.shape[1] // 2
+ q1 = q[:, block_seq_len:]
+
+ out = None
+ lse = None
+ next_k, next_v = None, None
+
+ def forward(q, k, v, causal):
+ params = get_default_args(_flash_attn_forward).copy()
+ params.update(
+ {
+ "q": q,
+ "k": k,
+ "v": v,
+ "dropout_p": dropout_p,
+ "softmax_scale": softmax_scale,
+ "causal": causal,
+ "alibi_slopes": alibi_slopes,
+ "return_softmax": True and dropout_p > 0,
+ }
+ )
+ if "window_size" in params:
+ params.update({"window_size": window_size})
+ else:
+ params.update(
+ {
+ "window_size_left": window_size[0],
+ "window_size_right": window_size[1],
+ }
+ )
+ outputs = _flash_attn_forward(**params)
+ if len(outputs) == 8:
+ block_out, _, _, _, _, block_lse, _, _ = outputs
+ else:
+ assert len(outputs) == 4
+ block_out, block_lse, _, _ = outputs
+ return block_out, block_lse
+
+ for step in range(comm.world_size):
+ if step + 1 != comm.world_size:
+ next_k, next_v = comm.send_recv_kv(k, v)
+
+ if step == 0:
+ # Do softmax(QK^T / sqrt(d_k))V on the currently hold K and V
+ # and record the output and the LSE
+ block_out, block_lse = forward(q, k, v, causal=True)
+ out, lse = update_out_and_lse(out, lse, block_out, block_lse)
+ elif step <= comm.revert_rank:
+ k0 = k[:, :block_seq_len]
+ v0 = v[:, :block_seq_len]
+ block_out, block_lse = forward(q, k0, v0, causal=False)
+ out, lse = update_out_and_lse(out, lse, block_out, block_lse)
+ else:
+ block_out, block_lse = forward(q1, k, v, causal=False)
+ out, lse = update_out_and_lse(
+ out,
+ lse,
+ block_out,
+ block_lse,
+ slice_=(slice(None), slice(block_seq_len, None)),
+ )
+
+ if step + 1 != comm.world_size:
+ comm.wait()
+ k, v = next_k, next_v
+
+ out = out.to(q.dtype)
+ lse = lse.squeeze(dim=-1).transpose(1, 2)
+ return out, lse
+
+
+def zigzag_ring_flash_attn_backward(
+ process_group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ layer_idx: int,
+ softmax_lse,
+ softmax_scale,
+ dropout_p=0,
+ causal=True,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+):
+ assert causal == True, "zigzag ring is meaningless for causal=False"
+ bsz, seq_len, num_heads, head_dim = q.shape
+
+ kv_comm = RingComm(process_group, zigzag=True)
+ d_kv_comm = RingComm(process_group, zigzag=True)
+
+ dq, dk, dv = None, None, None
+ next_dk, next_dv = None, None
+ next_k, next_v = None, None
+ dk_comm_buffer, dv_comm_buffer = None, None
+
+ dout1 = dout.chunk(2, dim=1)[1]
+ q1 = q.chunk(2, dim=1)[1]
+ out1 = out.chunk(2, dim=1)[1]
+ softmax_lse1 = softmax_lse.chunk(2, dim=2)[1].contiguous()
+ block_seq_len = q.shape[1] // 2
+
+ # repeatly allocating buffer may be slow...
+ dq_buffer = torch.empty(q.shape, dtype=q.dtype, device=q.device)
+ dk_buffer = torch.empty(k.shape, dtype=k.dtype, device=k.device)
+ dv_buffer = torch.empty(v.shape, dtype=v.dtype, device=v.device)
+
+ def backward(dout_, q_, k_, v_, out_, softmax_lse_, causal_):
+ seqlen_q = q_.shape[1]
+ seqlen_kv = k_.shape[1]
+ params = get_default_args(_flash_attn_backward).copy()
+ params.update(
+ {
+ "dout": dout_,
+ "q": q_,
+ "k": k_,
+ "v": v_,
+ "out": out_,
+ "softmax_lse": softmax_lse_,
+ "dq": dq_buffer[:, :seqlen_q],
+ "dk": dk_buffer[:, :seqlen_kv],
+ "dv": dv_buffer[:, :seqlen_kv],
+ "dropout_p": dropout_p,
+ "softmax_scale": softmax_scale,
+ "causal": causal_,
+ "alibi_slopes": alibi_slopes,
+ "deterministic": deterministic,
+ }
+ )
+ if "window_size" in params:
+ params.update({"window_size": window_size})
+ else:
+ params.update(
+ {
+ "window_size_left": window_size[0],
+ "window_size_right": window_size[1],
+ }
+ )
+ params.update(
+ {"rng_state": torch.zeros((2,), dtype=torch.int64, device=q.device)}
+ )
+
+ _flash_attn_backward(**params)
+
+ for step in range(kv_comm.world_size):
+ if step + 1 != kv_comm.world_size:
+ next_k, next_v = kv_comm.send_recv_kv(k, v)
+
+ # -----------------------------------------------------------
+ if step == 0:
+ backward(dout, q, k, v, out, softmax_lse, causal_=True)
+ dq = dq_buffer.to(torch.float32)
+ dk = dk_buffer.to(torch.float32)
+ dv = dv_buffer.to(torch.float32)
+ else:
+ if step <= kv_comm.revert_rank:
+ k0 = k[:, :block_seq_len]
+ v0 = v[:, :block_seq_len]
+ backward(dout, q, k0, v0, out, softmax_lse, causal_=False)
+ else:
+ backward(dout1, q1, k, v, out1, softmax_lse1, causal_=False)
+
+ d_kv_comm.wait()
+ dk_comm_buffer, dv_comm_buffer = dk, dv
+ dk, dv = next_dk, next_dv
+
+ if step <= kv_comm.revert_rank:
+ dq += dq_buffer
+ dk[:, :block_seq_len] += dk_buffer[:, :block_seq_len]
+ dv[:, :block_seq_len] += dv_buffer[:, :block_seq_len]
+ else:
+ dq[:, block_seq_len:] += dq_buffer[:, :block_seq_len]
+ dk += dk_buffer
+ dv += dv_buffer
+
+ # -----------------------------------------------------------
+ if step + 1 != kv_comm.world_size:
+ kv_comm.wait()
+ k, v = next_k, next_v
+
+ next_dk, next_dv = d_kv_comm.send_recv_kv(
+ dk,
+ dv,
+ dk_comm_buffer,
+ dv_comm_buffer,
+ )
+
+ d_kv_comm.wait()
+ return dq.to(q.dtype), next_dk.to(q.dtype), next_dv.to(q.dtype)
+
+
+"""
+In nnscaler, sequence are stored in the initial order, e.g., [0 1 2 3 4 5 6 7].
+However, zigzag ring flash attention requires the sequence to be in the order of [0 7 2 5 3 4 1 6].
+As a result:
+- in forward, we need to shuffle q, k, v and recover the out
+- in backward, we need to shuffle dout and recover the dq, dk, dv
+"""
+
+
+class ZigZagRingFlashAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_softmax,
+ group,
+ ):
+ assert alibi_slopes is None
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+
+ # ----------------------------------------------
+ # Shuffle
+ q = shuffle_zigzag_input(to_send=q, dim=1, process_group=group)
+ k = shuffle_zigzag_input(to_send=k, dim=1, process_group=group)
+ v = shuffle_zigzag_input(to_send=v, dim=1, process_group=group)
+ k, v = k.contiguous(), v.contiguous()
+
+ # ----------------------------------------------
+ # Compute
+ out, softmax_lse = zigzag_ring_flash_attn_forward(
+ group,
+ q,
+ k,
+ v,
+ layer_idx,
+ softmax_scale=softmax_scale,
+ dropout_p=dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=False,
+ )
+
+ # ----------------------------------------------
+ # Recover outputs
+ recovered_out = recover_zigzag_output(out, dim=1, process_group=group)
+ if return_softmax:
+ recovered_softmax_lse = recover_zigzag_output(
+ softmax_lse, dim=2, process_group=group
+ )
+
+ # ------------------------------
+ # Saving tensors
+ ctx.save_for_backward(q, k, v, out, softmax_lse)
+ ctx.dropout_p = dropout_p
+ ctx.softmax_scale = softmax_scale
+ ctx.causal = causal
+ ctx.window_size = window_size
+ ctx.alibi_slopes = alibi_slopes
+ ctx.deterministic = deterministic
+ ctx.group = group
+ ctx.layer_idx = layer_idx
+ ctx.return_softmax = return_softmax
+
+ # ----------------------------------------------
+ # Output and return
+ if return_softmax:
+ return (recovered_out, recovered_softmax_lse, None)
+ return recovered_out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ q, k, v, out, softmax_lse = ctx.saved_tensors
+ layer_idx = ctx.layer_idx
+ (
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_softmax,
+ group,
+ ) = (
+ ctx.dropout_p,
+ ctx.softmax_scale,
+ ctx.causal,
+ ctx.window_size,
+ ctx.alibi_slopes,
+ ctx.deterministic,
+ ctx.return_softmax,
+ ctx.group,
+ )
+
+ # ----------------------------------------------
+ # Shuffle
+ dout = shuffle_zigzag_input(to_send=dout, dim=1, process_group=group)
+
+ # ----------------------------------------------
+ # Compute
+ dq, dk, dv = zigzag_ring_flash_attn_backward(
+ group,
+ dout,
+ q,
+ k,
+ v,
+ out,
+ layer_idx,
+ softmax_lse,
+ softmax_scale=softmax_scale,
+ dropout_p=dropout_p,
+ causal=causal,
+ window_size=window_size,
+ alibi_slopes=alibi_slopes,
+ deterministic=deterministic,
+ )
+
+ # ----------------------------------------------
+ # Recover
+ dq = recover_zigzag_output(dq, dim=1, process_group=group)
+ dk = recover_zigzag_output(dk, dim=1, process_group=group)
+ dv = recover_zigzag_output(dv, dim=1, process_group=group)
+
+ return dq, dk, dv, None, None, None, None, None, None, None, None, None
+
+
+def zigzag_ring_flash_attn_qkvpacked_func(
+ qkv,
+ layer_idx,
+ dropout_p=0.0,
+ softmax_scale=None,
+ causal=False,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return ZigZagRingFlashAttnFunc.apply(
+ qkv[:, :, 0],
+ qkv[:, :, 1],
+ qkv[:, :, 2],
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def zigzag_ring_flash_attn_kvpacked_func(
+ q,
+ kv,
+ layer_idx,
+ dropout_p=0.0,
+ softmax_scale=None,
+ causal=False,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return ZigZagRingFlashAttnFunc.apply(
+ q,
+ kv[:, :, 0],
+ kv[:, :, 1],
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
+
+
+def zigzag_ring_flash_attn_func(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer_idx,
+ dropout_p=0.0,
+ softmax_scale=None,
+ causal=False,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=False,
+ group=None,
+):
+ return ZigZagRingFlashAttnFunc.apply(
+ q,
+ k,
+ v,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ group,
+ )
diff --git a/minference/minference_configuration.py b/minference/minference_configuration.py
index 204fae91..31175b6b 100644
--- a/minference/minference_configuration.py
+++ b/minference/minference_configuration.py
@@ -1,24 +1,43 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
-import os
-
-from .configs.model2path import MODEL2PATH
+from .configs.model2path import LEANKPATNS, MODEL2PATH
class MInferenceConfig:
MINFERENCE_ATTENTION_TYPES = [
"minference",
- "vllm",
+ "vllm_minference",
+ "tri_mix_minference",
]
- STASTIC_ATTENTION_TYPES = [
- "minference_with_dense",
- "static",
+ OTHER_ATTENTION_TYPES = [
+ # original implement
+ "hf",
+ "vllm",
+ # our custom implement
+ "dense",
+ "static", # minference w/ static
"dilated1",
"dilated2",
- "streaming",
+ "a_shape",
+ "tri_shape",
+ "vllm_a_shape",
+ "vllm_tri_shape",
"inf_llm",
- "hf",
+ "flexprefill",
+ "vllm_flexprefill",
+ "xattention",
+ "tri_mix",
+ ]
+ KV_TYPES = [
+ "dense",
+ "streamingllm",
+ "snapkv",
+ "pyramidkv",
+ "quest",
+ "retr_attn",
+ "kivi",
+ "leank",
]
def __init__(
@@ -29,15 +48,22 @@ def __init__(
starting_layer: int = -1,
kv_cache_cpu: bool = False,
kv_cache_cpu_device: str = "cpu",
- use_snapkv: bool = False,
+ kv_type: str = "dense",
is_search: bool = False,
attn_kwargs: dict = {},
**kwargs,
):
super(MInferenceConfig, self).__init__()
+ attn_type, kv_type = self.update_config_type(attn_type, kv_type)
assert (
- attn_type in self.MINFERENCE_ATTENTION_TYPES + self.STASTIC_ATTENTION_TYPES
- ), f"The attention_type {attn_type} you specified is not supported."
+ attn_type in self.MINFERENCE_ATTENTION_TYPES + self.OTHER_ATTENTION_TYPES
+ ), f"The attn_type {attn_type} you specified is not supported."
+ assert (
+ kv_type in self.KV_TYPES
+ ), f"The kv_type {kv_type} you specified is not supported."
+ print(
+ f"<---- MInference Config Detail ----> attn_type {attn_type}, kv_type {kv_type}"
+ )
self.attn_type = attn_type
self.config_path = self.update_config_path(config_path, model_name)
self.model_name = model_name
@@ -45,11 +71,19 @@ def __init__(
self.starting_layer = starting_layer
self.kv_cache_cpu = kv_cache_cpu
self.kv_cache_cpu_device = kv_cache_cpu_device
- self.use_snapkv = use_snapkv
- self.attn_kwargs = attn_kwargs
+ self.kv_type = kv_type
+ self.attn_kwargs = {
+ "is_search": is_search,
+ "starting_layer": starting_layer,
+ "config_path": config_path,
+ **attn_kwargs,
+ }
+ if kv_type == "leank":
+ model_name = model_name.split("/")[-1]
+ self.leank_path = LEANKPATNS[model_name]
- def update_config_path(self, config_path: str, model_name: str):
- if self.attn_type in self.STASTIC_ATTENTION_TYPES:
+ def update_config_path(self, config_path: str = None, model_name: str = None):
+ if self.attn_type in self.OTHER_ATTENTION_TYPES:
return ""
if config_path is not None:
return config_path
@@ -57,3 +91,21 @@ def update_config_path(self, config_path: str, model_name: str):
model_name in MODEL2PATH
), f"The model {model_name} you specified is not supported. You are welcome to add it and open a PR :)"
return MODEL2PATH[model_name]
+
+ def get(self, attr, default=None):
+ return getattr(self, attr, default)
+
+ def update_config_type(self, attn_type: str, kv_type: str):
+ if kv_type == "":
+ kv_type = "dense"
+ if attn_type == "minference_with_dense":
+ attn_type = "dense"
+ return attn_type, kv_type
+
+ @classmethod
+ def get_available_attn_types(cls):
+ return cls.MINFERENCE_ATTENTION_TYPES + cls.OTHER_ATTENTION_TYPES
+
+ @classmethod
+ def get_available_kv_types(cls):
+ return cls.KV_TYPES
diff --git a/minference/models_patch.py b/minference/models_patch.py
index d23141f4..75aec202 100644
--- a/minference/models_patch.py
+++ b/minference/models_patch.py
@@ -1,10 +1,17 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
+import json
import os
from .minference_configuration import MInferenceConfig
-from .patch import minference_patch, minference_patch_vllm, patch_hf
+from .patch import (
+ minference_patch,
+ minference_patch_vllm,
+ new_patch,
+ patch_hf,
+)
+from modules.leank import patch_leank
os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -17,7 +24,7 @@ def __init__(
config_path: str = None,
starting_layer: int = -1,
kv_cache_cpu: bool = False,
- use_snapkv: bool = False,
+ kv_type: str = "dense",
is_search: bool = False,
attn_kwargs: dict = {},
**kwargs,
@@ -29,7 +36,7 @@ def __init__(
config_path=config_path,
starting_layer=starting_layer,
kv_cache_cpu=kv_cache_cpu,
- use_snapkv=use_snapkv,
+ kv_type=kv_type,
is_search=is_search,
attn_kwargs=attn_kwargs,
**kwargs,
@@ -39,17 +46,89 @@ def __call__(self, model):
return self.patch_model(model)
def patch_model(self, model):
- if self.config.attn_type != "vllm":
+ if self.config.kv_type == "retr_attn":
+ self.config.attn_kwargs.setdefault(
+ "max_seq_length", model.config.max_position_embeddings
+ )
+ self.config.attn_kwargs.setdefault("max_new_tokens", 1024)
+ self.config.attn_kwargs.setdefault(
+ "num_layers", model.config.num_hidden_layers
+ )
+ self.config.attn_kwargs.setdefault("top_k", 4096)
+ self.config.attn_kwargs.setdefault("from_layer", 0)
+
+ if self.config.kv_type == "kivi":
+ self.config.attn_kwargs.setdefault("bits", 2)
+ self.config.attn_kwargs.setdefault("group_size", 32)
+ self.config.attn_kwargs.setdefault("residual_length", 32)
+
+ if self.config.kv_type in ["snapkv", "pyramidkv"]:
+ self.config.attn_kwargs.setdefault("window_size", 32)
+ self.config.attn_kwargs.setdefault("max_capacity_prompt", 4096)
+ self.config.attn_kwargs.setdefault("kernel_size", 5)
+ self.config.attn_kwargs.setdefault("pooling", "avgpool")
+
+ if self.config.kv_type == "quest":
+ self.config.attn_kwargs.setdefault("chunk_size", 16)
+ self.config.attn_kwargs.setdefault("token_budget", 1024)
+
+ if self.config.kv_type == "streamingllm":
+ self.config.attn_kwargs.setdefault("n_local", 3968)
+ self.config.attn_kwargs.setdefault("n_init", 128)
+
+ if self.config.kv_type == "leank":
+ self.config.attn_kwargs.setdefault("recent_size", 768)
+ self.config.attn_kwargs.setdefault("sink_size", 128)
+ self.config.attn_kwargs.setdefault("accumu_size", 128)
+ self.config.attn_kwargs.setdefault("leank_path", self.config.leank_path)
+ self.config.attn_kwargs.setdefault("round_to", 32)
+ patch_leank(model, self.config)
+
+ if self.config.attn_type == "flexprefill":
+ self.config.attn_kwargs.setdefault("gamma", 0.9)
+ self.config.attn_kwargs.setdefault("tau", 0.1)
+ self.config.attn_kwargs.setdefault("min_budget", None)
+ self.config.attn_kwargs.setdefault("max_budget", None)
+ self.config.attn_kwargs.setdefault("block_size", 128)
+
+ if "vllm" not in self.config.attn_type:
model.config.starting_layer = self.config.starting_layer
model.config.config_path = self.config.config_path
if self.config.attn_type == "minference":
- model.config.is_search = self.config.is_search
- model = minference_patch(model, self.config)
+ if not self.config.is_search:
+ with open(self.config.config_path, "r") as f:
+ self.config.attn_kwargs.setdefault("best_pattern", json.load(f))
+ model = new_patch(model, self.config)
+ elif self.config.attn_type == "tri_mix_minference":
+ with open(self.config.config_path, "r") as f:
+ self.config.attn_kwargs.setdefault("best_pattern", json.load(f))
+ self.config.attn_kwargs.setdefault("n_local", 512)
+ self.config.attn_kwargs.setdefault("n_init", 8)
+ self.config.attn_kwargs.setdefault("n_last", 128)
+ self.config.attn_kwargs.setdefault("starting_layer", 16)
+ model = new_patch(model, self.config)
- elif self.config.attn_type == "minference_with_dense":
- model.config.dense = True
- model = minference_patch(model, self.config)
+ elif self.config.attn_type == "a_shape":
+ self.config.attn_kwargs.setdefault("n_local", 3968)
+ self.config.attn_kwargs.setdefault("n_init", 128)
+ model = new_patch(model, self.config)
+
+ elif self.config.attn_type == "tri_shape":
+ self.config.attn_kwargs.setdefault("n_local", 3968)
+ self.config.attn_kwargs.setdefault("n_init", 128)
+ self.config.attn_kwargs.setdefault("n_last", 100)
+ model = new_patch(model, self.config)
+
+ elif self.config.attn_type == "tri_mix":
+ self.config.attn_kwargs.setdefault("n_local", 512)
+ self.config.attn_kwargs.setdefault("n_init", 8)
+ self.config.attn_kwargs.setdefault("n_last", 128)
+ self.config.attn_kwargs.setdefault("starting_layer", 16)
+ model = new_patch(model, self.config)
+
+ elif self.config.attn_type in ["flexprefill", "dense", "xattention"]:
+ model = new_patch(model, self.config)
elif self.config.attn_type == "dilated1":
model.config.dilated1 = True
@@ -63,22 +142,13 @@ def patch_model(self, model):
model.config.dilated2 = True
model = minference_patch(model, self.config)
- elif self.config.attn_type == "streaming":
- model.config.streaming = True
- model.config.streaming_kwargs = {
- "n_local": 3968,
- "n_init": 128,
- **self.config.attn_kwargs,
- }
- model = minference_patch(model, self.config)
-
elif self.config.attn_type == "streaming2":
model = patch_hf(
model,
- attn_type="streaming",
+ attn_type="a_shape",
attn_kwargs={"n_local": 3968, "n_init": 128, **self.config.attn_kwargs},
)
- elif self.config.attn_type == "hf":
+ elif self.config.attn_type in ["hf", "vllm"]:
pass
elif self.config.attn_type == "inf_llm":
model = patch_hf(
@@ -98,8 +168,37 @@ def patch_model(self, model):
**self.config.attn_kwargs,
},
)
- elif self.config.attn_type == "vllm":
- model = minference_patch_vllm(model, self.config.config_path)
+ elif self.config.attn_type == "vllm_minference":
+ model = minference_patch_vllm(
+ model, self.config.config_path, self.config.attn_kwargs
+ )
+ elif self.config.attn_type == "vllm_flexprefill":
+ patch_config = {
+ "flexprefill": True,
+ "flexprefill_kwargs": {},
+ **self.config.attn_kwargs,
+ }
+ model = minference_patch_vllm(model, self.config.config_path, patch_config)
+ elif self.config.attn_type == "vllm_a_shape":
+ patch_config = {
+ "a_shape": True,
+ "streaming_kwargs": {
+ "n_local": 3968,
+ "n_init": 128,
+ },
+ **self.config.attn_kwargs,
+ }
+ model = minference_patch_vllm(model, self.config.config_path, patch_config)
+ elif self.config.attn_type == "vllm_tri_shape":
+ patch_config = {
+ "tri_shape": True,
+ "streaming_kwargs": {
+ "n_local": 3968,
+ "n_init": 128,
+ },
+ **self.config.attn_kwargs,
+ }
+ model = minference_patch_vllm(model, self.config.config_path, patch_config)
else:
raise ValueError(
f"The attention type {self.config.attn_type} you specified is not supported."
diff --git a/minference/modules/flexprefill.py b/minference/modules/flexprefill.py
new file mode 100644
index 00000000..cb080250
--- /dev/null
+++ b/minference/modules/flexprefill.py
@@ -0,0 +1,1145 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://openreview.net/forum?id=OfjIlbelrT
+
+import math
+from typing import List, Optional, Union
+
+import torch
+import triton
+import triton.language as tl
+from einops import rearrange
+
+try:
+ from flash_attn import flash_attn_func
+except ImportError:
+ from ..ops.flash_attn_triton import _flash_attn_triton_decoding as flash_attn_func
+
+
+def torch_block_wise_attention(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ block_idx: torch.Tensor,
+ block_size: int,
+ grid_offset: int = 0,
+):
+ b, n, h, d = q.shape
+ assert k.shape == q.shape
+ assert v.shape == k.shape
+ num_block = math.ceil(grid_offset / block_size) + math.ceil(
+ (n - grid_offset) / block_size
+ )
+ # get topk block idx and build mask
+ mask = torch.zeros(b, h, num_block, num_block, dtype=torch.bool, device=q.device)
+ mask[
+ torch.arange(b).view(b, 1, 1).expand(b, h, block_idx.shape[-1]),
+ torch.arange(h).view(1, h, 1).expand(b, h, block_idx.shape[-1]),
+ block_idx // num_block,
+ block_idx % num_block,
+ ] = 1
+ act_blocks_per_row = torch.tril(mask).sum(-1)
+ mask = mask.repeat_interleave(block_size, -2).repeat_interleave(block_size, -1)
+ mask = mask[..., grid_offset : grid_offset + n, grid_offset : grid_offset + n]
+ mask = torch.tril(mask)
+ attn_weight = torch.einsum("bihd,bjhd->bhij", q, k) / math.sqrt(d)
+ attn_weight.masked_fill_(~mask, float("-inf"))
+ attn_weight = torch.softmax(attn_weight, dim=-1)
+ o = torch.einsum("bhij,bjhd->bhid", attn_weight, v)
+ o = o.transpose(1, 2)
+ return o
+
+
+@triton.jit
+def block_wise_decode_attention_kernel(
+ q_ptr, # shape: [batch_size, seq_len, num_heads, head_dim]
+ k_ptr,
+ v_ptr,
+ o_ptr,
+ block_idx_ptr, # shape: [batch_size, num_heads, num_activated_block]
+ # shape
+ BATCH_SIZE,
+ NUM_HEADS,
+ NUM_KV_HEADS,
+ GQA_GROUPS,
+ K_LEN,
+ HEAD_DIM: tl.constexpr,
+ NUM_BLOCK,
+ # softmax_scale
+ softmax_scale,
+ # gqa
+ gqa_interleave: tl.constexpr,
+ # stride
+ stride_qb,
+ stride_qn,
+ stride_qh,
+ stride_qd,
+ stride_kb,
+ stride_kn,
+ stride_kh,
+ stride_kd,
+ stride_vb,
+ stride_vn,
+ stride_vh,
+ stride_vd,
+ stride_ob,
+ stride_on,
+ stride_oh,
+ stride_od,
+ stride_bb,
+ stride_bh,
+ stride_bt,
+ # META parameters
+ BLOCK_SIZE_Q: tl.constexpr, # q block size
+ BLOCK_SIZE_K: tl.constexpr, # k block size
+):
+ pid_b = tl.program_id(0)
+ pid_h = tl.program_id(1)
+ if gqa_interleave:
+ pid_kh = pid_h % NUM_KV_HEADS
+ else:
+ pid_kh = pid_h // GQA_GROUPS
+ # get column block index ptr
+ block_idx_ptr = block_idx_ptr + pid_b * stride_bb + pid_h * stride_bh
+ # init qkv ptrs
+ q_ptrs = tl.make_block_ptr(
+ base=q_ptr + pid_b * stride_qb + pid_h * stride_qh,
+ shape=(1, HEAD_DIM),
+ strides=(stride_qn, stride_qd),
+ offsets=(0, 0),
+ block_shape=(BLOCK_SIZE_Q, HEAD_DIM),
+ order=(1, 0),
+ )
+ k_ptrs = tl.make_block_ptr(
+ base=k_ptr + pid_b * stride_kb + pid_kh * stride_kh,
+ shape=(HEAD_DIM, K_LEN),
+ strides=(stride_kd, stride_kn),
+ offsets=(0, 0),
+ block_shape=(HEAD_DIM, BLOCK_SIZE_K),
+ order=(0, 1),
+ )
+ v_ptrs = tl.make_block_ptr(
+ base=v_ptr + pid_b * stride_vb + pid_kh * stride_vh,
+ shape=(K_LEN, HEAD_DIM),
+ strides=(stride_vn, stride_vd),
+ offsets=(0, 0),
+ block_shape=(BLOCK_SIZE_K, HEAD_DIM),
+ order=(1, 0),
+ )
+ # load q
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
+ # init statistics
+ off_n = tl.arange(0, BLOCK_SIZE_K)
+ m_i = tl.full((BLOCK_SIZE_Q,), float("-inf"), dtype=tl.float32)
+ lse_i = tl.full((BLOCK_SIZE_Q,), float("-inf"), dtype=tl.float32)
+ acc_o = tl.full((BLOCK_SIZE_Q, HEAD_DIM), 0, dtype=tl.float32)
+ # flash attention
+ for i in range(0, NUM_BLOCK):
+ # get current block start index
+ c = tl.load(block_idx_ptr).to(tl.int32) * BLOCK_SIZE_K
+ block_idx_ptr = block_idx_ptr + stride_bt
+ # load k
+ k = tl.load(
+ tl.advance(k_ptrs, (0, c)), boundary_check=(1,), padding_option="zero"
+ )
+ # compute qk
+ qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_K), dtype=tl.float32)
+ qk += tl.where((off_n < K_LEN - c)[None, :], 0, float("-inf"))
+ qk += tl.dot(q, k) * softmax_scale
+ # compute m_ij and l_ij
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
+ p = tl.math.exp2(qk - m_ij[:, None])
+ l_ij = tl.sum(p, axis=1)
+ # scale acc_o
+ acc_o_scale = tl.math.exp2(m_i - m_ij)
+ acc_o = acc_o * acc_o_scale[:, None]
+ # load v and update acc_o
+ v = tl.load(
+ tl.advance(v_ptrs, (c, 0)), boundary_check=(0,), padding_option="zero"
+ )
+ p = p.to(v.dtype)
+ acc_o += tl.dot(p, v)
+ # update statistics
+ m_i = m_ij
+ lse_i = m_ij + tl.math.log2(tl.math.exp2(lse_i - m_ij) + l_ij)
+ # final scale
+ acc_o = acc_o * tl.math.exp2(m_i - lse_i)[:, None]
+ # save output
+ o_ptrs = tl.make_block_ptr(
+ base=o_ptr + pid_b * stride_ob + pid_h * stride_oh,
+ shape=(1, HEAD_DIM),
+ strides=(stride_on, stride_od),
+ offsets=(0, 0),
+ block_shape=(BLOCK_SIZE_Q, HEAD_DIM),
+ order=(1, 0),
+ )
+ tl.store(o_ptrs, acc_o.to(tl.bfloat16), boundary_check=(0,))
+
+
+def triton_block_wise_decode_attention(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ block_idx: torch.Tensor,
+ block_size: int,
+ softmax_scale: Optional[float] = None,
+ gqa_interleave: bool = False,
+) -> torch.Tensor:
+ """Block wise sparse attention (causal attention) implemented by openai triton (ver 3.0.0).
+
+ Args:
+ q (torch.Tensor): Query states, shape [batch_size, 1, num_heads, head_dim]
+ k (torch.Tensor): Key states, shape [batch_size, seq_len, num_heads, head_dim]
+ v (torch.Tensor): Value states, same as key
+ block_idx (torch.Tensor): Index of activated blocks, shape [batch_size, num_heads, activated_block_num]
+ block_size (int): Block size, only support 16, 32, 64 and 128.
+ softmax_scale (Optional[float], optional): Softmax scale. Defaults to 1/math.sqrt(head_dim)
+ gqa_interleave (bool): use interleave mode of gqa, default to False.
+
+ Returns:
+ torch.Tensor: Attention output, shape [batch_size, 1, num_heads, head_dim]
+ """
+ batch_size, q_len, num_q_heads, head_dim = q.shape
+ assert q_len == 1
+ batch_size, k_len, num_kv_heads, head_dim = k.shape
+ batch_size, num_q_heads, num_blocks = block_idx.shape
+ assert q.dtype == torch.bfloat16
+ assert head_dim in {16, 32, 64, 128}, "only support head_dim in {16, 32, 64, 128}"
+ assert block_size in {
+ 16,
+ 32,
+ 64,
+ 128,
+ }, "only support block size in {16, 32, 64, 128}"
+ assert num_blocks <= triton.cdiv(k_len, block_size)
+ # gqa
+ assert num_q_heads % num_kv_heads == 0
+ gqa_groups = num_q_heads // num_kv_heads
+ # softmax_scale
+ if softmax_scale is None:
+ softmax_scale = 1 / math.sqrt(head_dim) * math.log2(math.e)
+ else:
+ softmax_scale = softmax_scale * math.log2(math.e)
+ # sort idx and get block index bins
+ block_idx = block_idx.sort(-1).values
+ # launch attention kernel
+ o = torch.empty_like(q)
+ num_warps = 8
+ BLOCK_SIZE_Q = 16
+ BLOCK_SIZE_K = block_size
+ block_wise_decode_attention_kernel[(batch_size, num_q_heads)](
+ q,
+ k,
+ v,
+ o,
+ block_idx,
+ batch_size,
+ num_q_heads,
+ num_q_heads,
+ num_kv_heads,
+ gqa_groups,
+ k_len,
+ head_dim,
+ num_blocks,
+ softmax_scale,
+ gqa_interleave,
+ q.stride(0),
+ q.stride(1),
+ q.stride(2),
+ q.stride(3),
+ k.stride(0),
+ k.stride(1),
+ k.stride(2),
+ k.stride(3),
+ v.stride(0),
+ v.stride(1),
+ v.stride(2),
+ v.stride(3),
+ o.stride(0),
+ o.stride(1),
+ o.stride(2),
+ o.stride(3),
+ block_idx.stride(0),
+ block_idx.stride(1),
+ block_idx.stride(2),
+ BLOCK_SIZE_Q=BLOCK_SIZE_Q,
+ BLOCK_SIZE_K=BLOCK_SIZE_K,
+ num_warps=num_warps,
+ num_stages=3,
+ )
+ return o
+
+
+@triton.jit
+def count_kernel(
+ x_ptr,
+ y_ptr,
+ k,
+ r,
+ stride_xb,
+ stride_xh,
+ stride_xk,
+ stride_yb,
+ stride_yh,
+ stride_yr,
+ BLOCK_SIZE_K: tl.constexpr,
+ BLOCK_SIZE_R: tl.constexpr,
+):
+ pid_b = tl.program_id(0)
+ pid_h = tl.program_id(1)
+ # load x
+ x_ptr = x_ptr + pid_b * stride_xb + pid_h * stride_xh
+ off_k = tl.arange(0, BLOCK_SIZE_K)
+ x_ptrs = x_ptr + off_k * stride_xk
+ y = tl.zeros((BLOCK_SIZE_R,), dtype=tl.int32)
+ for i in range(0, k, BLOCK_SIZE_K):
+ x = tl.load(x_ptrs, off_k < k - i, -1)
+ x = x // r
+ x = tl.where(off_k < k - i, x, -1)
+ # count
+ # maybe triton bug: when BLOCK_SIZE_R == r, the count of values in bin [r-1, r) will be wrong
+ y += tl.histogram(x, BLOCK_SIZE_R)
+ # move ptr
+ x_ptrs = x_ptrs + BLOCK_SIZE_K * stride_xk
+ # cumsum
+ y = tl.cumsum(y, axis=0)
+ # store result
+ y_ptr = y_ptr + pid_b * stride_yb + pid_h * stride_yh + stride_yr
+ off_r = tl.arange(0, BLOCK_SIZE_R)
+ tl.store(y_ptr + off_r * stride_yr, y, off_r < r)
+
+
+def triton_column_count_cumsum(x: torch.Tensor, num_columns: int) -> torch.Tensor:
+ """count columns of each row for a given index tensor, then do cumsum
+
+ Args:
+ x (torch.Tensor): block index in a flatten 2d grid, shape [batch_size, num_heads, activated_block_num]
+ num_colums (int): number of columns in the grid
+
+ Returns:
+ torch.Tensor: cumsum of columns num in each row, shape [batch_size, num_heads, num_rows + 1 ]
+ For example, in a 4x4 block grid, activated blocks have index [0, 5, 8, 9, 13, 14], number of blocks in each row is [1, 1, 2, 2],
+ this function will return cumsum tensor [0, 1, 2, 4, 6]
+ """
+ x = x.to(torch.int32)
+ b, h, k = x.shape
+ r = num_columns
+ # torch implementation:
+ # y = torch.zeros(b,h,r*r,dtype=x.dtype,device=x.device)
+ # y[torch.arange(b,device=x.device)[:,None,None],torch.arange(h,device=x.device)[None,:,None],torch.where(x= 0, 0, float("-inf"))
+ qk += tl.where(off_m[:, None] >= (c + off_n)[None, :], 0, float("-inf"))
+ qk += tl.dot(q, k) * softmax_scale
+ # compute m_ij and l_ij
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
+ p = tl.math.exp2(qk - m_ij[:, None])
+ l_ij = tl.sum(p, axis=1)
+ # scale acc_o
+ acc_o_scale = tl.math.exp2(m_i - m_ij)
+ acc_o = acc_o * acc_o_scale[:, None]
+ # load v and update acc_o
+ v = tl.load(
+ tl.advance(v_ptrs, (c, 0)), boundary_check=(0,), padding_option="zero"
+ )
+ p = p.to(v.dtype)
+ acc_o += tl.dot(p, v)
+ # update statistics
+ m_i = m_ij
+ lse_i = m_ij + tl.math.log2(tl.math.exp2(lse_i - m_ij) + l_ij)
+ # final scale
+ acc_o = acc_o * tl.math.exp2(m_i - lse_i)[:, None]
+ # save output
+ o_ptrs = tl.make_block_ptr(
+ base=o_ptr + pid_b * stride_ob + pid_h * stride_oh,
+ shape=(Q_LEN, HEAD_DIM),
+ strides=(stride_on, stride_od),
+ offsets=(pid_q * BLOCK_SIZE_Q - grid_offset, 0),
+ block_shape=(BLOCK_SIZE_Q, HEAD_DIM),
+ order=(1, 0),
+ )
+ tl.store(o_ptrs, acc_o.to(tl.bfloat16), boundary_check=(0,))
+
+
+def triton_block_wise_prefill_attention(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ block_idx: Union[torch.Tensor, List[List[torch.Tensor]]],
+ block_size: int,
+ grid_offset: int = 0,
+ softmax_scale: Optional[float] = None,
+ gqa_interleave: bool = False,
+) -> torch.Tensor:
+ """Block wise sparse attention (causal attention) implemented by openai triton (ver 3.0.0).
+
+ Args:
+ q (torch.Tensor): Query states, shape [batch_size, seq_lens, num_heads, head_dim]
+ k (torch.Tensor): Key states, same as query
+ v (torch.Tensor): Value states, same as query
+ block_idx (torch.Tensor): Index of activated blocks, shape [batch_size, num_heads, activated_block_num], which is the index of the flattened block grid.
+ For example, in a 4x4 block grid, if you want to activate 5 blocks: (0,0), (1,1), (2,0), (3,1), (3,2), the index will be: [0, 5, 8, 13, 14]
+ block_size (int): Block size, only support 16, 32, 64 and 128.
+ grid_offset (int): Move the grid that divides the block to the lower left corner by grid_offset, default to 0.
+ softmax_scale (Optional[float], optional): Softmax scale. Defaults to 1/math.sqrt(head_dim)
+ gqa_interleave (bool): use interleave mode of gqa, default to False.
+
+ Returns:
+ torch.Tensor: Attention output, shape [batch_size, seq_lens, num_heads, head_dim]
+ """
+ batch_size, q_len, num_q_heads, head_dim = q.shape
+ batch_size, k_len, num_kv_heads, head_dim = k.shape
+ assert q.dtype == torch.bfloat16
+ assert q_len == k_len
+ assert head_dim in {16, 32, 64, 128}, "only support head_dim in {16, 32, 64, 128}"
+ assert block_size in {
+ 32,
+ 64,
+ 128,
+ }, "only support block size in {16, 32, 64, 128}"
+ total_q_blocks = triton.cdiv(grid_offset, block_size) + triton.cdiv(
+ q_len - grid_offset, block_size
+ )
+ total_k_blocks = triton.cdiv(grid_offset, block_size) + triton.cdiv(
+ k_len - grid_offset, block_size
+ )
+ # pad block_idx if get list[list[tensor]]
+ if not isinstance(block_idx, torch.Tensor):
+ assert (
+ isinstance(block_idx, list)
+ and isinstance(block_idx[0], list)
+ and isinstance(block_idx[0][0], torch.Tensor)
+ )
+ assert len(block_idx) == batch_size and len(block_idx[0]) == num_q_heads
+ block_idx = [item.view(-1, 1) for sublist in block_idx for item in sublist]
+ block_idx = torch.nn.utils.rnn.pad_sequence(
+ block_idx,
+ batch_first=True,
+ padding_value=total_k_blocks * (total_k_blocks + 1),
+ )
+ block_idx = block_idx.view(batch_size, num_q_heads, -1)
+ batch_size, num_q_heads, num_block = block_idx.shape
+ assert q_len == k_len
+ assert num_block <= total_q_blocks * (total_q_blocks + 1) // 2
+ # gqa
+ assert num_q_heads % num_kv_heads == 0
+ gqa_groups = num_q_heads // num_kv_heads
+ # softmax_scale
+ if softmax_scale is None:
+ softmax_scale = 1 / math.sqrt(head_dim) * math.log2(math.e)
+ else:
+ softmax_scale = softmax_scale * math.log2(math.e)
+ # sort idx and get block index bins
+ block_idx = block_idx.sort(-1).values
+ idx_bins = triton_column_count_cumsum(block_idx, total_k_blocks)
+ # launch attention kernel
+ o = torch.empty_like(q)
+ num_warps = 8
+ num_stages = 3 if block_size >= 128 else 5
+ block_wise_prefill_attention_kernel[(batch_size, num_q_heads, total_q_blocks)](
+ q,
+ k,
+ v,
+ o,
+ block_idx,
+ idx_bins,
+ batch_size,
+ num_q_heads,
+ num_kv_heads,
+ gqa_groups,
+ q_len,
+ k_len,
+ head_dim,
+ total_q_blocks,
+ grid_offset,
+ softmax_scale,
+ gqa_interleave,
+ q.stride(0),
+ q.stride(1),
+ q.stride(2),
+ q.stride(3),
+ k.stride(0),
+ k.stride(1),
+ k.stride(2),
+ k.stride(3),
+ v.stride(0),
+ v.stride(1),
+ v.stride(2),
+ v.stride(3),
+ o.stride(0),
+ o.stride(1),
+ o.stride(2),
+ o.stride(3),
+ block_idx.stride(0),
+ block_idx.stride(1),
+ block_idx.stride(2),
+ idx_bins.stride(0),
+ idx_bins.stride(1),
+ idx_bins.stride(2),
+ BLOCK_SIZE_Q=block_size,
+ BLOCK_SIZE_K=block_size,
+ num_warps=num_warps,
+ num_stages=num_stages,
+ )
+ return o
+
+
+def triton_block_wise_attention(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ block_idx: torch.Tensor,
+ block_size: int,
+ grid_offset: int = 0,
+ softmax_scale: Optional[float] = None,
+ gqa_interleave: bool = False,
+) -> torch.Tensor:
+ """Block wise sparse attention (causal attention) implemented by openai triton (ver 3.0.0).
+
+ Args:
+ q (torch.Tensor): Query states, shape [batch_size, seq_lens, num_heads, head_dim]
+ k (torch.Tensor): Key states, same as query
+ v (torch.Tensor): Value states, same as query
+ block_idx (torch.Tensor): Index of activated blocks, shape [batch_size, num_heads, activated_block_num], which is the index of the flattened block grid.
+ For example, in a 4x4 block grid, if you want to activate 5 blocks: (0,0), (1,1), (2,0), (3,1), (3,2), the index will be: [0, 5, 8, 13, 14]
+ block_size (int): Block size, only support 16, 32, 64 and 128.
+ grid_offset (int): Move the grid that divides the block to the lower left corner by grid_offset, default to 0.
+ softmax_scale (Optional[float], optional): Softmax scale. Defaults to 1/math.sqrt(head_dim)
+ gqa_interleave (bool): use interleave mode of gqa, default to False.
+
+ Returns:
+ torch.Tensor: Attention output, shape [batch_size, seq_lens, num_heads, head_dim]
+ """
+ if q.shape[1] > 1:
+ return triton_block_wise_prefill_attention(
+ q,
+ k,
+ v,
+ block_idx,
+ block_size,
+ grid_offset,
+ softmax_scale,
+ gqa_interleave,
+ )
+ else:
+ return triton_block_wise_decode_attention(
+ q, k, v, block_idx, block_size, softmax_scale, gqa_interleave
+ )
+
+
+@triton.jit
+def bnhd_pool_kernel(
+ x_ptr,
+ y_ptr,
+ # pool type. avg: 0, max: 1, min: 2, max abs: 3, sum: 4
+ pool_type: tl.constexpr,
+ # shape
+ batch_size,
+ seq_len,
+ num_heads,
+ head_dim: tl.constexpr,
+ # stride
+ stride_xb,
+ stride_xn,
+ stride_xh,
+ stride_xd,
+ stride_yb,
+ stride_yn,
+ stride_yh,
+ stride_yd,
+ # META parameters
+ BLOCK_SIZE_N: tl.constexpr,
+ BLOCK_SIZE_H: tl.constexpr, # {16, 32, 64, 128, 256, 512}
+ BLOCK_SIZE_D: tl.constexpr, # {16, 32, 64, 128, 256, 512}
+):
+ pid_b = tl.program_id(0)
+ pid_n = tl.program_id(1)
+ pid_h = tl.program_id(2)
+
+ x_ptr = (
+ x_ptr
+ + pid_b * stride_xb
+ + pid_n * BLOCK_SIZE_N * stride_xn
+ + pid_h * BLOCK_SIZE_H * stride_xh
+ )
+
+ off_n = tl.arange(0, BLOCK_SIZE_N)
+ off_h = tl.arange(0, BLOCK_SIZE_H)
+ off_d = tl.arange(0, BLOCK_SIZE_D)
+
+ cur_block_size_n = min(seq_len - pid_n * BLOCK_SIZE_N, BLOCK_SIZE_N)
+
+ x_mask = (
+ (off_n < seq_len - pid_n * BLOCK_SIZE_N)[:, None, None]
+ & (off_h < num_heads - pid_h * BLOCK_SIZE_H)[None, :, None]
+ & (off_d < head_dim)[None, None, :]
+ )
+ x = tl.load(
+ x_ptr
+ + off_n[:, None, None] * stride_xn
+ + off_h[None, :, None] * stride_xh
+ + off_d[None, None, :] * stride_xd,
+ mask=x_mask,
+ other=0,
+ )
+ if pool_type == 0:
+ y = tl.sum(x, axis=0) / cur_block_size_n
+ elif pool_type == 1:
+ y = tl.max(x, axis=0)
+ elif pool_type == 2:
+ y = tl.min(x, axis=0)
+ elif pool_type == 3:
+ y = tl.max(tl.abs(x), axis=0)
+ elif pool_type == 4:
+ y = tl.sum(x, axis=0)
+ else:
+ y = tl.sum(x, axis=0) / cur_block_size_n
+ y_ptr = (
+ y_ptr + pid_b * stride_yb + pid_n * stride_yn + pid_h * BLOCK_SIZE_H * stride_yh
+ )
+ y_mask = (off_h < num_heads - pid_h * BLOCK_SIZE_H)[:, None] & (off_d < head_dim)[
+ None, :
+ ]
+ tl.store(
+ y_ptr + off_h[:, None] * stride_yh + off_d[None, :] * stride_yd, y, mask=y_mask
+ )
+
+
+def triton_bnhd_pool(x: torch.Tensor, kernel_size: int, pool_type: str = "avg"):
+ x = x.to("cuda")
+ b, n, h, d = x.shape
+ assert d in {16, 32, 64, 128}
+ assert kernel_size in {16, 32, 64, 128, 256, 512}
+ m = triton.cdiv(n, kernel_size)
+ y = torch.zeros(b, m, h, d, device=x.device, dtype=x.dtype)
+
+ if pool_type == "last":
+ if n % kernel_size == 0:
+ return x[:, kernel_size - 1 :: kernel_size, ...]
+ else:
+ return torch.cat(
+ (x[:, kernel_size - 1 :: kernel_size, ...], x[:, -1:, ...]), dim=1
+ )
+
+ block_size_h = triton.next_power_of_2(h)
+ while kernel_size * block_size_h * d > 128 * 128 * 128:
+ block_size_h = block_size_h // 2
+
+ block_size_d = triton.next_power_of_2(d)
+ pool_str_to_type = {"avg": 0, "max": 1, "min": 2, "maxabs": 3, "sum": 4}
+ pool_type = pool_str_to_type[pool_type]
+
+ grid = lambda META: (
+ b,
+ triton.cdiv(n, META["BLOCK_SIZE_N"]),
+ triton.cdiv(h, META["BLOCK_SIZE_H"]),
+ )
+ bnhd_pool_kernel[grid](
+ x,
+ y,
+ pool_type,
+ b,
+ n,
+ h,
+ d,
+ x.stride(0),
+ x.stride(1),
+ x.stride(2),
+ x.stride(3),
+ y.stride(0),
+ y.stride(1),
+ y.stride(2),
+ y.stride(3),
+ BLOCK_SIZE_N=kernel_size,
+ BLOCK_SIZE_H=block_size_h,
+ BLOCK_SIZE_D=block_size_d,
+ )
+ return y
+
+
+@triton.jit
+def bhn_sumpool_kernel(
+ x_ptr,
+ y_ptr,
+ # shape
+ batch_size,
+ num_heads,
+ seq_len,
+ # stride
+ stride_xb,
+ stride_xh,
+ stride_xn,
+ stride_yb,
+ stride_yh,
+ stride_yn,
+ # META parameters
+ BLOCK_SIZE_N: tl.constexpr,
+ BLOCK_SIZE_H: tl.constexpr, # {16, 32, 64, 128, 256, 512}
+):
+ pid_b = tl.program_id(0)
+ pid_h = tl.program_id(1)
+ pid_n = tl.program_id(2)
+ x_ptr = (
+ x_ptr
+ + pid_b * stride_xb
+ + pid_h * BLOCK_SIZE_H * stride_xh
+ + pid_n * BLOCK_SIZE_N * stride_xn
+ )
+ off_h = tl.arange(0, BLOCK_SIZE_H)
+ off_n = tl.arange(0, BLOCK_SIZE_N)
+ x_mask = (off_n < seq_len - pid_n * BLOCK_SIZE_N)[None, :] & (
+ off_h < num_heads - pid_h * BLOCK_SIZE_H
+ )[:, None]
+ x = tl.load(
+ x_ptr + off_h[:, None] * stride_xh + off_n[None, :] * stride_xn,
+ mask=x_mask,
+ other=0,
+ )
+ y = tl.sum(x, axis=1)
+ y_ptr = (
+ y_ptr + pid_b * stride_yb + pid_h * BLOCK_SIZE_H * stride_yh + pid_n * stride_yn
+ )
+ y_mask = off_h < num_heads - pid_h * BLOCK_SIZE_H
+ tl.store(y_ptr + off_h * stride_yh, y, mask=y_mask)
+
+
+def triton_bhn_sumpool(x: torch.Tensor, kernel_size: int):
+ b, h, n = x.shape
+ assert kernel_size in {16, 32, 64, 128, 256, 512}
+ m = triton.cdiv(n, kernel_size)
+ y = torch.empty(b, h, m, device=x.device, dtype=x.dtype)
+ block_size_h = triton.next_power_of_2(h)
+ grid = lambda META: (
+ b,
+ triton.cdiv(h, META["BLOCK_SIZE_H"]),
+ triton.cdiv(n, META["BLOCK_SIZE_N"]),
+ )
+ bhn_sumpool_kernel[grid](
+ x,
+ y,
+ b,
+ h,
+ n,
+ x.stride(0),
+ x.stride(1),
+ x.stride(2),
+ y.stride(0),
+ y.stride(1),
+ y.stride(2),
+ BLOCK_SIZE_N=kernel_size,
+ BLOCK_SIZE_H=block_size_h,
+ )
+ return y
+
+
+def torch_bhn_sumpool(x: torch.Tensor, kernel_size: int):
+ b, h, n = x.shape
+ x = torch.nn.functional.pad(
+ x,
+ (
+ 0,
+ math.ceil(n / kernel_size) * kernel_size - n,
+ ),
+ value=0,
+ )
+ x = x.view(b, h, -1, kernel_size).sum(-1)
+ return x
+
+
+def score_cover_topk(x: torch.Tensor, score: float):
+ cumsum_x = torch.cumsum(torch.sort(x, dim=-1, descending=True).values, dim=-1)
+ topk = torch.sum(cumsum_x <= score, dim=-1) + 1
+ return topk
+
+
+def score_cover_idx(x: torch.Tensor, score: float, padding_value=0):
+ x, idx = torch.sort(x, dim=-1, descending=True)
+ cumsum_x = torch.cumsum(x, dim=-1)
+ idx[cumsum_x > score] = padding_value
+ return idx
+
+
+def sum_all_diagonal_matrix(mat: torch.tensor):
+ b, h, n, m = mat.shape
+ mat_padded = torch.nn.functional.pad(mat, (n - 1, 0), value=0)
+ mat_strided = mat_padded.as_strided(
+ (b, h, m, n), (h * n * (n + m - 1), n * (n + m - 1), 1, n + m)
+ )
+ sum_diags = torch.sum(mat_strided, -1)
+ return sum_diags
+
+
+def transform_veritcal_slash_idx(v_idx, s_idx, num_blocks):
+ batch_size, num_heads, _ = v_idx.shape
+ range_blocks = torch.arange(num_blocks, device=s_idx.device)[None, None, :, None]
+ # vertical
+ v_idx = (
+ torch.arange(0, num_blocks, device=v_idx.device)[None, None, :, None]
+ * num_blocks
+ + v_idx[:, :, None, :]
+ ).view(batch_size, num_heads, -1)
+ v_idx[v_idx // num_blocks < v_idx % num_blocks] = 0
+ # slash
+ s_idx = (
+ range_blocks * num_blocks + range_blocks + s_idx[:, :, None, :] * num_blocks
+ ).view(batch_size, num_heads, -1)
+ s_idx[s_idx >= num_blocks * num_blocks] = 0
+ # union
+ vs_idx = torch.cat((s_idx, v_idx), dim=-1)
+ block_idx = [
+ [torch.unique(vs_idx[b, h]) for h in range(num_heads)]
+ for b in range(batch_size)
+ ]
+ return block_idx
+
+
+causal_mask = None
+
+
+def get_block_vertical_slash_from_qk(
+ qk: torch.Tensor,
+ block_size: int,
+):
+ batch_size, num_heads, last_q_len, seq_len = qk.shape
+ # slash shape: [batch_size, num_heads, seq_len] -> [batch_size, num_heads, num_blocks]
+ slash = sum_all_diagonal_matrix(qk)
+ slash = torch_bhn_sumpool(slash, block_size)
+ slash = slash / last_q_len
+ # vertical shape: [batch_size, num_heads, seq_len] -> [batch_size, num_heads, num_blocks]
+ vertical = qk.sum(-2)
+ vertical = torch_bhn_sumpool(vertical, block_size)
+ vertical = vertical / last_q_len
+ return vertical, slash
+
+
+def square_root_js_divergence(p: torch.Tensor, q: torch.Tensor):
+ m = (p + q) / 2
+ return torch.sqrt(
+ 0.5 * (p * torch.log(p / m)).sum(-1) + 0.5 * (q * torch.log(q / m)).sum(-1)
+ )
+
+
+def get_active_blocks(
+ q,
+ k,
+ v,
+ block_size,
+ gamma,
+ min_budget,
+ max_budget,
+ tau=0,
+ gqa_interleave=False,
+):
+ batch_size, seq_len, num_heads, head_dim = q.shape
+ gqa_groups = num_heads // k.shape[2]
+ num_blocks = math.ceil(seq_len / block_size)
+ max_budget = min(max_budget, num_blocks)
+ # last qk attention, qk shape: [batch_size, num_heads, block_size, seq_len]
+ last_q = q[:, -block_size:, :, :] / math.sqrt(head_dim)
+ if not gqa_interleave:
+ qk = torch.einsum(
+ "bihgd, bjhgd -> bhgij",
+ last_q.view(last_q.shape[0], last_q.shape[1], -1, gqa_groups, head_dim),
+ k.view(k.shape[0], k.shape[1], -1, 1, head_dim),
+ )
+ else:
+ qk = torch.einsum(
+ "bihgd, bjhgd -> bhgij",
+ last_q.view(last_q.shape[0], last_q.shape[1], gqa_groups, -1, head_dim),
+ k.view(k.shape[0], k.shape[1], 1, -1, head_dim),
+ )
+ global causal_mask
+ if causal_mask is None:
+ causal_mask = torch.arange(0, block_size, device=last_q.device)
+ causal_mask = causal_mask[:, None] >= causal_mask[None, :]
+ causal_mask = causal_mask[None, None, None, ...]
+ qk[..., -block_size:].masked_fill_(
+ ~causal_mask[..., :block_size, :block_size].to(qk.device), float("-inf")
+ )
+ # softmax
+ qk = torch.nn.functional.softmax(qk, dim=-1, dtype=torch.float32)
+ qk = rearrange(qk, "b h g i j -> b (h g) i j")
+ slash = sum_all_diagonal_matrix(qk) / qk.shape[-2]
+ vertical = qk.mean(-2)
+ # get vertical slash size to make sure attention score >= gamma. shape: [batch_size, num_heads]
+ num_vertical_blocks = score_cover_topk(vertical, gamma) // 128 + 1
+ num_slash_blocks = score_cover_topk(slash, gamma) // 128 + 1
+ num_vertical_blocks[num_vertical_blocks < min_budget] = min_budget
+ num_vertical_blocks[num_vertical_blocks > max_budget] = max_budget
+ num_slash_blocks[num_slash_blocks < min_budget] = min_budget
+ num_slash_blocks[num_slash_blocks > max_budget] = max_budget
+ # block avg pool
+ vertical = torch_bhn_sumpool(vertical, block_size)
+ slash = torch_bhn_sumpool(slash, block_size)
+ # get block sparse mask
+ if not gqa_interleave:
+ avg_k = triton_bnhd_pool(k, block_size).repeat_interleave(gqa_groups, 2)
+ else:
+ avg_k = triton_bnhd_pool(k, block_size).repeat(1, 1, gqa_groups, 1)
+ avg_qk = torch.einsum(
+ "bihd, bjhd -> bhij", last_q.mean(1, keepdim=True), avg_k
+ ).squeeze(2)
+ avg_qk = torch.softmax(avg_qk, dim=-1, dtype=torch.float32)
+ kl_div = square_root_js_divergence(avg_qk, vertical)
+ block_sparse_mask = kl_div < tau
+ num_vertical_blocks[block_sparse_mask] = min_budget
+ num_slash_blocks[block_sparse_mask] = min_budget
+ # keep first vertical and slash block
+ vertical[..., :1] = torch.inf
+ slash[..., -1:] = torch.inf
+ # get slash topk
+ num_slash_blocks = num_slash_blocks.view(batch_size * num_heads)
+ slash = slash.view(batch_size * num_heads, -1)
+ slash_topk = (num_blocks - 1) - slash.topk(
+ min(num_slash_blocks.max().item(), num_blocks), -1
+ ).indices
+ slash_topk[
+ torch.arange(slash_topk.shape[-1], device=num_slash_blocks.device)[None, :]
+ >= num_slash_blocks[:, None]
+ ] = 0
+ slash_topk = slash_topk.view(batch_size, num_heads, -1)
+ # get vertical topk
+ num_vertical_blocks = num_vertical_blocks.view(batch_size * num_heads)
+ vertical = vertical.view(batch_size * num_heads, -1)
+ vertical_topk = vertical.topk(
+ min(num_vertical_blocks.max().item(), num_blocks), -1
+ ).indices
+ vertical_topk[
+ torch.arange(vertical_topk.shape[-1], device=num_vertical_blocks.device)[
+ None, :
+ ]
+ >= num_vertical_blocks[:, None]
+ ] = 0
+ vertical_topk = vertical_topk.view(batch_size, num_heads, -1)
+ # transform vertical slash index
+ block_idx = transform_veritcal_slash_idx(vertical_topk, slash_topk, num_blocks)
+ # get block sparse topk
+ block_causal_mask = None
+ for b, h in block_sparse_mask.nonzero():
+ if block_causal_mask is None:
+ block_causal_mask = torch.tril(
+ torch.ones(num_blocks, num_blocks, device=q.device, dtype=torch.bool)
+ )
+ pad_q = math.ceil(seq_len / block_size) * block_size - seq_len
+ avg_q = (
+ torch.nn.functional.pad(q[b, :, h, :], (0, 0, 0, pad_q), value=0)
+ .view(num_blocks, block_size, head_dim)
+ .mean(1)
+ )
+ avg_q[-1, :] = avg_q[-1, :] * block_size / (block_size - pad_q)
+ attn = torch.einsum(
+ "id, jd -> ij", avg_q / math.sqrt(head_dim), avg_k[b, :, h, :]
+ ).masked_fill_(~block_causal_mask, float("-inf"))
+ attn = torch.softmax(attn, dim=-1, dtype=torch.float32).view(-1)
+ block_topk = score_cover_idx(attn, gamma * num_blocks)
+ block_idx[b][h] = torch.unique(torch.cat((block_idx[b][h], block_topk), dim=-1))
+ return block_idx
+
+
+def flex_prefill_attention(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ gamma: float,
+ tau: float = 0,
+ min_budget: int = None,
+ max_budget: int = None,
+ gqa_interleave: bool = False,
+ softmax_scale: Optional[float] = None,
+ block_size: int = 128,
+):
+ batch_size, seq_len, num_heads, head_dim = q.shape
+ assert q.shape[1] == k.shape[1]
+ assert head_dim in {16, 32, 64, 128}
+ assert block_size in {16, 32, 64, 128}
+ num_blocks = math.ceil(seq_len / block_size)
+ min_budget = 1 if min_budget is None else min_budget
+ max_budget = 2147483647 if max_budget is None else max_budget
+ if seq_len <= max(2 * block_size, math.ceil(min_budget / block_size) * block_size):
+ return flash_attn_func(q, k, v, softmax_scale=softmax_scale, causal=True)
+ # get vertical slash index
+ block_idx = get_active_blocks(
+ q,
+ k,
+ v,
+ block_size,
+ gamma,
+ math.ceil(min_budget / block_size),
+ math.ceil(max_budget / block_size),
+ tau,
+ gqa_interleave,
+ )
+ attn_out = triton_block_wise_attention(
+ q,
+ k,
+ v,
+ block_idx,
+ block_size,
+ softmax_scale=softmax_scale,
+ gqa_interleave=gqa_interleave,
+ )
+ return attn_out
+
+
+def flexprefill_forward(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ config,
+):
+ gamma = config["attn_forward_config"].get("gamma", 0.9)
+ tau = config["attn_forward_config"].get("tau", 0.1)
+ min_budget = config["attn_forward_config"].get("min_budget", None)
+ max_budget = config["attn_forward_config"].get("max_budget", None)
+ block_size = config["attn_forward_config"].get("block_size", 128)
+ q = q.transpose(1, 2).contiguous()
+ k = k.transpose(1, 2).contiguous()
+ v = v.transpose(1, 2).contiguous()
+ out = flex_prefill_attention(
+ q, k, v, gamma, tau, min_budget, max_budget, block_size
+ )
+ return out.transpose(1, 2)
+
+
+if __name__ == "__main__":
+ torch.manual_seed(0)
+ B, N, H, D = 1, 64000, 32, 64
+ gamma = 0.9
+ tau = 0.1
+
+ q = torch.randn(B, N, H, D, device="cuda", dtype=torch.bfloat16)
+ k = torch.randn(B, N, H, D, device="cuda", dtype=torch.bfloat16)
+ v = torch.randn(B, N, H, D, device="cuda", dtype=torch.bfloat16)
+
+ flex_prefill_output = flex_prefill_attention(q, k, v, gamma, tau)
diff --git a/minference/modules/forward.py b/minference/modules/forward.py
new file mode 100644
index 00000000..42b62518
--- /dev/null
+++ b/minference/modules/forward.py
@@ -0,0 +1,257 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from typing import Optional, Tuple
+
+import torch
+from transformers.cache_utils import Cache
+from transformers.modeling_flash_attention_utils import _flash_attention_forward
+from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv
+
+from ..modules.flexprefill import flexprefill_forward
+from ..modules.kivi import kivi_forward
+from ..modules.leank import leank_forward
+from ..modules.minference_forward import minference_prefill_forward
+from ..modules.quest import quest_decode_kernel
+from ..modules.retr_attn import retr_attn
+from ..modules.tri_mix import tri_mix_forward, tri_mix_minference_forward
+from ..modules.xattention import xattention_forward
+from ..ops.streaming_kernel import a_shape_kernel, tri_shape_kernel
+
+
+def attn_forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.LongTensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[
+ Tuple[torch.Tensor, torch.Tensor]
+ ] = None, # will become mandatory in v4.46
+ past_key_value: Cache = None,
+ prefill_forward=None,
+ decoding_forward=None,
+ customized_rope_func=None,
+ attn_forward_config=None,
+ **kwargs,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ output_attentions = False
+
+ bsz, q_len, _ = hidden_states.size()
+
+ if "num_heads" not in self.__dict__:
+ self.is_transformers_v448_or_later = True
+ self.num_heads = self.config.num_attention_heads
+ self.num_key_value_heads = self.config.num_key_value_heads
+ if "q_proj" in self.__dict__["_modules"]:
+ query_states = self.q_proj(hidden_states)
+ key_states = self.k_proj(hidden_states)
+ value_states = self.v_proj(hidden_states)
+ else:
+ qkv = self.qkv_proj(hidden_states)
+ query_pos = self.num_heads * self.head_dim
+ key_value_pos = query_pos // self.num_key_value_groups
+ query_states, key_states, value_states = torch.split(
+ qkv, [query_pos, key_value_pos, key_value_pos], -1
+ )
+
+ # [bsz, q_len, num_heads, head_dim]
+ query_states = query_states.view(
+ bsz, q_len, self.num_heads, self.head_dim
+ ).transpose(1, 2)
+ key_states = key_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+ value_states = value_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+
+ if position_embeddings is None:
+ cos, sin = self.rotary_emb(value_states, position_ids)
+ else:
+ cos, sin = position_embeddings
+
+ if cos.device != query_states.device:
+ cos = cos.to(query_states.device)
+ sin = sin.to(query_states.device)
+
+ if customized_rope_func is not None: # eg, glm-4 rope
+ query_states, key_states = customized_rope_func(
+ query_states, key_states, cos, sin
+ )
+ else:
+ query_states, key_states = apply_rotary_pos_emb(
+ query_states, key_states, cos, sin
+ )
+
+ if past_key_value is not None:
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
+ cache_kwargs = {
+ "sin": sin,
+ "cos": cos,
+ "cache_position": cache_position,
+ "attn_forward_config": attn_forward_config,
+ "attention_mask": attention_mask,
+ "num_key_value_groups": self.num_key_value_groups,
+ "query_states": query_states,
+ "update_global_past_kv": getattr(self, "update_global_past_kv", True),
+ }
+ if decoding_forward == leank_forward:
+ (
+ key_states_full,
+ key_states_mid,
+ value_states_mid,
+ value_states_full,
+ ) = past_key_value.update(
+ key_states,
+ value_states,
+ self.layer_idx,
+ self.full_attn_channels,
+ self.boundaries,
+ self.counts,
+ cache_kwargs,
+ )
+ else:
+ (
+ key_states,
+ value_states,
+ ) = past_key_value.update( # DynamicCache/KvcompressCache
+ key_states,
+ value_states,
+ self.layer_idx,
+ cache_kwargs,
+ )
+ if decoding_forward == leank_forward and prefill_forward is None:
+ pass
+ elif query_states.size(1) != key_states.size(1):
+ key_states = repeat_kv(key_states, query_states.size(1) // key_states.size(1))
+ value_states = repeat_kv(
+ value_states, query_states.size(1) // value_states.size(1)
+ )
+
+ dropout_rate = self.attention_dropout if self.training else 0.0
+
+ if not use_cache or q_len == past_key_value.get_seq_length(
+ self.layer_idx
+ ): # use no cache or prefilling
+ # if q_len != 1: # prefilling
+ if prefill_forward is not None: # eg, a-shape/tri-shape/minference
+ prefill_kwargs = {
+ "attention_mask": attention_mask,
+ "layer_idx": self.layer_idx,
+ "num_hidden_layers": self.config.num_hidden_layers,
+ "attn_forward_config": attn_forward_config,
+ }
+ attn_output = prefill_forward( # [bsz, num_heads, q_len, head_dim]
+ query_states,
+ key_states,
+ value_states,
+ prefill_kwargs,
+ )
+ attn_output = attn_output.transpose(1, 2).contiguous()
+
+ else: # if not specified, use flash attention
+ attn_output = _flash_attention_forward( # [bsz, q_len, num_heads, head_dim]
+ query_states.transpose(1, 2),
+ key_states.transpose(1, 2),
+ value_states.transpose(1, 2),
+ attention_mask,
+ q_len,
+ position_ids=position_ids,
+ dropout=dropout_rate,
+ sliding_window=getattr(self, "sliding_window", None),
+ is_causal=self.is_causal,
+ )
+
+ else: # decoding
+ # assert q_len == 1
+ if decoding_forward is not None: # eg, retr_attn
+ decoding_kwargs = {
+ "layer_idx": self.layer_idx,
+ "attn_forward_config": attn_forward_config,
+ "position_ids": position_ids,
+ "num_key_value_groups": self.num_key_value_groups,
+ }
+ if decoding_forward == leank_forward:
+ decoding_kwargs.update(
+ {
+ "last_length": self.last_length,
+ "kernel": self.kernel,
+ "boundaries": self.boundaries,
+ "counts": self.counts,
+ "full_size": self.sink_size
+ + self.recent_size
+ + self.accumu_size,
+ "kernel_config": self.kernel_config,
+ "full_attn_channels": self.full_attn_channels,
+ "attention_mask": attention_mask,
+ }
+ )
+ attn_output, last_length, kernel = decoding_forward(
+ query_states,
+ key_states_full,
+ key_states_mid,
+ value_states_full,
+ value_states_mid,
+ **decoding_kwargs,
+ )
+ self.last_length = last_length
+ self.kernel = kernel
+ else:
+ attn_output = decoding_forward( # [bsz, num_heads, q_len, head_dim]
+ query_states,
+ key_states,
+ value_states,
+ decoding_kwargs,
+ )
+ attn_output = attn_output.transpose(
+ 1, 2
+ ) # [bsz, q_len, num_heads, head_dim]
+ else:
+ attn_output = _flash_attention_forward(
+ query_states.transpose(1, 2),
+ key_states.transpose(1, 2),
+ value_states.transpose(1, 2),
+ attention_mask,
+ q_len,
+ position_ids=position_ids,
+ dropout=dropout_rate,
+ sliding_window=getattr(self, "sliding_window", None),
+ is_causal=self.is_causal,
+ )
+
+ assert attn_output.size(1) == q_len
+ attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
+ attn_output = self.o_proj(attn_output)
+
+ if not output_attentions:
+ attn_weights = None
+
+ if "is_transformers_v448_or_later" in self.__dict__:
+ return attn_output, attn_weights
+ return attn_output, attn_weights, past_key_value
+
+
+prefill_forwards = { # None = use flash attention
+ "dense": None,
+ "a_shape": a_shape_kernel,
+ "tri_shape": tri_shape_kernel,
+ "minference": minference_prefill_forward,
+ "flexprefill": flexprefill_forward,
+ "xattention": xattention_forward,
+ "tri_mix": tri_mix_forward,
+ "tri_mix_minference": tri_mix_minference_forward,
+}
+
+decoding_forwards = {
+ "dense": None,
+ "snapkv": None,
+ "pyramidkv": None,
+ "quest": quest_decode_kernel,
+ "streamingllm": None,
+ "retr_attn": retr_attn,
+ "kivi": kivi_forward,
+ "leank": leank_forward,
+}
diff --git a/minference/modules/kivi.py b/minference/modules/kivi.py
new file mode 100644
index 00000000..4ae5f2df
--- /dev/null
+++ b/minference/modules/kivi.py
@@ -0,0 +1,1075 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://github.com/jy-yuan/KIVI/tree/main/quant
+
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import triton
+import triton.language as tl
+from transformers.models.llama.modeling_llama import Cache, repeat_kv
+from transformers.utils.import_utils import _is_package_available
+
+if _is_package_available("kivi_gemv"):
+ import kivi_gemv
+
+
+def quant_and_pack_kcache(k: torch.FloatTensor, group_size: int, bits: int):
+ assert len(k.shape) == 4
+ shape = k.shape
+ B, nh, T, D = shape
+ # ================== Get Scale & Zeros ===============
+ assert T % group_size == 0
+ num_groups = T // group_size
+ new_shape = (B, nh, num_groups, group_size, D)
+ # Quantize
+ max_int = 2**bits - 1
+ data = k.view(new_shape)
+ mn = torch.min(data, dim=-2, keepdim=True)[0]
+ mx = torch.max(data, dim=-2, keepdim=True)[0]
+ scale = (mx - mn) / max_int
+ data = data - mn
+ data.div_(scale)
+ data = data.clamp_(0, max_int).round_().to(torch.int32)
+ data = data.view(shape)
+ code = pack_tensor(data, bits, pack_dim=2)
+ return code, scale, mn
+
+
+def quant_and_pack_vcache(v: torch.FloatTensor, group_size: int, bits: int):
+ shape = v.shape
+ assert len(shape) == 4
+ assert v.shape[-1] % group_size == 0
+ num_groups = shape[-1] // group_size
+ new_shape = shape[:-1] + (num_groups, group_size)
+ # Quantize
+ max_int = 2**bits - 1
+ data = v.view(new_shape)
+ mn = torch.min(data, dim=-1, keepdim=True)[0]
+ mx = torch.max(data, dim=-1, keepdim=True)[0]
+ scale = (mx - mn) / max_int
+ data = data - mn
+ data.div_(scale)
+ data = data.clamp_(0, max_int).round_().to(torch.int32)
+ data = data.view(shape)
+ # Pack
+ code = pack_tensor(data, bits, pack_dim=3)
+ return code, scale, mn
+
+
+def unpack_and_dequant_kcache(
+ k_code: torch.FloatTensor,
+ scale: torch.FloatTensor,
+ mn: torch.FloatTensor,
+ group_size: int,
+ bits: int,
+):
+ pack_dim = 2
+ assert bits in [2, 4, 8]
+ assert len(k_code.shape) == 4
+ data = unpack_tensor(k_code, bits, pack_dim=pack_dim)
+ shape = data.shape
+ num_groups = shape[pack_dim] // group_size
+ data = data.view(
+ shape[:pack_dim]
+ + (
+ num_groups,
+ group_size,
+ )
+ + shape[pack_dim + 1 :]
+ )
+ data = data.to(torch.float16)
+ data = data * scale + mn
+ return data.view(shape)
+
+
+def unpack_and_dequant_vcache(
+ v_code: torch.FloatTensor,
+ scale: torch.FloatTensor,
+ mn: torch.FloatTensor,
+ group_size: int,
+ bits: int,
+):
+ assert bits in [2, 4, 8]
+ assert len(v_code.shape) == 4
+ data = unpack_tensor(v_code, bits, pack_dim=3)
+ shape = data.shape
+ num_groups = shape[-1] // group_size
+ data = data.view(
+ shape[:-1]
+ + (
+ num_groups,
+ group_size,
+ )
+ )
+ data = data.to(torch.float16)
+ data = data * scale + mn
+ return data.view(shape)
+
+
+def pack_tensor(data, bits, pack_dim):
+ # Pack
+ shape = data.shape
+ feat_per_int = 32 // bits
+ assert bits in [2, 4, 8], "Only 2, 4, 8 bits are supported"
+ assert (
+ shape[pack_dim] % feat_per_int == 0
+ ), "Dimension length must be divisible by number of features per int"
+ # BS, nh, T, nd // 16 # 16 is for 2bit
+ code = torch.zeros(
+ shape[:pack_dim] + (shape[pack_dim] // feat_per_int,) + shape[pack_dim + 1 :],
+ dtype=torch.int32,
+ device=data.device,
+ )
+ i = 0
+ row = 0
+ unpacked_indices = [slice(None)] * len(data.shape)
+ packed_indices = [slice(None)] * len(data.shape)
+ while row < code.shape[pack_dim]:
+ packed_indices[pack_dim] = row
+ for j in range(i, i + (32 // bits)):
+ unpacked_indices[pack_dim] = j
+ code[packed_indices] |= data[unpacked_indices] << (bits * (j - i))
+ i += 32 // bits
+ row += 1
+ return code
+
+
+def unpack_tensor(v_code: torch.FloatTensor, bits: int, pack_dim: int):
+ assert bits in [2, 4, 8]
+ shape = v_code.shape
+ feat_per_int = 32 // bits
+ new_shape = (
+ shape[:pack_dim] + (shape[pack_dim] * feat_per_int,) + shape[pack_dim + 1 :]
+ )
+ unpacked_v_code = torch.zeros(new_shape, dtype=torch.int8, device=v_code.device)
+ i = torch.arange(new_shape[pack_dim], device=v_code.device) // feat_per_int
+ j = torch.arange(new_shape[pack_dim], device=v_code.device) % feat_per_int
+ num = 0xFF >> (8 - bits)
+ packed_indices = [slice(None)] * len(new_shape)
+ packed_indices[pack_dim] = i
+ if pack_dim == 2:
+ unpacked_v_code = (
+ (v_code[packed_indices] >> (j * bits)[None, None, :, None]).to(torch.int16)
+ ) & num
+ elif pack_dim == 3:
+ unpacked_v_code = ((v_code[packed_indices] >> (j * bits)).to(torch.int16)) & num
+ else:
+ raise NotImplementedError
+ return unpacked_v_code
+
+
+@triton.jit
+def _pack_along_last_dim(
+ bits: tl.constexpr,
+ intensor_ptr,
+ code_ptr,
+ N,
+ num_feats: tl.constexpr,
+ feat_per_int: tl.constexpr,
+ BLOCK_SIZE_N: tl.constexpr,
+):
+ num_int_per_y_dim = num_feats // feat_per_int
+ bid = tl.program_id(axis=0)
+ yid = tl.program_id(axis=1)
+ offs_N = bid * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+ block_start = (
+ intensor_ptr + offs_N * num_feats + yid * feat_per_int
+ ) # offset of the first element at current tile
+ packed = tl.zeros((BLOCK_SIZE_N,), dtype=tl.int32)
+ for i in range(feat_per_int):
+ ptr = block_start + i
+ element = tl.load(ptr, mask=offs_N < N, other=0.0)
+ element = element << (i * bits)
+ # Combine the value using bitwise OR
+ packed = packed | element
+ tl.store(code_ptr + offs_N * num_int_per_y_dim + yid, packed, mask=offs_N < N)
+
+
+@triton.jit
+def _minmax_along_last_dim(
+ x_ptr,
+ mn_ptr,
+ mx_ptr,
+ total_elements: tl.constexpr,
+ N: tl.constexpr,
+ num_groups: tl.constexpr,
+ group_size: tl.constexpr,
+ BLOCK_SIZE_N: tl.constexpr,
+):
+ bid = tl.program_id(axis=0)
+ offsets_b = bid * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+ offsets = offsets_b[:, None] * group_size + tl.arange(0, group_size)[None, :]
+ mask = offsets < total_elements
+ x = tl.load(x_ptr + offsets, mask=mask)
+ mx_val = tl.max(x, axis=1)
+ mn_val = tl.min(x, axis=1)
+ # tl.device_print('shape', mn_val[:, None].shape)
+ tl.store(mn_ptr + offsets_b, mn_val, mask=offsets_b < N * num_groups)
+ tl.store(mx_ptr + offsets_b, mx_val, mask=offsets_b < N * num_groups)
+
+
+def triton_quantize_and_pack_along_last_dim(
+ data: torch.Tensor, group_size: int, bit: int
+):
+ assert len(data.shape) == 4
+ shape = data.shape
+ B, nh, D, T = shape
+ # ================== Get Scale & Zeros ===============
+ assert T % group_size == 0
+ num_groups = T // group_size
+ new_shape = (B * nh * D, num_groups, group_size)
+ scale_mn_shape = B, nh, D, num_groups
+ # Quantize
+ data = data.reshape(new_shape)
+ mx = torch.empty((B * nh * D, num_groups), device=data.device, dtype=data.dtype)
+ mn = torch.empty((B * nh * D, num_groups), device=data.device, dtype=data.dtype)
+ BLOCK_SIZE_N = 128
+ grid = lambda meta: (triton.cdiv(data.shape[0] * data.shape[1], BLOCK_SIZE_N),)
+ with torch.cuda.device(data.device):
+ _minmax_along_last_dim[grid](
+ data,
+ mn,
+ mx,
+ data.numel(),
+ data.shape[0],
+ num_groups,
+ group_size,
+ BLOCK_SIZE_N=BLOCK_SIZE_N,
+ num_warps=8,
+ )
+ # mn = torch.min(data, dim=-1, keepdim=True)[0].squeeze(-1)
+ # mx = torch.max(data, dim=-1, keepdim=True)[0].squeeze(-1)
+ scale = (mx - mn) / (2**bit - 1)
+ data = data - mn.unsqueeze(-1)
+ data.div_(scale.unsqueeze(-1))
+ data = data.clamp_(0, 2**bit - 1).round_().to(torch.int32)
+ data = data.view(-1, T)
+ feat_per_int = 32 // bit
+ packshape = (
+ np.prod(shape[:-1]),
+ shape[-1] // feat_per_int,
+ )
+ code = torch.zeros(*packshape, device=data.device, dtype=torch.int32)
+ grid = lambda meta: (
+ triton.cdiv(data.shape[0], BLOCK_SIZE_N),
+ data.shape[1] // feat_per_int,
+ )
+ with torch.cuda.device(data.device):
+ _pack_along_last_dim[grid](
+ bit,
+ data,
+ code,
+ data.shape[0],
+ data.shape[1],
+ feat_per_int,
+ BLOCK_SIZE_N=BLOCK_SIZE_N,
+ num_warps=8,
+ )
+ return (
+ code.view(B, nh, D, -1),
+ scale.reshape(scale_mn_shape),
+ mn.reshape(scale_mn_shape),
+ )
+
+
+@triton.jit
+def qbvm_kernel(
+ bits,
+ a_ptr,
+ b_ptr,
+ c_ptr,
+ scales_ptr,
+ zeros_ptr,
+ M,
+ N,
+ K,
+ stride_abatch,
+ stride_am,
+ stride_ak,
+ stride_bbatch,
+ stride_bk,
+ stride_bn,
+ stride_cbatch,
+ stride_cm,
+ stride_cn,
+ stride_scales_b,
+ stride_scales_k,
+ stride_scales_g,
+ stride_zeros_b,
+ stride_zeros_k,
+ stride_zeros_g,
+ groupsize,
+ BLOCK_SIZE_N: tl.constexpr,
+ BLOCK_SIZE_K: tl.constexpr,
+):
+ """
+ Compute the batch matrix multiplication C = A x B.
+ A is of shape (B, 1, K) float16
+ B is of shape (B, K, N//feat_per_int) int32
+ C is of shape (B, 1, N) float16
+ scales is of shape (B, K, G) float16
+ zeros is of shape (B, K, G) float16
+ groupsize is an int specifying the size of groups for scales and zeros.
+ G is N // groupsize.
+ Set NO_GROUPS to groupsize == K, in which case G = 1 and the kernel is more efficient.
+
+ WARNING: This kernel assumes that K is a multiple of BLOCK_SIZE_K.
+ WARNING: This kernel assumes that N is a multiple of BLOCK_SIZE_N.
+ WARNING: This kernel assumes that groupsize is a multiple of BLOCK_SIZE_K.
+ """
+ pid_batch = tl.program_id(axis=0)
+ pid = tl.program_id(axis=1)
+ feat_per_int = 32 // bits
+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
+ num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)
+ pid_n = pid % num_pid_n
+ offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+ offs_k = tl.arange(0, BLOCK_SIZE_K)
+ a_batch_offset = pid_batch * stride_abatch
+ b_batch_offset = pid_batch * stride_bbatch
+ c_batch_offset = pid_batch * stride_cbatch
+ a_ptr = a_ptr + a_batch_offset
+ b_ptr = b_ptr + b_batch_offset
+ c_ptr = c_ptr + c_batch_offset
+ a_ptrs = a_ptr + (offs_k[:, None] * stride_ak) # (BLOCK_SIZE_K, 1)
+ # a_mask = (offs_am[:, None] < M)
+ # b_ptrs is set up such that it repeats elements along the N axis feat_per_int times
+ b_ptrs = b_ptr + (
+ offs_k[:, None] * stride_bk + (offs_bn[None, :] // feat_per_int) * stride_bn
+ ) # (BLOCK_SIZE_K, BLOCK_SIZE_N)
+ # shifter is used to extract the # bits bits of each element in the 32-bit word from B
+ shifter = (offs_bn % feat_per_int) * bits
+ scales_ptr = (
+ scales_ptr
+ + pid_batch * stride_scales_b
+ + ((offs_bn[None, :] // groupsize)) * stride_scales_g
+ ) # (BLOCK_SIZE_N,)
+ zeros_ptr = (
+ zeros_ptr
+ + pid_batch * stride_zeros_b
+ + ((offs_bn[None, :] // groupsize)) * stride_zeros_g
+ ) # (BLOCK_SIZE_N,)
+
+ # Now calculate a block of output of shape (BLOCK_SIZE_M, BLOCK_SIZE_N)
+ # M is along the batch dimension, N is along the outfeatures dimension, K is along the infeatures dimension
+ # So this loop is along the infeatures dimension (K)
+ # It's calculating BLOCK_SIZE_M batches in parallel, and for each batch, BLOCK_SIZE_N outfeatures in parallel
+ # accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
+ accumulator = tl.zeros((BLOCK_SIZE_N,), dtype=tl.float32)
+ num = 0xFF >> (8 - bits)
+ for pid_k in range(0, num_pid_k):
+ offs_bk = offs_k[:, None] + pid_k * BLOCK_SIZE_K
+ # offs_k[None, :] < K - pid_k * BLOCK_SIZE_K
+ a = tl.load(a_ptrs, mask=offs_bk < K, other=0.0) # (1, BLOCK_SIZE_K)
+ b = tl.load(b_ptrs, mask=offs_bk < K, other=0.0) # (BLOCK_SIZE_K, BLOCK_SIZE_N)
+ ptr = scales_ptr + offs_bk * stride_scales_k
+ scales = tl.load(
+ ptr, mask=offs_bk < K, other=0.0
+ ) # (BLOCK_SIZE_K, BLOCK_SIZE_N)
+ ptr = zeros_ptr + offs_bk * stride_zeros_k
+ zeros = tl.load(
+ ptr, mask=offs_bk < K, other=0.0
+ ) # (BLOCK_SIZE_K, BLOCK_SIZE_N)
+ # Now we need to unpack b into 32-bit values
+ # tl.device_print("scale ",scales.dtype)
+ # tl.device_print("zeros ",zeros.dtype)
+ b = (b >> shifter[None, :]) & num # For 4-bit values, bit_op_num is 0xF
+ b = b * scales + zeros # Scale and shift
+ accumulator += tl.sum(a * b, 0) # tl.dot(a, b)
+ # if pid_m == 0 and pid_n == 0:
+ # tl.device_print("hello ", tl.dot(a, b).shape)
+ a_ptrs += BLOCK_SIZE_K * stride_ak
+ b_ptrs += BLOCK_SIZE_K * stride_bk
+ c = accumulator # .to(tl.float16)
+ # c = accumulator
+ # Store the result
+ offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+ c_ptrs = c_ptr + stride_cn * offs_cn
+ c_mask = offs_cn < N
+ tl.store(c_ptrs, c, mask=c_mask)
+
+
+def understand_code():
+ M, N, K = 512, 256, 256
+ BLOCK_SIZE_M, BLOCK_SIZE_N, GROUP_SIZE_M = 64, 64, 4
+ total_program_id = triton.cdiv(M, BLOCK_SIZE_M) * triton.cdiv(N, BLOCK_SIZE_N)
+ for pid in range(0, total_program_id):
+ num_pid_m = triton.cdiv(M, BLOCK_SIZE_M)
+ num_pid_n = triton.cdiv(N, BLOCK_SIZE_N)
+ num_pid_in_group = GROUP_SIZE_M * num_pid_n
+ group_id = pid // num_pid_in_group
+ first_pid_m = group_id * GROUP_SIZE_M
+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
+ pid_m = first_pid_m + (pid % group_size_m)
+ pid_n = (pid % num_pid_in_group) // group_size_m
+ print(f"pid={pid}, pid_m={pid_m}, pid_n={pid_n}")
+
+
+def triton_bmm_fA_qB_outer(
+ group_size: int,
+ fA: torch.FloatTensor,
+ qB: torch.IntTensor,
+ scales: torch.FloatTensor,
+ zeros: torch.FloatTensor,
+ bits: int,
+) -> torch.FloatTensor:
+ """
+ Compute the matrix multiplication C = query x key.
+ Where key is quantized into 2-bit values.
+
+ fA is of shape (B, nh, M, K) float16
+ qB is of shape (B, nh, K, N // feat_per_int) int32
+ scales is of shape (B, nh, K, G) float16
+ zeros is of shape (B, nh, K, G) float16
+
+ groupsize is the number of outer dimensions in each group.
+ G = N // groupsize
+
+ Returns C of shape (B, nh, M, N) float16
+ """
+ assert len(fA.shape) == 4 and len(qB.shape) == 4
+ B, nh, M, K = fA.shape
+ feat_per_int = 32 // bits
+ # flatten to a 3D tensor
+ fA = fA.view(-1, M, K)
+ N = qB.shape[-1] * feat_per_int
+ qB = qB.reshape(-1, K, qB.shape[-1])
+ # This is based on the possible BLOCK_SIZE_Ks
+ # assert K % 16 == 0 and K % 32 == 0 and K % 64 == 0 and K % 128 == 0, "K must be a multiple of 16, 32, 64, and 128"
+ # This is based on the possible BLOCK_SIZE_Ns
+ assert (
+ N % 16 == 0 and N % 32 == 0 and N % 64 == 0
+ ), "N must be a multiple of 16, 32, 64, 128, and 256"
+ # This is based on the possible BLOCK_SIZE_Ks
+ assert group_size % 64 == 0, "groupsize must be a multiple of 64, and 128"
+ flatten_B = B * nh
+ c = torch.empty((flatten_B, M, N), device="cuda", dtype=torch.float16)
+ # print(f'M {M} N {N} K {K}')
+ grid = lambda META: (
+ flatten_B,
+ triton.cdiv(N, META["BLOCK_SIZE_N"]),
+ )
+ scales = scales.view(flatten_B, scales.shape[-2], scales.shape[-1])
+ zeros = zeros.view(flatten_B, zeros.shape[-2], zeros.shape[-1])
+ if N > K:
+ BLOCK_SIZE_N = 128
+ BLOCK_SIZE_K = 32
+ num_warps = 4 #
+ else:
+ BLOCK_SIZE_N = 32
+ BLOCK_SIZE_K = 128
+ num_warps = 2
+ num_stages = 7 if K > 64 else 3 #
+ qbvm_kernel[grid](
+ bits,
+ fA,
+ qB,
+ c,
+ scales,
+ zeros,
+ M,
+ N,
+ K,
+ fA.stride(0),
+ fA.stride(1),
+ fA.stride(2),
+ qB.stride(0),
+ qB.stride(1),
+ qB.stride(2),
+ c.stride(0),
+ c.stride(1),
+ c.stride(2),
+ scales.stride(0),
+ scales.stride(1),
+ scales.stride(2),
+ zeros.stride(0),
+ zeros.stride(1),
+ scales.stride(2),
+ group_size,
+ BLOCK_SIZE_N,
+ BLOCK_SIZE_K,
+ num_warps=num_warps,
+ num_stages=num_stages,
+ )
+ return c.view(B, nh, c.shape[-2], c.shape[-1])
+
+
+def cuda_bmm_fA_qB_outer(
+ group_size: int,
+ fA: torch.FloatTensor,
+ qB: torch.IntTensor,
+ scales: torch.FloatTensor,
+ zeros: torch.FloatTensor,
+ bits: int,
+ mqa: bool = False,
+) -> torch.FloatTensor:
+ """
+ Compute the matrix multiplication C = query x key.
+ Where key is quantized into 2-bit values.
+
+ fA is of shape (B, nh, M, K) float16
+ qB is of shape (B, nh, K, N // feat_per_int) int32
+ scales is of shape (B, nh, K, G) float16
+ zeros is of shape (B, nh, K, G) float16
+
+ groupsize is the number of outer dimensions in each group.
+ G = N // groupsize
+
+ Returns C of shape (B, nh, M, N) float16
+ """
+ assert len(fA.shape) == 4 and len(qB.shape) == 4
+ B, nh, M, K = fA.shape
+ feat_per_int = 32 // bits
+ # flatten to a 3D tensor
+ fA = fA.view(-1, M, K).contiguous()
+ N = qB.shape[-1] * feat_per_int
+ qB = qB.reshape(-1, K, qB.shape[-1]).transpose(1, 2).contiguous()
+ # This is based on the possible BLOCK_SIZE_Ks
+ # assert K % 16 == 0 and K % 32 == 0 and K % 64 == 0 and K % 128 == 0, "K must be a multiple of 16, 32, 64, and 128"
+ # This is based on the possible BLOCK_SIZE_Ns
+ # assert N % 16 == 0 and N % 32 == 0 and N % 64 == 0, "N must be a multiple of 16, 32, 64, 128, and 256"
+ # This is based on the possible BLOCK_SIZE_Ks
+ # assert group_size % 64 == 0, "groupsize must be a multiple of 64, and 128"
+ flatten_B = B * nh
+ if mqa:
+ flatten_B = B
+ scales = (
+ scales.view(flatten_B, scales.shape[-2], scales.shape[-1])
+ .transpose(1, 2)
+ .contiguous()
+ )
+ zeros = (
+ zeros.view(flatten_B, zeros.shape[-2], zeros.shape[-1])
+ .transpose(1, 2)
+ .contiguous()
+ )
+ assert bits in [2, 4]
+ try:
+ c = kivi_gemv.gemv_forward_cuda_outer_dim(
+ fA, qB, scales, zeros, bits, group_size, nh, mqa
+ )
+ except:
+ assert False, "Please install kivi. Refer to https://github.com/jy-yuan/KIVI"
+ c = c.view(B, nh, c.shape[-2], c.shape[-1])
+ return c
+
+
+class KiviCache(Cache):
+ def __init__(self, config):
+ super().__init__()
+ self.group_size = config.attn_kwargs.get("group_size", 32)
+ self.bits = config.attn_kwargs.get("bits", 2)
+ self.residual_length = config.attn_kwargs.get("residual_length", 32)
+ self.group_size = config.attn_kwargs.get("group_size", 32)
+
+ self.k_bits = self.bits
+ self.v_bits = self.bits
+
+ self.kv_cache = []
+ self._seen_tokens = 0
+
+ self.temp_key_cache = []
+ self.temp_value_cache = []
+
+ def update(
+ self,
+ key_states,
+ value_states,
+ layer_idx,
+ cache_kwargs,
+ ):
+ update_global_past_kv = cache_kwargs.get("update_global_past_kv", True)
+ query_states = cache_kwargs["query_states"]
+
+ if layer_idx == 0:
+ self._seen_tokens += key_states.size(-2)
+
+ prefilling = False
+ if len(self.kv_cache) == layer_idx:
+ prefilling = True
+
+ if key_states.shape[-2] % self.residual_length != 0:
+ if key_states.shape[-2] < self.residual_length:
+ key_states_quant = None
+ key_states_full = key_states
+ else:
+ key_states_quant = key_states[
+ :, :, : -(key_states.shape[-2] % self.residual_length), :
+ ].contiguous()
+ key_states_full = key_states[
+ :, :, -(key_states.shape[-2] % self.residual_length) :, :
+ ].contiguous()
+ else:
+ key_states_quant = key_states
+ key_states_full = None
+
+ if key_states_quant is not None:
+ (
+ key_states_quant_trans,
+ key_scale_trans,
+ key_mn_trans,
+ ) = triton_quantize_and_pack_along_last_dim(
+ key_states_quant.transpose(2, 3).contiguous(),
+ self.group_size,
+ self.k_bits,
+ )
+ else:
+ key_states_quant_trans = None
+ key_scale_trans = None
+ key_mn_trans = None
+
+ if value_states.shape[-2] <= self.residual_length:
+ value_states_quant = None
+ value_states_full = value_states
+ value_scale = None
+ value_mn = None
+ else:
+ value_states_quant = value_states[
+ :, :, : -self.residual_length, :
+ ].contiguous()
+ value_states_full = value_states[
+ :, :, -self.residual_length :, :
+ ].contiguous()
+ (
+ value_states_quant,
+ value_scale,
+ value_mn,
+ ) = triton_quantize_and_pack_along_last_dim(
+ value_states_quant, self.group_size, self.v_bits
+ )
+
+ self.kv_cache.append(
+ (
+ key_states_quant_trans,
+ key_states_full,
+ key_scale_trans,
+ key_mn_trans,
+ value_states_quant,
+ value_states_full,
+ value_scale,
+ value_mn,
+ )
+ )
+
+ return (
+ repeat_kv(key_states, query_states.size(1) // key_states.size(1)),
+ repeat_kv(value_states, query_states.size(1) // value_states.size(1)),
+ )
+
+ else: # decoding
+ key_states_quant_trans = self.kv_cache[layer_idx][0]
+ key_states_full = self.kv_cache[layer_idx][1]
+ key_scale_trans = self.kv_cache[layer_idx][2]
+ key_mn_trans = self.kv_cache[layer_idx][3]
+
+ value_states_quant = self.kv_cache[layer_idx][4]
+ value_states_full = self.kv_cache[layer_idx][5]
+ value_scale = self.kv_cache[layer_idx][6]
+ value_mn = self.kv_cache[layer_idx][7]
+
+ if not update_global_past_kv:
+ if len(self.temp_key_cache) == layer_idx:
+ self.temp_key_cache.append(key_states)
+ self.temp_value_cache.append(value_states)
+ else:
+ self.temp_key_cache[layer_idx] = torch.cat(
+ [self.temp_key_cache[layer_idx], key_states], dim=-2
+ )
+ self.temp_value_cache[layer_idx] = torch.cat(
+ [self.temp_value_cache[layer_idx], value_states], dim=-2
+ )
+
+ if key_states_full is not None:
+ key_states_full_rt = torch.cat(
+ [key_states_full, self.temp_key_cache[layer_idx]], dim=-2
+ )
+ else:
+ key_states_full_rt = self.temp_key_cache[layer_idx]
+ value_states_full_rt = torch.cat(
+ [value_states_full, self.temp_value_cache[layer_idx]], dim=-2
+ )
+ else:
+ if key_states_full is not None:
+ key_states_full = torch.cat([key_states_full, key_states], dim=2)
+ else:
+ key_states_full = key_states
+
+ value_states_full = torch.cat([value_states_full, value_states], dim=2)
+ key_states_full_rt = key_states_full
+ value_states_full_rt = value_states_full
+
+ self.kv_cache[layer_idx] = (
+ key_states_quant_trans,
+ key_states_full,
+ key_scale_trans,
+ key_mn_trans,
+ value_states_quant,
+ value_states_full,
+ value_scale,
+ value_mn,
+ )
+
+ key_out = (
+ key_states_quant_trans,
+ key_states_full_rt, # key_states_full
+ key_scale_trans,
+ key_mn_trans,
+ )
+ value_out = (
+ value_states_quant,
+ value_states_full_rt,
+ value_scale,
+ value_mn,
+ )
+ return key_out, value_out
+
+ def legacy_update(
+ self,
+ key_states,
+ value_states,
+ layer_idx,
+ cache_kwargs,
+ ):
+ update_global_past_kv = cache_kwargs.get("update_global_past_kv", True)
+ query_states = cache_kwargs["query_states"]
+
+ if layer_idx == 0:
+ self._seen_tokens += key_states.size(-2)
+
+ prefilling = False
+ if len(self.kv_cache) == layer_idx:
+ prefilling = True
+
+ if key_states.shape[-2] % self.residual_length != 0:
+ if key_states.shape[-2] < self.residual_length:
+ key_states_quant = None
+ key_states_full = key_states
+ else:
+ key_states_quant = key_states[
+ :, :, : -(key_states.shape[-2] % self.residual_length), :
+ ].contiguous()
+ key_states_full = key_states[
+ :, :, -(key_states.shape[-2] % self.residual_length) :, :
+ ].contiguous()
+ else:
+ key_states_quant = key_states
+ key_states_full = None
+
+ if key_states_quant is not None:
+ (
+ key_states_quant_trans,
+ key_scale_trans,
+ key_mn_trans,
+ ) = triton_quantize_and_pack_along_last_dim(
+ key_states_quant.transpose(2, 3).contiguous(),
+ self.group_size,
+ self.k_bits,
+ )
+ else:
+ key_states_quant_trans = None
+ key_scale_trans = None
+ key_mn_trans = None
+
+ if value_states.shape[-2] <= self.residual_length:
+ value_states_quant = None
+ value_states_full = value_states
+ value_scale = None
+ value_mn = None
+ else:
+ value_states_quant = value_states[
+ :, :, : -self.residual_length, :
+ ].contiguous()
+ value_states_full = value_states[
+ :, :, -self.residual_length :, :
+ ].contiguous()
+ (
+ value_states_quant,
+ value_scale,
+ value_mn,
+ ) = triton_quantize_and_pack_along_last_dim(
+ value_states_quant, self.group_size, self.v_bits
+ )
+
+ self.kv_cache.append(
+ (
+ key_states_quant_trans,
+ key_states_full,
+ key_scale_trans,
+ key_mn_trans,
+ value_states_quant,
+ value_states_full,
+ value_scale,
+ value_mn,
+ )
+ )
+
+ return (
+ repeat_kv(key_states, query_states.size(-1) // key_states.size(-1)),
+ repeat_kv(value_states, query_states.size(-1) // value_states.size(-1)),
+ )
+
+ else: # decoding
+ key_states_quant_trans = self.kv_cache[layer_idx][0]
+ key_states_full = self.kv_cache[layer_idx][1]
+ key_scale_trans = self.kv_cache[layer_idx][2]
+ key_mn_trans = self.kv_cache[layer_idx][3]
+
+ value_states_quant = self.kv_cache[layer_idx][4]
+ value_states_full = self.kv_cache[layer_idx][5]
+ value_scale = self.kv_cache[layer_idx][6]
+ value_mn = self.kv_cache[layer_idx][7]
+
+ if not update_global_past_kv:
+ if len(self.temp_key_cache) == layer_idx:
+ self.temp_key_cache.append(key_states)
+ self.temp_value_cache.append(value_states)
+ else:
+ self.temp_key_cache[layer_idx] = torch.cat(
+ [self.temp_key_cache[layer_idx], key_states], dim=-2
+ )
+ self.temp_value_cache[layer_idx] = torch.cat(
+ [self.temp_value_cache[layer_idx], value_states], dim=-2
+ )
+
+ key_states_full_rt = torch.cat(
+ [key_states_full, self.temp_key_cache[layer_idx]], dim=-2
+ )
+ value_states_full_rt = torch.cat(
+ [value_states_full, self.temp_value_cache[layer_idx]], dim=-2
+ )
+
+ else:
+ if key_states_full is not None:
+ key_states_full = torch.cat([key_states_full, key_states], dim=2)
+ else:
+ key_states_full = key_states
+
+ if key_states_full.shape[-2] > self.residual_length:
+ key_states_quant = key_states_full[
+ :, :, : -(key_states_full.shape[-2] % self.residual_length), :
+ ].contiguous()
+ key_states_full = key_states_full[
+ :, :, -(key_states_full.shape[-2] % self.residual_length) :, :
+ ].contiguous()
+
+ (
+ key_states_quant_trans_new,
+ key_scale_trans_new,
+ key_mn_trans_new,
+ ) = triton_quantize_and_pack_along_last_dim(
+ key_states_quant.transpose(2, 3).contiguous(),
+ self.group_size,
+ self.k_bits,
+ )
+ if key_states_quant_trans is not None:
+ key_states_quant_trans = torch.cat(
+ [key_states_quant_trans, key_states_quant_trans_new], dim=3
+ )
+ key_scale_trans = torch.cat(
+ [key_scale_trans, key_scale_trans_new], dim=3
+ )
+ key_mn_trans = torch.cat(
+ [key_mn_trans, key_mn_trans_new], dim=3
+ )
+ else:
+ key_states_quant_trans = key_states_quant_trans_new
+ key_scale_trans = key_scale_trans_new
+ key_mn_trans = key_mn_trans_new
+
+ value_states_full = torch.cat([value_states_full, value_states], dim=2)
+ value_full_length = value_states_full.shape[-2]
+
+ if value_full_length > self.residual_length:
+ value_states_to_quant = value_states_full[
+ :, :, : -(value_full_length % self.residual_length), :
+ ].contiguous()
+ value_states_full = value_states_full[
+ :, :, -(value_full_length % self.residual_length) :, :
+ ].contiguous()
+
+ (
+ value_states_quant_new,
+ scale,
+ mn,
+ ) = triton_quantize_and_pack_along_last_dim(
+ value_states_to_quant.contiguous(),
+ self.group_size,
+ self.v_bits,
+ )
+ if value_states_quant is not None:
+ value_states_quant = torch.cat(
+ [value_states_quant, value_states_quant_new], dim=2
+ )
+ value_scale = torch.cat([value_scale, scale], dim=2)
+ value_mn = torch.cat([value_mn, mn], dim=2)
+ else:
+ value_states_quant = value_states_quant_new
+ value_scale = scale
+ value_mn = mn
+ key_states_full_rt = key_states_full
+ value_states_full_rt = value_states_full
+
+ self.kv_cache[layer_idx] = (
+ key_states_quant_trans,
+ key_states_full,
+ key_scale_trans,
+ key_mn_trans,
+ value_states_quant,
+ value_states_full,
+ value_scale,
+ value_mn,
+ )
+
+ key_out = (
+ key_states_quant_trans,
+ key_states_full_rt, # key_states_full
+ key_scale_trans,
+ key_mn_trans,
+ )
+ value_out = (
+ value_states_quant,
+ value_states_full_rt,
+ value_scale,
+ value_mn,
+ )
+ return key_out, value_out
+
+ def get_seq_length(self, layer_idx=0):
+ if len(self.kv_cache) <= layer_idx:
+ return 0
+ return self._seen_tokens
+
+ def clear_temp_kv_cache(self):
+ if self.temp_key_cache:
+ self._seen_tokens -= self.temp_key_cache[-1].shape[
+ -2
+ ] # seq_len of temp_kv_cache
+ self.temp_key_cache = []
+ self.temp_value_cache = []
+
+
+def kivi_forward(query_states, key_states, value_states, decoding_kwargs):
+ group_size = decoding_kwargs["attn_forward_config"].get("group_size", 32)
+ k_bits = decoding_kwargs["attn_forward_config"].get("bits", 2)
+ v_bits = decoding_kwargs["attn_forward_config"].get("bits", 2)
+ head_dim = query_states.size(-1)
+ q_len = query_states.shape[-2]
+
+ key_states_quant_trans = key_states[0]
+ key_states_full = key_states[1]
+ key_scale_trans = key_states[2].to(torch.float16)
+ key_mn_trans = key_states[3].to(torch.float16)
+
+ value_states_quant = value_states[0]
+ value_states_full = value_states[1]
+ value_scale = value_states[2].to(torch.float16)
+ value_mn = value_states[3].to(torch.float16)
+
+ num_key_value_groups = query_states.size(1) // key_states_quant_trans.size(1)
+ key_states_quant_trans = repeat_kv(key_states_quant_trans, num_key_value_groups)
+ key_scale_trans = repeat_kv(key_scale_trans, num_key_value_groups)
+ key_mn_trans = repeat_kv(key_mn_trans, num_key_value_groups)
+ key_states_full = repeat_kv(key_states_full, num_key_value_groups)
+
+ value_states_quant = repeat_kv(value_states_quant, num_key_value_groups)
+ value_scale = repeat_kv(value_scale, num_key_value_groups)
+ value_mn = repeat_kv(value_mn, num_key_value_groups)
+ value_states_full = repeat_kv(value_states_full, num_key_value_groups)
+
+ if key_states_quant_trans is not None:
+ if q_len == 1:
+ att_qkquant = cuda_bmm_fA_qB_outer(
+ group_size,
+ query_states.to(torch.float16),
+ key_states_quant_trans,
+ key_scale_trans,
+ key_mn_trans,
+ k_bits,
+ )
+ else: # cuda_bmm_fA_qB_outer will lead to nan when query_states.shape[-2] > 1, need a fix in the kernel side
+ attn_qkquants_t = []
+ for i in range(q_len):
+ att_qkquant_i = cuda_bmm_fA_qB_outer(
+ group_size,
+ query_states[:, :, i : i + 1, :].to(torch.float16),
+ key_states_quant_trans,
+ key_scale_trans,
+ key_mn_trans,
+ k_bits,
+ )
+ attn_qkquants_t.append(att_qkquant_i)
+ att_qkquant = torch.cat(attn_qkquants_t, dim=2)
+ else:
+ att_qkquant = None
+
+ if torch.isnan(att_qkquant).any():
+ print("NaN in att_qkquant")
+ att_qkfull = torch.matmul(query_states, key_states_full.transpose(2, 3))
+
+ if att_qkquant is not None:
+ attn_weights = torch.cat(
+ [att_qkquant.to(query_states.dtype), att_qkfull], dim=-1
+ ) / math.sqrt(head_dim)
+ else:
+ attn_weights = att_qkfull / math.sqrt(head_dim)
+
+ value_full_length = value_states_full.shape[-2]
+ if q_len == 1:
+ attn_weights = nn.functional.softmax(
+ attn_weights, dim=-1, dtype=torch.float32
+ ).to(query_states.dtype)
+ else:
+ attn_weights += (
+ torch.triu(
+ torch.ones_like(attn_weights), diagonal=value_full_length - q_len + 1
+ )
+ * torch.finfo(attn_weights.dtype).min
+ )
+ attn_weights = nn.functional.softmax(
+ attn_weights, dim=-1, dtype=torch.float32
+ ).to(query_states.dtype)
+
+ value_full_length = value_states_full.shape[-2]
+ if value_states_quant is not None:
+ if q_len == 1:
+ attn_output = cuda_bmm_fA_qB_outer(
+ group_size,
+ attn_weights[:, :, :, :-value_full_length].to(torch.float16),
+ value_states_quant,
+ value_scale,
+ value_mn,
+ v_bits,
+ ).to(query_states.dtype)
+ attn_output += torch.matmul(
+ attn_weights[:, :, :, -value_full_length:], value_states_full
+ )
+ else:
+ # if q_len > 1:
+ attn_outputs = []
+ for i in range(q_len):
+ attn_output_i = cuda_bmm_fA_qB_outer(
+ group_size,
+ attn_weights[:, :, i : i + 1, :-value_full_length].to(
+ torch.float16
+ ),
+ value_states_quant,
+ value_scale,
+ value_mn,
+ v_bits,
+ ).to(query_states.dtype)
+ attn_outputs.append(attn_output_i)
+ attn_output = torch.cat(attn_outputs, dim=2)
+ attn_output += torch.matmul(
+ attn_weights[:, :, :, -value_full_length:], value_states_full
+ )
+ # torch.testing.assert_close(attn_output, attn_output_t)
+ else:
+ attn_output = torch.matmul(attn_weights, value_states_full)
+
+ if torch.isnan(attn_output).any():
+ print("NaN in attn_output")
+
+ return attn_output
diff --git a/minference/modules/kvcompression.py b/minference/modules/kvcompression.py
new file mode 100644
index 00000000..f499a76b
--- /dev/null
+++ b/minference/modules/kvcompression.py
@@ -0,0 +1,463 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import functools
+from typing import List
+
+from transformers.cache_utils import Cache
+from transformers.modeling_flash_attention_utils import _flash_attention_forward
+from transformers.models.glm.modeling_glm import (
+ apply_rotary_pos_emb as apply_rotary_pos_emb_glm4,
+)
+
+from .kivi import KiviCache
+from .leank import LeanKCache
+from .pyramidkv import PyramidKVCluster
+from .quest import *
+from .retr_attn import RetrAttnCache
+from .snapkv import SnapKVCluster, StreamingLLMKVCluster
+
+
+def prepare_inputs_for_generation_kvcompression(
+ method: str, config, original_prepare_inputs_for_generation
+):
+ @functools.wraps(original_prepare_inputs_for_generation)
+ def new_prepare_inputs_for_generation(self, *args, **kwargs):
+ outputs = original_prepare_inputs_for_generation(*args, **kwargs)
+ use_cache = kwargs.get("use_cache", True)
+ if use_cache and not isinstance(
+ outputs.get("past_key_values", None), method_to_cache_obj[method]
+ ):
+ cache_obj: Cache = method_to_cache_obj[method]
+ config.num_layers = self.config.num_hidden_layers
+ outputs["past_key_values"] = cache_obj(config)
+ if (
+ hasattr(self, "_supports_num_logits_to_keep")
+ and self._supports_num_logits_to_keep()
+ ):
+ outputs["num_logits_to_keep"] = 1
+ if (
+ hasattr(self, "_supports_logits_to_keep")
+ and self._supports_logits_to_keep()
+ ):
+ outputs["logits_to_keep"] = 1
+ return outputs
+
+ return new_prepare_inputs_for_generation
+
+
+def snapkv_forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.LongTensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[
+ Tuple[torch.Tensor, torch.Tensor]
+ ] = None, # will become mandatory in v4.46
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ output_attentions = False
+
+ bsz, q_len, _ = hidden_states.size()
+
+ if "q_proj" in self.__dict__["_modules"]:
+ query_states = self.q_proj(hidden_states)
+ key_states = self.k_proj(hidden_states)
+ value_states = self.v_proj(hidden_states)
+ else:
+ qkv = self.qkv_proj(hidden_states)
+ query_pos = self.num_heads * self.head_dim
+ key_value_pos = query_pos // self.num_key_value_groups
+ query_states, key_states, value_states = torch.split(
+ qkv, [query_pos, key_value_pos, key_value_pos], -1
+ )
+
+ # [bsz, q_len, num_heads, head_dim]
+ query_states = query_states.view(
+ bsz, q_len, self.num_heads, self.head_dim
+ ).transpose(1, 2)
+ key_states = key_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+ value_states = value_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+
+ if position_embeddings is None:
+ cos, sin = self.rotary_emb(value_states, position_ids)
+ else:
+ cos, sin = position_embeddings
+
+ if query_states.shape[-1] != cos.shape[-1]: # glm-4 rope
+ query_states, key_states = apply_rotary_pos_emb_glm4(
+ query_states, key_states, cos, sin
+ )
+ else:
+ query_states, key_states = apply_rotary_pos_emb(
+ query_states, key_states, cos, sin
+ )
+
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
+ if past_key_value is not None:
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
+ cache_kwargs = {
+ "sin": sin,
+ "cos": cos,
+ "cache_position": cache_position,
+ "query_states": query_states,
+ "attention_mask": attention_mask,
+ "num_key_value_groups": self.num_key_value_groups,
+ }
+ key_states, value_states = past_key_value.update( # kvcompress
+ key_states,
+ value_states,
+ self.layer_idx,
+ cache_kwargs,
+ )
+
+ query_states = query_states.transpose(1, 2)
+ key_states = key_states.transpose(1, 2)
+ value_states = value_states.transpose(1, 2)
+
+ dropout_rate = self.attention_dropout if self.training else 0.0
+
+ # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+ # therefore the input hidden states gets silently casted in float32. Hence, we need
+ # cast them back in the correct dtype just to be sure everything works as expected.
+ # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+ # in fp32. (LlamaRMSNorm handles it correctly)
+
+ input_dtype = query_states.dtype
+ attn_output = _flash_attention_forward(
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ q_len,
+ position_ids=position_ids,
+ dropout=dropout_rate,
+ sliding_window=getattr(self, "sliding_window", None),
+ is_causal=self.is_causal,
+ )
+
+ attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
+ attn_output = self.o_proj(attn_output)
+
+ if not output_attentions:
+ attn_weights = None
+
+ return attn_output, attn_weights, past_key_value
+
+
+class BaseKVCache(Cache):
+ def __init__(self):
+ super().__init__()
+ # Used in `generate` to keep tally of how many tokens the cache has seen
+ self._seen_tokens = 0
+ self.key_cache: List[torch.Tensor] = []
+ self.value_cache: List[torch.Tensor] = []
+
+ self.kv_clusters = {}
+ self.kv_cluster_granularity = "layer"
+
+ self.temp_key_cache = []
+ self.temp_value_cache = []
+
+ def get_kv_cluster_class_config(self, layer_idx: int, head_idx: int = None):
+ raise NotImplementedError(
+ "Make sure to implement `get_kv_cluster_class_config` in a subclass."
+ )
+
+ def get_kv_cluster_class(self, layer_idx: int, head_idx=None):
+ cluster_name, cluster_class, cluster_config = self.get_kv_cluster_class_config(
+ layer_idx, head_idx
+ )
+ if cluster_name not in self.kv_clusters:
+ self.kv_clusters[cluster_name] = cluster_class(**cluster_config)
+ return self.kv_clusters[cluster_name]
+
+ def compresssed_kv(
+ self,
+ key_states,
+ query_states,
+ value_states,
+ attention_mask,
+ num_key_value_groups,
+ layer_idx: int,
+ ):
+ if self.kv_cluster_granularity == "layer":
+ kv_cluster = self.get_kv_cluster_class(layer_idx)
+
+ key_compress, value_compress = kv_cluster.update_kv(
+ key_states,
+ query_states,
+ value_states,
+ attention_mask,
+ num_key_value_groups,
+ )
+ self.key_cache.append(key_compress)
+ self.value_cache.append(value_compress)
+ else:
+ assert (
+ False
+ ), f"kv_cluster_granularity {self.kv_cluster_granularity} not supported"
+
+ def update(
+ self,
+ key_states,
+ value_states,
+ layer_idx,
+ cache_kwargs,
+ ):
+ # if prefill, then compress; if decode, then update
+ # [bsz, num_heads, q_len, head_dim]
+
+ update_global_past_kv = cache_kwargs.get("update_global_past_kv", True)
+ query_states = cache_kwargs["query_states"]
+ attention_mask = cache_kwargs["attention_mask"]
+ num_key_value_groups = cache_kwargs["num_key_value_groups"]
+
+ if key_states.size(1) != query_states.size(1): # GQA
+ key_states = repeat_kv(key_states, num_key_value_groups)
+ value_states = repeat_kv(value_states, num_key_value_groups)
+
+ if layer_idx == 0:
+ self._seen_tokens += key_states.shape[-2]
+
+ q_len = query_states.shape[-2]
+ initializing_kv_cluster = False
+ if (
+ len(self.key_cache) == layer_idx
+ ): # initialize kv_cluster, ie, the first query/context
+ initializing_kv_cluster = True
+ self.compresssed_kv(
+ key_states,
+ query_states,
+ value_states,
+ attention_mask,
+ num_key_value_groups,
+ layer_idx,
+ )
+ else: # the follow up queries/contexts
+ if update_global_past_kv:
+ self.key_cache[layer_idx] = torch.cat(
+ [self.key_cache[layer_idx], key_states], dim=-2
+ )
+ self.value_cache[layer_idx] = torch.cat(
+ [self.value_cache[layer_idx], value_states], dim=-2
+ )
+ else: # add KVs to temp_kv_cache
+ if len(self.temp_key_cache) == layer_idx:
+ self.temp_key_cache.append(key_states)
+ self.temp_value_cache.append(value_states)
+ else:
+ self.temp_key_cache[layer_idx] = torch.cat(
+ [self.temp_key_cache[layer_idx], key_states], dim=-2
+ )
+ self.temp_value_cache[layer_idx] = torch.cat(
+ [self.temp_value_cache[layer_idx], value_states], dim=-2
+ )
+
+ torch.cuda.empty_cache()
+ if not initializing_kv_cluster: # return the compressed KV cache
+ if self.temp_key_cache: # concat global past_kv and temp_kv_cache
+ key_states = torch.cat(
+ [self.key_cache[layer_idx], self.temp_key_cache[layer_idx]], dim=-2
+ )
+ value_states = torch.cat(
+ [self.value_cache[layer_idx], self.temp_value_cache[layer_idx]],
+ dim=-2,
+ )
+ else:
+ key_states = self.key_cache[layer_idx]
+ value_states = self.value_cache[layer_idx]
+ key_states = repeat_kv(key_states, query_states.size(1) // key_states.size(1))
+ value_states = repeat_kv(
+ value_states, query_states.size(1) // value_states.size(1)
+ )
+ return key_states, value_states
+
+ def get_seq_length(self, layer_idx=0):
+ if len(self.key_cache) <= layer_idx:
+ return 0
+ return self._seen_tokens
+
+ def to_legacy_cache(self):
+ legacy_cache = ()
+ for layer_idx in range(len(self.key_cache)):
+ legacy_cache += ((self.key_cache[layer_idx], self.value_cache[layer_idx]),)
+ return legacy_cache
+
+ @classmethod
+ def from_legacy_cache(cls, past_key_values):
+ cache = cls()
+ for layer_idx in range(len(past_key_values)):
+ key_states, value_states = past_key_values[layer_idx]
+ cache.update(key_states, value_states, layer_idx)
+ return cache
+
+ def clear_temp_kv_cache(self):
+ if self.temp_key_cache:
+ self._seen_tokens -= self.temp_key_cache[-1].shape[
+ -2
+ ] # seq_len of temp_kv_cache
+ self.temp_key_cache = []
+ self.temp_value_cache = []
+
+
+class SnapKVCache(BaseKVCache):
+ def __init__(self, config):
+ super().__init__()
+ self.window_size = config.attn_kwargs.get("window_size", 32)
+ self.max_capacity_prompt = config.attn_kwargs.get("max_capacity_prompt", 4096)
+ self.kernel_size = config.attn_kwargs.get("kernel_size", 5)
+ self.pooling = config.attn_kwargs.get("pooling", "avgpool")
+
+ def get_kv_cluster_class_config(self, layer_idx: int, head_idx: int = None):
+ cluster_config = {
+ "window_size": self.window_size,
+ "max_capacity_prompt": self.max_capacity_prompt,
+ "kernel_size": self.kernel_size,
+ "pooling": self.pooling,
+ }
+ cluster_name = ",".join(["snapv"] + [str(i) for i in cluster_config.values()])
+ return cluster_name, SnapKVCluster, cluster_config
+
+
+class PyramidKVCache(SnapKVCache):
+ def __init__(self, config):
+ super().__init__(config)
+ self.num_layers = config.num_layers
+
+ def get_kv_cluster_class_config(self, layer_idx: int, head_idx: int = None):
+ cluster_config = {
+ "num_hidden_layers": self.num_layers,
+ "window_size": self.window_size,
+ "max_capacity_prompt": self.max_capacity_prompt,
+ "kernel_size": self.kernel_size,
+ "pooling": self.pooling,
+ "layer_idx": layer_idx,
+ }
+ cluster_name = ",".join(
+ ["pyramidv"] + [str(i) for i in cluster_config.values()]
+ )
+ return cluster_name, PyramidKVCluster, cluster_config
+
+
+class StreamingLLMKVCache(SnapKVCache):
+ def __init__(self, config):
+ n_local = config.attn_kwargs.get("n_local", 3968)
+ n_init = config.attn_kwargs.get("n_init", 128)
+ config.attn_kwargs["window_size"] = n_local
+ config.attn_kwargs["max_capacity_prompt"] = n_local + n_init
+ super().__init__(config)
+
+ def get_kv_cluster_class_config(self, layer_idx: int, head_idx: int = None):
+ cluster_config = {
+ "window_size": self.window_size,
+ "max_capacity_prompt": self.max_capacity_prompt,
+ }
+ cluster_name = ",".join(
+ ["streamingllm"] + [str(i) for i in cluster_config.values()]
+ )
+ return cluster_name, StreamingLLMKVCluster, cluster_config
+
+
+class DynamicCacheWithRepeat(DynamicCache):
+ def __init__(self, config, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.temp_key_cache = []
+ self.temp_value_cache = []
+
+ def update(
+ self,
+ key_states: torch.Tensor,
+ value_states: torch.Tensor,
+ layer_idx: int,
+ cache_kwargs=None,
+ ):
+ update_global_past_kv = cache_kwargs.get("update_global_past_kv", True)
+ if layer_idx == 0:
+ self._seen_tokens += key_states.shape[-2]
+
+ if update_global_past_kv: # add KVs to global past_kv
+ assert len(self.temp_key_cache) == 0 and len(self.temp_value_cache) == 0, (
+ "when you updating global past_kv, make sure the temp_kv_cache is empty. "
+ "User past_key_values.clear_temp_kv_cache() to empty the temp_kv_cache"
+ )
+
+ # prefilling
+ if len(self.key_cache) == layer_idx:
+ self.key_cache.append(key_states)
+ self.value_cache.append(value_states)
+ else: # decoding
+ self.key_cache[layer_idx] = torch.cat(
+ [self.key_cache[layer_idx], key_states], dim=-2
+ )
+ self.value_cache[layer_idx] = torch.cat(
+ [self.value_cache[layer_idx], value_states], dim=-2
+ )
+ else: # add KVs to temp_kv_cache, this is used when you have a common context but different query, the KVs of the query will be added to temp_kv_cache, and will be cleaned in the next query
+ if len(self.temp_key_cache) == layer_idx:
+ self.temp_key_cache.append(key_states)
+ self.temp_value_cache.append(value_states)
+ else: # decoding
+ self.temp_key_cache[layer_idx] = torch.cat(
+ [self.temp_key_cache[layer_idx], key_states], dim=-2
+ )
+ self.temp_value_cache[layer_idx] = torch.cat(
+ [self.temp_value_cache[layer_idx], value_states], dim=-2
+ )
+
+ if self.temp_key_cache: # concat global past_kv and temp_kv_cache
+ key_states, value_states = torch.cat(
+ [self.key_cache[layer_idx], self.temp_key_cache[layer_idx]], dim=-2
+ ), torch.cat(
+ [self.value_cache[layer_idx], self.temp_value_cache[layer_idx]], dim=-2
+ )
+ else:
+ key_states, value_states = (
+ self.key_cache[layer_idx],
+ self.value_cache[layer_idx],
+ )
+
+ # repeat kv if needed
+ query_states = cache_kwargs.get("query_states", None)
+ if query_states is not None:
+ key_states = repeat_kv(
+ key_states, query_states.size(1) // key_states.size(1)
+ )
+ value_states = repeat_kv(
+ value_states, query_states.size(1) // value_states.size(1)
+ )
+ return key_states, value_states
+
+ def get_seq_length(self, layer_idx=0):
+ if len(self.key_cache) <= layer_idx:
+ return 0
+ return self._seen_tokens
+
+ def clear_temp_kv_cache(self):
+ if self.temp_key_cache:
+ self._seen_tokens -= self.temp_key_cache[-1].shape[
+ -2
+ ] # seq_len of temp_kv_cache
+ self.temp_key_cache = []
+ self.temp_value_cache = []
+
+
+method_to_cache_obj = {
+ "": DynamicCacheWithRepeat,
+ "dense": DynamicCacheWithRepeat,
+ "snapkv": SnapKVCache,
+ "pyramidkv": PyramidKVCache,
+ "streamingllm": StreamingLLMKVCache,
+ "quest": DynamicCacheWithRepeat,
+ "retr_attn": RetrAttnCache,
+ "kivi": KiviCache,
+ "leank": LeanKCache,
+}
diff --git a/minference/modules/leank.py b/minference/modules/leank.py
new file mode 100644
index 00000000..82ce1c31
--- /dev/null
+++ b/minference/modules/leank.py
@@ -0,0 +1,442 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from typing import List, Optional, Union
+
+import tilelang
+import tilelang.language as T
+import torch
+from transformers.cache_utils import DynamicCache
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.models.llama.modeling_llama import LlamaForCausalLM
+from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM
+from transformers.processing_utils import Unpack
+
+
+def reorder_linear_weights(
+ linear_module: torch.nn.Linear,
+ channel_mask: torch.Tensor,
+ repeat_num,
+ reorder_channel,
+ boundaries: List[int],
+):
+ assert reorder_channel in ["in", "out"]
+ channel_mask = torch.repeat_interleave(channel_mask, repeats=repeat_num).to(
+ linear_module.weight.device
+ )
+ reordered_weight = torch.zeros_like(linear_module.weight)
+ if linear_module.bias is not None:
+ reordered_bias = torch.zeros_like(linear_module.bias)
+
+ l, r = 0, 0
+ for boundary in boundaries:
+ mask = channel_mask == boundary
+ r += int(mask.sum())
+ if reorder_channel == "in":
+ reordered_weight[:, l:r] = linear_module.weight.data[:, mask.bool()]
+ else:
+ reordered_weight[l:r] = linear_module.weight.data[mask.bool()]
+ if linear_module.bias is not None:
+ reordered_bias[l:r] = linear_module.bias.data[mask.bool()]
+ l = r
+
+ linear_module.weight.data = reordered_weight
+ if linear_module.bias is not None:
+ linear_module.bias.data = reordered_bias
+ return linear_module
+
+
+def reorder_channel_mask(
+ channel_mask: torch.Tensor,
+ layer_full_attn_channels: torch.Tensor,
+ boundaries: List[int],
+):
+ new_rst = torch.zeros_like(layer_full_attn_channels)
+ l, r = 0, 0
+ for boundary in boundaries:
+ mask = channel_mask == boundary
+ r += int(mask.sum())
+ new_rst[l:r] = layer_full_attn_channels[mask]
+ l = r
+ return new_rst
+
+
+def mask_channels_key(key_states, mask, remained_count):
+ bs, nh, seq_len, dim = key_states.shape
+ return (
+ key_states.transpose(1, 2)
+ .reshape(bs, seq_len, dim * nh)[..., mask.flatten().bool()]
+ .reshape(bs, seq_len, nh, remained_count)
+ .transpose(1, 2)
+ )
+
+
+def mask_channels_query(query_states, mask, remained_count):
+ bs, nh, dim = query_states.shape
+ ng = mask.shape[0]
+ heads_per_group = nh // ng
+ return (
+ query_states.reshape(bs, ng, heads_per_group, -1)
+ .transpose(1, 2)
+ .reshape(bs, heads_per_group, -1)[:, :, mask.flatten().bool()]
+ .reshape(bs, heads_per_group, ng, remained_count)
+ .transpose(1, 2)
+ .reshape(bs, -1, remained_count)
+ )
+
+
+def get_round_seqlen_and_split_hueristic(seqlen):
+ supported_lengths = [4096, 8192] + [2**i + 8192 for i in range(13, 18)]
+ num_split_hueristic = [2, 2, 4, 4, 8, 8, 16]
+ last_length, last_n_split = supported_lengths[-1], num_split_hueristic[-1]
+ for l, ns in zip(supported_lengths[::-1], num_split_hueristic[::-1]):
+ if seqlen > l:
+ return last_length, last_n_split
+ last_length = l
+ last_n_split = ns
+ return last_length, last_n_split
+
+
+class LeanKCache(DynamicCache):
+ def __init__(self, config):
+ super().__init__()
+ self.sink_size = config.attn_kwargs.get("sink_size", 128)
+ self.recent_size = config.attn_kwargs.get("recent_size", 768)
+ self.full_size = self.sink_size + self.recent_size
+ self.accumu_size = config.attn_kwargs.get("accumu_size", 128)
+ # key caches
+ self.key_cache_mid = []
+ self.key_cache_full = []
+ # value caches
+ self.value_cache_full = []
+ self.value_cache_mid = []
+ self.mid_seq_len = -1
+ import gc
+
+ gc.collect() # fix tilelang kernel compile related issues
+
+ def update(
+ self,
+ key_states,
+ value_states,
+ layer_idx,
+ mask,
+ boundaries,
+ counts,
+ cache_kwargs,
+ ):
+ # Update the cache
+ if key_states is not None:
+ if len(self.key_cache_full) <= layer_idx:
+ # adding cache for the first time
+ self.key_cache_full.append(
+ torch.cat(
+ (
+ key_states[:, :, : self.sink_size],
+ key_states[:, :, -self.recent_size :],
+ ),
+ dim=-2,
+ )
+ )
+ self.value_cache_full.append(
+ torch.cat(
+ (
+ value_states[:, :, : self.sink_size],
+ value_states[:, :, -self.recent_size :],
+ ),
+ dim=-2,
+ )
+ )
+
+ self.key_cache_mid.append({})
+ self.value_cache_mid.append({})
+
+ l, r = boundaries[0], -1
+ for boundary, count in zip(boundaries[1:], counts[1:]):
+ r = boundary
+ # Tilelang 0.1.5 requires contiguous inputs
+ self.key_cache_mid[layer_idx][count] = mask_channels_key(
+ key_states[:, l:r, self.sink_size : -self.recent_size],
+ mask[l:r],
+ count,
+ ).contiguous()
+ self.value_cache_mid[layer_idx][count] = value_states[
+ :, l:r, self.sink_size : -self.recent_size
+ ].contiguous()
+ l = r
+
+ self.mid_seq_len = value_states.shape[-2] - self.full_size
+
+ else:
+ self.key_cache_full[layer_idx] = torch.cat(
+ [self.key_cache_full[layer_idx], key_states], dim=-2
+ )
+ self.value_cache_full[layer_idx] = torch.cat(
+ [self.value_cache_full[layer_idx], value_states], dim=-2
+ )
+
+ if (
+ self.key_cache_full[layer_idx].shape[-2]
+ >= self.full_size + self.accumu_size
+ ):
+ self.value_cache_full[layer_idx] = torch.cat(
+ (
+ self.value_cache_full[layer_idx][:, :, : self.sink_size],
+ self.value_cache_full[layer_idx][:, :, -self.recent_size :],
+ ),
+ dim=-2,
+ )
+
+ l, r = boundaries[0], -1
+ for boundary, count in zip(boundaries[1:], counts[1:]):
+ r = boundary
+ self.key_cache_mid[layer_idx][count] = torch.cat(
+ (
+ self.key_cache_mid[layer_idx][count],
+ mask_channels_key(
+ self.key_cache_full[layer_idx][
+ :, l:r, self.sink_size : -self.recent_size
+ ],
+ mask[l:r],
+ count,
+ ),
+ ),
+ dim=2,
+ ).contiguous()
+ self.value_cache_mid[layer_idx][count] = torch.cat(
+ (
+ self.value_cache_mid[layer_idx][count],
+ self.value_cache_full[layer_idx][
+ :, l:r, self.sink_size : -self.recent_size
+ ],
+ ),
+ dim=-2,
+ ).contiguous()
+ l = r
+
+ self.mid_seq_len += self.accumu_size
+
+ self.key_cache_full[layer_idx] = torch.cat(
+ (
+ self.key_cache_full[layer_idx][:, :, : self.sink_size],
+ self.key_cache_full[layer_idx][:, :, -self.recent_size :],
+ ),
+ dim=-2,
+ )
+
+ torch.cuda.empty_cache()
+ return (
+ self.key_cache_full[layer_idx],
+ self.key_cache_mid[layer_idx],
+ self.value_cache_mid[layer_idx],
+ self.value_cache_full[layer_idx],
+ )
+
+ def get_seq_length(self, layer_idx: Optional[int] = 0) -> int:
+ is_empty_layer = (
+ len(self.key_cache_full) == 0 # no cache in any layer
+ or len(self.key_cache_full) <= layer_idx # the layer has no cache
+ )
+ if is_empty_layer:
+ return 0
+ return self.value_cache_full[layer_idx].shape[-2] + self.mid_seq_len
+
+
+def leank_forward(
+ query_states: torch.Tensor,
+ key_states_full: torch.Tensor,
+ key_states_mid: list[torch.Tensor],
+ value_states_full: torch.Tensor,
+ value_states_mid: torch.Tensor,
+ **kwargs: Unpack[FlashAttentionKwargs],
+) -> torch.Tensor:
+ bsz, ng, full_kvlen, dim = key_states_full.shape
+ dtype = key_states_full.dtype
+ device = key_states_full.device
+
+ nh = query_states.shape[1]
+ heads_per_group = nh // ng
+
+ if len(key_states_mid) == 0:
+ kv_seqlen = 0
+ else:
+ kv_seqlen = next(iter(key_states_mid.items()))[1].shape[-2]
+ maskmid = torch.ones((bsz, kv_seqlen), dtype=torch.uint8, device=device)
+
+ need_compile = False
+ rouned_seq_len, num_split = get_round_seqlen_and_split_hueristic(kv_seqlen)
+ last_length = kwargs["last_length"]
+ if last_length != rouned_seq_len:
+ last_length = rouned_seq_len
+ if kwargs["layer_idx"] == 0:
+ print(f"--- Compiling LeanK decoding kernel, might take some time ---")
+ need_compile = True
+
+ if kwargs["attention_mask"] is not None:
+ mask = kwargs["attention_mask"].to(torch.uint8)
+ else:
+ mask = torch.ones((bsz, full_kvlen), dtype=torch.uint8, device=device)
+ glse = torch.full((bsz, nh, num_split + 1), -torch.inf, dtype=dtype, device=device)
+ Output_partial = torch.zeros(
+ bsz, nh, num_split + 1, dim, dtype=dtype, device=device
+ )
+
+ args = []
+ number_groups = []
+
+ args.append(query_states.squeeze(-2))
+ args.append(key_states_full)
+ args.append(value_states_full)
+
+ args.append(mask)
+ args.append(glse)
+ args.append(Output_partial)
+
+ boundaries = kwargs["boundaries"]
+ counts = kwargs["counts"]
+ l, r = boundaries[0], -1
+ for boundary, count in zip(boundaries[1:], counts[1:]):
+ r = boundary
+ args.append(
+ mask_channels_query(
+ query_states[:, l * heads_per_group : r * heads_per_group, 0],
+ kwargs["full_attn_channels"][l:r],
+ count,
+ )
+ )
+ args.append(key_states_mid[count])
+ args.append(value_states_mid[count])
+ args.append(glse[:, l * heads_per_group : r * heads_per_group].contiguous())
+ args.append(
+ Output_partial[:, l * heads_per_group : r * heads_per_group].contiguous()
+ )
+ number_groups.append(r - l)
+ l = r
+
+ n_groups = len(number_groups)
+ if n_groups > 0:
+ args.append(maskmid)
+ if n_groups < 4:
+ number_groups += [0] * (4 - n_groups)
+
+ kernel_kwargs = [T.symbolic("batch"), nh]
+ kernel_kwargs += [i * heads_per_group for i in number_groups]
+ kernel_kwargs += [ng]
+ kernel_kwargs += number_groups
+ kernel_kwargs += [
+ last_length,
+ T.symbolic("cur_len"),
+ kwargs["full_size"],
+ T.symbolic("cur_full_len"),
+ dim,
+ ]
+ kernel_kwargs += counts[1:]
+ if n_groups < 4:
+ kernel_kwargs += [0] * (4 - n_groups)
+ kernel_kwargs += [n_groups]
+ kernel_kwargs += ["float16" if dtype == torch.float16 else "bfloat16"]
+
+ if need_compile or kwargs["kernel"] is None:
+ try:
+ from minference.ops.leank_flash_decoding import leank_flashattn
+ except:
+ raise ImportError(
+ "LeanK Flash Decoding kernel is not available. Please install the required Tilelang package."
+ )
+ program = leank_flashattn(*kernel_kwargs)(
+ num_split=num_split, **kwargs["kernel_config"]
+ )
+ kernel = tilelang.compile(
+ program, out_idx=[5 * (n_groups + 1) + 1 + (n_groups > 0)]
+ )
+ else:
+ kernel = kwargs["kernel"]
+
+ attn_output = kernel(*args)
+ return attn_output.unsqueeze(2), last_length, kernel
+
+
+def patch_leank(
+ model: Union[LlamaForCausalLM, Qwen2ForCausalLM],
+ config,
+):
+ device = next(model.parameters()).device
+ dtype = next(model.parameters()).dtype
+ align = config.attn_kwargs.get("round_to", 32)
+ supported_dims = [
+ x
+ for x in range(
+ 0, model.config.hidden_size // model.config.num_attention_heads + 1, align
+ )
+ ]
+ leank_pattern = torch.load(config.attn_kwargs.get("leank_path"))
+
+ for idx, layer in enumerate(model.model.layers):
+ module = layer.self_attn
+ module.sink_size = config.attn_kwargs.get("sink_size", 128)
+ module.recent_size = config.attn_kwargs.get("recent_size", 768)
+ module.accumu_size = config.attn_kwargs.get("recent_size", 128)
+ layer_full_attn_channels = (
+ leank_pattern[idx].to(device).to(dtype).reshape(-1, module.head_dim)
+ )
+ channel_mask = layer_full_attn_channels.sum(dim=-1)
+ module.q_proj = reorder_linear_weights(
+ module.q_proj,
+ channel_mask,
+ module.num_key_value_groups * module.head_dim,
+ "out",
+ supported_dims,
+ )
+ module.k_proj = reorder_linear_weights(
+ module.k_proj,
+ channel_mask,
+ module.head_dim,
+ "out",
+ supported_dims,
+ )
+ module.v_proj = reorder_linear_weights(
+ module.v_proj,
+ channel_mask,
+ module.head_dim,
+ "out",
+ supported_dims,
+ )
+ module.o_proj = reorder_linear_weights(
+ module.o_proj,
+ channel_mask,
+ module.num_key_value_groups * module.head_dim,
+ "in",
+ supported_dims,
+ )
+ layer_full_attn_channels = reorder_channel_mask(
+ channel_mask,
+ layer_full_attn_channels,
+ supported_dims,
+ )
+ module.register_buffer(
+ "full_attn_channels",
+ layer_full_attn_channels,
+ )
+
+ dim_cnt = (
+ (layer_full_attn_channels.sum(dim=-1) == supported_dims[0]).sum().item()
+ )
+ boundaries = [dim_cnt]
+ counts = [supported_dims[0]]
+ for d in supported_dims[1:]:
+ nheads = (layer_full_attn_channels.sum(dim=-1) == d).sum().item()
+ if nheads > 0:
+ dim_cnt += nheads
+ boundaries.append(dim_cnt)
+ counts.append(d)
+ setattr(module, "boundaries", boundaries)
+ setattr(module, "counts", counts)
+
+ module.kernel_config = {
+ "block_N": 64,
+ "block_H": 64,
+ "num_stages": 2,
+ "threads": 128,
+ }
+ module.last_length = -1
+ module.kernel = None
diff --git a/minference/modules/minference_forward.py b/minference/modules/minference_forward.py
index d34bc844..d353be7d 100644
--- a/minference/modules/minference_forward.py
+++ b/minference/modules/minference_forward.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import inspect
@@ -7,15 +7,18 @@
import warnings
from importlib import import_module
-from transformers.models.llama.modeling_llama import *
+from transformers.models.llama.modeling_llama import rotate_half
from transformers.utils import is_flash_attn_2_available
from transformers.utils.import_utils import _is_package_available
if _is_package_available("vllm"):
try:
from vllm import _custom_ops as vllm_ops
+ from vllm.attention.backends.abstract import AttentionType
+ from vllm.attention.backends.utils import get_num_prefill_decode_query_kv_tokens
from vllm.attention.ops.paged_attn import PagedAttention
- from vllm_flash_attn import flash_attn_with_kvcache
+ from vllm.distributed import get_tensor_model_parallel_rank
+ from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
except:
import vllm
vllm_version = vllm.__version__
@@ -25,7 +28,10 @@
from ..ops.block_sparse_flash_attention import block_sparse_attention
from ..ops.pit_sparse_flash_attention_v2 import vertical_slash_sparse_attention
from ..ops.streaming_kernel import streaming_forward, streaming_forward2
-from .snap_kv import *
+from .flexprefill import flexprefill_forward
+from .kvcompression import *
+from .quest import quest_forward
+from .snapkv import *
try:
from flash_attn import flash_attn_func
@@ -53,7 +59,7 @@ def set_rope_type(self):
ROPE_TYPE = "position_ids"
def get_cos_sin(self, value_states, kv_seq_len, position_ids):
- if self.rotary_emb.inv_freq is not None and value_states.device != self.rotary_emb.inv_freq.device:
+ if "inv_freq" in self.rotary_emb.__dict__ is not None and value_states.device != self.rotary_emb.inv_freq.device:
value_states = value_states.to(self.rotary_emb.inv_freq.device)
position_ids = position_ids.to(self.rotary_emb.inv_freq.device)
if value_states.device != position_ids.device:
@@ -63,9 +69,12 @@ def get_cos_sin(self, value_states, kv_seq_len, position_ids):
elif ROPE_TYPE == "seq_len,position_ids":
cos, sin = self.rotary_emb(value_states, position_ids=position_ids, seq_len=kv_seq_len)
elif ROPE_TYPE == "max_seq_len":
+ if position_ids is not None and position_ids[0][0] < 0:
+ kv_seq_len -= position_ids[0][0].item()
+ position_ids = position_ids - position_ids[0][0]
cos = self.rotary_emb(kv_seq_len)
if position_ids is not None:
- cos = cos[position_ids]
+ cos = cos[position_ids.to(cos.device)]
else:
cos = cos[None, :kv_seq_len]
sin = None
@@ -285,7 +294,6 @@ def repeat(self, q, k, v, attention_mask):
return attn_output
def gather_last_q_vertical_slash_topk_v4(self, q, k, v, head_id):
- kv_seq_len = k.size(2)
def vertical_and_slash(attn_weights, vertical_size, slash_size):
last_q = 64
@@ -335,10 +343,10 @@ def stream_llm(attn_weights, vertical_size, slash_size):
def block_sparse(attn_weights, topk_ratio, slash_size=None, block_size=8):
block_num = (q_len -1) // block_size + 1
- block_q = torch.zeros(1,1,block_num * block_size,head_dim).to(q)
+ block_q = torch.zeros(1,1,block_num * block_size, head_dim).to(q)
block_q[:,:,:q_len] = q
block_q = block_q.reshape(1,1,block_num,block_size,-1).mean(-2)
- block_k = torch.zeros(1,1,block_num * block_size,head_dim).to(k)
+ block_k = torch.zeros(1,1,block_num * block_size, head_dim).to(k)
block_k[:,:,:q_len] = k
block_k = block_k.reshape(1,1,block_num,block_size,-1).mean(-2)
@@ -433,6 +441,19 @@ def block_sparse_kernel(q, k, v, vertical_size=None, slash_size=None):
topk = 100
return block_sparse_attention(q, k, v, topk)
+ def tri_shape_kernel(q, k, v, n_init, n_local, n_last=100):
+ n_last = min(n_last, q.size(2) - 1)
+ q1, q2 = q[:,:,:-n_last], q[:,:,-n_last:]
+ y1 = streaming_forward(q1, k[:,:,:-n_last], v[:,:,:-n_last], n_init, n_local)
+
+ qk = torch.einsum(f'bhmk, bhnk -> bhmn', q2, k) / math.sqrt(self.head_dim)
+ arange = torch.arange(n_last, device="cuda")
+ mask = arange[None, None, :, None] >= arange[None, None, None, :]
+ qk[:, :, :, -n_last:] = torch.where(mask, qk[:, :, :, -n_last:], -torch.inf)
+ qk = torch.nn.functional.softmax(qk, dim=-1, dtype=torch.float32).to(q.dtype)
+ y2 = torch.einsum(f'bhmn, bhnk -> bhmk', qk, v)
+ return torch.cat([y1, y2], dim=2)
+
q_len = q.shape[2]
bsz = q.shape[0]
@@ -445,8 +466,10 @@ def block_sparse_kernel(q, k, v, vertical_size=None, slash_size=None):
return dialted(q, k, v, 'dilated2')
if self.config.to_dict().get("dense", False):
return dense(q, k, v)
- if self.config.to_dict().get("streaming", False):
+ if self.config.to_dict().get("a_shape", False):
return streaming_forward(q, k, v, self.config.streaming_kwargs["n_init"], self.config.streaming_kwargs["n_local"])
+ if self.config.to_dict().get("tri_shape", False):
+ return tri_shape_kernel(q, k, v, self.config.streaming_kwargs["n_init"], self.config.streaming_kwargs["n_local"])
ty, vertical_size, slash_size, _ = self.best_pattern.get(head_id, ("vertical_and_slash", 1000, 6096, 1))
@@ -567,6 +590,89 @@ def forward(
return forward
+
+def minference_prefill_kernel(
+ q, k, v, head_id, layer_idx,
+ config,
+):
+ head_dim = q.size(-1)
+ def vertical_and_slash_kernel(q, k, v, vertical_size, slash_size):
+ vertical_size, slash_size = min(q_len, max(vertical_size, 30)), min(q_len, max(slash_size, 50))
+ last_q = min(64, q_len)
+ qk = torch.einsum(f'bhmk, bhnk -> bhmn', q[:,:,-last_q:,:], k) / math.sqrt(head_dim)
+ qk[:, :, :, -last_q:] = torch.where(LAST_Q_MASK[...,-last_q:,-last_q:].to(q.device), qk[:, :, :, -last_q:], -torch.inf)
+ qk = torch.nn.functional.softmax(qk, dim=-1, dtype=torch.float32)
+ vertical = qk.sum(-2, keepdim=True)
+ vertical[...,:30] = torch.inf
+ vertical_topk = torch.topk(vertical, vertical_size, -1).indices
+
+ slash = sum_all_diagonal_matrix(qk)[...,:-last_q + 1]
+ slash[...,-100:] = torch.inf
+ slash_topk = slash
+ slash = (q_len - 1) - torch.topk(slash, slash_size, -1).indices
+
+ return vertical_slash_sparse_attention(q, k, v, vertical_topk, slash)
+
+ def block_sparse_kernel(q, k, v, vertical_size=None, slash_size=None):
+ topk = 100
+ return block_sparse_attention(q, k, v, topk)
+
+ q_len = q.shape[2]
+ ty, vertical_size, slash_size, _ = config["best_pattern"][layer_idx].get(str(head_id), ("vertical_and_slash", 1000, 6096, 1))
+
+ if "minference_ratio" in config:
+ vertical_size = int(vertical_size * config.get("minference_ratio", 1))
+ slash_size = int(slash_size * config.get("minference_ratio", 1))
+ fc = {
+ "stream_llm": streaming_forward,
+ "vertical_and_slash": vertical_and_slash_kernel,
+ "block_sparse": block_sparse_kernel,
+ }[ty]
+ return fc(q, k, v, vertical_size, slash_size)
+
+def minference_prefill_forward(
+ query_states, key_states, value_states,
+ prefill_kwargs,
+):
+ starting_layer = prefill_kwargs["attn_forward_config"].get("starting_layer", 0)
+ is_search = prefill_kwargs["attn_forward_config"].get("is_search", False)
+ config_path = prefill_kwargs["attn_forward_config"].get("config_path", None)
+ layer_idx = prefill_kwargs["layer_idx"]
+ num_hidden_layers = prefill_kwargs["num_hidden_layers"]
+
+ # Load or initialize search configs
+ config_list = []
+ if is_search:
+ if config_path and os.path.isfile(config_path):
+ with open(config_path, "r") as f:
+ config_list = json.load(f)
+ if len(config_list) == num_hidden_layers:
+ raise RuntimeError(f"Search already completed; see {config_path}.")
+ print(f"--- Searching layer {layer_idx} ---")
+
+ bsz, _, q_len, head_dim = query_states.shape
+ output = torch.empty_like(query_states)
+ new_config: dict[int, dict] = {}
+
+ for head in range(query_states.size(1)):
+ q = query_states[:, head, :, :].unsqueeze(1)
+ k = key_states[:, head, :, :].unsqueeze(1)
+ v = value_states[:, head, :, :].unsqueeze(1)
+ if is_search and layer_idx >= len(config_list):
+ new_config[head] = search_pattern(q, k, head)
+ if layer_idx >= starting_layer and not is_search:
+ attn_output = minference_prefill_kernel(q, k, v, head, layer_idx, prefill_kwargs["attn_forward_config"])
+ else:
+ attn_output = flash_attn_func(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1,2), 0.0, softmax_scale=None, causal=q_len != 1).view(bsz, 1, q_len, head_dim)
+ output[:, head:head + 1] = attn_output
+
+ # Save updated search configs
+ if is_search and new_config and config_path:
+ config_list.append(new_config)
+ with open(config_path, "w") as f:
+ json.dump(config_list, f, indent=2)
+ return output
+
def minference_kv_cache_cpu_forward():
def forward(
self,
@@ -666,104 +772,23 @@ def forward(
return forward
-def minference_with_snapkv_forward():
- def forward(
- self,
- hidden_states,
- attention_mask,
- position_ids,
- past_key_value,
- output_attentions,
- use_cache,
- **kwargs,
- ):
- self.init_minference_parameters()
- self.ne_inf = torch.finfo(hidden_states.dtype).min
-
- init_snapkv(self)
-
- bsz, q_len, _ = hidden_states.size()
-
- query_states = self.q_proj(hidden_states)
- key_states = self.k_proj(hidden_states)
- value_states = self.v_proj(hidden_states)
-
- query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
- key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
- value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
-
- kv_seq_len = key_states.shape[-2]
- if past_key_value is not None:
- if self.layer_idx is None:
- raise ValueError(
- f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
- "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
- "with a layer index."
- )
-
- if hasattr(self, "kv_seq_len"): #[SnapKV] add kv_seq_len
- if self.kv_seq_len != 0:
- kv_seq_len += self.kv_seq_len
- else:
- kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
- else:
- kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
- set_rope_type(self)
- cos, sin = get_cos_sin(self, value_states, kv_seq_len, position_ids)
- if ROPE_TYPE == "max_seq_len":
- if cos.device != query_states.device:
- cos = cos.to(query_states.device)
- query_states = apply_rotary_pos_emb(query_states, cos)
- key_states = apply_rotary_pos_emb(key_states, cos)
- else:
- if position_ids is not None and position_ids.device != cos.device:
- position_ids = position_ids.to(cos.device)
- query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
- key_states = repeat_kv(key_states, self.num_key_value_groups)
- value_states = repeat_kv(value_states, self.num_key_value_groups)
-
- if past_key_value is not None:
- cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
- if key_states.shape[-2] == kv_seq_len: # [SnapKV] add kv_cluster
- self.kv_seq_len = kv_seq_len # [SnapKV] register kv_seq_len
- key_states_compress, value_states_compress = self.kv_cluster.update_kv(key_states, query_states, value_states, attention_mask, self.num_key_value_groups)
- past_key_value.update(key_states_compress, value_states_compress, self.layer_idx, cache_kwargs)
- else:
- self.kv_seq_len += q_len
- key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
-
- if self.layer_idx >= self.starting_layer:
- assert query_states.size(1) == key_states.size(1) == value_states.size(1)
- output = torch.empty_like(query_states)
- for head in range(query_states.size(1)):
- q = query_states[:, head, :, :].unsqueeze(1)
- k = key_states[:, head, :, :].unsqueeze(1)
- v = value_states[:, head, :, :].unsqueeze(1)
- output[:, head:head + 1] = self.gather_last_q_vertical_slash_topk_v4(q, k, v, head)
-
- attn_output = output.transpose(1, 2).contiguous()
- attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
- attn_output = self.o_proj(attn_output)
- return attn_output, None, past_key_value
-
- else:
- output = torch.empty_like(query_states)
- for head in range(query_states.size(1)):
- q = query_states[:, head, :, :].unsqueeze(1)
- k = key_states[:, head, :, :].unsqueeze(1)
- v = value_states[:, head, :, :].unsqueeze(1)
- if is_flash_attn_2_available():
- attn_output = flash_attn_func(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1,2), 0.0, softmax_scale=None, causal=q_len != 1).view(bsz, 1, q.shape[2], self.head_dim)
- else:
- attn_output = gather_qkv(q, k, v, attention_mask)
- output[:, head:head + 1] = attn_output
- attn_output = output.transpose(1, 2).contiguous()
- attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
- attn_output = self.o_proj(attn_output)
+def kvcompress_forward(
+ original_forward,
+ method: str = "snapkv",
+ config: dict = {},
+):
+ if config.attn_type in ["minference"]:
+ return minference_forward()
- return attn_output, None, past_key_value
+ forward_map = {
+ "snapkv": snapkv_forward,
+ "pyramidkv": snapkv_forward,
+ "streaming": snapkv_forward,
+ "quest": quest_forward,
+ "dense": snapkv_forward,
+ }
- return forward
+ return forward_map[method]
def gather_last_q_vertical_slash_topk_vllm(self, q, k, v, head_id):
kv_seq_len = k.size(2)
@@ -772,9 +797,9 @@ def gather_last_q_vertical_slash_topk_vllm(self, q, k, v, head_id):
def vertical_and_slash_kernel(q, k, v, vertical_size, slash_size):
vertical_size, slash_size = min(q_len, max(vertical_size, 30)), min(q_len, max(slash_size, 50))
last_q = min(64, q_len)
- qk = torch.einsum(f'bhmk, bhnk -> bhmn', q[:,:,-last_q:,:], k)
+ qk = torch.einsum(f'bhmk, bhnk -> bhmn', q[:,:,-last_q:,:], k) / math.sqrt(q.shape[-1])
- qk[:, :, :, -last_q:] = torch.where(LAST_Q_MASK[...,-last_q:,-last_q:], qk[:, :, :, -last_q:], -torch.inf)
+ qk[:, :, :, -last_q:] = torch.where(LAST_Q_MASK[...,-last_q:,-last_q:].to(q.device), qk[:, :, :, -last_q:], -torch.inf)
qk = torch.nn.functional.softmax(qk, dim=-1, dtype=torch.float32)
vertical = qk.sum(-2, keepdim=True)
vertical[...,:30] = torch.inf
@@ -794,14 +819,35 @@ def block_sparse_kernel(q, k, v, vertical_size=None, slash_size=None):
def dense(q, k, v, vertical_size=None, slash_size=None):
return flash_attn_func(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1,2), 0.0, softmax_scale=None, causal=q_len != 1).view(bsz, 1, q_len, head_dim)
+ def tri_shape_kernel(q, k, v, n_init, n_local, n_last=100):
+ q1, q2 = q[:,:,:-n_last], q[:,:,-n_last:]
+ y1 = streaming_forward(q1, k[:,:,:-n_last], v[:,:,:-n_last], n_init, n_local)
+ qk = torch.einsum(f'bhmk, bhnk -> bhmn', q2, k) / math.sqrt(q2.shape[-1])
+ arange = torch.arange(n_last, device="cuda")
+ mask = arange[None, None, :, None] >= arange[None, None, None, :]
+ qk[:, :, :, -n_last:] = torch.where(mask, qk[:, :, :, -n_last:], -torch.inf)
+ qk = torch.nn.functional.softmax(qk, dim=-1, dtype=torch.float32).to(q.dtype)
+ y2 = torch.einsum(f'bhmn, bhnk -> bhmk', qk, v)
+ return torch.cat([y1, y2], dim=2)
+
q_len = q.shape[2]
bsz = q.shape[0]
- ty, vertical_size, slash_size, _ = self.best_pattern[head_id]
+ ty, vertical_size, slash_size, _ = self.best_pattern.get(head_id, ("vertical_and_slash", 1000, 6096, 1))
+ if "minference_ratio" in self.patch_config:
+ vertical_size = int(vertical_size * self.patch_config.get("minference_ratio", 1))
+ slash_size = int(slash_size * self.patch_config.get("minference_ratio", 1))
if q_len == 1:
return dense(q, k, v)
+ if self.patch_config.get("flexprefill", False):
+ return flexprefill_forward(q, k, v, {"attn_forward_config": self.patch_config["flexprefill_kwargs"]})
+ if self.patch_config.get("a_shape", False):
+ return streaming_forward(q, k, v, self.patch_config["streaming_kwargs"]["n_init"], self.patch_config["streaming_kwargs"]["n_local"])
+ if self.patch_config.get("tri_shape", False):
+ return tri_shape_kernel(q, k, v, self.patch_config["streaming_kwargs"]["n_init"], self.patch_config["streaming_kwargs"]["n_local"])
+
fc = {
"stream_llm": streaming_forward,
"vertical_and_slash": vertical_and_slash_kernel,
@@ -811,7 +857,8 @@ def dense(q, k, v, vertical_size=None, slash_size=None):
def minference_vllm_forward(
pattern_config,
- vllm_version = "0.4.1"
+ vllm_version = "0.4.1",
+ patch_config = {},
):
def forward(
self,
@@ -834,6 +881,7 @@ def forward(
Returns:
shape = [num_tokens, num_heads * head_size]
"""
+ self.patch_config = patch_config
self.best_pattern = {int(ii): jj for ii, jj in pattern_config[layer_idx].items()}
def repeat_kv(hidden_states, n_rep):
sqlen, num_head, head_dim = hidden_states.shape
@@ -851,6 +899,7 @@ def minference_prefill_func(
v = repeat_kv(v, q.size(-2) // v.size(-2))
output = torch.empty_like(q)
+ head_idx_st = get_tensor_model_parallel_rank() * q.size(-2)
for head in range(q.size(-2)):
q_head = q[:, head, :].unsqueeze(1)
k_head = k[:, head, :].unsqueeze(1)
@@ -865,7 +914,7 @@ def minference_prefill_func(
k_head = k_head.transpose(1, 2)
v_head = v_head.transpose(1, 2)
- out = self.gather_last_q_vertical_slash_topk_vllm(q_head, k_head, v_head, head)
+ out = self.gather_last_q_vertical_slash_topk_vllm(q_head, k_head, v_head, head + head_idx_st)
out = out.transpose(1, 2).squeeze(0).contiguous()
output[:, head:head+1, :] = out
@@ -987,6 +1036,7 @@ def forward_vllm_042(
Returns:
shape = [num_tokens, num_heads * head_size]
"""
+ self.patch_config = patch_config
self.best_pattern = {int(ii): jj for ii, jj in pattern_config[layer_idx].items()}
def repeat_kv(hidden_states, n_rep):
sqlen, num_head, head_dim = hidden_states.shape
@@ -1004,6 +1054,7 @@ def minference_prefill_func(
v = repeat_kv(v, q.size(-2) // v.size(-2))
output = torch.empty_like(q)
+ head_idx_st = get_tensor_model_parallel_rank() * q.size(-2)
for head in range(q.size(-2)):
q_head = q[:, head, :].unsqueeze(1)
k_head = k[:, head, :].unsqueeze(1)
@@ -1018,7 +1069,7 @@ def minference_prefill_func(
k_head = k_head.transpose(1, 2)
v_head = v_head.transpose(1, 2)
- out = self.gather_last_q_vertical_slash_topk_vllm(q_head, k_head, v_head, head)
+ out = self.gather_last_q_vertical_slash_topk_vllm(q_head, k_head, v_head, head + head_idx_st)
out = out.transpose(1, 2).squeeze(0).contiguous()
output[:, head:head+1, :] = out
@@ -1119,15 +1170,17 @@ def minference_prefill_func(
# Reshape the output tensor.
return output.view(num_tokens, hidden_size)
- def forward_vllm_043(
+ def forward_vllm_080(
self,
+ layer,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata,
- kv_scale: float,
- layer_idx: int,
+ output: Optional[torch.Tensor] = None,
+ output_scale: Optional[torch.Tensor] = None,
+ layer_idx: int = 0,
) -> torch.Tensor:
"""Forward pass with FlashAttention.
@@ -1141,8 +1194,8 @@ def forward_vllm_043(
shape = [num_tokens, num_heads * head_size]
"""
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
+ self.patch_config = patch_config
self.best_pattern = {int(ii): jj for ii, jj in pattern_config[layer_idx].items()}
- assert kv_scale == 1.0, "kv_scale is not supported in FlashAttention."
def repeat_kv(hidden_states, n_rep):
sqlen, num_head, head_dim = hidden_states.shape
@@ -1160,6 +1213,7 @@ def minference_prefill_func(
v = repeat_kv(v, q.size(-2) // v.size(-2))
output = torch.empty_like(q)
+ head_idx_st = get_tensor_model_parallel_rank() * q.size(-2)
for head in range(q.size(-2)):
q_head = q[:, head, :].unsqueeze(1)
k_head = k[:, head, :].unsqueeze(1)
@@ -1174,7 +1228,7 @@ def minference_prefill_func(
k_head = k_head.transpose(1, 2)
v_head = v_head.transpose(1, 2)
- out = self.gather_last_q_vertical_slash_topk_vllm(q_head, k_head, v_head, head)
+ out = self.gather_last_q_vertical_slash_topk_vllm(q_head, k_head, v_head, head + head_idx_st)
out = out.transpose(1, 2).squeeze(0).contiguous()
output[:, head:head+1, :] = out
@@ -1186,41 +1240,64 @@ def minference_prefill_func(
key = key.view(-1, self.num_kv_heads, self.head_size)
value = value.view(-1, self.num_kv_heads, self.head_size)
- if kv_cache is not None:
+ attn_type = self.attn_type
+ kv_cache_dtype: str = self.kv_cache_dtype
+ softmax_scale: float = self.scale
+ window_size = self.sliding_window
+ alibi_slopes: Optional[torch.Tensor] = self.alibi_slopes
+ logits_soft_cap: Optional[float] = self.logits_soft_cap
+ fp8_attention = kv_cache_dtype.startswith("fp8")
+
+ if kv_cache.numel() > 0:
key_cache = kv_cache[0]
value_cache = kv_cache[1]
+ # We skip updating the KV cache under two conditions:
+ # a. When the Attention Type is ENCODER. In this phase, we compute
+ # only the encoder attention without updating the cache.
+ # b. When both Key and Value are None. This occurs during
+ # cross-attention computation in the decoding phase, where the
+ # KV cache is already populated with the cross-attention
+ # tensor. Thus, we skip cache updates during this time.
+ if (attn_type != AttentionType.ENCODER) and (key is not None) and (
+ value is not None):
+ if attn_type == AttentionType.ENCODER_DECODER:
+ # Update cross-attention KV cache (prefill-only)
+ updated_slot_mapping = attn_metadata.cross_slot_mapping
+ else:
+ # Update self-attention KV cache (prefill/decode)
+ updated_slot_mapping = attn_metadata.slot_mapping
+
+ # Reshape the input keys and values and store them in the cache.
+ # If kv_cache is not provided, the new key and value tensors are
+ # not cached. This happens during the initial memory
+ # profiling run.
+ torch.ops._C_cache_ops.reshape_and_cache_flash(
+ key,
+ value,
+ kv_cache[0],
+ kv_cache[1],
+ updated_slot_mapping.flatten(), # type: ignore[union-attr]
+ kv_cache_dtype,
+ layer._k_scale,
+ layer._v_scale,
+ )
- # Reshape the input keys and values and store them in the cache.
- # If kv_cache is not provided, the new key and value tensors are
- # not cached. This happens during the initial memory profiling run.
- vllm_ops.reshape_and_cache_flash(
- key,
- value,
- key_cache,
- value_cache,
- attn_metadata.slot_mapping.flatten(),
- self.kv_cache_dtype,
- )
-
- num_prefill_tokens = attn_metadata.num_prefill_tokens
- num_decode_tokens = attn_metadata.num_decode_tokens
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens
-
- output = torch.empty_like(query)
- # Query for decode. KV is not needed because it is already cached.
- decode_query = query[num_prefill_tokens:]
+ (num_prefill_query_tokens, num_prefill_kv_tokens,
+ num_decode_query_tokens) = \
+ get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
+ decode_query = query[num_prefill_query_tokens:]
+ decode_output = output[num_prefill_query_tokens:]
# QKV for prefill.
- query = query[:num_prefill_tokens]
- key = key[:num_prefill_tokens]
- value = value[:num_prefill_tokens]
+ query = query[:num_prefill_query_tokens]
+ prefill_output = output[:num_prefill_query_tokens]
+ assert query.shape[0] == num_prefill_query_tokens
+ assert decode_query.shape[0] == num_decode_query_tokens
- assert query.shape[0] == num_prefill_tokens
- assert decode_query.shape[0] == num_decode_tokens
+ output = torch.empty_like(query)
if prefill_meta := attn_metadata.prefill_metadata:
# Prompt run.
- if (kv_cache is None or prefill_meta.block_tables is None
+ if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
or prefill_meta.block_tables.numel() == 0):
# normal attention
# When block_tables are not filled, it means q and k are the
@@ -1239,13 +1316,13 @@ def minference_prefill_func(
# alibi_slopes=self.alibi_slopes,
# )
out = minference_prefill_func(query, key, value)
- assert output[:num_prefill_tokens].shape == out.shape
- output[:num_prefill_tokens] = out
+ assert output[:num_prefill_query_tokens].shape == out.shape
+ output[:num_prefill_query_tokens] = out
else:
# prefix-enabled attention
assert prefill_meta.seq_lens is not None
max_seq_len = max(prefill_meta.seq_lens)
- output[:num_prefill_tokens] = flash_attn_varlen_func(
+ output[:num_prefill_query_tokens] = flash_attn_varlen_func(
q=query,
k=key_cache,
v=value_cache,
@@ -1261,7 +1338,7 @@ def minference_prefill_func(
if decode_meta := attn_metadata.decode_metadata:
# Decoding run.
- output[num_prefill_tokens:] = flash_attn_with_kvcache(
+ output[num_prefill_query_tokens:] = flash_attn_with_kvcache(
decode_query.unsqueeze(1),
key_cache,
value_cache,
@@ -1280,5 +1357,5 @@ def minference_prefill_func(
elif vllm_version == "0.4.2":
return forward_vllm_042
elif vllm_version >= "0.4.3":
- return forward_vllm_043
+ return forward_vllm_080
assert False, "Only support 'vllm>=0.4.1'. Please update your vllm version."
diff --git a/minference/modules/pyramidkv.py b/minference/modules/pyramidkv.py
new file mode 100644
index 00000000..7eceaa95
--- /dev/null
+++ b/minference/modules/pyramidkv.py
@@ -0,0 +1,183 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://github.com/Zefan-Cai/PyramidKV/blob/main/pyramidkv/pyramidkv_utils.py
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class PyramidKVCluster:
+ def __init__(
+ self,
+ num_hidden_layers=32,
+ window_size=64,
+ max_capacity_prompt=256 + 64,
+ kernel_size=5,
+ pooling="avgpool",
+ beta=20,
+ num_layers=80,
+ layer_idx=None,
+ ):
+ self.layer_idx = layer_idx
+ self.num_hidden_layers = num_hidden_layers
+
+ self.steps = -1
+ self.beta = beta
+
+ self.window_size = window_size
+ self.max_capacity_prompt = max_capacity_prompt
+ assert self.max_capacity_prompt - self.window_size > 0
+ self.kernel_size = kernel_size
+ self.pooling = pooling
+
+ def reset(
+ self,
+ window_size=64,
+ max_capacity_prompt=256 + 64,
+ kernel_size=5,
+ pooling="avgpool",
+ ):
+ self.window_size = window_size
+ self.max_capacity_prompt = max_capacity_prompt
+ assert self.max_capacity_prompt - self.window_size > 0
+ self.kernel_size = kernel_size
+ self.pooling = pooling
+
+ def update_kv(
+ self,
+ key_states,
+ query_states,
+ value_states,
+ attention_mask,
+ num_key_value_groups,
+ ):
+ # check if prefix phase
+ assert key_states.shape[-2] == query_states.shape[-2]
+ bsz, num_heads, q_len, head_dim = query_states.shape
+
+ # TODO
+ # window_sizes = 32
+ min_num = (self.max_capacity_prompt - self.window_size) // self.beta
+ max_num = (self.max_capacity_prompt - self.window_size) * 2 - min_num
+
+ if max_num >= q_len - self.window_size:
+ max_num = q_len - self.window_size
+ min_num = (self.max_capacity_prompt - self.window_size) * 2 - max_num
+
+ steps = (max_num - min_num) // (self.num_hidden_layers - 1)
+ max_capacity_prompt = max_num - self.layer_idx * steps
+
+ # print(f"PyramidKV max_capacity_prompt {max_capacity_prompt}")
+ if q_len < self.max_capacity_prompt:
+ return key_states, value_states
+ elif q_len < (self.max_capacity_prompt - self.window_size) * 2:
+ attn_weights = torch.matmul(
+ query_states[..., -self.window_size :, :], key_states.transpose(2, 3)
+ ) / math.sqrt(head_dim)
+ mask = torch.full(
+ (self.window_size, self.window_size),
+ torch.finfo(attn_weights.dtype).min,
+ device=attn_weights.device,
+ )
+ mask_cond = torch.arange(mask.size(-1), device=attn_weights.device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(attn_weights.device)
+ attention_mask = mask[None, None, :, :]
+
+ attn_weights[
+ :, :, -self.window_size :, -self.window_size :
+ ] += attention_mask
+
+ attn_weights = nn.functional.softmax(
+ attn_weights, dim=-1, dtype=torch.float32
+ ).to(query_states.dtype)
+ attn_weights_sum = attn_weights[
+ :, :, -self.window_size :, : -self.window_size
+ ].sum(dim=-2)
+ if self.pooling == "avgpool":
+ attn_cache = F.avg_pool1d(
+ attn_weights_sum,
+ kernel_size=self.kernel_size,
+ padding=self.kernel_size // 2,
+ stride=1,
+ )
+ elif self.pooling == "maxpool":
+ attn_cache = F.max_pool1d(
+ attn_weights_sum,
+ kernel_size=self.kernel_size,
+ padding=self.kernel_size // 2,
+ stride=1,
+ )
+ else:
+ raise ValueError("Pooling method not supported")
+ indices = attn_cache.topk(
+ self.max_capacity_prompt - self.window_size, dim=-1
+ ).indices
+ indices = indices.unsqueeze(-1).expand(-1, -1, -1, head_dim)
+ k_past_compress = key_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ v_past_compress = value_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ k_cur = key_states[:, :, -self.window_size :, :]
+ v_cur = value_states[:, :, -self.window_size :, :]
+ key_states = torch.cat([k_past_compress, k_cur], dim=2)
+ value_states = torch.cat([v_past_compress, v_cur], dim=2)
+ return key_states, value_states
+ else:
+ attn_weights = torch.matmul(
+ query_states[..., -self.window_size :, :], key_states.transpose(2, 3)
+ ) / math.sqrt(head_dim)
+ mask = torch.full(
+ (self.window_size, self.window_size),
+ torch.finfo(attn_weights.dtype).min,
+ device=attn_weights.device,
+ )
+ mask_cond = torch.arange(mask.size(-1), device=attn_weights.device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(attn_weights.device)
+ attention_mask = mask[None, None, :, :]
+
+ attn_weights[
+ :, :, -self.window_size :, -self.window_size :
+ ] += attention_mask
+
+ attn_weights = nn.functional.softmax(
+ attn_weights, dim=-1, dtype=torch.float32
+ ).to(query_states.dtype)
+ attn_weights_sum = attn_weights[
+ :, :, -self.window_size :, : -self.window_size
+ ].sum(dim=-2)
+ if self.pooling == "avgpool":
+ attn_cache = F.avg_pool1d(
+ attn_weights_sum,
+ kernel_size=self.kernel_size,
+ padding=self.kernel_size // 2,
+ stride=1,
+ )
+ elif self.pooling == "maxpool":
+ attn_cache = F.max_pool1d(
+ attn_weights_sum,
+ kernel_size=self.kernel_size,
+ padding=self.kernel_size // 2,
+ stride=1,
+ )
+ else:
+ raise ValueError("Pooling method not supported")
+ indices = attn_cache.topk(max_capacity_prompt, dim=-1).indices
+ indices = indices.unsqueeze(-1).expand(-1, -1, -1, head_dim)
+ k_past_compress = key_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ v_past_compress = value_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ k_cur = key_states[:, :, -self.window_size :, :]
+ v_cur = value_states[:, :, -self.window_size :, :]
+ key_states = torch.cat([k_past_compress, k_cur], dim=2)
+ value_states = torch.cat([v_past_compress, v_cur], dim=2)
+ return key_states, value_states
diff --git a/minference/modules/quest.py b/minference/modules/quest.py
new file mode 100644
index 00000000..fe23544a
--- /dev/null
+++ b/minference/modules/quest.py
@@ -0,0 +1,589 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://github.com/mit-han-lab/Quest/blob/main/evaluation/quest_attention.py
+
+import math
+import types
+from typing import Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from transformers import DynamicCache
+from transformers.models.llama.modeling_llama import (
+ LlamaAttention,
+ apply_rotary_pos_emb,
+ repeat_kv,
+)
+
+
+def local_heavy_hitter_mask(attn_weights, token_budget, chunk_size):
+ # attn_weights (BS, head, query, keys)
+
+ # expend attn_weights to be divisible by chunk_size
+ seq_length = attn_weights.shape[-1]
+ padding_length = chunk_size - ((seq_length - 1) % chunk_size + 1)
+ attn_weights = torch.cat(
+ [
+ attn_weights,
+ torch.ones(
+ (
+ attn_weights.shape[0],
+ attn_weights.shape[1],
+ attn_weights.shape[2],
+ padding_length,
+ ),
+ device=attn_weights.device,
+ )
+ * torch.tensor(torch.finfo(attn_weights.dtype).min),
+ ],
+ dim=-1,
+ )
+
+ # chunk attn_weights into chunk_size tokens
+ chunk_attn_weights = attn_weights.reshape(
+ attn_weights.shape[0],
+ attn_weights.shape[1],
+ attn_weights.shape[2],
+ attn_weights.shape[3] // chunk_size,
+ chunk_size,
+ ).amax(dim=-1)
+
+ _, topk = chunk_attn_weights.topk(
+ k=min(max(3, token_budget // chunk_size), chunk_attn_weights.size(-1)), dim=-1
+ )
+ # repeat topk chunk_size times and recover the original indexes (* chunk_size + arange(chunk_size))
+ topk = topk.unsqueeze(-1).repeat(
+ 1, 1, 1, 1, chunk_size
+ ) * chunk_size + torch.arange(chunk_size, device=topk.device)
+ topk = topk.reshape(topk.shape[0], topk.shape[1], topk.shape[2], -1)
+ mask_bottom = torch.zeros_like(attn_weights, dtype=torch.bool)
+ mask_bottom.scatter_(-1, topk, True)
+
+ # remove the padding
+ mask_bottom = mask_bottom[:, :, :, :seq_length]
+
+ return mask_bottom
+
+
+def quest_forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[DynamicCache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ **kwargs,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, _ = hidden_states.size()
+
+ if q_len > 1 or self.layer_idx < 2:
+ return self.flash_forward(
+ hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ **kwargs,
+ )
+
+ query_states = (
+ self.q_proj(hidden_states)
+ .view(bsz, q_len, self.num_heads, self.head_dim)
+ .transpose(1, 2)
+ )
+ key_states = (
+ self.k_proj(hidden_states)
+ .view(bsz, q_len, self.num_key_value_heads, self.head_dim)
+ .transpose(1, 2)
+ )
+ value_states = (
+ self.v_proj(hidden_states)
+ .view(bsz, q_len, self.num_key_value_heads, self.head_dim)
+ .transpose(1, 2)
+ )
+
+ kv_seq_len = key_states.shape[-2]
+ if past_key_value is not None:
+ kv_seq_len += past_key_value.get_seq_length(self.layer_idx)
+ # cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+ cos, sin = self.rotary_emb(value_states, position_ids)
+ query_states, key_states = apply_rotary_pos_emb(
+ query_states, key_states, cos, sin, position_ids
+ )
+ # [bsz, nh, t, hd]
+
+ if past_key_value is not None:
+ # reuse k, v, self_attention
+ key_states, value_states = past_key_value.update(
+ key_states, value_states, self.layer_idx
+ )
+
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(
+ self.head_dim
+ )
+
+ sign = (query_states > 0) + (~(query_states > 0)) * -1
+ max_key = key_states * sign
+ postive_query = query_states * sign
+
+ # expend max_key to be divisible by chunk_size
+ seq_length = max_key.shape[-2]
+ padding_length = self.chunk_size - ((seq_length - 1) % self.chunk_size + 1)
+ max_key = torch.cat(
+ [
+ max_key,
+ torch.ones(
+ (max_key.shape[0], max_key.shape[1], padding_length, max_key.shape[3]),
+ device=max_key.device,
+ )
+ * torch.tensor(torch.finfo(max_key.dtype).min),
+ ],
+ dim=-2,
+ )
+
+ # chunk max_key into chunk_size tokens
+ chunk_max_key = max_key.reshape(
+ max_key.shape[0],
+ max_key.shape[1],
+ max_key.shape[2] // self.chunk_size,
+ self.chunk_size,
+ max_key.shape[3],
+ ).amax(dim=-2)
+
+ # duplicate chunk_max_key chunk_size times
+ chunk_max_key = chunk_max_key.unsqueeze(-2).repeat(1, 1, 1, self.chunk_size, 1)
+ # reshape chunk_max_key to the original shape
+ chunk_max_key = chunk_max_key.reshape(
+ chunk_max_key.shape[0], chunk_max_key.shape[1], -1, chunk_max_key.shape[-1]
+ )[:, :, :seq_length, :]
+
+ quantized_weight = torch.matmul(
+ postive_query.float(),
+ chunk_max_key.transpose(2, 3),
+ )
+
+ if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+ raise ValueError(
+ f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+ f" {attn_weights.size()}"
+ )
+
+ if attention_mask is not None:
+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+ raise ValueError(
+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+ )
+ attn_weights = attn_weights + attention_mask
+ attn_weights = torch.max(
+ attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)
+ )
+ quantized_weight = quantized_weight + attention_mask
+ quantized_weight = torch.max(
+ quantized_weight, torch.tensor(torch.finfo(quantized_weight.dtype).min)
+ )
+
+ token_budget = min(kv_seq_len, self.token_budget)
+
+ attn_weights_for_selection = quantized_weight
+
+ if token_budget > 0:
+ mask_bottom = local_heavy_hitter_mask(
+ attn_weights_for_selection, token_budget, self.chunk_size
+ ) # Default: No padding applied to input
+ else:
+ mask_bottom = torch.zeros_like(attn_weights_for_selection, dtype=torch.bool)
+
+ mask_bottom = torch.tril(mask_bottom, diagonal=position_ids[0][0].item())
+ attn_weights[~mask_bottom] = torch.tensor(torch.finfo(attn_weights.dtype).min)
+
+ # upcast attention to fp32
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ query_states.dtype
+ )
+ attn_output = torch.matmul(attn_weights, value_states)
+
+ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+ raise ValueError(
+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+ f" {attn_output.size()}"
+ )
+
+ attn_output = attn_output.transpose(1, 2)
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+ attn_output = self.o_proj(attn_output)
+
+ if not output_attentions:
+ attn_weights = None
+
+ return attn_output, attn_weights, past_key_value
+
+
+def quest_decode_kernel(
+ query_states,
+ key_states,
+ value_states,
+ decoding_kwargs,
+):
+ chunk_size = decoding_kwargs["attn_forward_config"].get("chunk_size", 16)
+ token_budget = decoding_kwargs["attn_forward_config"].get("token_budget", 1024)
+ attention_mask = decoding_kwargs.get("attention_mask", None)
+ position_ids = decoding_kwargs.get("position_ids", None)
+ kv_seq_len = key_states.size(-2)
+ bsz, _, q_len, _ = query_states.shape
+
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(
+ query_states.size(-1)
+ )
+
+ sign = (query_states > 0) + (
+ ~(query_states > 0)
+ ) * -1 # [bsz, nh, q_len, kv_seq_len]
+ if sign.size(-2) != 1:
+ sign = sign.sum(dim=-2, keepdim=True)
+ max_key = key_states * sign
+ postive_query = query_states * sign
+
+ # expend max_key to be divisible by chunk_size
+ seq_length = max_key.shape[-2]
+ padding_length = chunk_size - ((seq_length - 1) % chunk_size + 1)
+ max_key = torch.cat(
+ [
+ max_key,
+ torch.ones(
+ (max_key.shape[0], max_key.shape[1], padding_length, max_key.shape[3]),
+ device=max_key.device,
+ )
+ * torch.tensor(torch.finfo(max_key.dtype).min),
+ ],
+ dim=-2,
+ )
+
+ # chunk max_key into chunk_size tokens
+ chunk_max_key = max_key.reshape(
+ max_key.shape[0],
+ max_key.shape[1],
+ max_key.shape[2] // chunk_size,
+ chunk_size,
+ max_key.shape[3],
+ ).amax(dim=-2)
+
+ # duplicate chunk_max_key chunk_size times
+ chunk_max_key = chunk_max_key.unsqueeze(-2).repeat(1, 1, 1, chunk_size, 1)
+ # reshape chunk_max_key to the original shape
+ chunk_max_key = chunk_max_key.reshape(
+ chunk_max_key.shape[0], chunk_max_key.shape[1], -1, chunk_max_key.shape[-1]
+ )[:, :, :seq_length, :]
+
+ quantized_weight = torch.matmul( # [bsz, nh, q_len, kv_seq_len]
+ postive_query.float(),
+ chunk_max_key.transpose(2, 3),
+ )
+
+ if attention_mask is not None:
+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+ raise ValueError(
+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+ )
+ attn_weights = attn_weights + attention_mask
+ attn_weights = torch.max(
+ attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)
+ )
+ quantized_weight = quantized_weight + attention_mask
+ quantized_weight = torch.max(
+ quantized_weight, torch.tensor(torch.finfo(quantized_weight.dtype).min)
+ )
+
+ token_budget = min(kv_seq_len, token_budget)
+ attn_weights_for_selection = quantized_weight
+
+ if token_budget > 0:
+ mask_bottom = local_heavy_hitter_mask(
+ attn_weights_for_selection, token_budget, chunk_size
+ ) # Default: No padding applied to input
+ else:
+ mask_bottom = torch.zeros_like(attn_weights_for_selection, dtype=torch.bool)
+
+ mask_bottom = torch.tril(mask_bottom, diagonal=position_ids[0][0].item())
+ attn_weights[~mask_bottom] = torch.tensor(torch.finfo(attn_weights.dtype).min)
+
+ # upcast attention to fp32
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ query_states.dtype
+ )
+ attn_output = torch.matmul(attn_weights, value_states)
+
+ return attn_output
+
+
+def forward_yarn(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ is_padded_inputs: Optional[bool] = False,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ bsz, q_len, h_size = hidden_states.size()
+
+ # Prefill stage utilizes flash attention
+ if q_len > 1 or self.layer_id < 2:
+ return self.flash_forward(
+ hidden_states,
+ attention_mask,
+ position_ids,
+ past_key_value,
+ output_attentions,
+ use_cache,
+ is_padded_inputs,
+ )
+
+ has_layer_past = past_key_value is not None
+
+ if has_layer_past:
+ past_kv = past_key_value[0]
+ past_len = past_key_value[1]
+ else:
+ past_len = 0
+
+ if self.config.pretraining_tp > 1:
+ key_value_slicing = (
+ self.num_key_value_heads * self.head_dim
+ ) // self.config.pretraining_tp
+ query_slices = self.q_proj.weight.split(
+ (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0
+ )
+ key_slices = self.k_proj.weight.split(key_value_slicing, dim=0)
+ value_slices = self.v_proj.weight.split(key_value_slicing, dim=0)
+
+ q = [
+ F.linear(hidden_states, query_slices[i])
+ for i in range(self.config.pretraining_tp)
+ ]
+ q = torch.cat(q, dim=-1)
+
+ k = [
+ F.linear(hidden_states, key_slices[i])
+ for i in range(self.config.pretraining_tp)
+ ]
+ k = torch.cat(k, dim=-1)
+
+ v = [
+ F.linear(hidden_states, value_slices[i])
+ for i in range(self.config.pretraining_tp)
+ ]
+ v = torch.cat(v, dim=-1)
+
+ else:
+ q = self.q_proj(hidden_states)
+ k = self.k_proj(hidden_states)
+ v = self.v_proj(hidden_states)
+
+ q = q.view(bsz, q_len, self.num_heads, self.head_dim)
+ k = k.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
+ v = v.view(bsz, q_len, self.num_key_value_heads, self.head_dim)
+
+ q, k = self.rotary_emb(q, k, past_len)
+
+ @torch.jit.script
+ def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+ """
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+ """
+ batch, slen, _, num_key_value_heads, head_dim = hidden_states.shape
+ if n_rep == 1:
+ return hidden_states
+ hidden_states = hidden_states[:, :, :, :, None, :].expand(
+ batch, slen, 2, num_key_value_heads, n_rep, head_dim
+ )
+ return hidden_states.reshape(
+ batch, slen, 2, num_key_value_heads * n_rep, head_dim
+ )
+
+ kv = torch.stack([k, v], 2)
+ kv = repeat_kv(kv, self.num_key_value_groups)
+
+ # Cache QKV values
+ if has_layer_past:
+ new_len = past_len + q.size(1)
+ if new_len > past_kv.size(1):
+ past_kv = torch.cat(
+ [
+ past_kv,
+ torch.empty(
+ bsz,
+ 256,
+ 2,
+ kv.size(3),
+ kv.size(4),
+ dtype=kv.dtype,
+ device=kv.device,
+ ),
+ ],
+ 1,
+ )
+ past_kv[:, past_len:new_len] = kv
+ kv = past_kv[:, :new_len]
+ else:
+ past_kv = kv
+
+ k, v = kv.split(1, dim=2)
+ k = k.squeeze(2)
+ v = v.squeeze(2)
+
+ past_key_value = (past_kv, past_len + q.size(1)) if use_cache else None
+
+ q = q.transpose(1, 2)
+ k = k.transpose(1, 2)
+ v = v.transpose(1, 2)
+
+ kv_seq_len = k.shape[-2]
+
+ attn_weights = torch.matmul(q, k.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+ sign = (q > 0) + (~(q > 0)) * -1
+ max_key = k * sign
+ postive_query = q * sign
+
+ # expend max_key to be divisible by chunk_size
+ seq_length = max_key.shape[-2]
+ padding_length = self.chunk_size - ((seq_length - 1) % self.chunk_size + 1)
+ max_key = torch.cat(
+ [
+ max_key,
+ torch.ones(
+ (max_key.shape[0], max_key.shape[1], padding_length, max_key.shape[3]),
+ device=max_key.device,
+ )
+ * torch.tensor(torch.finfo(max_key.dtype).min),
+ ],
+ dim=-2,
+ )
+
+ # chunk max_key into chunk_size tokens
+ chunk_max_key = max_key.reshape(
+ max_key.shape[0],
+ max_key.shape[1],
+ max_key.shape[2] // self.chunk_size,
+ self.chunk_size,
+ max_key.shape[3],
+ ).amax(dim=-2)
+
+ # duplicate chunk_max_key chunk_size times
+ chunk_max_key = chunk_max_key.unsqueeze(-2).repeat(1, 1, 1, self.chunk_size, 1)
+ # reshape chunk_max_key to the original shape
+ chunk_max_key = chunk_max_key.reshape(
+ chunk_max_key.shape[0], chunk_max_key.shape[1], -1, chunk_max_key.shape[-1]
+ )[:, :, :seq_length, :]
+
+ quantized_weight = torch.matmul(
+ postive_query.float(),
+ chunk_max_key.transpose(2, 3),
+ )
+
+ if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+ raise ValueError(
+ f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+ f" {attn_weights.size()}"
+ )
+
+ assert q_len == 1, "Prefill stage utilizes flash attention."
+
+ token_budget = min(kv_seq_len, self.token_budget)
+
+ attn_weights_for_selection = quantized_weight
+ # attn_weights_for_selection = attn_weights
+
+ if token_budget > 0:
+ mask_bottom = local_heavy_hitter_mask(
+ attn_weights_for_selection, token_budget, self.chunk_size
+ ) # Default: No padding applied to input
+ else:
+ mask_bottom = torch.zeros_like(attn_weights_for_selection, dtype=torch.bool)
+
+ # Attention mask for multi-stage Q&A, todo
+ mask_bottom = torch.tril(mask_bottom, diagonal=k.shape[-2] - q.shape[-2])
+ attn_weights[~mask_bottom] = torch.tensor(torch.finfo(attn_weights.dtype).min)
+
+ # upcast attention to fp32
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ q.dtype
+ )
+ attn_output = torch.matmul(attn_weights, v)
+
+ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+ raise ValueError(
+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+ f" {attn_output.size()}"
+ )
+
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+ if self.config.pretraining_tp > 1:
+ attn_output = attn_output.split(
+ self.hidden_size // self.config.pretraining_tp, dim=2
+ )
+ o_proj_slices = self.o_proj.weight.split(
+ self.hidden_size // self.config.pretraining_tp, dim=1
+ )
+ attn_output = sum(
+ [
+ F.linear(attn_output[i], o_proj_slices[i])
+ for i in range(self.config.pretraining_tp)
+ ]
+ )
+ else:
+ attn_output = self.o_proj(attn_output)
+
+ if not output_attentions:
+ attn_weights = None
+
+ return attn_output, attn_weights, past_key_value
+
+
+global layer_id
+layer_id = 32
+
+
+def enable_quest_attention_eval(model, args):
+ for name, module in reversed(model._modules.items()):
+ if len(list(module.children())) > 0:
+ enable_quest_attention_eval(
+ module,
+ args,
+ )
+
+ global layer_id
+ if isinstance(module, LlamaAttention):
+ # For longchat model
+ layer_id -= 1
+ model._modules[name].layer_id = layer_id
+ model._modules[name].flash_forward = model._modules[name].forward
+ model._modules[name].forward = types.MethodType(
+ quest_forward, model._modules[name]
+ )
+
+ model._modules[name].token_budget = args.token_budget
+ model._modules[name].chunk_size = args.chunk_size
+ elif module.__class__.__name__ == "LlamaAttention":
+ # For yarn model
+ layer_id -= 1
+ model._modules[name].layer_id = layer_id
+ model._modules[name].flash_forward = model._modules[name].forward
+ model._modules[name].forward = types.MethodType(
+ forward_yarn, model._modules[name]
+ )
+
+ model._modules[name].token_budget = args.token_budget
+ model._modules[name].chunk_size = args.chunk_size
diff --git a/minference/modules/retr_attn.py b/minference/modules/retr_attn.py
new file mode 100644
index 00000000..1fc4bbf9
--- /dev/null
+++ b/minference/modules/retr_attn.py
@@ -0,0 +1,545 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import math
+import os
+import threading
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+from transformers.cache_utils import Cache
+from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv
+from transformers.utils.import_utils import _is_package_available
+
+try:
+ from flash_attn import flash_attn_func
+except ImportError:
+ from ..ops.flash_attn_triton import _flash_attn_triton_decoding as flash_attn_func
+
+if _is_package_available("papyfaiss"):
+ import papyfaiss
+
+
+class VectorDB_KV_Cache:
+ def __init__(self, max_length, temp_cache_size):
+ self.key_cache = []
+ self.value_cache = []
+ self.update_thread = None
+ self.gpu_key_cache = []
+ self.gpu_value_cache = []
+ self.temp_gpu_key_cache = []
+ self.temp_gpu_value_cache = []
+ self.start_size = 128
+ self.recent_size = 512
+ self.temp_cache = temp_cache_size # set equal to max generation length
+ self.cache_size = 640
+ self.seen1 = None
+ self.seen2 = None
+ self.temp_seen = 0
+ self.max_size = max_length
+
+ def async_update(
+ self, key_states, value_states, layer_idx, do_strllm_until_layer, core
+ ):
+ def update_task(key_states, value_states, layer_idx):
+ self.key_vec_cache_update(key_states, layer_idx, core)
+ self.val_vec_cache_update(value_states, layer_idx, do_strllm_until_layer)
+
+ assert self.start_size + self.recent_size == self.cache_size
+
+ start_size = self.start_size
+ recent_size = self.recent_size
+ cache_size = self.cache_size
+
+ bs, head_num, qlen, dim = key_states.shape
+
+ if layer_idx <= do_strllm_until_layer:
+ assert 0 < cache_size <= qlen # cache_size is not 0 when using pattern
+
+ key_tensor = torch.zeros(
+ (bs, head_num, cache_size, dim),
+ dtype=key_states.dtype,
+ device=key_states.device,
+ )
+ value_tensor = torch.zeros(
+ (bs, head_num, cache_size, dim),
+ dtype=key_states.dtype,
+ device=key_states.device,
+ )
+
+ key_tensor[:, :, :, :] = torch.cat(
+ (
+ key_states[:, :, :start_size, :],
+ key_states[:, :, qlen - recent_size : qlen, :],
+ ),
+ dim=2,
+ )
+ value_tensor[:, :, :, :] = torch.cat(
+ (
+ value_states[:, :, :start_size, :],
+ value_states[:, :, qlen - recent_size : qlen, :],
+ ),
+ dim=2,
+ )
+
+ self.gpu_key_cache.append(key_tensor)
+ self.gpu_value_cache.append(value_tensor)
+
+ key_states = key_states[:, :, start_size : qlen - recent_size, :]
+ value_states = value_states[:, :, start_size : qlen - recent_size, :]
+
+ else:
+ self.gpu_key_cache.append(None)
+ self.gpu_value_cache.append(None)
+
+ self.temp_gpu_key_cache.append(
+ torch.zeros(
+ (bs, head_num, self.temp_cache, dim),
+ dtype=key_states.dtype,
+ device=key_states.device,
+ )
+ )
+ self.temp_gpu_value_cache.append(
+ torch.zeros(
+ (bs, head_num, self.temp_cache, dim),
+ dtype=key_states.dtype,
+ device=key_states.device,
+ )
+ )
+
+ if 0 < layer_idx < do_strllm_until_layer:
+ self.sync()
+
+ key_states = key_states.to(torch.float32).detach().cpu().numpy()
+ value_states = value_states.detach().cpu()
+
+ self.update_thread = threading.Thread(
+ target=update_task, args=(key_states, value_states, layer_idx)
+ )
+ self.update_thread.start()
+
+ def sync(self):
+ self.update_thread.join()
+
+ # sync update, with load data
+ def sync_key_update(
+ self, key_states, layer_idx, do_strllm_until_layer, core, insert_db
+ ):
+ if insert_db:
+ start_size = self.start_size
+ recent_size = self.recent_size
+ if layer_idx <= do_strllm_until_layer and recent_size > 0:
+ bs, head_num, _, dim = key_states.shape
+ key_tensor = torch.zeros(
+ (bs, head_num, 1, dim),
+ dtype=key_states.dtype,
+ device=key_states.device,
+ )
+ key_tensor = self.gpu_key_cache[layer_idx][
+ :, :, start_size : start_size + 1, :
+ ]
+ tempk = torch.cat(
+ [
+ self.gpu_key_cache[layer_idx][:, :, :start_size, :],
+ self.gpu_key_cache[layer_idx][:, :, start_size + 1 :, :],
+ ],
+ dim=-2,
+ )
+ self.gpu_key_cache[layer_idx] = torch.cat([tempk, key_states], dim=-2)
+ else:
+ key_tensor = key_states
+
+ if key_tensor.dtype == torch.bfloat16:
+ key_states = key_tensor.float().detach().cpu().numpy()
+ else:
+ key_states = key_tensor.detach().cpu().numpy()
+ key_cache = self.key_vec_cache_update(key_states, layer_idx, core)
+ return key_cache, self.gpu_key_cache[layer_idx]
+ else:
+ if layer_idx == 0:
+ self.temp_seen += 1
+ self.temp_gpu_key_cache[layer_idx][
+ :, :, self.temp_seen - 1 : self.temp_seen, :
+ ] = key_states
+ return self.key_cache[layer_idx], torch.cat(
+ (
+ self.gpu_key_cache[layer_idx],
+ self.temp_gpu_key_cache[layer_idx][:, :, : self.temp_seen, :],
+ ),
+ dim=-2,
+ )
+
+ def sync_value_update(
+ self, value_states, layer_idx, do_strllm_until_layer, insert_db
+ ):
+ if insert_db:
+ start_size = self.start_size
+ recent_size = self.recent_size
+ if layer_idx <= do_strllm_until_layer and recent_size > 0:
+ bs, head_num, _, dim = value_states.shape
+ value_tensor = torch.zeros(
+ (bs, head_num, 1, dim),
+ dtype=value_states.dtype,
+ device=value_states.device,
+ )
+ value_tensor = self.gpu_value_cache[layer_idx][
+ :, :, start_size : start_size + 1, :
+ ]
+ tempv = torch.cat(
+ [
+ self.gpu_value_cache[layer_idx][:, :, :start_size, :],
+ self.gpu_value_cache[layer_idx][:, :, start_size + 1 :, :],
+ ],
+ dim=-2,
+ )
+ self.gpu_value_cache[layer_idx] = torch.cat(
+ [tempv, value_states], dim=-2
+ )
+ else:
+ value_tensor = value_states
+
+ value_states = value_tensor.detach().cpu()
+ value_cache = self.val_vec_cache_update(
+ value_states, layer_idx, do_strllm_until_layer
+ )
+ return value_cache, self.gpu_value_cache[layer_idx]
+ else:
+ self.temp_gpu_value_cache[layer_idx][
+ :, :, self.temp_seen - 1 : self.temp_seen, :
+ ] = value_states
+ return self.value_cache[layer_idx], torch.cat(
+ (
+ self.gpu_value_cache[layer_idx],
+ self.temp_gpu_value_cache[layer_idx][:, :, : self.temp_seen, :],
+ ),
+ dim=-2,
+ )
+
+ # papyfaiss update vectordb cache
+ def key_vec_cache_update(self, key_states, layer_idx, core):
+ def cpp_add(task_id, key, key_cache):
+ key_cache.add(key[task_id], task_id, core)
+
+ if len(self.key_cache) <= layer_idx:
+ # Flat index
+ try:
+ index = papyfaiss.FlatIndex(
+ head_num=key_states.shape[1], dim=key_states.shape[3]
+ )
+ except:
+ assert (
+ False
+ ), "Please install papyfaiss. Refer to https://github.com/microsoft/RetrievalAttention"
+ # index = papyfaiss.FlatIndexSQ(head_num=key_states.shape[1], dim=key_states.shape[3])
+ # IVF index
+ # index = papyfaiss.IVFIndexSQ(head_num=key_states.shape[1], dim=key_states.shape[3], n_centroids=512, quant="SQ8", use_gpu=True)
+ # index.set_nprobe(150, 150)
+ self.key_cache.append(index)
+
+ # pool = ThreadPool(core)
+ # tasks = range(key_states.shape[1])
+ # pool.map(lambda task_id: cpp_add(task_id, key_states[0], self.key_cache[layer_idx]), tasks)
+ # pool.close()
+ # pool.join()
+ self.key_cache[layer_idx].paraadd(key_states[0], core)
+ else:
+ self.key_cache[layer_idx].paraadd(key_states[0], core)
+
+ return self.key_cache[layer_idx]
+
+ def val_vec_cache_update(
+ self,
+ value_states, # (batch_size, nheads, seqlen, head_dim)
+ layer_idx: int,
+ do_strllm_until_layer,
+ ):
+ if len(self.value_cache) <= layer_idx: # prompt phase
+ self.value_cache.append(value_states)
+ return self.value_cache[layer_idx]
+ else: # decode phase
+ self.value_cache[layer_idx] = torch.cat(
+ (self.value_cache[layer_idx], value_states[:, :, :, :]), dim=-2
+ )
+ return self.value_cache[layer_idx]
+
+
+class RetrAttnCache(Cache):
+ def __init__(self, config):
+ super().__init__()
+ self.max_length = config.attn_kwargs["max_seq_length"]
+ self.temp_cache_size = config.attn_kwargs["max_new_tokens"]
+ self._seen_tokens = 0
+
+ self.core = os.cpu_count() // 2
+ self.do_strllm_until_layer = config.attn_kwargs["num_layers"] - 1
+ self.num_layers = config.attn_kwargs["num_layers"]
+ self.vector_db_cache = VectorDB_KV_Cache(self.max_length, self.temp_cache_size)
+
+ def update(self, key_states, value_states, layer_idx, cache_kwargs):
+ query_states = cache_kwargs.get("query_states", None)
+ q_len = query_states.size(-2)
+ insert_db = cache_kwargs.get("update_global_past_kv", True)
+
+ if layer_idx == 0:
+ self._seen_tokens += key_states.size(-2)
+
+ # if query_states.size(-2) != 1: # prefill
+ # if len(self.vector_db_cache.value_cache) == layer_idx: # initializing vector_db_cache
+ if q_len == self._seen_tokens:
+ self.vector_db_cache.async_update(
+ key_states.contiguous(),
+ value_states,
+ layer_idx,
+ self.do_strllm_until_layer,
+ self.core,
+ )
+ if layer_idx == self.num_layers - 1:
+ self.vector_db_cache.sync()
+
+ key_states = repeat_kv(
+ key_states, query_states.size(1) // key_states.size(1)
+ )
+ value_states = repeat_kv(
+ value_states, query_states.size(1) // value_states.size(1)
+ )
+ # assert len(self.vector_db_cache.value_cache) == layer_idx + 1
+ return key_states, value_states
+ else:
+ if q_len == 1: # the decoding
+ key_cache, gpu_key_cache = self.vector_db_cache.sync_key_update(
+ key_states,
+ layer_idx,
+ self.do_strllm_until_layer,
+ self.core,
+ insert_db,
+ )
+ value_cache, gpu_value_cache = self.vector_db_cache.sync_value_update(
+ value_states, layer_idx, self.do_strllm_until_layer, insert_db
+ )
+ else: # the follow-up queries
+ for i in range(q_len): # insert query token one by one
+ key_cache, gpu_key_cache = self.vector_db_cache.sync_key_update(
+ key_states[:, :, i : i + 1, :],
+ layer_idx,
+ self.do_strllm_until_layer,
+ self.core,
+ insert_db,
+ )
+ (
+ value_cache,
+ gpu_value_cache,
+ ) = self.vector_db_cache.sync_value_update(
+ value_states[:, :, i : i + 1, :],
+ layer_idx,
+ self.do_strllm_until_layer,
+ insert_db,
+ )
+ return (key_cache, gpu_key_cache), (value_cache, gpu_value_cache)
+
+ def get_seq_length(self, layer_idx=0):
+ if len(self.vector_db_cache.gpu_key_cache) <= layer_idx:
+ return 0
+ return self._seen_tokens
+
+ def clear_temp_kv_cache(self):
+ self._seen_tokens -= self.vector_db_cache.temp_seen
+ self.vector_db_cache.temp_seen = 0
+
+
+def retr_attn(
+ q,
+ key_cache,
+ value_cache,
+ decoding_kwargs,
+):
+ key_cache, gpu_key_cache = key_cache
+ value_cache, gpu_value_cache = value_cache
+
+ kv_heads = gpu_key_cache.size(1)
+ num_key_value_groups = decoding_kwargs.get(
+ "num_key_value_groups", q.size(1) // kv_heads
+ )
+ layer_idx = decoding_kwargs.get("layer_idx", None)
+ top_k = decoding_kwargs["attn_forward_config"].get("top_k", 2000)
+ from_layer = decoding_kwargs["attn_forward_config"].get("from_layer", 0)
+ core = os.cpu_count() // 2
+
+ device = q.device
+ q_dtype = q.dtype
+
+ qq = q.clone()
+ if q.dtype == torch.bfloat16:
+ q = q.float().detach().cpu().numpy()
+ else:
+ q = q.detach().cpu().numpy()
+
+ # retrieval
+ top_k = top_k if layer_idx >= from_layer else 124_000
+ if key_cache.get_index_type() == "Flat":
+ distances, indices = key_cache.search(q[0], num_key_value_groups, top_k, core)
+ elif "IVF" in key_cache.get_index_type():
+ distances, indices = key_cache.search(
+ q[0], layer_idx, num_key_value_groups, top_k, core
+ )
+
+ retrieval_attn_weights = distances.reshape(
+ (q.shape[0], q.shape[1], q.shape[2], top_k)
+ ) / math.sqrt(q.shape[3])
+ retrieval_indices = indices.reshape((q.shape[1], q.shape[2], top_k))
+
+ retrieval_attn_weights = torch.from_numpy(retrieval_attn_weights)
+ retrieval_lse = torch.log(torch.sum(torch.exp(retrieval_attn_weights), dim=-1)).to(
+ device
+ )
+ retrieval_attn_weights = nn.functional.softmax(
+ retrieval_attn_weights, dim=-1, dtype=torch.float32
+ )
+ retrieval_attn_weights = nn.functional.dropout(
+ retrieval_attn_weights, p=0, training=False
+ )
+
+ retrieval_out = torch.full(
+ (q.shape[0], q.shape[1], q.shape[2], q.shape[3]), 0, dtype=torch.float32
+ )
+ for head_idx in range(q.shape[1]):
+ for q_idx in range(q.shape[2]):
+ query2key = head_idx // num_key_value_groups
+ retrieval_attn_values = value_cache[0][query2key][
+ retrieval_indices[head_idx][q_idx]
+ ] # (bs, nhead, true_k, head_dim)
+ retrieval_out[:, head_idx:, q_idx, :] = torch.matmul(
+ retrieval_attn_weights[:, head_idx, q_idx, :],
+ retrieval_attn_values.to(torch.float32),
+ )
+
+ retrieval_out = retrieval_out.to(q_dtype).to(device)
+ if gpu_key_cache is not None:
+ flash_out, flash_lse, _ = flash_attn_func(
+ qq.transpose(1, 2),
+ gpu_key_cache.transpose(1, 2),
+ gpu_value_cache.transpose(1, 2),
+ dropout_p=0,
+ causal=False,
+ window_size=(-1, -1),
+ alibi_slopes=None,
+ deterministic=False,
+ return_attn_probs=True,
+ )
+
+ flash_lse = flash_lse.transpose(-2, -1).unsqueeze(dim=-1)
+ retrieval_lse = retrieval_lse.transpose(-2, -1).unsqueeze(dim=-1)
+ new_lse = retrieval_lse + torch.log(1 + torch.exp(flash_lse - retrieval_lse))
+
+ final_out = (
+ torch.exp(retrieval_lse - new_lse) * retrieval_out.transpose(1, 2)
+ + torch.exp(flash_lse - new_lse) * flash_out
+ )
+ retrieval_out = final_out.to(q_dtype)
+
+ retrieval_out = retrieval_out.transpose(1, 2)
+ return retrieval_out
+
+
+def llama_retr_flash_attention_forward(
+ self,
+ top_k: int,
+ from_layer: int,
+ total_layers: int,
+ insert_db,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.LongTensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ output_attentions = False
+
+ bsz, q_len, _ = hidden_states.size()
+
+ query_states = self.q_proj(hidden_states)
+ key_states = self.k_proj(hidden_states)
+ value_states = self.v_proj(hidden_states)
+
+ query_states = query_states.view(
+ bsz, q_len, self.num_heads, self.head_dim
+ ).transpose(1, 2)
+ key_states = key_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+ value_states = value_states.view(
+ bsz, q_len, self.num_key_value_heads, self.head_dim
+ ).transpose(1, 2)
+
+ cos, sin = self.rotary_emb(value_states, position_ids)
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+ past_key_value = getattr(self, "past_key_value", past_key_value)
+
+ dropout_rate = self.attention_dropout if self.training else 0.0
+
+ do_strllm_until_layer = total_layers - 1
+ core = os.cpu_count() // 2
+
+ # prefill phase, using flash attn
+ if key_states.shape[2] != 1:
+ # (1) async update
+ # past_key_value.sync_key_update(key_states, self.layer_idx)
+ # past_key_value.sync_value_update(value_states, self.layer_idx)
+ past_key_value.async_update(
+ key_states.contiguous(),
+ value_states,
+ self.layer_idx,
+ do_strllm_until_layer,
+ core,
+ )
+
+ # (2) computation
+ attn_output = flash_attn_func(
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ q_len,
+ causal=True,
+ dropout=dropout_rate,
+ )
+
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+ attn_output = self.o_proj(attn_output)
+
+ # (3) sync
+ if self.layer_idx == total_layers - 1:
+ past_key_value.sync()
+
+ # decoding phase
+ else:
+ key_cache, gpu_key_cache = past_key_value.sync_key_update(
+ key_states, self.layer_idx, do_strllm_until_layer, core, insert_db
+ )
+ value_cache, gpu_value_cache = past_key_value.sync_value_update(
+ value_states, self.layer_idx, do_strllm_until_layer, insert_db
+ )
+
+ attn_output = retr_attn(
+ query_states,
+ key_cache,
+ value_cache,
+ gpu_key_cache,
+ gpu_value_cache,
+ self.num_key_value_groups,
+ self.layer_idx,
+ top_k,
+ from_layer,
+ core,
+ )
+
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+ if attn_output.dtype != torch.bfloat16:
+ attn_output = attn_output.to(torch.bfloat16)
+ attn_output = self.o_proj(attn_output)
+
+ return attn_output, None, past_key_value
diff --git a/minference/modules/snap_kv.py b/minference/modules/snap_kv.py
deleted file mode 100644
index d6cf9e81..00000000
--- a/minference/modules/snap_kv.py
+++ /dev/null
@@ -1,426 +0,0 @@
-# Copyright (c) 2024 Microsoft
-# Licensed under The MIT License [see LICENSE for details]
-# Refer to the code in https://github.com/FasterDecoding/SnapKV/blob/main/snapkv/monkeypatch/snapkv_utils.py
-
-import math
-import time
-import warnings
-from importlib.metadata import version
-from typing import List, Optional, Tuple, Union
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-import transformers
-from transformers.cache_utils import Cache, DynamicCache
-from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv
-from transformers.utils import logging
-
-logger = logging.get_logger(__name__)
-
-
-# https://github.com/huggingface/transformers/blob/v4.37-release/src/transformers/models/llama/modeling_llama.py
-def llama_flash_attn2_forward(
- self,
- hidden_states: torch.Tensor,
- attention_mask: Optional[torch.LongTensor] = None,
- position_ids: Optional[torch.LongTensor] = None,
- past_key_value: Optional[Cache] = None,
- output_attentions: bool = False,
- use_cache: bool = False,
- **kwargs,
-) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
- # [SnapKV] register kv_cluster
- init_snapkv(self)
- # LlamaFlashAttention2 attention does not support output_attentions
- if "padding_mask" in kwargs:
- warnings.warn(
- "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
- )
-
- # overwrite attention_mask with padding_mask
- attention_mask = kwargs.pop("padding_mask")
-
- output_attentions = False
-
- bsz, q_len, _ = hidden_states.size()
-
- query_states = self.q_proj(hidden_states)
- key_states = self.k_proj(hidden_states)
- value_states = self.v_proj(hidden_states)
-
- # Flash attention requires the input to have the shape
- # batch_size x seq_length x head_dim x hidden_dim
- # therefore we just need to keep the original shape
- query_states = query_states.view(
- bsz, q_len, self.num_heads, self.head_dim
- ).transpose(1, 2)
- key_states = key_states.view(
- bsz, q_len, self.num_key_value_heads, self.head_dim
- ).transpose(1, 2)
- value_states = value_states.view(
- bsz, q_len, self.num_key_value_heads, self.head_dim
- ).transpose(1, 2)
-
- kv_seq_len = key_states.shape[-2]
- # if past_key_value is not None:
- # kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
- if past_key_value is not None:
- if self.layer_idx is None:
- raise ValueError(
- f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
- "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
- "with a layer index."
- )
- if hasattr(self, "kv_seq_len"): # [SnapKV] add kv_seq_len
- if self.kv_seq_len != 0:
- kv_seq_len += self.kv_seq_len
- else:
- kv_seq_len += past_key_value.get_usable_length(
- kv_seq_len, self.layer_idx
- )
- else:
- kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
-
- cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
- query_states, key_states = apply_rotary_pos_emb(
- query_states, key_states, cos, sin, position_ids
- )
- # [SnapKV] move to ahead
- key_states = repeat_kv(key_states, self.num_key_value_groups)
- value_states = repeat_kv(value_states, self.num_key_value_groups)
-
- if past_key_value is not None:
- cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
- # key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
- # print('kv_seq_len:', kv_seq_len)
- # print('key_states.shape:', key_states.shape)
- if key_states.shape[-2] == kv_seq_len: # [SnapKV] add kv_cluster
- self.kv_seq_len = kv_seq_len # [SnapKV] register kv_seq_len
- key_states_compress, value_states_compress = self.kv_cluster.update_kv(
- key_states,
- query_states,
- value_states,
- attention_mask,
- self.num_key_value_groups,
- )
- past_key_value.update(
- key_states_compress, value_states_compress, self.layer_idx, cache_kwargs
- )
- else:
- self.kv_seq_len += q_len
- key_states, value_states = past_key_value.update(
- key_states, value_states, self.layer_idx, cache_kwargs
- )
-
- # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
- # to be able to avoid many of these transpose/reshape/view.
- query_states = query_states.transpose(1, 2)
- key_states = key_states.transpose(1, 2)
- value_states = value_states.transpose(1, 2)
-
- dropout_rate = self.attention_dropout if self.training else 0.0
-
- # In PEFT, usually we cast the layer norms in float32 for training stability reasons
- # therefore the input hidden states gets silently casted in float32. Hence, we need
- # cast them back in the correct dtype just to be sure everything works as expected.
- # This might slowdown training & inference so it is recommended to not cast the LayerNorms
- # in fp32. (LlamaRMSNorm handles it correctly)
-
- input_dtype = query_states.dtype
- if input_dtype == torch.float32:
- if torch.is_autocast_enabled():
- target_dtype = torch.get_autocast_gpu_dtype()
- # Handle the case where the model is quantized
- elif hasattr(self.config, "_pre_quantization_dtype"):
- target_dtype = self.config._pre_quantization_dtype
- else:
- target_dtype = self.q_proj.weight.dtype
-
- logger.warning_once(
- f"The input hidden states seems to be silently casted in float32, this might be related to"
- f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
- f" {target_dtype}."
- )
-
- query_states = query_states.to(target_dtype)
- key_states = key_states.to(target_dtype)
- value_states = value_states.to(target_dtype)
-
- attn_output = self._flash_attention_forward(
- query_states,
- key_states,
- value_states,
- attention_mask,
- q_len,
- dropout=dropout_rate,
- )
-
- attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
- attn_output = self.o_proj(attn_output)
-
- if not output_attentions:
- attn_weights = None
-
- return attn_output, attn_weights, past_key_value
-
-
-def prepare_inputs_for_generation_llama(
- self,
- input_ids,
- past_key_values=None,
- attention_mask=None,
- inputs_embeds=None,
- **kwargs,
-):
- if past_key_values is None: # [SnapKV]
- for layer in self.model.layers:
- layer.self_attn.kv_seq_len = 0
- if past_key_values is not None:
- if isinstance(past_key_values, Cache):
- cache_length = past_key_values.get_seq_length()
- past_length = past_key_values.seen_tokens
- max_cache_length = past_key_values.get_max_length()
- else:
- # cache_length = past_length = past_key_values[0][0].shape[2]
- # max_cache_length = None
- cache_length = past_length = self.model.layers[0].self_attn.kv_seq_len
- max_cache_length = None
- # Keep only the unprocessed tokens:
- # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
- # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
- # input)
- if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
- input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
- # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
- # input_ids based on the past_length.
- elif past_length < input_ids.shape[1]:
- input_ids = input_ids[:, past_length:]
- # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
-
- # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
- if (
- max_cache_length is not None
- and attention_mask is not None
- and cache_length + input_ids.shape[1] > max_cache_length
- ):
- attention_mask = attention_mask[:, -max_cache_length:]
-
- position_ids = kwargs.get("position_ids", None)
- if attention_mask is not None and position_ids is None:
- # create position_ids on the fly for batch generation
- position_ids = attention_mask.long().cumsum(-1) - 1
- position_ids.masked_fill_(attention_mask == 0, 1)
- if past_key_values:
- position_ids = position_ids[:, -input_ids.shape[1] :]
-
- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
- if inputs_embeds is not None and past_key_values is None:
- model_inputs = {"inputs_embeds": inputs_embeds}
- else:
- model_inputs = {"input_ids": input_ids}
-
- model_inputs.update(
- {
- "position_ids": position_ids,
- "past_key_values": past_key_values,
- "use_cache": kwargs.get("use_cache"),
- "attention_mask": attention_mask,
- }
- )
- return model_inputs
-
-
-llama_flash_attn2_forward_4_37 = llama_flash_attn2_forward
-prepare_inputs_for_generation_llama_4_37 = prepare_inputs_for_generation_llama
-
-
-@torch.no_grad()
-def rope_forward(self, x, seq_len):
- # x: [bs, num_attention_heads, seq_len, head_size]
- position_ids = torch.arange(seq_len, device=x.device).unsqueeze(0)
- inv_freq_expanded = (
- self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
- )
- position_ids_expanded = position_ids[:, None, :].float()
- # Force float32 since bfloat16 loses precision on long contexts
- # See https://github.com/huggingface/transformers/pull/29285
- device_type = x.device.type
- device_type = (
- device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
- )
- with torch.autocast(device_type=device_type, enabled=False):
- freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(
- 1, 2
- )
- emb = torch.cat((freqs, freqs), dim=-1)
- cos = emb.cos()
- sin = emb.sin()
- return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
-
-
-##################
-
-# perform qk calculation and get indices
-# this version will not update in inference mode
-
-
-# Copied from transformers.models.llama.modeling_llama.repeat_kv
-def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
- """
- This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
- num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
- """
- batch, num_key_value_heads, slen, head_dim = hidden_states.shape
- if n_rep == 1:
- return hidden_states
- hidden_states = hidden_states[:, :, None, :, :].expand(
- batch, num_key_value_heads, n_rep, slen, head_dim
- )
- return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
-
-
-class SnapKVCluster:
- def __init__(
- self,
- window_size=64,
- max_capacity_prompt=256 + 64,
- kernel_size=5,
- pooling="avgpool",
- ):
- self.window_size = window_size
- self.max_capacity_prompt = max_capacity_prompt
- assert self.max_capacity_prompt - self.window_size > 0
- self.kernel_size = kernel_size
- self.pooling = pooling
-
- def reset(
- self,
- window_size=64,
- max_capacity_prompt=256 + 64,
- kernel_size=5,
- pooling="avgpool",
- ):
- self.window_size = window_size
- self.max_capacity_prompt = max_capacity_prompt
- assert self.max_capacity_prompt - self.window_size > 0
- self.kernel_size = kernel_size
- self.pooling = pooling
-
- def update_kv(
- self,
- key_states,
- query_states,
- value_states,
- attention_mask,
- num_key_value_groups,
- ):
- # check if prefix phase
- assert key_states.shape[-2] == query_states.shape[-2]
- bsz, num_heads, q_len, head_dim = query_states.shape
- if q_len < self.max_capacity_prompt:
- return key_states, value_states
- else:
- attn_weights = torch.matmul(
- query_states[..., -self.window_size :, :], key_states.transpose(2, 3)
- ) / math.sqrt(head_dim)
- mask = torch.full(
- (self.window_size, self.window_size),
- torch.finfo(attn_weights.dtype).min,
- device=attn_weights.device,
- )
- mask_cond = torch.arange(mask.size(-1), device=attn_weights.device)
- mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
- mask = mask.to(attn_weights.device)
- attention_mask = mask[None, None, :, :]
-
- attn_weights[
- :, :, -self.window_size :, -self.window_size :
- ] += attention_mask
-
- attn_weights = nn.functional.softmax(
- attn_weights, dim=-1, dtype=torch.float32
- ).to(query_states.dtype)
- attn_weights_sum = attn_weights[
- :, :, -self.window_size :, : -self.window_size
- ].sum(dim=-2)
- if self.pooling == "avgpool":
- attn_cache = F.avg_pool1d(
- attn_weights_sum,
- kernel_size=self.kernel_size,
- padding=self.kernel_size // 2,
- stride=1,
- )
- elif self.pooling == "maxpool":
- attn_cache = F.max_pool1d(
- attn_weights_sum,
- kernel_size=self.kernel_size,
- padding=self.kernel_size // 2,
- stride=1,
- )
- else:
- raise ValueError("Pooling method not supported")
- indices = attn_cache.topk(
- self.max_capacity_prompt - self.window_size, dim=-1
- ).indices
- indices = indices.unsqueeze(-1).expand(-1, -1, -1, head_dim)
- k_past_compress = key_states[:, :, : -self.window_size, :].gather(
- dim=2, index=indices
- )
- v_past_compress = value_states[:, :, : -self.window_size, :].gather(
- dim=2, index=indices
- )
- k_cur = key_states[:, :, -self.window_size :, :]
- v_cur = value_states[:, :, -self.window_size :, :]
- key_states = torch.cat([k_past_compress, k_cur], dim=2)
- value_states = torch.cat([v_past_compress, v_cur], dim=2)
- return key_states, value_states
-
-
-def init_snapkv(self):
- if not hasattr(self, "kv_cluster"):
- if not hasattr(self.config, "window_size"):
- self.config.window_size = 64
- if not hasattr(self.config, "max_capacity_prompt"):
- self.config.max_capacity_prompt = 4096
- if not hasattr(self.config, "kernel_size"):
- self.config.kernel_size = 13
- if not hasattr(self.config, "pooling"):
- self.config.pooling = "avgpool"
- self.kv_cluster = SnapKVCluster(
- window_size=self.config.window_size,
- max_capacity_prompt=self.config.max_capacity_prompt,
- kernel_size=self.config.kernel_size,
- pooling=self.config.pooling,
- )
-
-
-############
-
-
-def check_version():
- try:
- transformers_version = version("transformers")
- except Exception as e:
- print(f"Transformers not installed: {e}")
- return transformers_version
-
-
-def replace_llama():
- transformers_version = check_version()
- version_list = ["4.37"]
- warning_flag = True
- for version in version_list:
- if version in transformers_version:
- warning_flag = False
- break
- if warning_flag:
- warnings.warn(
- f"Transformers version {transformers_version} might not be compatible with SnapKV. SnapKV is tested with Transformers version {version_list}."
- )
- transformers.models.llama.modeling_llama.LlamaForCausalLM.prepare_inputs_for_generation = (
- prepare_inputs_for_generation_llama_4_37
- )
- transformers.models.llama.modeling_llama.LlamaFlashAttention2.forward = (
- llama_flash_attn2_forward_4_37
- )
diff --git a/minference/modules/snapkv.py b/minference/modules/snapkv.py
new file mode 100644
index 00000000..01d33295
--- /dev/null
+++ b/minference/modules/snapkv.py
@@ -0,0 +1,172 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://github.com/FasterDecoding/SnapKV/blob/main/snapkv/monkeypatch/snapkv_utils.py,
+# https://github.com/Zefan-Cai/PyramidKV/blob/main/pyramidkv/pyramidkv_utils.py
+
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class SnapKVCluster:
+ def __init__(
+ self,
+ window_size=64,
+ max_capacity_prompt=256 + 64,
+ kernel_size=5,
+ pooling="avgpool",
+ ):
+ self.window_size = window_size
+ self.max_capacity_prompt = max_capacity_prompt
+ assert self.max_capacity_prompt - self.window_size > 0
+ self.kernel_size = kernel_size
+ self.pooling = pooling
+
+ def reset(
+ self,
+ window_size=64,
+ max_capacity_prompt=256 + 64,
+ kernel_size=5,
+ pooling="avgpool",
+ ):
+ self.window_size = window_size
+ self.max_capacity_prompt = max_capacity_prompt
+ assert self.max_capacity_prompt - self.window_size > 0
+ self.kernel_size = kernel_size
+ self.pooling = pooling
+
+ def update_kv(
+ self,
+ key_states,
+ query_states,
+ value_states,
+ attention_mask,
+ num_key_value_groups,
+ ):
+ # check if prefix phase
+ assert key_states.shape[-2] == query_states.shape[-2]
+ bsz, num_heads, q_len, head_dim = query_states.shape
+ if q_len < self.max_capacity_prompt:
+ return key_states, value_states
+ else:
+ attn_weights = torch.matmul(
+ query_states[..., -self.window_size :, :], key_states.transpose(2, 3)
+ ) / math.sqrt(head_dim)
+ mask = torch.full(
+ (self.window_size, self.window_size),
+ torch.finfo(attn_weights.dtype).min,
+ device=attn_weights.device,
+ )
+ mask_cond = torch.arange(mask.size(-1), device=attn_weights.device)
+ mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+ mask = mask.to(attn_weights.device)
+ attention_mask = mask[None, None, :, :]
+
+ attn_weights[
+ :, :, -self.window_size :, -self.window_size :
+ ] += attention_mask
+
+ attn_weights = nn.functional.softmax(
+ attn_weights, dim=-1, dtype=torch.float32
+ ).to(query_states.dtype)
+ attn_weights_sum = attn_weights[
+ :, :, -self.window_size :, : -self.window_size
+ ].sum(dim=-2)
+ if self.pooling == "avgpool":
+ attn_cache = F.avg_pool1d(
+ attn_weights_sum,
+ kernel_size=self.kernel_size,
+ padding=self.kernel_size // 2,
+ stride=1,
+ )
+ elif self.pooling == "maxpool":
+ attn_cache = F.max_pool1d(
+ attn_weights_sum,
+ kernel_size=self.kernel_size,
+ padding=self.kernel_size // 2,
+ stride=1,
+ )
+ else:
+ raise ValueError("Pooling method not supported")
+ indices = attn_cache.topk(
+ self.max_capacity_prompt - self.window_size, dim=-1
+ ).indices
+ indices = indices.unsqueeze(-1).expand(-1, -1, -1, head_dim)
+ k_past_compress = key_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ v_past_compress = value_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ k_cur = key_states[:, :, -self.window_size :, :]
+ v_cur = value_states[:, :, -self.window_size :, :]
+ key_states = torch.cat([k_past_compress, k_cur], dim=2)
+ value_states = torch.cat([v_past_compress, v_cur], dim=2)
+ return key_states, value_states
+
+
+class StreamingLLMKVCluster:
+ def __init__(
+ self,
+ window_size=4096 - 128,
+ max_capacity_prompt=4096,
+ kernel_size=5,
+ pooling="avgpool",
+ ):
+ self.window_size = window_size
+ self.max_capacity_prompt = max_capacity_prompt
+ assert self.max_capacity_prompt - self.window_size > 0
+ self.kernel_size = kernel_size
+ self.pooling = pooling
+
+ def reset(
+ self,
+ window_size=4096 - 128,
+ max_capacity_prompt=4096,
+ kernel_size=5,
+ pooling="avgpool",
+ ):
+ self.window_size = window_size
+ self.max_capacity_prompt = max_capacity_prompt
+ assert self.max_capacity_prompt - self.window_size > 0
+ self.kernel_size = kernel_size
+ self.pooling = pooling
+
+ def update_kv(
+ self,
+ key_states,
+ query_states,
+ value_states,
+ attention_mask,
+ num_key_value_groups,
+ ):
+ # check if prefix phase
+ assert key_states.shape[-2] == query_states.shape[-2]
+ bsz, num_heads, q_len, head_dim = query_states.shape
+
+ if q_len < self.max_capacity_prompt:
+ return key_states, value_states
+ else:
+ indices = torch.tensor(
+ range(self.max_capacity_prompt - self.window_size), dtype=torch.int64
+ ).to(key_states.device)
+ indices = (
+ indices.unsqueeze(0)
+ .unsqueeze(0)
+ .unsqueeze(-1)
+ .repeat(bsz, num_heads, 1, head_dim)
+ )
+
+ k_past_compress = key_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ v_past_compress = value_states[:, :, : -self.window_size, :].gather(
+ dim=2, index=indices
+ )
+ k_cur = key_states[:, :, -self.window_size :, :]
+ v_cur = value_states[:, :, -self.window_size :, :]
+ key_states = torch.cat([k_past_compress, k_cur], dim=2)
+ value_states = torch.cat([v_past_compress, v_cur], dim=2)
+ return key_states, value_states
diff --git a/minference/modules/tri_mix.py b/minference/modules/tri_mix.py
new file mode 100644
index 00000000..85452f70
--- /dev/null
+++ b/minference/modules/tri_mix.py
@@ -0,0 +1,53 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from ..ops.streaming_kernel import tri_shape_kernel
+
+try:
+ from flash_attn import flash_attn_func
+except ImportError:
+ from ..ops.flash_attn_triton import _flash_attn_triton_decoding as flash_attn_func
+
+import copy
+
+import torch
+
+from ..modules.flexprefill import flexprefill_forward
+from ..modules.minference_forward import minference_prefill_forward
+
+
+def tri_mix_forward(query_states, key_states, value_states, prefill_kwargs):
+ starting_layer = prefill_kwargs["attn_forward_config"].get("starting_layer", 0)
+ layer_idx = prefill_kwargs["layer_idx"]
+
+ bsz, head_num, q_len, head_dim = query_states.shape
+ if layer_idx < starting_layer:
+ # flash attention
+ result = flash_attn_func(
+ query_states.transpose(1, 2),
+ key_states.transpose(1, 2),
+ value_states.transpose(1, 2),
+ 0.0,
+ softmax_scale=None,
+ causal=q_len != 1,
+ ).transpose(1, 2)
+ else:
+ result = tri_shape_kernel(
+ query_states, key_states, value_states, prefill_kwargs
+ )
+
+ return result
+
+
+def tri_mix_minference_forward(q, k, v, prefill_kwargs):
+ layer_idx = prefill_kwargs["layer_idx"]
+ starting_layer = prefill_kwargs["attn_forward_config"].get("starting_layer", 0)
+
+ if layer_idx < starting_layer:
+ # minference
+ minference_prefill_kwargs = copy.deepcopy(prefill_kwargs)
+ minference_prefill_kwargs["attn_forward_config"]["starting_layer"] = 0
+ result = minference_prefill_forward(q, k, v, minference_prefill_kwargs)
+ else:
+ result = tri_shape_kernel(q, k, v, prefill_kwargs)
+ return result
diff --git a/minference/modules/xattention.py b/minference/modules/xattention.py
new file mode 100644
index 00000000..c833bc63
--- /dev/null
+++ b/minference/modules/xattention.py
@@ -0,0 +1,599 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://github.com/mit-han-lab/x-attention
+
+import math
+
+import torch
+import torch.nn.functional as F
+
+from ..ops.xattention_fa import flat_group_gemm_fuse_reshape, softmax_fuse_block_sum
+
+
+def find_blocks_chunked(
+ input_tensor,
+ current_index,
+ threshold,
+ num_to_choose,
+ decoding: bool,
+ mode: str = "both",
+ causal=True,
+):
+ """
+ Finds and selects relevant blocks of attention for transformer-based models based on a
+ threshold or a predefined number of blocks.
+
+ Parameters:
+ - input_tensor (torch.Tensor): The input tensor of shape (batch_size, head_num, chunk_num, block_num).
+ - current_index (int): The current index in the sequence processing.
+ - threshold (float or None): A threshold value used to determine the minimum attention weight sum.
+ - num_to_choose (int or None): The number of blocks to be selected, ensuring sufficient information retrieval.
+ - decoding (bool): If True, operates in decoding mode; otherwise, it's in encoding mode.
+ - mode (str): Defines the processing mode, either 'both', 'prefill', or 'decode'.
+ - causal (bool): If True, applies causal masking to prevent future information leakage.
+
+ Returns:
+ - torch.Tensor: A boolean mask of shape (batch_size, head_num, chunk_num, block_num),
+ indicating which blocks should be attended to.
+ """
+ assert threshold is None or num_to_choose is None
+ batch_size, head_num, chunk_num, block_num = input_tensor.shape
+ # 0 -- -- -- -- current_index
+ # 0 -- -- -- -- -- current_index+1
+ # 0 -- -- -- -- -- ----------- current_index + chunk_num - 1
+ if mode == "prefill" and decoding:
+ return torch.ones_like(input_tensor, dtype=torch.bool)
+ if mode == "decode" and not decoding:
+ mask = torch.ones_like(input_tensor, dtype=torch.bool)
+ if causal:
+ mask[:, :, :, current_index : current_index + chunk_num] = torch.tril(
+ torch.ones(
+ 1, head_num, chunk_num, chunk_num, device=input_tensor.device
+ )
+ )
+ mask[:, :, current_index + chunk_num :, :] = 0
+ return torch.cat(
+ [
+ torch.ones_like(input_tensor, dtype=torch.bool)[
+ :, :, 0 : current_index + 1
+ ],
+ torch.zeros_like(input_tensor, dtype=torch.bool)[
+ :, :, current_index + 1 :
+ ],
+ ],
+ dim=-1,
+ )
+ else:
+ return mask
+ input_tensor = input_tensor.to(float)
+
+ if threshold is not None:
+ total_sum = input_tensor.sum(dim=-1, keepdim=True)
+ if isinstance(threshold, torch.Tensor):
+ threshold = threshold.to(float)
+ required_sum = total_sum * threshold.unsqueeze(0).unsqueeze(-1).unsqueeze(
+ -1
+ ).expand((batch_size, head_num, chunk_num, 1)).to(input_tensor.device)
+ else:
+ required_sum = total_sum * threshold
+ if causal:
+ mask = torch.zeros_like(input_tensor, dtype=torch.bool)
+ mask[:, :, :, 0] = 1
+ mask[:, :, :, current_index : current_index + chunk_num] = (
+ torch.eye(chunk_num, device=mask.device)
+ .unsqueeze(0)
+ .unsqueeze(0)
+ .expand(1, head_num, chunk_num, chunk_num)
+ )
+ other_values = input_tensor.masked_fill(mask, 0)
+ sorted_values, _ = torch.sort(other_values, dim=-1, descending=True)
+ sorted_values = sorted_values.to(input_tensor.device)
+
+ sorted_values = torch.cat(
+ [
+ torch.zeros(
+ (batch_size, head_num, chunk_num, 1), device=input_tensor.device
+ ),
+ torch.where(mask, input_tensor, 0).sum(dim=-1, keepdim=True),
+ sorted_values[:, :, :, :-2],
+ ],
+ dim=-1,
+ )
+
+ _, index = torch.sort(
+ torch.where(mask, 100000 * (1 + input_tensor), input_tensor),
+ dim=-1,
+ descending=True,
+ )
+ cumulative_sum_without_self = torch.cat(
+ [
+ torch.zeros(
+ (batch_size, head_num, chunk_num, 1), device=input_tensor.device
+ ),
+ sorted_values[:, :, :, 0:-1],
+ ],
+ dim=-1,
+ ).cumsum(dim=-1)
+
+ index_mask = cumulative_sum_without_self < required_sum
+ index = torch.where(index_mask, index, 0)
+ mask = mask.view(batch_size, head_num * chunk_num, block_num)
+ index = index.view(batch_size, head_num * chunk_num, block_num)
+ mask[
+ :,
+ torch.arange(mask.shape[1], device=mask.device).unsqueeze(dim=-1),
+ index,
+ ] = True
+ mask = mask.view(batch_size, head_num, chunk_num, block_num)
+ # assert(bool((torch.where(mask,input_tensor,0).sum(dim=-1,keepdim=True) >= required_sum*0.99).all()))
+ else:
+ mask = torch.zeros_like(input_tensor, dtype=torch.bool)
+ sorted_values, index = torch.sort(input_tensor, dim=-1, descending=True)
+ sorted_values = sorted_values.to(input_tensor.device)
+ cumulative_sum_without_self = torch.cat(
+ [
+ torch.zeros(
+ (batch_size, head_num, chunk_num, 1), device=input_tensor.device
+ ),
+ sorted_values[:, :, :, 0:-1],
+ ],
+ dim=-1,
+ ).cumsum(dim=-1)
+ index_mask = cumulative_sum_without_self < required_sum
+ index = torch.where(index_mask, index, 0)
+ mask = mask.view(batch_size, head_num * chunk_num, block_num)
+ index = index.view(batch_size, head_num * chunk_num, block_num)
+ mask[
+ :,
+ torch.arange(mask.shape[1], device=mask.device).unsqueeze(dim=-1),
+ index,
+ ] = True
+ mask = mask.view(batch_size, head_num, chunk_num, block_num)
+ else:
+ raise NotImplementedError("block num chunk prefill not impleted")
+
+ try:
+ if causal:
+ assert (~mask[:, :, :, current_index + chunk_num :]).all()
+ except:
+ mask[:, :, :, current_index + chunk_num :] = False
+
+ if causal:
+ if decoding:
+ assert mask[:, :, :, 0].all() and mask[:, :, :, -1].all()
+ else:
+ lambda_mask = torch.zeros_like(
+ input_tensor, dtype=bool, device=input_tensor.device
+ )
+ lambda_mask[:, :, :, 0] = 1
+ lambda_mask[:, :, :, current_index : current_index + chunk_num] = (
+ torch.eye(chunk_num, device=lambda_mask.device)
+ .unsqueeze(0)
+ .unsqueeze(0)
+ .expand(1, head_num, chunk_num, chunk_num)
+ )
+ assert torch.where(lambda_mask, mask, True).all()
+
+ return mask
+
+
+def xattn_estimate(
+ query_states: torch.Tensor,
+ key_states: torch.Tensor,
+ block_size,
+ stride,
+ norm=1,
+ softmax=True,
+ threshold=0.9,
+ chunk_size=16384,
+ select_mode="inverse",
+ use_triton=True,
+ causal=True,
+ kdb: int = 1,
+ keep_sink=False,
+ keep_recent=False,
+) -> torch.Tensor:
+ batch_size, num_kv_head, k_len, head_dim = key_states.shape
+ batch_size, num_q_head, q_len, head_dim = query_states.shape
+ assert num_q_head == num_kv_head
+
+ k_num_to_pad = ((k_len + chunk_size - 1) // chunk_size) * chunk_size - k_len
+ q_num_to_pad = ((q_len + chunk_size - 1) // chunk_size) * chunk_size - q_len
+ k_chunk_num = (k_len + k_num_to_pad) // chunk_size
+ k_block_num = (k_len + k_num_to_pad) // block_size
+ q_chunk_num = (q_len + q_num_to_pad) // chunk_size
+ q_block_num = (q_len + q_num_to_pad) // block_size
+
+ if k_num_to_pad > 0:
+ pad_key_states = F.pad(key_states, (0, 0, 0, k_num_to_pad), value=0).to("cuda")
+ else:
+ pad_key_states = key_states
+ if q_num_to_pad > 0:
+ pad_query_states = F.pad(query_states, (0, 0, 0, q_num_to_pad), value=0).to(
+ "cuda"
+ )
+ else:
+ pad_query_states = query_states
+
+ assert num_kv_head == num_q_head
+ attn_sum_list = []
+ simple_mask_list = []
+
+ if use_triton and (
+ "100" not in torch.cuda.get_device_properties(torch.cuda.current_device()).name
+ ):
+ use_triton = False
+ print(
+ "setting use triton to false. Triton kernel not surpported on this device"
+ )
+
+ reshaped_chunk_size = chunk_size // stride
+ reshaped_block_size = block_size // stride
+ k_reshaped_num_to_pad = k_num_to_pad // stride
+ k_reshaped_seq_len = (k_len + k_num_to_pad) // stride
+ q_reshaped_num_to_pad = q_num_to_pad // stride
+ num_blocks_per_chunk = reshaped_chunk_size // reshaped_block_size
+ if not use_triton:
+ if select_mode == "random":
+ perm_idx = torch.randperm(stride)
+ reshaped_key = torch.cat(
+ [(pad_key_states[:, :, k::stride, :]) for k in range(stride)], dim=-1
+ )
+ reshaped_query = torch.cat(
+ [
+ pad_query_states[:, :, perm_idx[i] :: stride, :]
+ for i in range(stride)
+ ],
+ dim=-1,
+ )
+ elif select_mode == "inverse" or select_mode == "":
+ reshaped_key = torch.cat(
+ [(pad_key_states[:, :, k::stride, :]) for k in range(stride)], dim=-1
+ )
+ reshaped_query = torch.cat(
+ [
+ (pad_query_states[:, :, (stride - 1 - q) :: (stride * kdb), :])
+ for q in range(stride)
+ ],
+ dim=-1,
+ )
+ elif select_mode == "slash":
+ reshaped_key = torch.cat(
+ [(pad_key_states[:, :, k::stride, :]) for k in range(stride)], dim=-1
+ )
+ reshaped_query = torch.cat(
+ [(pad_query_states[:, :, q::stride, :]) for q in range(stride)], dim=-1
+ )
+ elif select_mode == "double":
+ reshaped_key = torch.cat(
+ [(pad_key_states[:, :, k::stride, :]) for k in range(stride)], dim=-1
+ )
+ reshaped_key = reshaped_key + torch.cat(
+ [reshaped_key[:, :, :, head_dim:], reshaped_key[:, :, :, 0:head_dim]],
+ dim=-1,
+ )
+ reshaped_query = torch.cat(
+ [
+ (pad_query_states[:, :, (stride - 1 - q) :: stride, :])
+ for q in range(stride)
+ ],
+ dim=-1,
+ )
+ elif select_mode == "triple":
+ reshaped_key = torch.cat(
+ [(pad_key_states[:, :, k::stride, :]) for k in range(stride)], dim=-1
+ )
+ reshaped_key = reshaped_key + torch.cat(
+ [reshaped_key[:, :, :, head_dim:], reshaped_key[:, :, :, 0:head_dim]],
+ dim=-1,
+ )
+ reshaped_key = reshaped_key + torch.cat(
+ [reshaped_key[:, :, :, -head_dim:], reshaped_key[:, :, :, 0:-head_dim]],
+ dim=-1,
+ )
+ reshaped_query = torch.cat(
+ [
+ (pad_query_states[:, :, (stride - 1 - q) :: stride, :])
+ for q in range(stride)
+ ],
+ dim=-1,
+ )
+ assert reshaped_key.shape[-2] == k_reshaped_seq_len
+
+ for chunk_idx in range(q_chunk_num):
+ if use_triton:
+ if kdb != 1:
+ raise ValueError("use_triton and kdb cannot be used together")
+ attn_weights_slice = flat_group_gemm_fuse_reshape(
+ pad_query_states[
+ :,
+ :,
+ (chunk_idx * reshaped_chunk_size)
+ * stride : (chunk_idx * reshaped_chunk_size + reshaped_chunk_size)
+ * stride,
+ :,
+ ],
+ pad_key_states,
+ stride,
+ (k_block_num - q_block_num) * reshaped_block_size
+ + chunk_idx * reshaped_chunk_size,
+ (k_block_num - q_block_num) * reshaped_block_size
+ + chunk_idx * reshaped_chunk_size
+ + reshaped_chunk_size,
+ is_causal=causal,
+ )
+ attn_sum = softmax_fuse_block_sum(
+ attn_weights_slice,
+ reshaped_block_size,
+ min(4096, reshaped_block_size),
+ (k_block_num - q_block_num) * reshaped_block_size
+ + chunk_idx * reshaped_chunk_size,
+ (k_block_num - q_block_num) * reshaped_block_size
+ + chunk_idx * reshaped_chunk_size
+ + reshaped_chunk_size,
+ k_reshaped_seq_len - k_reshaped_num_to_pad,
+ 1.4426950408889634 / math.sqrt(head_dim) / stride / norm,
+ is_causal=causal,
+ )
+ else:
+ chunked_query = reshaped_query[
+ :,
+ :,
+ (chunk_idx * reshaped_chunk_size)
+ // kdb : (chunk_idx * reshaped_chunk_size + reshaped_chunk_size)
+ // kdb,
+ :,
+ ]
+ attn_weights_slice = torch.matmul(
+ chunked_query,
+ reshaped_key.transpose(2, 3),
+ ).to("cuda")
+
+ attn_weights_slice = (
+ attn_weights_slice / math.sqrt(head_dim) / stride / norm
+ )
+
+ if causal:
+ causal_mask = torch.zeros(
+ (
+ batch_size,
+ num_q_head,
+ reshaped_chunk_size,
+ reshaped_chunk_size * k_chunk_num,
+ ),
+ device=key_states.device,
+ )
+ causal_mask[:, :, :, (-k_reshaped_num_to_pad):] = float("-inf")
+ chunk_start = chunk_idx * reshaped_chunk_size
+ chunk_end = chunk_start + reshaped_chunk_size
+ causal_mask[:, :, :, chunk_start:chunk_end] = torch.triu(
+ torch.ones(
+ 1,
+ num_q_head,
+ reshaped_chunk_size,
+ reshaped_chunk_size,
+ device=key_states.device,
+ )
+ * float("-inf"),
+ diagonal=1,
+ )
+
+ if chunk_idx == q_chunk_num - 1 and q_reshaped_num_to_pad != 0:
+ causal_mask[:, :, (-(q_reshaped_num_to_pad // kdb)) :, :] = float(
+ "-inf"
+ )
+
+ causal_mask[:, :, :, chunk_end:] = float("-inf")
+ causal_mask = causal_mask[:, :, kdb - 1 :: kdb, :]
+ attn_weights_slice = attn_weights_slice + causal_mask.to(
+ attn_weights_slice.device
+ )
+
+ if softmax:
+ attn_weights_slice = F.softmax(
+ attn_weights_slice, dim=-1, dtype=torch.float32
+ ).to(pad_query_states.dtype)
+ else:
+ attn_weights_slice = torch.exp(attn_weights_slice).to(
+ pad_query_states.dtype
+ )
+ attn_weights_slice = F.dropout(attn_weights_slice, p=0, training=False)
+
+ if chunk_idx == q_chunk_num - 1 and q_reshaped_num_to_pad != 0:
+ attn_weights_slice[:, :, (-(q_reshaped_num_to_pad // kdb)) :, :] = 0
+
+ attn_sum = (
+ attn_weights_slice.view(
+ batch_size,
+ num_kv_head,
+ num_blocks_per_chunk,
+ reshaped_block_size // kdb,
+ -1,
+ reshaped_block_size,
+ )
+ .sum(dim=-1)
+ .sum(dim=-2)
+ .to("cuda")
+ )
+ del chunked_query
+
+ simple_mask = find_blocks_chunked(
+ attn_sum,
+ k_block_num - q_block_num + chunk_idx * num_blocks_per_chunk,
+ threshold,
+ None,
+ decoding=False,
+ mode="prefill",
+ causal=causal,
+ )
+
+ attn_sum_list.append(attn_sum)
+ simple_mask_list.append(simple_mask)
+
+ del attn_weights_slice
+
+ if not use_triton:
+ del reshaped_query, reshaped_key
+ attn_sums = torch.cat(attn_sum_list, dim=-2)
+ simple_masks = torch.cat(simple_mask_list, dim=-2)
+
+ if causal:
+ simple_masks[:, :, -q_block_num:, -q_block_num:] = torch.where(
+ torch.tril(
+ torch.ones(
+ q_block_num, q_block_num, dtype=bool, device=key_states.device
+ ),
+ diagonal=0,
+ ),
+ simple_masks[:, :, -q_block_num:, -q_block_num:],
+ False,
+ )
+ if keep_sink:
+ simple_masks[:, :, 0, :] = True
+ if keep_recent:
+ eye_matrix = torch.eye(q_block_num, device=simple_masks.device, dtype=bool)
+ eye_matrix_expanded = (
+ eye_matrix.unsqueeze(0)
+ .unsqueeze(0)
+ .expand(1, num_kv_head, q_block_num, q_block_num)
+ )
+ simple_masks[:, :, -q_block_num:, -q_block_num:] = torch.where(
+ eye_matrix_expanded, True, simple_masks[:, :, -q_block_num:, -q_block_num:]
+ )
+
+ return attn_sums, simple_masks
+
+
+def Xattention_prefill(
+ query_states: torch.Tensor,
+ key_states: torch.Tensor,
+ value_states: torch.Tensor,
+ stride,
+ norm=1,
+ threshold=0.8,
+ block_size=128,
+ use_triton=True,
+ causal=True,
+ kdb=1,
+ chunk_size=None,
+ keep_sink=False,
+ keep_recent=False,
+):
+ try:
+ from block_sparse_attn import block_sparse_attn_func
+ except:
+ assert (
+ False
+ ), "Please install papyfaiss. Refer to https://github.com/mit-han-lab/Block-Sparse-Attention"
+
+ batch_size, num_heads, k_len, head_dim = key_states.shape
+ _, _, q_len, _ = query_states.shape
+
+ q_block_num = (q_len + block_size - 1) // block_size
+ k_block_num = (k_len + block_size - 1) // block_size
+ if chunk_size is None:
+ chunk_size = int(
+ max(
+ min(
+ max(2048, 1 << (k_len - 1).bit_length()),
+ 128 * 1024 * 2048 // (1 << (k_len - 1).bit_length()),
+ ),
+ 2048,
+ )
+ )
+ attn_sums, approx_simple_mask = xattn_estimate(
+ query_states,
+ key_states,
+ block_size=block_size,
+ stride=stride,
+ norm=norm,
+ threshold=threshold,
+ select_mode="inverse",
+ use_triton=use_triton,
+ causal=causal,
+ chunk_size=chunk_size,
+ kdb=kdb,
+ keep_sink=keep_sink,
+ keep_recent=keep_recent,
+ )
+
+ if query_states.device != key_states.device:
+ key_states = key_states.to(query_states.device)
+ if query_states.device != value_states.device:
+ value_states = value_states.to(query_states.device)
+ if approx_simple_mask.device != query_states.device:
+ approx_simple_mask = approx_simple_mask.to(query_states.device)
+
+ ####################
+ assert block_size == 128
+ assert batch_size == 1
+ query_states = query_states.transpose(1, 2).view(q_len, num_heads, head_dim)
+ key_states = key_states.transpose(1, 2).view(k_len, num_heads, head_dim)
+ value_states = value_states.transpose(1, 2).view(k_len, num_heads, head_dim)
+ q_cu_seq_lens = torch.tensor(
+ [0, q_len], dtype=torch.int32, device=query_states.device
+ )
+ k_cu_seq_lens = torch.tensor(
+ [0, k_len], dtype=torch.int32, device=query_states.device
+ )
+ head_mask_type = torch.tensor(
+ [1 for _ in range(num_heads)], device=query_states.device, dtype=torch.int32
+ )
+ assert head_mask_type.device == query_states.device
+ assert q_cu_seq_lens.device == query_states.device
+ assert k_cu_seq_lens.device == query_states.device
+ assert key_states.device == query_states.device
+ assert value_states.device == query_states.device
+ assert approx_simple_mask.device == query_states.device
+
+ attn_output = block_sparse_attn_func(
+ query_states,
+ key_states,
+ value_states,
+ q_cu_seq_lens,
+ k_cu_seq_lens,
+ head_mask_type,
+ None,
+ approx_simple_mask[:, :, :q_block_num, :k_block_num].contiguous(),
+ q_len,
+ k_len,
+ p_dropout=0.0,
+ deterministic=True,
+ is_causal=causal,
+ )
+ attn_output = attn_output.view(batch_size, q_len, num_heads, head_dim).transpose(
+ 1, 2
+ )
+ ################################
+
+ del query_states
+ num_to_compute = (k_block_num + 1) * k_block_num / 2 * num_heads
+
+ # print(f"approximated prefilling Computation: {approx_simple_mask.sum() / num_to_compute}")
+ del approx_simple_mask, attn_sums
+ return attn_output
+
+
+def xattention_forward(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ config,
+):
+ stride = config["attn_forward_config"].get("stride", 8)
+ norm = config["attn_forward_config"].get("norm", 1)
+ threshold = config["attn_forward_config"].get("threshold", 0.9)
+ block_size = config["attn_forward_config"].get("block_size", 128)
+ chunk_size = config["attn_forward_config"].get("chunk_size", 2048)
+
+ out = Xattention_prefill(
+ q,
+ k,
+ v,
+ stride=stride,
+ norm=norm,
+ threshold=threshold,
+ block_size=block_size,
+ chunk_size=chunk_size,
+ )
+ return out
diff --git a/minference/ops/leank_flash_decoding.py b/minference/ops/leank_flash_decoding.py
new file mode 100644
index 00000000..4e25696d
--- /dev/null
+++ b/minference/ops/leank_flash_decoding.py
@@ -0,0 +1,724 @@
+# Copyright (c) 2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import itertools
+
+import tilelang
+import tilelang.language as T
+import torch
+import torch.nn.functional as F
+from tilelang.autotuner import *
+
+torch.random.manual_seed(0)
+
+def get_configs():
+ block_N = [64, 128]
+ block_H = [64, 32]
+ num_split = [2, 4, 8, 16]
+ num_stages = [1, 2, 3]
+ threads = [128]
+ _configs = list(itertools.product(block_N, block_H, num_split, num_stages, threads))
+
+ configs = [{
+ 'block_N': c[0],
+ 'block_H': c[1],
+ 'num_split': c[2],
+ 'num_stages': c[3],
+ 'threads': c[4]
+ } for c in _configs]
+ return configs
+
+def leank_flashattn(batch, heads, heads1, heads2, heads3, heads4, groups, groups1, groups2, groups3, groups4, seqlen_kv, true_seq_len, seqlen_fullkv, true_full_len, dim, dim1, dim2, dim3, dim4, ndim, dtype, tune=False):
+ scale = (1.0 / dim)**0.5 * 1.44269504 # log2(e)
+ shape_q1 = [batch, heads1, dim1]
+ shape_k1 = [batch, groups1, true_seq_len, dim1]
+ shape_v1 = [batch, groups1, true_seq_len, dim]
+
+ shape_q2 = [batch, heads2, dim2]
+ shape_k2 = [batch, groups2, true_seq_len, dim2]
+ shape_v2 = [batch, groups2, true_seq_len, dim]
+
+ shape_q3 = [batch, heads3, dim3]
+ shape_k3 = [batch, groups3, true_seq_len, dim3]
+ shape_v3 = [batch, groups3, true_seq_len, dim]
+
+ shape_q4 = [batch, heads4, dim]
+ shape_k4 = [batch, groups4, true_seq_len, dim4]
+ shape_v4 = [batch, groups4, true_seq_len, dim]
+
+ shape_full_q = [batch, heads, dim]
+ shape_full_k = [batch, groups, true_full_len, dim]
+ shape_full_v = [batch, groups, true_full_len, dim]
+
+ shape_o = [batch, heads, dim]
+
+ accum_dtype = "float"
+ kv_group_num = heads // groups
+
+ def kernel_func(block_N, block_H, num_split, num_stages, threads):
+ part_shape = [batch, heads, num_split + 1, dim]
+ part_shape1 = [batch, heads1, num_split + 1, dim]
+ part_shape2 = [batch, heads2, num_split + 1, dim]
+ part_shape3 = [batch, heads3, num_split + 1, dim]
+ part_shape4 = [batch, heads4, num_split + 1, dim]
+ valid_block_H = min(block_H, kv_group_num)
+
+ @T.macro
+ def flash_attn_split1(
+ Q: T.Tensor(shape_q1, dtype),
+ K: T.Tensor(shape_k1, dtype),
+ V: T.Tensor(shape_v1, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ glse1: T.Tensor([batch, heads1, num_split + 1], dtype),
+ Output_partial1: T.Tensor(part_shape1, dtype),
+ ):
+ with T.Kernel(
+ batch, heads1 // valid_block_H, num_split, threads=threads) as (bx, by, bz):
+ Q_shared = T.alloc_shared([block_H, dim1], dtype)
+ K_shared = T.alloc_shared([block_N, dim1], dtype)
+ V_shared = T.alloc_shared([block_N, dim], dtype)
+ O_shared = T.alloc_shared([valid_block_H, dim], dtype)
+ acc_s = T.alloc_fragment([block_H, block_N], accum_dtype)
+ acc_s_cast = T.alloc_fragment([block_H, block_N], dtype)
+ mask_local = T.alloc_fragment([block_N], "uint8")
+ acc_o = T.alloc_fragment([block_H, dim], accum_dtype)
+ scores_max = T.alloc_fragment([block_H], accum_dtype)
+
+ scores_max_prev = T.alloc_fragment([block_H], accum_dtype)
+ scores_scale = T.alloc_fragment([block_H], accum_dtype)
+ scores_sum = T.alloc_fragment([block_H], accum_dtype)
+ logsum = T.alloc_fragment([block_H], accum_dtype)
+
+ bid = bx
+ hid = by
+ sid = bz
+ cur_kv_head = hid // (kv_group_num // valid_block_H)
+
+ T.copy(Q[bid, hid * valid_block_H: hid * valid_block_H + block_H, :], Q_shared)
+ T.fill(acc_o, 0)
+ T.fill(logsum, 0)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ per_block_len = T.ceildiv(seqlen_kv, num_split)
+ this_block_end = T.min(per_block_len * (sid + 1), true_seq_len)
+ this_block_begin = per_block_len * sid
+
+ if this_block_begin < true_seq_len:
+ loop_range = T.ceildiv(per_block_len, block_N)
+ for k in T.Pipelined(loop_range, num_stages=num_stages):
+ if per_block_len * sid + k * block_N < true_seq_len:
+ T.copy(
+ K[bid, cur_kv_head, per_block_len * sid +
+ k * block_N: per_block_len * sid + (k + 1) * block_N, :], K_shared)
+ for i in T.Parallel(block_N):
+ if per_block_len * sid + k * block_N + i < this_block_end:
+ mask_local[i] = mask_mid[bid, per_block_len * sid +
+ k * block_N + i]
+ else:
+ mask_local[i] = 0
+ T.clear(acc_s)
+ T.gemm(
+ Q_shared,
+ K_shared,
+ acc_s,
+ transpose_B=True,
+ policy=T.GemmWarpPolicy.FullRow, )
+
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.if_then_else(mask_local[j] != 0, acc_s[i, j],
+ -T.infinity(accum_dtype))
+
+ T.copy(scores_max, scores_max_prev)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ T.reduce_max(acc_s, scores_max, dim=1, clear=False)
+ for i in T.Parallel(block_H):
+ scores_scale[i] = T.exp2(scores_max_prev[i] * scale - scores_max[i] * scale)
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.exp2(acc_s[i, j] * scale - scores_max[i] * scale)
+ T.reduce_sum(acc_s, scores_sum, dim=1)
+ for i in T.Parallel(block_H):
+ logsum[i] = logsum[i] * scores_scale[i] + scores_sum[i]
+ T.copy(acc_s, acc_s_cast)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] *= scores_scale[i]
+ T.copy(
+ V[bid, cur_kv_head, per_block_len * sid +
+ k * block_N:per_block_len * sid + (k + 1) * block_N, :], V_shared)
+ T.gemm(acc_s_cast, V_shared, acc_o, policy=T.GemmWarpPolicy.FullRow)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] /= logsum[i]
+ for i in T.Parallel(block_H):
+ logsum[i] = T.log2(logsum[i]) + scores_max[i] * scale
+
+ for i in T.Parallel(block_H):
+ if i < valid_block_H:
+ glse1[bid, hid * valid_block_H + i, sid] = logsum[i]
+ T.copy(acc_o[:valid_block_H, :], O_shared)
+ T.copy(O_shared, Output_partial1[bid, hid * valid_block_H:(hid + 1) * valid_block_H,
+ sid, :])
+
+ @T.macro
+ def flash_attn_split2(
+ Q2: T.Tensor(shape_q2, dtype),
+ K2: T.Tensor(shape_k2, dtype),
+ V2: T.Tensor(shape_v2, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ glse2: T.Tensor([batch, heads2, num_split + 1], dtype),
+ Output_partial2: T.Tensor(part_shape2, dtype),
+ ):
+ with T.Kernel(
+ batch, heads1 // valid_block_H, num_split, threads=threads) as (bx, by, bz):
+ Q_shared = T.alloc_shared([block_H, dim2], dtype)
+ K_shared = T.alloc_shared([block_N, dim2], dtype)
+ V_shared = T.alloc_shared([block_N, dim], dtype)
+ O_shared = T.alloc_shared([valid_block_H, dim], dtype)
+ acc_s = T.alloc_fragment([block_H, block_N], accum_dtype)
+ acc_s_cast = T.alloc_fragment([block_H, block_N], dtype)
+ mask_local = T.alloc_fragment([block_N], "uint8")
+ acc_o = T.alloc_fragment([block_H, dim], accum_dtype)
+ scores_max = T.alloc_fragment([block_H], accum_dtype)
+
+ scores_max_prev = T.alloc_fragment([block_H], accum_dtype)
+ scores_scale = T.alloc_fragment([block_H], accum_dtype)
+ scores_sum = T.alloc_fragment([block_H], accum_dtype)
+ logsum = T.alloc_fragment([block_H], accum_dtype)
+
+ bid = bx
+ hid = by
+ sid = bz
+ cur_kv_head = hid // (kv_group_num // valid_block_H)
+
+ T.copy(Q2[bid, hid * valid_block_H: hid * valid_block_H + block_H, :], Q_shared)
+ T.fill(acc_o, 0)
+ T.fill(logsum, 0)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ per_block_len = T.ceildiv(seqlen_kv, num_split)
+ this_block_end = T.min(per_block_len * (sid + 1), true_seq_len)
+ this_block_begin = per_block_len * sid
+
+ if this_block_begin < true_seq_len:
+ loop_range = T.ceildiv(per_block_len, block_N)
+ for k in T.Pipelined(loop_range, num_stages=2):
+ if per_block_len * sid + k * block_N < true_seq_len:
+ T.copy(
+ K2[bid, cur_kv_head, per_block_len * sid +
+ k * block_N: per_block_len * sid + (k + 1) * block_N, :], K_shared)
+ for i in T.Parallel(block_N):
+ if per_block_len * sid + k * block_N + i < this_block_end:
+ mask_local[i] = mask_mid[bid, per_block_len * sid +
+ k * block_N + i]
+ else:
+ mask_local[i] = 0
+ T.clear(acc_s)
+ T.gemm(
+ Q_shared,
+ K_shared,
+ acc_s,
+ transpose_B=True,
+ policy=T.GemmWarpPolicy.FullRow, )
+
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.if_then_else(mask_local[j] != 0, acc_s[i, j],
+ -T.infinity(accum_dtype))
+
+ T.copy(scores_max, scores_max_prev)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ T.reduce_max(acc_s, scores_max, dim=1, clear=False)
+ for i in T.Parallel(block_H):
+ scores_scale[i] = T.exp2(scores_max_prev[i] * scale - scores_max[i] * scale)
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.exp2(acc_s[i, j] * scale - scores_max[i] * scale)
+ T.reduce_sum(acc_s, scores_sum, dim=1)
+ for i in T.Parallel(block_H):
+ logsum[i] = logsum[i] * scores_scale[i] + scores_sum[i]
+ T.copy(acc_s, acc_s_cast)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] *= scores_scale[i]
+ T.copy(
+ V2[bid, cur_kv_head, per_block_len * sid +
+ k * block_N:per_block_len * sid + (k + 1) * block_N, :], V_shared)
+ T.gemm(acc_s_cast, V_shared, acc_o, policy=T.GemmWarpPolicy.FullRow)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] /= logsum[i]
+ for i in T.Parallel(block_H):
+ logsum[i] = T.log2(logsum[i]) + scores_max[i] * scale
+
+ for i in T.Parallel(block_H):
+ if i < valid_block_H:
+ glse2[bid, hid * valid_block_H + i, sid] = logsum[i]
+ T.copy(acc_o[:valid_block_H, :], O_shared)
+ T.copy(O_shared, Output_partial2[bid, hid * valid_block_H:(hid + 1) * valid_block_H,
+ sid, :])
+
+ @T.macro
+ def flash_attn_split3(
+ Q3: T.Tensor(shape_q3, dtype),
+ K3: T.Tensor(shape_k3, dtype),
+ V3: T.Tensor(shape_v3, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ glse3: T.Tensor([batch, heads3, num_split + 1], dtype),
+ Output_partial3: T.Tensor(part_shape3, dtype),
+ ):
+ with T.Kernel(
+ batch, heads1 // valid_block_H, num_split, threads=threads) as (bx, by, bz):
+ Q_shared = T.alloc_shared([block_H, dim3], dtype)
+ K_shared = T.alloc_shared([block_N, dim3], dtype)
+ V_shared = T.alloc_shared([block_N, dim], dtype)
+ O_shared = T.alloc_shared([valid_block_H, dim], dtype)
+ acc_s = T.alloc_fragment([block_H, block_N], accum_dtype)
+ acc_s_cast = T.alloc_fragment([block_H, block_N], dtype)
+ mask_local = T.alloc_fragment([block_N], "uint8")
+ acc_o = T.alloc_fragment([block_H, dim], accum_dtype)
+ scores_max = T.alloc_fragment([block_H], accum_dtype)
+
+ scores_max_prev = T.alloc_fragment([block_H], accum_dtype)
+ scores_scale = T.alloc_fragment([block_H], accum_dtype)
+ scores_sum = T.alloc_fragment([block_H], accum_dtype)
+ logsum = T.alloc_fragment([block_H], accum_dtype)
+
+ bid = bx
+ hid = by
+ sid = bz
+ cur_kv_head = hid // (kv_group_num // valid_block_H)
+
+ T.copy(Q3[bid, hid * valid_block_H: hid * valid_block_H + block_H, :], Q_shared)
+ T.fill(acc_o, 0)
+ T.fill(logsum, 0)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ per_block_len = T.ceildiv(seqlen_kv, num_split)
+ this_block_end = T.min(per_block_len * (sid + 1), true_seq_len)
+ this_block_begin = per_block_len * sid
+
+ if this_block_begin < true_seq_len:
+ loop_range = T.ceildiv(per_block_len, block_N)
+ for k in T.Pipelined(loop_range, num_stages=num_stages):
+ if per_block_len * sid + k * block_N < true_seq_len:
+ T.copy(
+ K3[bid, cur_kv_head, per_block_len * sid +
+ k * block_N: per_block_len * sid + (k + 1) * block_N, :], K_shared)
+ for i in T.Parallel(block_N):
+ if per_block_len * sid + k * block_N + i < this_block_end:
+ mask_local[i] = mask_mid[bid, per_block_len * sid +
+ k * block_N + i]
+ else:
+ mask_local[i] = 0
+ T.clear(acc_s)
+ T.gemm(
+ Q_shared,
+ K_shared,
+ acc_s,
+ transpose_B=True,
+ policy=T.GemmWarpPolicy.FullRow, )
+
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.if_then_else(mask_local[j] != 0, acc_s[i, j],
+ -T.infinity(accum_dtype))
+
+ T.copy(scores_max, scores_max_prev)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ T.reduce_max(acc_s, scores_max, dim=1, clear=False)
+ for i in T.Parallel(block_H):
+ scores_scale[i] = T.exp2(scores_max_prev[i] * scale - scores_max[i] * scale)
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.exp2(acc_s[i, j] * scale - scores_max[i] * scale)
+ T.reduce_sum(acc_s, scores_sum, dim=1)
+ for i in T.Parallel(block_H):
+ logsum[i] = logsum[i] * scores_scale[i] + scores_sum[i]
+ T.copy(acc_s, acc_s_cast)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] *= scores_scale[i]
+ T.copy(
+ V3[bid, cur_kv_head, per_block_len * sid +
+ k * block_N: per_block_len * sid + (k + 1) * block_N, :], V_shared)
+ T.gemm(acc_s_cast, V_shared, acc_o, policy=T.GemmWarpPolicy.FullRow)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] /= logsum[i]
+ for i in T.Parallel(block_H):
+ logsum[i] = T.log2(logsum[i]) + scores_max[i] * scale
+
+ for i in T.Parallel(block_H):
+ if i < valid_block_H:
+ glse3[bid, hid * valid_block_H + i, sid] = logsum[i]
+ T.copy(acc_o[:valid_block_H, :], O_shared)
+ T.copy(O_shared, Output_partial3[bid, hid * valid_block_H:(hid + 1) * valid_block_H,
+ sid, :])
+
+ @T.macro
+ def flash_attn_split4(
+ Q4: T.Tensor(shape_q4, dtype),
+ K4: T.Tensor(shape_k4, dtype),
+ V4: T.Tensor(shape_v4, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ glse4: T.Tensor([batch, heads4, num_split + 1], dtype),
+ Output_partial4: T.Tensor(part_shape4, dtype),
+ ):
+ with T.Kernel(
+ batch, heads1 // valid_block_H, num_split, threads=threads) as (bx, by, bz):
+ Q_shared = T.alloc_shared([block_H, dim4], dtype)
+ K_shared = T.alloc_shared([block_N, dim4], dtype)
+ V_shared = T.alloc_shared([block_N, dim], dtype)
+ O_shared = T.alloc_shared([valid_block_H, dim], dtype)
+ acc_s = T.alloc_fragment([block_H, block_N], accum_dtype)
+ acc_s_cast = T.alloc_fragment([block_H, block_N], dtype)
+ mask_local = T.alloc_fragment([block_N], "uint8")
+ acc_o = T.alloc_fragment([block_H, dim], accum_dtype)
+ scores_max = T.alloc_fragment([block_H], accum_dtype)
+
+ scores_max_prev = T.alloc_fragment([block_H], accum_dtype)
+ scores_scale = T.alloc_fragment([block_H], accum_dtype)
+ scores_sum = T.alloc_fragment([block_H], accum_dtype)
+ logsum = T.alloc_fragment([block_H], accum_dtype)
+
+ bid = bx
+ hid = by
+ sid = bz
+ cur_kv_head = hid // (kv_group_num // valid_block_H)
+
+ T.copy(Q4[bid, hid * valid_block_H: hid * valid_block_H + block_H, :], Q_shared)
+ T.fill(acc_o, 0)
+ T.fill(logsum, 0)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ per_block_len = T.ceildiv(seqlen_kv, num_split)
+ this_block_end = T.min(per_block_len * (sid + 1), true_seq_len)
+ this_block_begin = per_block_len * sid
+
+ if this_block_begin < true_seq_len:
+ loop_range = T.ceildiv(per_block_len, block_N)
+ for k in T.Pipelined(loop_range, num_stages=num_stages):
+ if per_block_len * sid + k * block_N < true_seq_len:
+ T.copy(
+ K4[bid, cur_kv_head, per_block_len * sid +
+ k * block_N: per_block_len * sid + (k + 1) * block_N, :], K_shared)
+ for i in T.Parallel(block_N):
+ if per_block_len * sid + k * block_N + i < this_block_end:
+ mask_local[i] = mask_mid[bid, per_block_len * sid +
+ k * block_N + i]
+ else:
+ mask_local[i] = 0
+ T.clear(acc_s)
+ T.gemm(
+ Q_shared,
+ K_shared,
+ acc_s,
+ transpose_B=True,
+ policy=T.GemmWarpPolicy.FullRow, )
+
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.if_then_else(mask_local[j] != 0, acc_s[i, j],
+ -T.infinity(accum_dtype))
+
+ T.copy(scores_max, scores_max_prev)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ T.reduce_max(acc_s, scores_max, dim=1, clear=False)
+ for i in T.Parallel(block_H):
+ scores_scale[i] = T.exp2(scores_max_prev[i] * scale - scores_max[i] * scale)
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.exp2(acc_s[i, j] * scale - scores_max[i] * scale)
+ T.reduce_sum(acc_s, scores_sum, dim=1)
+ for i in T.Parallel(block_H):
+ logsum[i] = logsum[i] * scores_scale[i] + scores_sum[i]
+ T.copy(acc_s, acc_s_cast)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] *= scores_scale[i]
+ T.copy(
+ V4[bid, cur_kv_head, per_block_len * sid +
+ k * block_N: per_block_len * sid + (k + 1) * block_N, :], V_shared)
+ T.gemm(acc_s_cast, V_shared, acc_o, policy=T.GemmWarpPolicy.FullRow)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] /= logsum[i]
+ for i in T.Parallel(block_H):
+ logsum[i] = T.log2(logsum[i]) + scores_max[i] * scale
+
+ for i in T.Parallel(block_H):
+ if i < valid_block_H:
+ glse4[bid, hid * valid_block_H + i, sid] = logsum[i]
+ T.copy(acc_o[:valid_block_H, :], O_shared)
+ T.copy(O_shared, Output_partial4[bid, hid * valid_block_H: (hid + 1) * valid_block_H,
+ sid, :])
+
+ @T.macro
+ def flash_attn_split_full(
+ Q_full: T.Tensor(shape_full_q, dtype),
+ K_full: T.Tensor(shape_full_k, dtype),
+ V_full: T.Tensor(shape_full_v, dtype),
+ mask: T.Tensor([batch, true_full_len], "uint8"),
+ glse: T.Tensor([batch, heads, num_split + 1], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ ):
+ with T.Kernel(
+ batch, heads // valid_block_H, threads=threads) as (bx, by):
+ Q_shared = T.alloc_shared([block_H, dim], dtype)
+ K_shared = T.alloc_shared([block_N, dim], dtype)
+ V_shared = T.alloc_shared([block_N, dim], dtype)
+ O_shared = T.alloc_shared([valid_block_H, dim], dtype)
+ acc_s = T.alloc_fragment([block_H, block_N], accum_dtype)
+ acc_s_cast = T.alloc_fragment([block_H, block_N], dtype)
+ mask_local = T.alloc_fragment([block_N], "uint8")
+ acc_o = T.alloc_fragment([block_H, dim], accum_dtype)
+ scores_max = T.alloc_fragment([block_H], accum_dtype)
+
+ scores_max_prev = T.alloc_fragment([block_H], accum_dtype)
+ scores_scale = T.alloc_fragment([block_H], accum_dtype)
+ scores_sum = T.alloc_fragment([block_H], accum_dtype)
+ logsum = T.alloc_fragment([block_H], accum_dtype)
+
+ bid = bx
+ hid = by
+ sid = num_split
+ cur_kv_head = hid // (kv_group_num // valid_block_H)
+
+ T.copy(Q_full[bid, hid * valid_block_H: hid * valid_block_H + block_H, :], Q_shared)
+ T.fill(acc_o, 0)
+ T.fill(logsum, 0)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+
+ loop_range = T.ceildiv(seqlen_fullkv, block_N)
+ for k in T.Pipelined(loop_range, num_stages=num_stages):
+ if k * block_N < true_full_len:
+ T.copy(
+ K_full[bid, cur_kv_head,
+ k * block_N: (k + 1) * block_N, :], K_shared)
+ for i in T.Parallel(block_N):
+ if k * block_N + i < true_full_len:
+ mask_local[i] = mask[bid, k * block_N + i]
+ else:
+ mask_local[i] = 0
+ T.clear(acc_s)
+ T.gemm(
+ Q_shared,
+ K_shared,
+ acc_s,
+ transpose_B=True,
+ policy=T.GemmWarpPolicy.FullRow, )
+
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.if_then_else(mask_local[j] != 0, acc_s[i, j],
+ -T.infinity(accum_dtype))
+
+ T.copy(scores_max, scores_max_prev)
+ T.fill(scores_max, -T.infinity(accum_dtype))
+ T.reduce_max(acc_s, scores_max, dim=1, clear=False)
+ for i in T.Parallel(block_H):
+ scores_scale[i] = T.exp2(scores_max_prev[i] * scale - scores_max[i] * scale)
+ for i, j in T.Parallel(block_H, block_N):
+ acc_s[i, j] = T.exp2(acc_s[i, j] * scale - scores_max[i] * scale)
+ T.reduce_sum(acc_s, scores_sum, dim=1)
+ for i in T.Parallel(block_H):
+ logsum[i] = logsum[i] * scores_scale[i] + scores_sum[i]
+ T.copy(acc_s, acc_s_cast)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] *= scores_scale[i]
+ T.copy(
+ V_full[bid, cur_kv_head,
+ k * block_N: (k + 1) * block_N, :], V_shared)
+ T.gemm(acc_s_cast, V_shared, acc_o, policy=T.GemmWarpPolicy.FullRow)
+ for i, j in T.Parallel(block_H, dim):
+ acc_o[i, j] /= logsum[i]
+ for i in T.Parallel(block_H):
+ logsum[i] = T.log2(logsum[i]) + scores_max[i] * scale
+
+ for i in T.Parallel(block_H):
+ if i < valid_block_H:
+ glse[bid, hid * valid_block_H + i, sid] = logsum[i]
+ T.copy(acc_o[:valid_block_H, :], O_shared)
+ T.copy(O_shared, Output_partial[bid, hid * valid_block_H:(hid + 1) * valid_block_H,
+ sid, :])
+
+ @T.macro
+ def combine(
+ glse: T.Tensor([batch, heads, num_split + 1], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ with T.Kernel(heads, batch, threads=128) as (by, bz):
+ po_local = T.alloc_fragment([dim], dtype)
+ o_accum_local = T.alloc_fragment([dim], accum_dtype)
+ lse_local = T.alloc_fragment([num_split + 1, 128], dtype)
+ lse_local_split = T.alloc_local([1], accum_dtype)
+ lse_logsum_local = T.alloc_local([1], accum_dtype)
+ lse_max_local = T.alloc_fragment([128], accum_dtype)
+ scale_local = T.alloc_local([1], accum_dtype)
+
+ T.annotate_layout({
+ lse_logsum_local:
+ T.Fragment(lse_logsum_local.shape, forward_thread_fn=lambda i: i),
+ lse_max_local:
+ T.Fragment(lse_max_local.shape, forward_thread_fn=lambda i: i),
+ lse_local:
+ T.Fragment(lse_local.shape, forward_fn=lambda i, j: (j, i)),
+ })
+
+ T.clear(lse_logsum_local)
+ T.clear(o_accum_local)
+ for k, j in T.Parallel(num_split + 1, 128):
+ lse_local[k, j] = glse[bz, by, k]
+ T.reduce_max(lse_local, lse_max_local, dim=0, clear=True)
+ for k in T.Pipelined(num_split + 1, num_stages=1):
+ lse_local_split[0] = glse[bz, by, k]
+ lse_logsum_local[0] += T.exp2(lse_local_split[0] - lse_max_local[0])
+ lse_logsum_local[0] = T.log2(lse_logsum_local[0]) + lse_max_local[0]
+ for k in T.serial(num_split + 1):
+ for i in T.Parallel(dim):
+ po_local[i] = Output_partial[bz, by, k, i]
+ lse_local_split[0] = glse[bz, by, k]
+ scale_local[0] = T.exp2(lse_local_split[0] - lse_logsum_local[0])
+ for i in T.Parallel(dim):
+ o_accum_local[i] += po_local[i] * scale_local[0]
+ for i in T.Parallel(dim):
+ Output[bz, by, i] = o_accum_local[i]
+
+ @T.prim_func
+ def flashattn_gqa_decode_split_stream(
+ Q_full: T.Tensor(shape_full_q, dtype),
+ K_full: T.Tensor(shape_full_k, dtype),
+ V_full: T.Tensor(shape_full_v, dtype),
+ mask: T.Tensor([batch, true_full_len], "uint8"),
+ glse: T.Tensor([batch, heads, num_split + 1], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ flash_attn_split_full(Q_full, K_full, V_full, mask, glse, Output_partial)
+ combine(glse, Output_partial, Output)
+
+ @T.prim_func
+ def flashattn_gqa_decode_split_1group(
+ Q_full: T.Tensor(shape_full_q, dtype),
+ K_full: T.Tensor(shape_full_k, dtype),
+ V_full: T.Tensor(shape_full_v, dtype),
+ mask: T.Tensor([batch, true_full_len], "uint8"),
+ glse: T.Tensor([batch, heads, num_split + 1], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Q: T.Tensor(shape_q1, dtype),
+ K: T.Tensor(shape_k1, dtype),
+ V: T.Tensor(shape_v1, dtype),
+ glse1: T.Tensor([batch, heads1, num_split + 1], dtype),
+ Output_partial1: T.Tensor(part_shape1, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ flash_attn_split1(Q, K, V, mask_mid, glse1, Output_partial1)
+ flash_attn_split_full(Q_full, K_full, V_full, mask, glse, Output_partial)
+ combine(glse, Output_partial, Output)
+
+ @T.prim_func
+ def flashattn_gqa_decode_split_2groups(
+ Q_full: T.Tensor(shape_full_q, dtype),
+ K_full: T.Tensor(shape_full_k, dtype),
+ V_full: T.Tensor(shape_full_v, dtype),
+ mask: T.Tensor([batch, true_full_len], "uint8"),
+ glse: T.Tensor([batch, heads, num_split + 1], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Q: T.Tensor(shape_q1, dtype),
+ K: T.Tensor(shape_k1, dtype),
+ V: T.Tensor(shape_v1, dtype),
+ glse1: T.Tensor([batch, heads1, num_split + 1], dtype),
+ Output_partial1: T.Tensor(part_shape1, dtype),
+ Q2: T.Tensor(shape_q2, dtype),
+ K2: T.Tensor(shape_k2, dtype),
+ V2: T.Tensor(shape_v2, dtype),
+ glse2: T.Tensor([batch, heads2, num_split + 1], dtype),
+ Output_partial2: T.Tensor(part_shape2, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ flash_attn_split1(Q, K, V, mask_mid, glse1, Output_partial1)
+ flash_attn_split2(Q2, K2, V2, mask_mid, glse2, Output_partial2)
+ flash_attn_split_full(Q_full, K_full, V_full, mask, glse, Output_partial)
+ combine(glse, Output_partial, Output)
+
+ @T.prim_func
+ def flashattn_gqa_decode_split_3groups(
+ Q_full: T.Tensor(shape_full_q, dtype),
+ K_full: T.Tensor(shape_full_k, dtype),
+ V_full: T.Tensor(shape_full_v, dtype),
+ mask: T.Tensor([batch, true_full_len], "uint8"),
+ glse: T.Tensor([batch, heads, num_split + 1], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Q: T.Tensor(shape_q1, dtype),
+ K: T.Tensor(shape_k1, dtype),
+ V: T.Tensor(shape_v1, dtype),
+ glse1: T.Tensor([batch, heads1, num_split + 1], dtype),
+ Output_partial1: T.Tensor(part_shape1, dtype),
+ Q2: T.Tensor(shape_q2, dtype),
+ K2: T.Tensor(shape_k2, dtype),
+ V2: T.Tensor(shape_v2, dtype),
+ glse2: T.Tensor([batch, heads2, num_split + 1], dtype),
+ Output_partial2: T.Tensor(part_shape2, dtype),
+ Q3: T.Tensor(shape_q3, dtype),
+ K3: T.Tensor(shape_k3, dtype),
+ V3: T.Tensor(shape_v3, dtype),
+ glse3: T.Tensor([batch, heads3, num_split + 1], dtype),
+ Output_partial3: T.Tensor(part_shape3, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ flash_attn_split1(Q, K, V, mask_mid, glse1, Output_partial1)
+ flash_attn_split2(Q2, K2, V2, mask_mid, glse2, Output_partial2)
+ flash_attn_split3(Q3, K3, V3, mask_mid, glse3, Output_partial3)
+ flash_attn_split_full(Q_full, K_full, V_full, mask, glse, Output_partial)
+ combine(glse, Output_partial, Output)
+
+ @T.prim_func
+ def flashattn_gqa_decode_split_4groups(
+ Q_full: T.Tensor(shape_full_q, dtype),
+ K_full: T.Tensor(shape_full_k, dtype),
+ V_full: T.Tensor(shape_full_v, dtype),
+ mask: T.Tensor([batch, true_full_len], "uint8"),
+ glse: T.Tensor([batch, heads, num_split + 1], dtype),
+ Output_partial: T.Tensor(part_shape, dtype),
+ Q: T.Tensor(shape_q1, dtype),
+ K: T.Tensor(shape_k1, dtype),
+ V: T.Tensor(shape_v1, dtype),
+ glse1: T.Tensor([batch, heads1, num_split + 1], dtype),
+ Output_partial1: T.Tensor(part_shape1, dtype),
+ Q2: T.Tensor(shape_q2, dtype),
+ K2: T.Tensor(shape_k2, dtype),
+ V2: T.Tensor(shape_v2, dtype),
+ glse2: T.Tensor([batch, heads2, num_split + 1], dtype),
+ Output_partial2: T.Tensor(part_shape2, dtype),
+ Q3: T.Tensor(shape_q3, dtype),
+ K3: T.Tensor(shape_k3, dtype),
+ V3: T.Tensor(shape_v3, dtype),
+ glse3: T.Tensor([batch, heads3, num_split + 1], dtype),
+ Output_partial3: T.Tensor(part_shape3, dtype),
+ Q4: T.Tensor(shape_q4, dtype),
+ K4: T.Tensor(shape_k4, dtype),
+ V4: T.Tensor(shape_v4, dtype),
+ glse4: T.Tensor([batch, heads4, num_split + 1], dtype),
+ Output_partial4: T.Tensor(part_shape4, dtype),
+ mask_mid: T.Tensor([batch, true_seq_len], "uint8"),
+ Output: T.Tensor(shape_o, dtype),
+ ):
+ flash_attn_split1(Q, K, V, mask_mid, glse1, Output_partial1)
+ flash_attn_split2(Q2, K2, V2, mask_mid, glse2, Output_partial2)
+ flash_attn_split3(Q3, K3, V3, mask_mid, glse3, Output_partial3)
+ flash_attn_split4(Q4, K4, V4, mask_mid, glse4, Output_partial4)
+ flash_attn_split_full(Q_full, K_full, V_full, mask, glse, Output_partial)
+ combine(glse, Output_partial, Output)
+
+ if ndim == 0:
+ return flashattn_gqa_decode_split_stream
+ elif ndim == 1:
+ return flashattn_gqa_decode_split_1group
+ elif ndim == 2:
+ return flashattn_gqa_decode_split_2groups
+ elif ndim == 3:
+ return flashattn_gqa_decode_split_3groups
+ elif ndim == 4:
+ return flashattn_gqa_decode_split_4groups
+
+
+ if tune:
+
+ @autotune(configs=get_configs(), warmup=10, rep=10)
+ @tilelang.jit(out_idx=[5 * (ndim + 1) + 1 + (ndim > 0)])
+ def kernel(block_N=None, block_H=None, num_split=None, num_stages=None, threads=None):
+ return kernel_func(block_N, block_H, num_split, num_stages, threads)
+
+ return kernel()
+
+ else:
+
+ def kernel(block_N, block_H, num_split, num_stages, threads):
+ return kernel_func(block_N, block_H, num_split, num_stages, threads)
+
+ return kernel
diff --git a/minference/ops/moba.py b/minference/ops/moba.py
new file mode 100644
index 00000000..8f401d0b
--- /dev/null
+++ b/minference/ops/moba.py
@@ -0,0 +1,634 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+"""A clean version of moba implementation for educational purposes"""
+import math
+from typing import Callable, Optional, Tuple, Union
+
+import torch
+from einops import rearrange
+from flash_attn import flash_attn_func, flash_attn_varlen_func
+from flash_attn.flash_attn_interface import (
+ _flash_attn_varlen_backward,
+ _flash_attn_varlen_forward,
+)
+
+from .op_utils.moba_utils import calc_chunks
+
+
+def hf_to_fa(x: torch.Tensor):
+ """
+ Args:
+ x (torch.Tensor): [batch, heads, seqlen, head_dim]
+
+ Returns:
+ torch.Tensor: [batch * seqlen, heads, head_dim]
+ """
+ return x.permute(0, 2, 1, 3).reshape(-1, x.shape[1], x.shape[3])
+
+
+def moba_attn_varlen_naive(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ cu_seqlens: torch.Tensor,
+ max_seqlen: int,
+ moba_chunk_size: int,
+ moba_topk: int,
+) -> torch.Tensor:
+ """Implement the moba brute-force setting for reference
+
+ Args:
+ q (torch.Tensor): [seqlen, head, head_dim]
+ k (torch.Tensor): [seqlen, head, head_dim]
+ v (torch.Tensor): [seqlen, head, head_dim]
+ cu_seqlens (torch.Tensor): the cumulative sequence length tensor, same definition in flash attn
+ max_seqlen (int): the max sequence length of the batch, same definition in flash attn
+
+ Returns:
+ attn_output (torch.Tensor): [seqlen, head, head_dim]
+ """
+
+ # qkv shape = [ S, H, D ]
+ batch = cu_seqlens.numel() - 1
+ softmax_scale = q.shape[-1] ** (-0.5)
+
+ o = torch.zeros_like(q)
+ for batch_idx in range(batch):
+ batch_start = cu_seqlens[batch_idx].item()
+ batch_end = cu_seqlens[batch_idx + 1].item()
+ # get qkv of this batch
+ q_ = q[batch_start:batch_end]
+ k_ = k[batch_start:batch_end]
+ v_ = v[batch_start:batch_end]
+ o_ = o[batch_start:batch_end]
+ # calc key gate weight
+ key_gate_weight = []
+ batch_size = batch_end - batch_start
+ num_block = math.ceil(batch_size / moba_chunk_size)
+ for block_idx in range(0, num_block):
+ block_start = block_idx * moba_chunk_size
+ block_end = min(batch_size, block_start + moba_chunk_size)
+ key_gate_weight.append(k_[block_start:block_end].mean(dim=0, keepdim=True))
+ key_gate_weight = torch.cat(key_gate_weight, dim=0) # [ N, H, D ]
+ # calc & mask gate
+ # use fp32 to avoid precision issue in bf16
+ q_ = q_.type(torch.float32)
+ key_gate_weight = key_gate_weight.type(torch.float32)
+ gate = torch.einsum("shd,nhd->hsn", q_, key_gate_weight) # [ H, S, N ]
+ key_gate_weight = key_gate_weight.type_as(k)
+ q_ = q_.type_as(k)
+ for i in range(num_block):
+ # select the future Qs that can attend to KV chunk i
+ gate[:, : (i + 1) * moba_chunk_size, i] = float("-inf")
+ gate[:, i * moba_chunk_size : (i + 1) * moba_chunk_size, i] = float("inf")
+ # gate_top_k_idx = gate_top_k_val = [ H S K ]
+ gate_top_k_val, gate_top_k_idx = torch.topk(
+ gate, k=min(moba_topk, num_block), dim=-1, largest=True, sorted=False
+ )
+ gate_top_k_val, _ = gate_top_k_val.min(dim=-1) # [ H, S ]
+ need_attend = gate >= gate_top_k_val.unsqueeze(-1)
+ # add gate_idx_mask in case of there is cornercases of same topk val been selected
+ gate_idx_mask = torch.zeros(
+ need_attend.shape, dtype=torch.bool, device=q.device
+ )
+ gate_idx_mask = gate_idx_mask.scatter_(dim=-1, index=gate_top_k_idx, value=True)
+ need_attend = torch.logical_and(need_attend, gate_idx_mask)
+ gate[need_attend] = 0
+ gate[~need_attend] = -float("inf")
+ gate = gate.repeat_interleave(moba_chunk_size, dim=-1)[
+ :, :, :batch_size
+ ] # [ H, S, S ]
+ gate.masked_fill_(
+ torch.ones_like(gate, dtype=torch.bool).tril().logical_not(), -float("inf")
+ )
+ # print(f"moba_naive | gate ({gate.shape}): {gate}")
+
+ # calc qk = qk^t
+ q_ = q_.type(torch.float32)
+ k_ = k_.type(torch.float32)
+ v_ = v_.type(torch.float32)
+ qk = torch.einsum("xhd,yhd->hxy", q_, k_)
+ # mask
+ qk += gate
+ qk *= softmax_scale
+ # calc o
+ p = qk.softmax(dim=-1)
+ o_ += torch.einsum("hxy,yhd->xhd", p, v_)
+ o = o.type_as(q)
+
+ return o
+
+
+
+
+class MixedAttention(torch.autograd.Function):
+
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ self_attn_cu_seqlen,
+ moba_q,
+ moba_kv,
+ moba_cu_seqlen_q,
+ moba_cu_seqlen_kv,
+ max_seqlen,
+ moba_chunk_size,
+ moba_q_sh_indices,
+ return_lse,
+ ):
+ ctx.max_seqlen = max_seqlen
+ ctx.moba_chunk_size = moba_chunk_size
+ ctx.softmax_scale = softmax_scale = q.shape[-1] ** (-0.5)
+
+ # self attn
+ self_attn_out_sh, self_attn_lse_hs, _, _ = (
+ _flash_attn_varlen_forward(
+ q=q,
+ k=k,
+ v=v,
+ cu_seqlens_q=self_attn_cu_seqlen,
+ cu_seqlens_k=self_attn_cu_seqlen,
+ max_seqlen_q=max_seqlen,
+ max_seqlen_k=max_seqlen,
+ softmax_scale=softmax_scale,
+ causal=True,
+ dropout_p=0.0,
+ )
+ )
+
+ moba_attn_out, moba_attn_lse_hs, _, _ = _flash_attn_varlen_forward(
+ q=moba_q,
+ k=moba_kv[:, 0],
+ v=moba_kv[:, 1],
+ cu_seqlens_q=moba_cu_seqlen_q,
+ cu_seqlens_k=moba_cu_seqlen_kv,
+ max_seqlen_q=max_seqlen,
+ max_seqlen_k=moba_chunk_size,
+ softmax_scale=softmax_scale,
+ causal=False,
+ dropout_p=0.0,
+ )
+
+ # convert lse shape hs -> sh ( follow the legacy mix attn logic )
+ self_attn_lse_sh = self_attn_lse_hs.t().contiguous()
+ moba_attn_lse = moba_attn_lse_hs.t().contiguous()
+
+ # output buffer [S, H, D], same shape as q
+ output = torch.zeros(
+ (q.shape[0], q.shape[1], q.shape[2]), device=q.device, dtype=torch.float32
+ )
+
+ # flatten vS & H for index ops
+ output_2d = output.view(-1, q.shape[2])
+
+ # calc mixed_lse
+ # minus max lse to avoid exp explosion
+ max_lse_1d = self_attn_lse_sh.view(-1)
+ max_lse_1d = max_lse_1d.index_reduce(
+ 0, moba_q_sh_indices, moba_attn_lse.view(-1), "amax"
+ )
+ self_attn_lse_sh = self_attn_lse_sh - max_lse_1d.view_as(self_attn_lse_sh)
+ moba_attn_lse = (
+ moba_attn_lse.view(-1)
+ .sub(max_lse_1d.index_select(0, moba_q_sh_indices))
+ .reshape_as(moba_attn_lse)
+ )
+
+ mixed_attn_se_sh = self_attn_lse_sh.exp()
+ moba_attn_se = moba_attn_lse.exp()
+
+ mixed_attn_se_sh.view(-1).index_add_(
+ 0, moba_q_sh_indices, moba_attn_se.view(-1)
+ )
+ mixed_attn_lse_sh = mixed_attn_se_sh.log()
+
+ # add attn output
+ factor = (self_attn_lse_sh - mixed_attn_lse_sh).exp() # [ vS, H ]
+ self_attn_out_sh = self_attn_out_sh * factor.unsqueeze(-1)
+ output_2d += self_attn_out_sh.reshape_as(output_2d)
+
+ # add moba output
+ mixed_attn_lse = (
+ mixed_attn_lse_sh.view(-1)
+ .index_select(0, moba_q_sh_indices)
+ .view_as(moba_attn_lse)
+ )
+ factor = (moba_attn_lse - mixed_attn_lse).exp() # [ vS, H ]
+ moba_attn_out = moba_attn_out * factor.unsqueeze(-1)
+ raw_attn_out = moba_attn_out.view(-1, moba_attn_out.shape[-1])
+ output_2d.index_add_(0, moba_q_sh_indices, raw_attn_out)
+ output = output.to(q.dtype)
+
+
+ # add back max lse
+ mixed_attn_lse_sh = mixed_attn_lse_sh + max_lse_1d.view_as(mixed_attn_se_sh)
+
+
+ ctx.save_for_backward(
+ output,
+ mixed_attn_lse_sh,
+ q,
+ k,
+ v,
+ self_attn_cu_seqlen,
+ moba_q,
+ moba_kv,
+ moba_cu_seqlen_q,
+ moba_cu_seqlen_kv,
+ moba_q_sh_indices,
+ )
+ ctx.return_lse = return_lse
+
+ if return_lse:
+ return output, mixed_attn_lse_sh
+ else:
+ return output
+
+ @staticmethod
+ def backward(ctx, d_output, *args):
+
+ max_seqlen = ctx.max_seqlen
+ moba_chunk_size = ctx.moba_chunk_size
+ softmax_scale = ctx.softmax_scale
+
+ (
+ output,
+ mixed_attn_vlse_sh,
+ q,
+ k,
+ v,
+ self_attn_cu_seqlen,
+ moba_q,
+ moba_kv,
+ moba_cu_seqlen_q,
+ moba_cu_seqlen_kv,
+ moba_q_sh_indices,
+ ) = ctx.saved_tensors
+
+ d_output = d_output.contiguous()
+
+ dq, dk, dv = torch.zeros_like(q), torch.zeros_like(k), torch.zeros_like(v)
+ _flash_attn_varlen_backward(
+ dout=d_output,
+ q=q,
+ k=k,
+ v=v,
+ out=output,
+ softmax_lse=mixed_attn_vlse_sh.t().contiguous(),
+ dq=dq,
+ dk=dk,
+ dv=dv,
+ cu_seqlens_q=self_attn_cu_seqlen,
+ cu_seqlens_k=self_attn_cu_seqlen,
+ max_seqlen_q=max_seqlen,
+ max_seqlen_k=max_seqlen,
+ softmax_scale=softmax_scale,
+ causal=True,
+ dropout_p=0.0,
+ window_size_left=-1,
+ window_size_right=-1,
+ softcap=0.0,
+ alibi_slopes=None,
+ deterministic=True,
+ )
+
+ headdim = q.shape[-1]
+ d_moba_output = (
+ d_output.view(-1, headdim).index_select(0, moba_q_sh_indices).unsqueeze(1)
+ )
+ moba_output = (
+ output.view(-1, headdim).index_select(0, moba_q_sh_indices).unsqueeze(1)
+ )
+
+ mixed_attn_vlse = (
+ mixed_attn_vlse_sh.view(-1).index_select(0, moba_q_sh_indices).view(1, -1)
+ )
+
+ dmq = torch.zeros_like(moba_q, dtype=moba_q.dtype, device=moba_q.device)
+ dmk = torch.zeros_like(moba_kv[:, 0], dtype=moba_kv.dtype, device=moba_kv.device)
+ dmv = torch.zeros_like(moba_kv[:, 1], dtype=moba_kv.dtype, device=moba_kv.device)
+ _flash_attn_varlen_backward(
+ dout=d_moba_output,
+ q=moba_q,
+ k=moba_kv[:, 0],
+ v=moba_kv[:, 1],
+ out=moba_output,
+ softmax_lse=mixed_attn_vlse,
+ dq=dmq,
+ dk=dmk,
+ dv=dmv,
+ cu_seqlens_q=moba_cu_seqlen_q,
+ cu_seqlens_k=moba_cu_seqlen_kv,
+ max_seqlen_q=max_seqlen,
+ max_seqlen_k=moba_chunk_size,
+ softmax_scale=softmax_scale,
+ causal=False,
+ dropout_p=0.0,
+ window_size_left=-1,
+ window_size_right=-1,
+ softcap=0.0,
+ alibi_slopes=None,
+ deterministic=True,
+ )
+
+ dmkv = torch.stack((dmk, dmv), dim=1)
+ return dq, dk, dv, None, dmq, dmkv, None, None, None, None, None, None
+
+
+def moba_attn_varlen(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ cu_seqlens: torch.Tensor,
+ max_seqlen: int,
+ moba_chunk_size: int,
+ moba_topk: int,
+ return_lse: bool=False,
+) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+ """An efficient version of moba implementation with triton kernels and flash-attn, the core logic:
+ 1. Calculate the chunks and the number of chunks, n = floor(data_size / chunk_size)
+ - tokens in the tail chunk are reserved for self attn
+ - tokens in other chunks will be processed in later steps
+ 2. K in each chunk will calculate mean value as the representative k, and Q will attend to these representative
+ k to get the gate logit, which will be used to select topk chunks
+ 3. Select the topk chunks and get the dense q for each kv chunk pair and do the varlen attention
+ 4. Combine the varlen attn and self attn results via online softmax to get the final result
+
+ Args:
+ q (torch.Tensor): [seqlen, head, head_dim]
+ k (torch.Tensor): [seqlen, head, head_dim]
+ v (torch.Tensor): [seqlen, head, head_dim]
+ cu_seqlens (torch.Tensor): the cumulative sequence length tensor, same definition in flash attn
+ max_seqlen (int): the max sequence length of the batch, same definition in flash attn
+
+ Returns:
+ attn_output (torch.Tensor): [seqlen, head, head_dim]
+ """
+ head_group_size = q.shape[1] // k.shape[1]
+ if head_group_size > 1:
+ k = torch.repeat_interleave(k, head_group_size, dim=1)
+ v = torch.repeat_interleave(v, head_group_size, dim=1)
+
+ # ---------------------------------------------------------------------------------------------
+ kv = torch.stack((k, v), dim=1) # stack along a new dimension -> [S, 2, H, D]
+
+ """ some basic variables """
+ # qkv shape = [ S, H, D ]
+ seqlen, num_head, head_dim = q.shape
+
+ """ prepare chunk meta """
+ (
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ ) = calc_chunks(cu_seqlens, moba_chunk_size)
+
+ # we will adjust selective topk to moba_topk - 1, as the last chunk is always chosen
+ moba_topk = min(moba_topk - 1, num_filtered_chunk)
+ need_moba_attn = moba_topk > 0
+
+ # corner case: if no moba attn needed, just return self attn
+ if not need_moba_attn:
+ return flash_attn_varlen_func(
+ q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen, causal=True
+ )
+
+ self_attn_cu_seqlen = cu_chunk
+
+ # filtered_kv is a dense matrix that only contains filtered chunk of kv
+ filtered_kv_indices = torch.arange(
+ 0, moba_chunk_size, dtype=torch.int32, device=q.device
+ )[None, :].repeat(num_filtered_chunk, 1)
+ filtered_kv_indices += cu_chunk[filtered_chunk_indices][:, None]
+
+ # select the elements of KV corresponding to all chunks that are not filtered out
+ filtered_kv = kv.index_select(0, filtered_kv_indices.view(-1))
+
+ """ calc key_gate_weight and gate """
+ # key_gate_weight [ F_N_CHUNK, HEAD, HEAD_DIM ]
+ key_gate_weight = (
+ filtered_kv[:, 0] # K
+ .view(num_filtered_chunk, moba_chunk_size, num_head, head_dim)
+ .mean(dim=1) # mean pooling along chunk size
+ .float()
+ )
+ q = q.type(torch.float32) # float logit on the fly for better gate logit perception
+ key_gate_weight = key_gate_weight.type(
+ torch.float32
+ ) # float logit for better gate logit perception
+ gate = torch.einsum(
+ "nhd,shd->nhs", key_gate_weight, q
+ ) # gate [ F_N_CHUNK, HEAD, SEQ ]
+ key_gate_weight = key_gate_weight.type_as(k)
+ q = q.type_as(k)
+
+ # pose process gate, masking unchosen batch and apply causal mask to current chunk
+ gate_seq_idx = torch.arange(0, seqlen, device=q.device, dtype=torch.int32)[
+ None, :
+ ].repeat(num_filtered_chunk, 1)
+ chunk_end = cu_chunk[filtered_chunk_indices + 1]
+ batch_end = cu_seqlens[chunk_to_batch[filtered_chunk_indices] + 1]
+ gate_chunk_end_mask = gate_seq_idx < chunk_end[:, None]
+ gate_batch_end_mask = gate_seq_idx >= batch_end[:, None]
+ gate_inf_mask = gate_chunk_end_mask | gate_batch_end_mask
+ gate.masked_fill_(gate_inf_mask.unsqueeze(1), -float("inf"))
+
+ """ find moba q that needs moba attn """
+ # find topk chunks
+ # gate_mask [ N_CHUNK, HEAD, SEQ ], true indicates that needs attention
+ _, gate_top_k_idx = torch.topk(gate, k=moba_topk, dim=0, largest=True, sorted=False)
+
+ # apply causal mask
+ gate_mask = torch.logical_not(gate.isinf())
+
+ # select topk chunks
+ gate_idx_mask = torch.zeros(gate_mask.shape, dtype=torch.bool, device=q.device)
+ gate_idx_mask = gate_idx_mask.scatter_(dim=0, index=gate_top_k_idx, value=True)
+
+ # gate_mask has the shape [ N_CHUNK, HEAD, SEQ ].
+ # For each chunk, the sequence-dimension indices will be True if it belongs to the top-K chunks
+ gate_mask = torch.logical_and(gate_mask, gate_idx_mask)
+ # ---------------------------------------------------------------------------------------------
+
+
+ # varlen trick: combining all q index that needs moba attn
+ # the result will be like [ C0H0 ][ C0H1 ][ C0H2 ][ ... ][ CnHm ]
+ # torch.nonzero (as_tuple=True): Returns a tuple of 1-D tensors, one for each dimension in input, each containing the indices (in that dimension) of all non-zero elements of input .
+ # if input has n-dimension, the resulting tuple will have n tensors of size z, where z is the number of non-zero elements in input.
+ # (i-th values of all n tuple elements represent the indices of the i-th non-zero element in each dimension)
+ # using index [-1] => indices of HS (combined sequence) dimension that contains non-zero elements
+ moba_q_indices = gate_mask.reshape(gate_mask.shape[0], -1) # [ N, HS ]
+
+ moba_q_indices = moba_q_indices.nonzero(as_tuple=True)[-1] # [HS indices] * N (total size: all non-zero elements in HS dimension)
+
+
+ # moba_seqlen_q indicates that how many q chunks are selected for each kv chunk - head
+ moba_seqlen_q = gate_mask.sum(dim=-1).flatten()
+
+ # select all q that needs moba attn based on the moba_q_indices
+ moba_q = rearrange(q, "s h d -> ( h s ) d").index_select(
+ 0, moba_q_indices
+ ) # [ selected_S, D ]
+ moba_q = moba_q.unsqueeze(1)
+
+ # moba_q_sh_indices represents the position in the origin q tensor of each q token inside moba_q
+ moba_q_sh_indices = moba_q_indices % seqlen * num_head + moba_q_indices // seqlen
+
+ """ prepare moba kv """
+ # Since moba_q is organized as HS * N, we need to reorganize kv to adapt to q
+
+ # cut off zero experts
+ q_zero_mask = moba_seqlen_q == 0
+ valid_expert_mask = ~q_zero_mask
+ zero_expert_count = q_zero_mask.sum()
+
+ # only keep the kv that has q select > 0
+ if zero_expert_count > 0:
+ moba_seqlen_q = moba_seqlen_q[valid_expert_mask]
+
+
+ # moba cu_seqlen for flash attn
+ moba_cu_seqlen_q = torch.cat(
+ (
+ torch.tensor([0], device=q.device, dtype=moba_seqlen_q.dtype),
+ moba_seqlen_q.cumsum(dim=0),
+ ),
+ dim=0,
+ ).to(torch.int32)
+
+ # -----------------------------------------------
+ moba_kv = rearrange(filtered_kv, "s x h d -> h s x d") # here `x` only stands for a dimension (stack dimension for KV)
+
+ moba_kv = moba_kv.split(moba_chunk_size, dim=1)
+ moba_kv = torch.cat(moba_kv, dim=0) # [num_selected_chunks, H x S // moba_chunk_size, D]
+
+ if zero_expert_count > 0:
+ assert valid_expert_mask.sum() == moba_kv.shape[0] - zero_expert_count
+ moba_kv = moba_kv[
+ valid_expert_mask
+ ] # cut off zero Q expert from kv , or the grad may be nan
+
+ moba_kv = moba_kv.flatten(start_dim=0, end_dim=1).unsqueeze(2)
+
+ moba_cu_seqlen_kv = (
+ torch.arange(
+ 0,
+ num_filtered_chunk * num_head + 1 - zero_expert_count,
+ dtype=torch.int32,
+ device=q.device,
+ )
+ * moba_chunk_size
+ )
+
+ # Shape check
+ assert (
+ moba_cu_seqlen_kv.shape == moba_cu_seqlen_q.shape
+ ), f"moba_cu_seqlen_kv.shape != moba_cu_seqlen_q.shape {moba_cu_seqlen_kv.shape} != {moba_cu_seqlen_q.shape}"
+
+ # Wrapping up the flash attn call and online softmax dlse inside MixedAttention class
+ return MixedAttention.apply(
+ q, k, v,
+ self_attn_cu_seqlen,
+ moba_q,
+ moba_kv,
+ moba_cu_seqlen_q,
+ moba_cu_seqlen_kv,
+ max_seqlen,
+ moba_chunk_size,
+ moba_q_sh_indices,
+ return_lse
+ )
+
+
+def moba_attn_func(
+ q: torch.Tensor, # [batch, q_len, q_heads, head_dim]
+ k: torch.Tensor,
+ v: torch.Tensor,
+ global_seq_len: int,
+ moba_chunk_size: int,
+ moba_topk: int,
+ **kwargs,
+):
+ batch_size = q.shape[0]
+ cu_seqlens = torch.cumsum(
+ torch.tensor([0] + [global_seq_len] * batch_size, device=q.device),
+ dim=0,
+ dtype=torch.int32,
+ )
+
+ q_3d, k_3d, v_3d = \
+ q.reshape(-1, q.shape[2], q.shape[3]), \
+ k.reshape(-1, k.shape[2], k.shape[3]), \
+ v.reshape(-1, v.shape[2], v.shape[3])
+
+ # output: [batch_size, global_seq_len, q_heads, head_dim]
+ return moba_attn_varlen(
+ q_3d, k_3d, v_3d,
+ cu_seqlens,
+ global_seq_len,
+ moba_chunk_size,
+ moba_topk,
+ ).view(q.shape)
+
+
+def moba_layer(
+ moba_impl: Callable,
+ moba_chunk_size: int,
+ moba_topk: int,
+ module: torch.nn.Module,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ *args,
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ **kwargs,
+) -> Tuple[torch.Tensor, None]:
+ """
+ Args:
+ query (torch.Tensor): [batch, q_heads, q_len, head_dim]
+ key (torch.Tensor): [batch, kv_heads, kv_len, head_dim]
+ value (torch.Tensor): [batch, kv_heads, kv_len, head_dim]
+
+ Returns:
+ attn_output (torch.Tensor): [batch, q_len, q_heads, head_dim]
+ attn_weights (None): not needed
+ """
+ assert module.is_causal
+ batch, q_heads, q_len, head_dim = query.shape
+ _, kv_heads, kv_len, _ = key.shape
+ if q_len == kv_len:
+ # prefill phase
+ query = hf_to_fa(query)
+ key = hf_to_fa(key)
+ value = hf_to_fa(value)
+ kv_replicas = q_heads // kv_heads
+ key = torch.repeat_interleave(key, kv_replicas, dim=1)
+ value = torch.repeat_interleave(value, kv_replicas, dim=1)
+ cu_seqlens_k = torch.cumsum(
+ torch.tensor([0] + [kv_len] * batch, device=query.device),
+ dim=0,
+ dtype=torch.int32,
+ )
+ out = moba_impl(
+ q=query,
+ k=key,
+ v=value,
+ cu_seqlens=cu_seqlens_k,
+ max_seqlen=kv_len,
+ moba_chunk_size=moba_chunk_size,
+ moba_topk=moba_topk,
+ )
+ else:
+ # decode phase
+ # TODO release paged attn implementation
+ query = query.transpose(1, 2)
+ key = key.transpose(1, 2)
+ value = value.transpose(1, 2)
+ out = flash_attn_func(query, key, value, dropout, scaling, True)
+ return out, None
diff --git a/minference/ops/op_utils/__init__.py b/minference/ops/op_utils/__init__.py
new file mode 100644
index 00000000..0d2017b4
--- /dev/null
+++ b/minference/ops/op_utils/__init__.py
@@ -0,0 +1,2 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
diff --git a/minference/ops/op_utils/moba_utils.py b/minference/ops/op_utils/moba_utils.py
new file mode 100644
index 00000000..3cb3b909
--- /dev/null
+++ b/minference/ops/op_utils/moba_utils.py
@@ -0,0 +1,369 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from dataclasses import dataclass
+from functools import lru_cache
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+import torch
+import torch.distributed as dist
+
+
+def tensor_4d_to_3d(tensor: torch.Tensor) -> torch.Tensor:
+ """Convert a 4D tensor to a 3D tensor by collapsing the first two dimensions."""
+ if tensor.ndim != 4:
+ raise ValueError("Input tensor must be 4D.")
+ return tensor.reshape(tensor.shape[0] * tensor.shape[1], tensor.shape[2], tensor.shape[3])
+
+
+@dataclass
+class MoBAConfig:
+ moba_chunk_size: int
+ moba_topk: int
+
+def shuffle_input_all(
+ to_send: torch.Tensor, # [S, H, D]
+ seq_offset: torch.Tensor, # [2]
+ gate_mask: torch.Tensor = None, # [num_chunks, H, S]
+ process_group: dist.ProcessGroup = None
+ ):
+ orig_ndim = to_send.ndim
+ if orig_ndim == 3: to_send = to_send.unsqueeze(0)
+
+ rank = dist.get_rank(process_group)
+ world_size = dist.get_world_size(process_group)
+
+ if not to_send.is_contiguous():
+ to_send = to_send.contiguous()
+ block_seq_len = to_send.shape[1] // 2
+
+ seq_offset_val = seq_offset.detach().cpu().item()
+ seq_offsets = torch.Tensor([seq_offset_val, seq_offset_val + block_seq_len]).to(to_send.device)
+
+ # We must use outplace, otherwise it will raise error at backward due to inplace operations.
+ # We can not change to_send directly and create a new tensor to store the result.
+ to_send_f = torch.zeros_like(to_send)
+ to_send_gate_mask = torch.zeros_like(gate_mask)
+ to_send_offset = seq_offsets[1]
+
+ # assume the input sequence length is 8, and computation runs on 4 GPUs
+ # the seq is represented as [0 1 2 3 4 5 6 7], world size is 4
+ # the input status before `shuffle_input` is
+ # - gpu A: [0 1]
+ # - gpu B: [2 3]
+ # - gpu C: [4 5]
+ # - gpu D: [6 7]
+ # the value of `to_send_slice` is
+ # - gpu A: [1]
+ # - gpu B: [3]
+ # - gpu C: [5]
+ # - gpu D: [7]
+ to_send_slice = to_send[:, block_seq_len:].contiguous()
+ to_send_gate_mask_slice = gate_mask[..., block_seq_len:].contiguous()
+
+ res = torch.zeros_like(to_send_slice)
+ res_gate_mask = torch.zeros_like(to_send_gate_mask_slice)
+ res_offset= torch.zeros_like(to_send_offset)
+
+ _ops = []
+ offset = ((dist.get_rank() // world_size) * world_size)
+ # rank src_rank
+ # 0 3
+ # 1 2
+ # 2 1
+ # 3 0
+ src_rank = (world_size - rank - 1) % world_size + offset
+ send_op = dist.P2POp(
+ dist.isend, to_send_slice, src_rank, group=process_group
+ )
+ send_gate_mask_op = dist.P2POp(
+ dist.isend, to_send_gate_mask_slice, src_rank, group=process_group
+ )
+ send_offset_op = dist.P2POp(
+ dist.isend, to_send_offset, src_rank, group=process_group
+ )
+ _ops.append(send_op)
+ _ops.append(send_gate_mask_op)
+ _ops.append(send_offset_op)
+
+ recv_op = dist.P2POp(
+ dist.irecv, res, src_rank, group=process_group)
+ recv_gate_mask_op = dist.P2POp(
+ dist.irecv, res_gate_mask, src_rank, group=process_group
+ )
+ recv_offset_op = dist.P2POp(
+ dist.irecv, res_offset, src_rank, group=process_group
+ )
+ _ops.append(recv_op)
+ _ops.append(recv_gate_mask_op)
+ _ops.append(recv_offset_op)
+
+ # response = dist.dist.batch_isend_irecv(_ops)
+ response = dist.batch_isend_irecv(_ops)
+ for resp in response:
+ resp.wait()
+
+ if rank >= world_size // 2: # D: 6 7, -> 1 6
+ to_send_f[:, block_seq_len:] = to_send[:, :block_seq_len]
+ to_send_f[:, :block_seq_len, ...] = res
+
+ to_send_gate_mask[..., block_seq_len:] = gate_mask[..., :block_seq_len]
+ to_send_gate_mask[..., :block_seq_len] = res_gate_mask
+
+ seq_offsets[1] = seq_offsets[0]
+ seq_offsets[0] = res_offset
+ else: # A: 0 1, -> 0 7
+ to_send_f[:, :block_seq_len] = to_send[:, :block_seq_len]
+ to_send_f[:, block_seq_len:, ...] = res
+
+ to_send_gate_mask[..., :block_seq_len] = gate_mask[..., :block_seq_len]
+ to_send_gate_mask[..., block_seq_len:] = res_gate_mask
+
+ seq_offsets[1] = res_offset
+
+ # after shuffle, the status of `to_send_f`
+ # GPU A: [0 7]
+ # GPU B: [2 5]
+ # GPU C: [3 4]
+ # GPU D: [1 6]
+ return (
+ to_send_f if orig_ndim != 3 else to_send_f.squeeze(0),
+ seq_offsets,
+ to_send_gate_mask,
+ )
+
+
+def shuffle_input_only(
+ to_send: torch.Tensor, # [S, H, D]
+ process_group: dist.ProcessGroup = None
+ ) -> torch.Tensor:
+ orig_ndim = to_send.ndim
+ if orig_ndim == 3: to_send = to_send.unsqueeze(0)
+
+ rank = dist.get_rank(process_group)
+ world_size = dist.get_world_size(process_group)
+
+ if not to_send.is_contiguous():
+ to_send = to_send.contiguous()
+ block_seq_len = to_send.shape[1] // 2
+
+ # We must use outplace, otherwise it will raise error at backward due to inplace operations.
+ # We can not change to_send directly and create a new tensor to store the result.
+ to_send_f = torch.zeros_like(to_send)
+
+ to_send_slice = to_send[:, block_seq_len:].contiguous()
+ res = torch.zeros_like(to_send_slice)
+
+ _ops = []
+ offset = ((dist.get_rank() // world_size) * world_size)
+
+ src_rank = (world_size - rank - 1) % world_size + offset
+ send_op = dist.P2POp(
+ dist.isend, to_send_slice, src_rank, group=process_group
+ )
+ _ops.append(send_op)
+
+ recv_op = dist.P2POp(
+ dist.irecv, res, src_rank, group=process_group)
+ _ops.append(recv_op)
+
+ # response = dist.dist.batch_isend_irecv(_ops)
+ response = dist.batch_isend_irecv(_ops)
+ for resp in response:
+ resp.wait()
+
+ if rank >= world_size // 2: # D: 6 7, -> 1 6
+ to_send_f[:, block_seq_len:] = to_send[:, :block_seq_len]
+ to_send_f[:, :block_seq_len, ...] = res
+ else: # A: 0 1, -> 0 7
+ to_send_f[:, :block_seq_len] = to_send[:, :block_seq_len]
+ to_send_f[:, block_seq_len:, ...] = res
+ return to_send_f if orig_ndim != 3 else to_send_f.squeeze(0)
+
+@lru_cache(maxsize=16)
+def calc_chunks(cu_seqlen, moba_chunk_size):
+ """calc chunks that needs moba attention"""
+
+ # batch_sizes[batch_idx] = batch size ( seqlen ) of batch idx
+ # example: [seq_len]
+ batch_sizes = cu_seqlen[1:] - cu_seqlen[:-1]
+
+ # batch_num_chunk[batch_idx] = how many chunk in batch idx
+ # example: [number of all chunks with chunk size equal to moba_chunk_size + 1 (the one with smaller size)]
+ batch_num_chunk = (batch_sizes + (moba_chunk_size - 1)) // moba_chunk_size
+
+ # cu_num_chunk[batch_idx] = first chunk id of this batch
+ # example: [1, 1]
+ cu_num_chunk = torch.ones(
+ batch_num_chunk.numel() + 1,
+ device=cu_seqlen.device,
+ dtype=batch_num_chunk.dtype,
+ )
+ # example: [1, 1 + num of chunks]
+ cu_num_chunk[1:] = batch_num_chunk.cumsum(dim=0)
+
+ # total chunk ( for all batch )
+ # example: 1 + num of chunks
+ num_chunk = cu_num_chunk[-1]
+
+ # chunk_sizes[chunk_idx] = chunk_size of chunk idx
+ chunk_sizes = torch.full(
+ (num_chunk + 1,), moba_chunk_size, dtype=torch.int32, device=cu_seqlen.device
+ )
+ chunk_sizes[0] = 0 # for calc cu chunk
+ batch_last_chunk_size = batch_sizes - (batch_num_chunk - 1) * moba_chunk_size
+ chunk_sizes[cu_num_chunk[1:]] = batch_last_chunk_size
+ # example chunk_sizes: [0, moba_chunk_size, ..., moba_chunk_size, batch_last_chunk_size]
+
+
+ # cu_chunk[chunk_idx] = the start chunk offset of chunk idx
+ # example: [0, moba_chunk_size, ..., seq_len]
+ cu_chunk = chunk_sizes.cumsum(dim=-1, dtype=torch.int32)
+
+
+ # chunk_to_batch[chunk_idx] = batch idx of the chunk idx
+ # example: [0, 0, 0, ...., 0]
+ chunk_to_batch = torch.zeros(
+ (num_chunk,), dtype=torch.int32, device=cu_seqlen.device
+ )
+
+ # example: [0, 0, 0, ... , 0] (if there are multiple samples in the batch, the index of the starting chunk of each batch from the 1st sample will be 1)
+ # but if there is only one batch, cu_num_chunk[1:-1] will be empty and no element will be assigned with 1 (all correspond to 0-th sample)
+ chunk_to_batch[cu_num_chunk[1:-1]] = 1
+
+ # example: [0, 0, 0, ..., 0]
+ chunk_to_batch = chunk_to_batch.cumsum(dim=0, dtype=torch.int32)
+
+ """ filter chunks that need moba attn """
+ # filter chunks ( remove last chunk of each batch )
+ # filtered_chunk_indices: chunk index list that excludes the last chunk of each batch
+ chunk_to_remove = cu_num_chunk[1:] - 1 # example: number of chunks (num_chunk - 1)
+ # print(f"calc_chunks | chunk_to_remove: {chunk_to_remove}")
+
+ chunk_to_remain = torch.ones(
+ (num_chunk, ), dtype=torch.bool, device=cu_seqlen.device
+ )
+ chunk_to_remain[chunk_to_remove] = False # example:
+ filtered_chunk_indices = chunk_to_remain.nonzero(as_tuple=True)[0]
+ num_filtered_chunk = len(filtered_chunk_indices)
+
+ return (
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch,
+ )
+
+
+def compute_moba_gate(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ seq_offset: torch.Tensor,
+ cu_seqlens: torch.Tensor,
+ moba_chunk_size: int,
+ moba_topk: int,
+):
+ if len(q.shape) == 4:
+ q, k, v = \
+ tensor_4d_to_3d(q), \
+ tensor_4d_to_3d(k), \
+ tensor_4d_to_3d(v)
+ seq_offset: int = seq_offset.detach().cpu().item()
+ seqlen_block, num_head, head_dim = q.shape
+ _, k_num_head, _ = k.shape
+ if num_head > k_num_head:
+ k = torch.repeat_interleave(k, num_head // k_num_head, dim=1)
+ v = torch.repeat_interleave(v, num_head // k_num_head, dim=1)
+
+ # ---------------------------------------------------------------------------------------------
+ kv = torch.stack((k, v), dim=1) # [ blk_S, 2, H, D ]
+
+ world_size = dist.get_world_size()
+ kv_list = [torch.zeros_like(kv, dtype=q.dtype, device=q.device) for _ in range(world_size)]
+ dist.all_gather(kv_list, kv)
+ kv_gathered = torch.cat(kv_list, dim=0) # [ S, 2, H, D ]
+
+
+ """ some basic variables """
+ # qkv shape = [ S, H, D ]
+ block_size = q.shape[0]
+ seqlen, _, num_head, head_dim = kv_gathered.shape
+
+ """ prepare chunk meta """
+ (
+ cu_chunk, # example: [0, moba_chunk_size, ..., seq_len]
+ filtered_chunk_indices, # example: [0, 1, 2, ..., num_filtered_chunk-1] (i.e. except the last chunk)
+ num_filtered_chunk, # example: num_filtered_chunk
+ chunk_to_batch, # example: [0, 0, ... ,0] (for batch_size=1) with size 1 + real num of chunks
+ ) = calc_chunks(cu_seqlens, moba_chunk_size)
+
+ # we will adjust selective topk to moba_topk - 1, as the last chunk is always chosen
+ moba_topk = min(moba_topk - 1, num_filtered_chunk)
+ assert moba_topk > 0, "moba_topk should be greater than 0"
+
+ # filtered_kv is a dense matrix that only contains filtered chunk of kv
+ filtered_kv_indices = torch.arange(
+ 0, moba_chunk_size, dtype=torch.int32, device=q.device
+ )[None, :].repeat(num_filtered_chunk, 1)
+ filtered_kv_indices += cu_chunk[filtered_chunk_indices][:, None]
+
+ # select the elements of KV corresponding to all chunks that are not filtered out
+ filtered_kv = kv_gathered.index_select(0, filtered_kv_indices.view(-1))
+
+ """ calc key_gate_weight and gate """
+ # key_gate_weight [ F_N_CHUNK, HEAD, HEAD_DIM ]
+ key_gate_weight = (
+ filtered_kv[:, 0] # K
+ .view(num_filtered_chunk, moba_chunk_size, num_head, head_dim)
+ .mean(dim=1) # mean pooling along chunk size
+ .float()
+ )
+ # print(f"Rank {dist.get_rank()} | compute_moba_gate | key_gate_weight shape: {key_gate_weight.shape}")
+
+ q = q.type(torch.float32) # float logit on the fly for better gate logit perception
+ key_gate_weight = key_gate_weight.type(
+ torch.float32
+ ) # float logit for better gate logit perception
+ gate = torch.einsum(
+ "nhd,shd->nhs", key_gate_weight, q
+ ) # gate [ F_N_CHUNK, HEAD, SEQ_BLOCK]
+ key_gate_weight = key_gate_weight.type_as(k)
+ q = q.type_as(k)
+
+ # pose process gate, masking unchosen batch and apply causal mask to current chunk
+ gate_seq_idx = torch.arange(
+ seq_offset, min(seq_offset + block_size, seqlen), device=q.device, dtype=torch.int32
+ )[None, :].repeat(num_filtered_chunk, 1)
+ chunk_end = cu_chunk[filtered_chunk_indices + 1]
+ batch_end = cu_seqlens[chunk_to_batch[filtered_chunk_indices] + 1]
+ gate_chunk_end_mask = gate_seq_idx < chunk_end[:, None]
+ gate_batch_end_mask = gate_seq_idx >= batch_end[:, None]
+ gate_inf_mask = gate_chunk_end_mask | gate_batch_end_mask
+ gate.masked_fill_(gate_inf_mask.unsqueeze(1), -float("inf"))
+ # print(f"Rank {dist.get_rank()} | compute_moba_gate | gate shape before topK: {gate.shape}")
+
+ """ find moba q that needs moba attn """
+ # find topk chunks
+ # gate_top_k_idx with shape [TOP_K, HEAD, SEQ_BLOCK]
+ _, gate_top_k_idx = torch.topk(gate, k=moba_topk, dim=0, largest=True, sorted=False)
+ # apply causal mask
+ gate_mask = torch.logical_not(gate.isinf())
+
+ # select topk chunks
+ gate_idx_mask = torch.zeros(gate_mask.shape, dtype=torch.bool, device=q.device)
+ gate_idx_mask = gate_idx_mask.scatter_(dim=0, index=gate_top_k_idx, value=True)
+
+ # [ F_N_CHUNK, HEAD, SEQ_BLOCK]
+ gate_mask = torch.logical_and(gate_mask, gate_idx_mask).contiguous()
+
+ return (
+ # gate_mask does not need to be gathered because
+ # each device only needs the gate_mask corresponding to the current query block
+ gate_mask,
+ cu_chunk,
+ filtered_chunk_indices,
+ num_filtered_chunk,
+ chunk_to_batch
+ )
diff --git a/minference/ops/op_utils/vertical_slash_utils.py b/minference/ops/op_utils/vertical_slash_utils.py
new file mode 100644
index 00000000..deea9cdd
--- /dev/null
+++ b/minference/ops/op_utils/vertical_slash_utils.py
@@ -0,0 +1,771 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+from typing import List
+
+import torch
+import torch.distributed as dist
+import triton
+import triton.language as tl
+
+
+@triton.jit
+def _triton_extract_kv_kernel(
+ local_k, local_v, bar_k, bar_v, v_idx, v_cnt,
+ stride_lz, stride_ln, stride_lh, stride_ld,
+ stride_bz, stride_bn, stride_bh, stride_bd,
+ stride_iz, stride_ih, stride_in,
+ stride_cz, stride_ch, stride_cr,
+ step, num_tokens, num_qo_heads, num_kv_heads,
+ BLOCK_N: tl.constexpr, BLOCK_D: tl.constexpr,
+):
+ start_n = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ v_cnt_ptr = v_cnt + batch_idx * stride_cz + qo_head_idx * stride_ch
+ min_n = tl.load(v_cnt_ptr + step * stride_cr)
+ max_n = tl.load(v_cnt_ptr + (step + 1) * stride_cr)
+ start_n = start_n * BLOCK_N
+ end_n = start_n + BLOCK_N
+ if start_n >= max_n or end_n <= min_n:
+ return
+
+ offs_d = tl.arange(0, BLOCK_D)
+ offs_n = start_n + tl.arange(0, BLOCK_N)
+ mask_n = (offs_n >= min_n) & (offs_n < max_n)
+
+ v_idx_ptr = v_idx + batch_idx * stride_iz + qo_head_idx * stride_ih
+ local_k_ptr = local_k + batch_idx * stride_lz + kv_head_idx * stride_lh + offs_d[None, :] * stride_ld
+ local_v_ptr = local_v + batch_idx * stride_lz + kv_head_idx * stride_lh + offs_d[None, :] * stride_ld
+ bar_k_ptr = bar_k + batch_idx * stride_bz + qo_head_idx * stride_bh + offs_d[None, :] * stride_bd
+ bar_v_ptr = bar_v + batch_idx * stride_bz + qo_head_idx * stride_bh + offs_d[None, :] * stride_bd
+
+ # idx = tl.load(v_idx_ptr + offs_n * stride_in, mask=mask_n, other=0) - step * num_tokens
+ idx = tl.load(v_idx_ptr + offs_n * stride_in, mask=mask_n, other=0) % num_tokens
+ k = tl.load(local_k_ptr + idx[:, None] * stride_ln, mask=mask_n[:, None], other=0.)
+ v = tl.load(local_v_ptr + idx[:, None] * stride_ln, mask=mask_n[:, None], other=0.)
+ tl.store(bar_k_ptr + offs_n[:, None] * stride_bn, k, mask=mask_n[:, None])
+ tl.store(bar_v_ptr + offs_n[:, None] * stride_bn, v, mask=mask_n[:, None])
+
+
+def extract_kv(
+ local_k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ local_v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ bar_k: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ bar_v: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ step: int,
+):
+ batch_size, max_v_size, num_qo_heads, head_dim = bar_k.shape
+ _, num_tokens, num_kv_heads, _ = local_k.shape
+ block_N = 128
+ block_D = head_dim
+ _triton_extract_kv_kernel[(triton.cdiv(max_v_size, block_N), num_qo_heads, batch_size)](
+ local_k, local_v, bar_k, bar_v, v_idx, v_cnt,
+ local_k.stride(0), local_k.stride(1), local_k.stride(2), local_k.stride(3),
+ bar_k.stride(0), bar_k.stride(1), bar_k.stride(2), bar_k.stride(3),
+ v_idx.stride(0), v_idx.stride(1), v_idx.stride(2),
+ v_cnt.stride(0), v_cnt.stride(1), v_cnt.stride(2),
+ step, num_tokens, num_qo_heads, num_kv_heads,
+ BLOCK_N=block_N, BLOCK_D=block_D,
+ num_warps=4, num_stages=1,
+ )
+
+
+@triton.jit
+def _triton_merge_kv_kernel(
+ local_k, local_v, bar_k, bar_v, v_idx, v_cnt,
+ stride_lz, stride_ln, stride_lh, stride_ld,
+ stride_bz, stride_bn, stride_bh, stride_bd,
+ stride_iz, stride_ih, stride_in,
+ stride_cz, stride_ch, stride_cr,
+ step, num_tokens, num_qo_heads, num_kv_heads,
+ BLOCK_N: tl.constexpr, BLOCK_D: tl.constexpr,
+):
+ start_n = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ v_cnt_ptr = v_cnt + batch_idx * stride_cz + qo_head_idx * stride_ch
+ min_n = tl.load(v_cnt_ptr + step * stride_cr)
+ max_n = tl.load(v_cnt_ptr + (step + 1) * stride_cr)
+ start_n = start_n * BLOCK_N
+ end_n = start_n + BLOCK_N
+ if start_n >= max_n or end_n <= min_n:
+ return
+
+ offs_d = tl.arange(0, BLOCK_D)
+ offs_n = start_n + tl.arange(0, BLOCK_N)
+ mask_n = (offs_n >= min_n) & (offs_n < max_n)
+
+ v_idx_ptr = v_idx + batch_idx * stride_iz + qo_head_idx * stride_ih
+ local_k_ptr = local_k + batch_idx * stride_lz + kv_head_idx * stride_lh + offs_d[None, :] * stride_ld
+ local_v_ptr = local_v + batch_idx * stride_lz + kv_head_idx * stride_lh + offs_d[None, :] * stride_ld
+ bar_k_ptr = bar_k + batch_idx * stride_bz + qo_head_idx * stride_bh + offs_d[None, :] * stride_bd
+ bar_v_ptr = bar_v + batch_idx * stride_bz + qo_head_idx * stride_bh + offs_d[None, :] * stride_bd
+
+ # idx = tl.load(v_idx_ptr + offs_n * stride_in, mask=mask_n, other=0) - step * num_tokens
+ idx = tl.load(v_idx_ptr + offs_n * stride_in, mask=mask_n, other=0) % num_tokens
+ k = tl.load(bar_k_ptr + offs_n[:, None] * stride_bn, mask=mask_n[:, None], other=0.).to(local_k.type.element_ty)
+ v = tl.load(bar_v_ptr + offs_n[:, None] * stride_bn, mask=mask_n[:, None], other=0.).to(local_v.type.element_ty)
+ tl.atomic_add(local_k_ptr + idx[:, None] * stride_ln, k, mask=mask_n[:, None], sem="relaxed")
+ tl.atomic_add(local_v_ptr + idx[:, None] * stride_ln, v, mask=mask_n[:, None], sem="relaxed")
+
+
+def merge_kv(
+ local_k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ local_v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ bar_k: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ bar_v: torch.Tensor, # [batch_size, max_v_size, num_qo_heads, head_dim]
+ v_idx: torch.Tensor, # [batch_size, num_qo_heads, max_v_size]
+ v_cnt: torch.Tensor, # [batch_size, num_qo_heads, world_size + 1]
+ step: int,
+):
+ batch_size, max_v_size, num_qo_heads, head_dim = bar_k.shape
+ _, num_tokens, num_kv_heads, _ = local_k.shape
+ block_N = 128
+ block_D = head_dim
+ _triton_merge_kv_kernel[(triton.cdiv(max_v_size, block_N), num_qo_heads, batch_size)](
+ local_k, local_v, bar_k, bar_v, v_idx, v_cnt,
+ local_k.stride(0), local_k.stride(1), local_k.stride(2), local_k.stride(3),
+ bar_k.stride(0), bar_k.stride(1), bar_k.stride(2), bar_k.stride(3),
+ v_idx.stride(0), v_idx.stride(1), v_idx.stride(2),
+ v_cnt.stride(0), v_cnt.stride(1), v_cnt.stride(2),
+ step, num_tokens, num_qo_heads, num_kv_heads,
+ BLOCK_N=block_N, BLOCK_D=block_D,
+ num_warps=4, num_stages=1,
+ )
+
+
+# triton.cdiv(world_size * num_blocks, BLOCK_N), num_heads, batch_size
+# block_mask: [batch_size, num_heads, num_blocks_global]
+@triton.jit
+def _calc_block_mask_kernel(
+ s_idx, block_mask,
+ stride_sz, stride_sh, stride_sk,
+ stride_bz, stride_bh, stride_bn,
+ max_s_size, num_tokens, granularity,
+ BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
+):
+ batch_idx = tl.program_id(2)
+ head_idx = tl.program_id(1)
+ group_idx = tl.program_id(0)
+
+ block_offs = tl.arange(0, BLOCK_N)
+ slash_offs = tl.arange(0, BLOCK_K)
+
+ s_idx_ptr = s_idx + batch_idx * stride_sz + head_idx * stride_sh
+ block_mask_ptr = block_mask + batch_idx * stride_bz + head_idx * stride_bh
+ block_idx = group_idx * BLOCK_N + block_offs
+
+ blocks = tl.zeros([BLOCK_N], dtype=tl.uint8)
+ for s_off in range(0, max_s_size, BLOCK_K):
+ s = tl.load(s_idx_ptr + (s_off + slash_offs) * stride_sk)
+ left = (num_tokens - granularity - s) // granularity
+ right = (num_tokens - 1 - s) // granularity
+
+ # mask is generated by checking if a block's index falls between the calculated ranges
+ blocks |= tl.max((block_idx[None, :] >= left[:, None]) & (block_idx[None, :] <= right[:, None]), 0).to(tl.uint8)
+
+ b_mask = (group_idx * BLOCK_N + block_offs) * granularity < num_tokens
+ tl.store(block_mask_ptr + (group_idx * BLOCK_N + block_offs) * stride_bn, blocks, mask=b_mask)
+
+
+@triton.jit
+def _striped_convert_indices_kernel(
+ last_row_mask, v_idx, v_cnt,
+ block_mask, bar_idx, bar_pos, bar_cnt,
+ stride_rz, stride_rh, stride_rn,
+ stride_vz, stride_vh, stride_vk,
+ stride_nz, stride_nh, stride_nt,
+ stride_bt, stride_bz, stride_bh, stride_bm, stride_bn,
+ stride_iz, stride_ih, stride_im, stride_ik,
+ stride_cz, stride_ch, stride_cm, stride_ct,
+ max_v_size, num_blocks, granularity, world_size, rank,
+ BLOCK_N: tl.constexpr,
+):
+ batch_idx = tl.program_id(2)
+ head_idx = tl.program_id(1)
+ block_idx_q_local = tl.program_id(0)
+
+ block_idx_q_global = block_idx_q_local * world_size + rank
+
+ num_tokens_local = num_blocks * granularity
+ num_blocks_global = world_size * num_blocks
+ shift = num_blocks_global - 1 - block_idx_q_global
+
+ block_offs = tl.arange(0, BLOCK_N)
+
+ last_row_mask_ptr = last_row_mask + batch_idx * stride_rz + head_idx * stride_rh
+ v_idx_ptr = v_idx + batch_idx * stride_vz + head_idx * stride_vh
+ v_cnt_ptr = v_cnt + batch_idx * stride_nz + head_idx * stride_nh
+ block_mask_ptr = block_mask + batch_idx * stride_bz + head_idx * stride_bh + block_idx_q_local * stride_bm
+ bar_idx_ptr = bar_idx + batch_idx * stride_iz + head_idx * stride_ih + block_idx_q_local * stride_im
+ bar_pos_ptr = bar_pos + batch_idx * stride_iz + head_idx * stride_ih + block_idx_q_local * stride_im
+ bar_cnt_ptr = bar_cnt + batch_idx * stride_cz + head_idx * stride_ch + block_idx_q_local * stride_cm
+
+ cnt_valid = 0
+ cnt_all = 0
+ v_off = 0
+ v = tl.load(v_idx_ptr + cnt_all * stride_vk)
+ cnt_all += 1
+
+ tl.store(bar_cnt_ptr, cnt_valid)
+ bar_cnt_ptr += stride_ct
+ if block_idx_q_local == tl.num_programs(0) - 1:
+ tl.store(v_cnt_ptr, cnt_all - 1)
+ v_cnt_ptr += stride_nt
+
+ for step in range(world_size):
+ for block_off_k in range(0, num_blocks, BLOCK_N):
+ block_idx_k_local = block_off_k + block_offs
+ block_idx_k_global = (block_off_k + block_offs) * world_size + step
+ mask_local = tl.load(
+ last_row_mask_ptr + (block_idx_k_global + shift) * stride_rn,
+ mask=(block_idx_k_global + shift < num_blocks_global),
+ other=0,
+ )
+ tl.store(
+ block_mask_ptr + block_idx_k_local * stride_bn,
+ mask_local,
+ mask=(block_idx_k_local < num_blocks),
+ )
+ block_left = v_off + block_idx_k_local * granularity
+ block_right = block_left + granularity
+ max_blocks = block_idx_q_local + 1 if step <= rank else block_idx_q_local
+ v_max = v_off + min(block_off_k + BLOCK_N, max_blocks) * granularity
+ while v < v_max and cnt_all < max_v_size:
+ if tl.max(((v >= block_left) & (v < block_right)) & (~mask_local), 0):
+ tl.store(bar_idx_ptr + cnt_valid * stride_ik, v - v_off)
+ tl.store(bar_pos_ptr + cnt_valid * stride_ik, cnt_all - 1)
+ cnt_valid += 1
+ v = tl.load(v_idx_ptr + cnt_all * stride_vk)
+ cnt_all += 1
+ block_mask_ptr += stride_bt
+ tl.store(bar_cnt_ptr, cnt_valid)
+ bar_cnt_ptr += stride_ct
+ v_off += num_tokens_local
+ if block_idx_q_local == tl.num_programs(0) - 1:
+ tl.store(v_cnt_ptr, cnt_all - 1)
+ v_cnt_ptr += stride_nt
+
+
+@triton.jit
+def _zigzag_convert_indices_kernel(
+ last_row_mask, v_idx, v_cnt,
+ block_mask, bar_idx, bar_pos, bar_cnt,
+ stride_rz, stride_rh, stride_rn,
+ stride_vz, stride_vh, stride_vk,
+ stride_nz, stride_nh, stride_nt,
+ stride_bt, stride_bz, stride_bh, stride_bm, stride_bn,
+ stride_iz, stride_ih, stride_im, stride_ik,
+ stride_cz, stride_ch, stride_cm, stride_ct,
+ max_v_size, num_blocks, granularity, world_size, rank,
+ BLOCK_N: tl.constexpr,
+):
+ batch_idx = tl.program_id(2)
+ head_idx = tl.program_id(1)
+ block_idx_q_local = tl.program_id(0)
+
+ if rank < world_size // 2:
+ revert_rank = rank * 2
+ else:
+ revert_rank = (world_size - 1 - rank) * 2 + 1
+ if block_idx_q_local < num_blocks // 2:
+ block_idx_q_global = revert_rank * (num_blocks // 2) + block_idx_q_local
+ else:
+ block_idx_q_global = (world_size * 2 - 1 - revert_rank) * (num_blocks // 2) + block_idx_q_local - (num_blocks // 2)
+
+ num_blocks_global = world_size * num_blocks
+ shift = num_blocks_global - 1 - block_idx_q_global
+
+ block_offs = tl.arange(0, BLOCK_N)
+
+ last_row_mask_ptr = last_row_mask + batch_idx * stride_rz + head_idx * stride_rh
+ v_idx_ptr = v_idx + batch_idx * stride_vz + head_idx * stride_vh
+ v_cnt_ptr = v_cnt + batch_idx * stride_nz + head_idx * stride_nh
+ block_mask_ptr = block_mask + batch_idx * stride_bz + head_idx * stride_bh + block_idx_q_local * stride_bm
+ bar_idx_ptr = bar_idx + batch_idx * stride_iz + head_idx * stride_ih + block_idx_q_local * stride_im
+ bar_pos_ptr = bar_pos + batch_idx * stride_iz + head_idx * stride_ih + block_idx_q_local * stride_im
+ bar_cnt_ptr = bar_cnt + batch_idx * stride_cz + head_idx * stride_ch + block_idx_q_local * stride_cm
+
+ cnt_valid = 0
+ cnt_all = 0
+ v = tl.load(v_idx_ptr + cnt_all * stride_vk)
+ cnt_all += 1
+
+ tl.store(bar_cnt_ptr, cnt_valid)
+ bar_cnt_ptr += stride_ct
+ if block_idx_q_local == tl.num_programs(0) - 1:
+ tl.store(v_cnt_ptr, cnt_all - 1)
+ v_cnt_ptr += stride_nt
+
+ for step in range(world_size):
+ v_off = step * num_blocks * granularity
+ v_end = v_off + num_blocks * granularity
+ for block_off_k in range(0, num_blocks, BLOCK_N):
+ block_idx_k_local = block_off_k + block_offs
+ # assert BLOCK_N <= num_blocks // 2
+ if block_off_k < num_blocks // 2:
+ v_off_global = step * (num_blocks // 2) * granularity
+ block_idx_k_global = step * (num_blocks // 2) + block_idx_k_local
+ else:
+ v_off_global = (world_size * 2 - 2 - step) * (num_blocks // 2) * granularity
+ block_idx_k_global = (world_size * 2 - 1 - step) * (num_blocks // 2) + block_idx_k_local - (num_blocks // 2)
+ mask_local = tl.load(
+ last_row_mask_ptr + (block_idx_k_global + shift) * stride_rn,
+ mask=(block_idx_k_global + shift < num_blocks_global),
+ other=0,
+ )
+ tl.store(
+ block_mask_ptr + block_idx_k_local * stride_bn,
+ mask_local,
+ mask=(block_idx_k_local < num_blocks),
+ )
+ # block_left = block_idx_k_global * granularity - v_off_global + v_off
+ # block_right = block_left + granularity
+ block_left = v_off + block_idx_k_local * granularity
+ block_right = block_left + granularity
+ v_max = (block_idx_q_global + 1) * granularity - v_off_global + v_off
+ while v < v_end and cnt_all <= max_v_size:
+ if v < v_max:
+ if tl.max(((v >= block_left) & (v < block_right)) & (~mask_local), 0):
+ tl.store(bar_idx_ptr + cnt_valid * stride_ik, v - v_off)
+ tl.store(bar_pos_ptr + cnt_valid * stride_ik, cnt_all - 1)
+ cnt_valid += 1
+ v = tl.load(v_idx_ptr + cnt_all * stride_vk)
+ cnt_all += 1
+ block_mask_ptr += stride_bt
+ tl.store(bar_cnt_ptr, cnt_valid)
+ bar_cnt_ptr += stride_ct
+ if block_idx_q_local == tl.num_programs(0) - 1:
+ tl.store(v_cnt_ptr, cnt_all - 1)
+ v_cnt_ptr += stride_nt
+
+
+def convert_indices(
+ v_idx: torch.Tensor, # [batch_size, num_heads, max_v_size]
+ s_idx: torch.Tensor, # [batch_size, num_heads, max_s_size]
+ world_size: int,
+ rank: int,
+ num_blocks: int,
+ granularity: int,
+ num_tokens: int = None,
+ stripe_transform: bool = False,
+ zigzag_transform: bool = False,
+):
+ num_blocks_global = world_size * num_blocks
+ if num_tokens is None:
+ # Note that for each invokation of `convert_indices`, `num_tokens` is None and becomes the **global number of tokens**
+ num_tokens = num_blocks_global * granularity
+ batch_size, num_heads, max_v_size = v_idx.shape
+ batch_size, num_heads, max_s_size = s_idx.shape
+ last_row_mask = torch.zeros((batch_size, num_heads, num_blocks_global), dtype=torch.bool, device=s_idx.device)
+
+ BLOCK_N, BLOCK_K = 128, 128
+ assert max_s_size <= BLOCK_K * BLOCK_K, f"max_s_size={max_s_size} > BLOCK_K * BLOCK_K={BLOCK_K * BLOCK_K}"
+ _calc_block_mask_kernel[(triton.cdiv(num_blocks_global, BLOCK_N), num_heads, batch_size)](
+ s_idx, last_row_mask,
+ s_idx.stride(0), s_idx.stride(1), s_idx.stride(2),
+ last_row_mask.stride(0), last_row_mask.stride(1), last_row_mask.stride(2),
+ max_s_size, num_tokens, granularity,
+ BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K,
+ num_warps=4, num_stages=2,
+ )
+
+ block_mask = torch.zeros((world_size, batch_size, num_heads, num_blocks, num_blocks), dtype=torch.bool, device=v_idx.device)
+ bar_idx = torch.zeros((batch_size, num_heads, num_blocks, max_v_size), dtype=torch.int32, device=v_idx.device)
+ bar_cnt = torch.empty((batch_size, num_heads, num_blocks, world_size + 1), dtype=torch.int32, device=v_idx.device)
+ v_cnt = torch.empty((batch_size, num_heads, world_size + 1), dtype=torch.int32, device=v_idx.device)
+ bar_pos = torch.zeros_like(bar_idx)
+ if zigzag_transform:
+ convert_indices_kernel = _zigzag_convert_indices_kernel
+ assert num_blocks % 2 == 0
+ BLOCK_N = max(num_blocks // 2, 128)
+ else:
+ convert_indices_kernel = _striped_convert_indices_kernel
+ BLOCK_N = 128
+ convert_indices_kernel[(num_blocks, num_heads, batch_size)](
+ last_row_mask, v_idx, v_cnt, block_mask, bar_idx, bar_pos, bar_cnt,
+ last_row_mask.stride(0), last_row_mask.stride(1), last_row_mask.stride(2),
+ v_idx.stride(0), v_idx.stride(1), v_idx.stride(2),
+ v_cnt.stride(0), v_cnt.stride(1), v_cnt.stride(2),
+ block_mask.stride(0), block_mask.stride(1), block_mask.stride(2), block_mask.stride(3), block_mask.stride(4),
+ bar_idx.stride(0), bar_idx.stride(1), bar_idx.stride(2), bar_idx.stride(3),
+ bar_cnt.stride(0), bar_cnt.stride(1), bar_cnt.stride(2), bar_cnt.stride(3),
+ max_v_size, num_blocks, granularity, world_size, rank, BLOCK_N=BLOCK_N,
+ num_warps=1, num_stages=1,
+ )
+
+ return block_mask, bar_idx, bar_cnt, bar_pos, v_cnt
+
+
+def _torch_convert_indices(
+ v_idx: torch.Tensor, # [batch_size, num_heads, max_v_size]
+ s_idx: torch.Tensor, # [batch_size, num_heads, max_s_size]
+ world_size: int,
+ rank: int,
+ num_blocks: int,
+ granularity: int,
+):
+ batch_size, num_heads, max_v_size = v_idx.shape
+ num_tokens = world_size * num_blocks * granularity
+ block_mask = torch.zeros((world_size, batch_size, num_heads, num_blocks, num_blocks), dtype=torch.bool, device=v_idx.device)
+ bar_idx = torch.zeros((batch_size, num_heads, num_blocks, max_v_size), dtype=torch.int32, device=v_idx.device)
+ bar_cnt = torch.zeros((batch_size, num_heads, num_blocks, world_size + 1), dtype=torch.int32, device=v_idx.device)
+ for batch_idx in range(batch_size):
+ for head_idx in range(num_heads):
+ for block_idx_q in range(num_blocks):
+ block_idx_q_global = block_idx_q * world_size + rank
+ cnt_all, cnt_valid = 0, 0
+ for step in range(world_size):
+ for block_idx_k in range(block_idx_q + 1):
+ block_idx_k_global = block_idx_k * world_size + step
+ s_min = max((block_idx_q_global - block_idx_k_global - 1) * granularity, 0)
+ s_max = (block_idx_q_global - block_idx_k_global + 1) * granularity
+ flag = torch.any((s_idx[batch_idx, head_idx] > s_min) & (s_idx[batch_idx, head_idx] < s_max))
+ block_mask[step, batch_idx, head_idx, block_idx_q, block_idx_k] = flag
+ v_min = (step * num_blocks + block_idx_k) * granularity
+ max_blocks = block_idx_q + 1 if step <= rank else block_idx_q
+ v_max = (step * num_blocks + min(block_idx_k + 1, max_blocks)) * granularity
+ while cnt_all < max_v_size and v_idx[batch_idx, head_idx, cnt_all] < v_min:
+ cnt_all += 1
+ while cnt_all < max_v_size and v_idx[batch_idx, head_idx, cnt_all] < v_max:
+ if not flag:
+ bar_idx[batch_idx, head_idx, block_idx_q, cnt_valid] = \
+ v_idx[batch_idx, head_idx, cnt_all] - step * num_blocks * granularity
+ cnt_valid += 1
+ cnt_all += 1
+ bar_cnt[batch_idx, head_idx, block_idx_q, step + 1] = cnt_valid
+ return block_mask, bar_idx, bar_cnt
+
+
+
+def sum_all_diagonal_matrix(mat: torch.Tensor):
+ b, h, m, n = mat.shape
+
+ # Pads the matrix on left and right (on the last dimension)
+ mat_padded = torch.nn.functional.pad(mat, (m, m), "constant", 0.) # shape: [b, h, m, 2 * m + n]
+ # Change the strides
+ mat_strided = mat_padded.as_strided((b, h, m, m + n), (m * (2 * m + n) * h, m * (2 * m + n), 2 * m + n + 1, 1))
+ # Sums the resulting matrix's columns
+ sum_diags = torch.sum(mat_strided, 2) # shape: [b, h, m + n]
+ return sum_diags[:, :, 1:].contiguous()
+
+def calc_index(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v_size: List[int],
+ s_size: List[int],
+ last_q_size: int = 64,
+ sink_tokens: int = 30,
+ sliding_window: int = 100,
+ group: dist.group = None,
+ stripe_transform: bool = False,
+ zigzag_transform: bool = False,
+ granularity: int = 128,
+):
+ # TODO: adapt naturely striped inputs
+ # TODO: flex-prefill (top-P)
+ # TODO: reduce bubble
+ # TODO: support total_num_tokens % world_size != 0
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ rank = dist.get_rank(group)
+ world_size = dist.get_world_size(group)
+
+ if all([type(x) is list for x in v_size]) and all([type(x) is list for x in s_size]):
+ flex_prefill = True
+ v_p = [x[0] for x in v_size]
+ v_size = [x[1] for x in v_size]
+ s_p = [x[0] for x in s_size]
+ s_size = [x[1] for x in s_size]
+ else:
+ flex_prefill = False
+ assert all([type(x) is int for x in v_size]) and all([type(x) is int for x in s_size])
+
+ max_v_size = min(triton.cdiv(max(v_size), 128), num_tokens // 128) * 128
+ max_s_size = min(triton.cdiv(max(s_size), 128), num_tokens // 128) * 128
+
+ last_rank = world_size - 1
+ if rank == last_rank:
+ last_q = q[:, -last_q_size:, :, :].detach().clone().reshape((batch_size, last_q_size, num_kv_heads, -1, head_dim))
+ else:
+ last_q = torch.zeros((batch_size, last_q_size, num_kv_heads, num_qo_heads // num_kv_heads, head_dim), device=q.device, dtype=q.dtype)
+
+ dist.broadcast(last_q, src=last_rank, group=group, async_op=False)
+
+ qk = torch.einsum('bmghd, bngd -> bghmn', last_q, k) * (k.shape[-1] ** -0.5)
+ qk = qk.reshape((batch_size, num_qo_heads, last_q_size, num_tokens))
+
+ if rank == last_rank:
+ # Causal Mask, requires num_tokens // world_size >= last_q
+ arange = torch.arange(last_q_size, device=k.device)
+ mask = arange[None, None, :, None] >= arange[None, None, None, :]
+ qk[:, :, :, -last_q_size:] = torch.where(mask, qk[:, :, :, -last_q_size:], -torch.inf)
+ if flex_prefill: # qk = torch.softmax(qk, dim=-1) / last_q_size
+ qk_max = torch.max(qk, dim=-1, keepdim=True).values
+ qk_max_list = [torch.empty_like(qk_max) for _ in range(world_size)]
+ dist.all_gather(qk_max_list, qk_max, group=group, async_op=False)
+ qk_max = torch.max(torch.stack(qk_max_list), dim=0).values
+ qk = torch.exp(qk - qk_max)
+ qk_sum = torch.sum(qk, dim=-1, keepdim=True)
+ qk_sum_list = [torch.empty_like(qk_sum) for _ in range(world_size)]
+ dist.all_gather(qk_sum_list, qk_sum, group=group, async_op=False)
+ qk_sum = torch.sum(torch.stack(qk_sum_list), dim=0)
+ qk /= (qk_sum * last_q_size)
+
+ v_gather_rank = 0
+ vertical = qk.sum(-2, keepdim=False) # [B, H, N_LOCAL]
+ if rank == 0 and not flex_prefill:
+ vertical[..., :sink_tokens] = torch.inf
+ if rank == v_gather_rank:
+ gathered_vertical = [torch.empty_like(vertical) for _ in range(world_size)]
+ else:
+ gathered_vertical = None
+ dist.gather(vertical, gathered_vertical, dst=v_gather_rank, group=group, async_op=False)
+
+ if rank == v_gather_rank:
+ vertical: torch.Tensor = torch.cat(gathered_vertical, dim=-1)
+ if stripe_transform:
+ vertical = vertical.reshape((batch_size, num_qo_heads, -1, world_size, granularity))
+ vertical = vertical.swapaxes(2, 3)
+ vertical = vertical.reshape((batch_size, num_qo_heads, -1))
+ elif zigzag_transform:
+ vertical = vertical.reshape((batch_size, num_qo_heads, 2, world_size, -1))
+ chunks = []
+ for step in range(world_size):
+ chunks.append(vertical[:, :, 0, step])
+ chunks.append(vertical[:, :, 1, world_size - 1 - step])
+ vertical = torch.concat(chunks, dim=2).reshape((batch_size, num_qo_heads, -1))
+
+ v_topk = torch.topk(vertical, max_v_size, -1, sorted=True)
+ v_indices = v_topk.indices.to(torch.int32)
+ if flex_prefill:
+ v_cumsum = v_topk.values.cumsum_(dim=-1)
+ v_size = (v_cumsum < torch.tensor(v_p, device=k.device)[None, :, None]).sum(dim=-1, keepdim=True)
+ else:
+ v_size = torch.tensor(v_size, device=k.device)[None, :, None]
+ v_arange = torch.arange(max_v_size, device=k.device)
+ v_indices.masked_fill_(v_arange[None, None, :] >= v_size, num_tokens * world_size)
+ v_indices = v_indices.sort(dim=-1, descending=False).values
+ else:
+ v_indices = torch.empty((batch_size, num_qo_heads, max_v_size), dtype=torch.int32, device=k.device)
+ dist.broadcast(v_indices, src=v_gather_rank, group=group, async_op=False) # async
+
+ s_gather_rank = 0
+ slash = sum_all_diagonal_matrix(qk) # shape: [B, H, N_LOCAL + LAST_Q_SIZE - 1]
+ if rank == world_size - 1 and not flex_prefill:
+ # -> index starting from the left bottom corner to right upper corner
+ # (sliding_window) from -(last_q_size-1) is the sliding window close to the main diagonal
+ slash[..., -(last_q_size - 1 + sliding_window):] = torch.inf
+
+
+ if rank == s_gather_rank:
+ gathered_slash = [torch.empty_like(slash) for _ in range(world_size)]
+ else:
+ gathered_slash = None
+ dist.gather(slash, gathered_slash, dst=s_gather_rank, group=group, async_op=False)
+
+ if rank == s_gather_rank:
+ slash = gathered_slash[0]
+ for next_slash in gathered_slash[1:]:
+ slash[..., -last_q_size + 1:] += next_slash[..., :last_q_size - 1]
+ slash = torch.cat((slash, next_slash[..., last_q_size - 1:]), dim=-1)
+
+ # slash presents the sum of attention from 0-th to (num_tokens_global - last_q_size - 1), where 0 represents the diagonal at bottom left corner
+ slash = slash[..., :-last_q_size + 1]
+ s_topk = torch.topk(slash, max_s_size, -1, sorted=True)
+
+ # s_indices contain indices starting from the right upper corner to left bottom corner
+ s_indices = (num_tokens * world_size - 1) - s_topk.indices.to(torch.int32)
+ if flex_prefill:
+ s_cumsum = s_topk.values.cumsum_(dim=-1)
+ s_size = (s_cumsum < torch.tensor(s_p, device=k.device)[None, :, None]).sum(dim=-1, keepdim=True)
+ else:
+ s_size = torch.tensor(s_size, device=k.device)[None, :, None]
+ s_arange = torch.arange(max_s_size, device=k.device)
+ s_indices.masked_fill_(s_arange[None, None, :] >= s_size, -1)
+ s_indices = s_indices.sort(dim=-1, descending=True).values
+ else:
+ s_indices = torch.empty((batch_size, num_qo_heads, max_s_size), dtype=torch.int32, device=k.device)
+ dist.broadcast(s_indices, src=s_gather_rank, group=group, async_op=False)
+
+ return v_indices.to(torch.int32), s_indices.to(torch.int32)
+
+def calc_index_local(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v_size: List[int],
+ s_size: List[int],
+ last_q_size: int = 64,
+ sink_tokens: int = 30,
+ sliding_window: int = 100,
+ group: dist.group = None,
+ stripe_transform: bool = False,
+ zigzag_transform: bool = False,
+ granularity: int = 128,
+):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+
+ if all([type(x) is list for x in v_size]) and all([type(x) is list for x in s_size]):
+ flex_prefill = True
+ v_p = [x[0] for x in v_size]
+ v_size = [x[1] for x in v_size]
+ s_p = [x[0] for x in s_size]
+ s_size = [x[1] for x in s_size]
+ else:
+ flex_prefill = False
+ assert all([type(x) is int for x in v_size]) and all([type(x) is int for x in s_size])
+
+ qk = torch.einsum(
+ f'bmghd, bngd -> bghmn',
+ q[:, -last_q_size:, :, :].reshape((batch_size, last_q_size, num_kv_heads, -1, head_dim)),
+ k,
+ ).reshape((batch_size, num_qo_heads, last_q_size, num_tokens)) * (head_dim ** -0.5)
+
+ arange = torch.arange(last_q_size, device=k.device)
+ mask = arange[None, None, :, None] >= arange[None, None, None, :]
+ qk[:, :, :, -last_q_size:] = torch.where(mask, qk[:, :, :, -last_q_size:], -torch.inf)
+ if flex_prefill:
+ qk = torch.softmax(qk, dim=-1) / last_q_size
+
+ max_v_size = min(max(v_size), num_tokens)
+ max_v_size = triton.cdiv(max_v_size, 128) * 128
+ vertical = qk.sum(-2, keepdim=False)
+ if not flex_prefill:
+ vertical[..., :sink_tokens] = torch.inf
+ if stripe_transform:
+ vertical = vertical.reshape((batch_size, num_qo_heads, -1, dist.get_world_size(group), granularity))
+ vertical = vertical.swapaxes(2, 3)
+ vertical = vertical.reshape((batch_size, num_qo_heads, -1))
+ elif zigzag_transform:
+ vertical = vertical.reshape((batch_size, num_qo_heads, 2, dist.get_world_size(group), -1))
+ chunks = []
+ for step in range(dist.get_world_size(group)):
+ chunks.append(vertical[:, :, 0, step])
+ chunks.append(vertical[:, :, 1, dist.get_world_size(group) - 1 - step])
+ vertical = torch.concat(chunks, dim=2).reshape((batch_size, num_qo_heads, -1))
+ v_topk = torch.topk(vertical, max_v_size, -1, sorted=True)
+ v_indices = v_topk.indices
+ if flex_prefill:
+ v_cumsum = v_topk.values.cumsum_(dim=-1)
+ v_size = (v_cumsum < torch.tensor(v_p, device=k.device)[None, :, None]).sum(dim=-1, keepdim=True)
+ else:
+ v_size = torch.tensor(v_size, device=k.device)[None, :, None]
+
+ max_s_size = min(max(s_size), num_tokens)
+ max_s_size = triton.cdiv(max_s_size, 128) * 128
+ slash = sum_all_diagonal_matrix(qk)[..., :-last_q_size + 1]
+ if not flex_prefill:
+ slash[..., -sliding_window:] = torch.inf
+ s_topk = torch.topk(slash, max_s_size, -1, sorted=True)
+ s_indices = (num_tokens - 1) - s_topk.indices
+ if flex_prefill:
+ s_cumsum = s_topk.values.cumsum_(dim=-1)
+ s_size = (s_cumsum < torch.tensor(s_p, device=k.device)[None, :, None]).sum(dim=-1, keepdim=True)
+ else:
+ s_size = torch.tensor(s_size, device=k.device)[None, :, None]
+
+ v_arange = torch.arange(max_v_size, device=k.device)
+ v_idx = v_indices.to(torch.int32).reshape((batch_size, num_qo_heads, -1))
+ v_idx.masked_fill_(v_arange[None, None, :] >= v_size, 2147483647)
+ v_idx = v_idx.sort(dim=-1, descending=False).values
+
+ s_arange = torch.arange(max_s_size, device=k.device)
+ s_idx = s_indices.to(torch.int32).reshape((batch_size, num_qo_heads, -1))
+ s_idx.masked_fill_(s_arange[None, None, :] >= s_size, -1)
+ s_idx = s_idx.sort(dim=-1, descending=True).values
+
+ return v_idx, s_idx
+
+def build_index_local(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v_size: List[int],
+ s_size: List[int],
+ num_tokens: int,
+ granularity: int,
+ world_size: int = 1,
+ rank: int = 0,
+):
+ if type(v_size) is list:
+ assert len(v_size) == q.shape[2]
+ assert len(s_size) == q.shape[2]
+ v_idx, s_idx = calc_index_local(q, k, v_size, s_size, last_q_size=64)
+ else:
+ v_idx, s_idx = v_size, s_size
+
+ num_blocks = triton.cdiv(num_tokens, granularity)
+ block_mask, bar_idx, bar_cnt, _, _ = convert_indices(v_idx, s_idx, world_size, rank, num_blocks, granularity)
+ block_mask = block_mask[rank]
+ return block_mask, bar_idx, bar_cnt
+
+def build_index(
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v_size: List[int],
+ s_size: List[int],
+ num_tokens: int, # num_tokens_local
+ granularity: int,
+ stripe_transform: bool = True,
+ zigzag_transform: bool = False,
+ group: dist.group = None,
+):
+ """
+ Input: (all inputs correspond to the local part for each rank)
+ q: shape [batch_size, num_tokens_local, num_qo_heads, head_dim]
+ k: shape [batch_size, num_tokens_local, num_kv_heads, head_dim]
+ v_size: shape [num_qo_heads]
+ s_size: shape [num_qo_heads]
+ num_tokens: number of tokens in the local part of QK
+ Returns:
+ block_mask: shape [world_size, batch_size, num_heads, num_blocks, num_blocks]
+ bar_idx: shape [batch_size, num_heads, num_blocks, max_v_size]
+ bar_cnt: shape [batch_size, num_heads, num_blocks, world_size + 1], each entry is the cumulative number of selected bars corresponding a rank
+ """
+ rank = dist.get_rank(group)
+ world_size = dist.get_world_size(group)
+
+ if isinstance(v_size, list):
+ v_idx, s_idx = calc_index(
+ q, k, v_size, s_size, last_q_size=64, group=group,
+ stripe_transform=stripe_transform,
+ zigzag_transform=zigzag_transform,
+ granularity=granularity
+ )
+ else:
+ v_idx, s_idx = v_size, s_size
+
+ num_blocks = triton.cdiv(num_tokens, granularity) # num_blocks_local
+
+ # Note that block_mask is a 5D tensor with shape [world_size, batch_size, num_heads, num_blocks, num_blocks]
+ # with each block_mask[i] is to a mask corresponding the num_tokens_local x num_tokens_local matmul for each step
+ block_mask, bar_idx, bar_cnt, bar_pos, v_cnt = convert_indices(
+ v_idx, s_idx, world_size, rank, num_blocks, granularity,
+ stripe_transform=stripe_transform,
+ zigzag_transform=zigzag_transform,
+ )
+ return block_mask, bar_idx, bar_cnt, bar_pos, v_idx, v_cnt
+
+def convert_blockmask(
+ blockmask: torch.Tensor, # [world_size, batch_size, num_heads, num_blocks, num_blocks]
+ block_size_M: int,
+ block_size_N: int,
+):
+ ratio = block_size_M // block_size_N
+ original_shape = blockmask.shape
+ blockmask = blockmask.to(dtype=torch.uint8)
+ blockmask = blockmask.unsqueeze(-1).tile([1] * len(original_shape) + [ratio]).reshape((*original_shape[:-1], -1))
+
+ # now block_mask is [world_size, batch_size, num_heads, num_blocks, num_blocks * ratio]
+ nonzero_val, nonzero_idx = blockmask.sort(dim=-1, stable=True, descending=True)
+
+ nonzero_rowcnt = blockmask.sum(dim=-1, dtype=torch.int32)
+ return nonzero_idx.contiguous().to(dtype=torch.int32), nonzero_rowcnt.contiguous()
diff --git a/minference/ops/op_utils/xattn_utils.py b/minference/ops/op_utils/xattn_utils.py
new file mode 100644
index 00000000..4d1cb889
--- /dev/null
+++ b/minference/ops/op_utils/xattn_utils.py
@@ -0,0 +1,593 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import torch
+import torch.distributed as dist
+import triton
+import triton.language as tl
+
+LN2 = 1 / 1.4426950408889634
+def create_causal_mask(batch_size, head_num, block_size, block_num, divide_block_num):
+ """
+ Creates a causal attention mask used in transformer-based models.
+
+ Parameters:
+ - batch_size (int): The number of sequences in the batch.
+ - head_num (int): The number of attention heads.
+ - block_size (int): The size of each block in the sequence.
+ - block_num (int): The total number of blocks in the sequence.
+ - divide_block_num (int): The block index at which causality is applied.
+
+ Returns:
+ - torch.Tensor: A mask tensor of shape (batch_size, head_num, block_size, total_size)
+ where total_size = block_size * block_num. The mask enforces causal attention by
+ setting certain positions to `-inf` to prevent information leakage from future tokens.
+ """
+ divide_block_num += 1
+ if divide_block_num < 1 or divide_block_num > block_num:
+ raise ValueError(
+ f"divide_block_num ({divide_block_num}) must be between 1 and block_num ({block_num})."
+ )
+
+ total_size = block_size * block_num
+ device = "cuda"
+ mask = torch.zeros(block_size, total_size, device=device)
+ if divide_block_num < block_num:
+ mask[:, divide_block_num * block_size :] = float("-inf")
+
+ if divide_block_num - 1 < block_num:
+ start_col = (divide_block_num - 1) * block_size
+ end_col = start_col + block_size
+ upper_tri_mask = torch.triu(
+ torch.full((block_size, block_size), float("-inf"), device=device),
+ diagonal=1,
+ )
+ mask[:, start_col:end_col] = upper_tri_mask
+
+ mask = mask.unsqueeze(0).unsqueeze(0)
+ mask = mask.expand(batch_size, head_num, block_size, total_size)
+ return mask
+
+def find_blocks_chunked(
+ input_tensor: torch.Tensor, # (batch_size, num_heads, num_block_q, num_block_k)
+ current_index, #
+ threshold, num_to_choose, decoding: bool, mode: str = "both", causal=True
+):
+ """
+ Finds and selects relevant blocks of attention for transformer-based models based on a
+ threshold or a predefined number of blocks.
+
+ Parameters:
+ - input_tensor (torch.Tensor): The input tensor of shape (batch_size, head_num, num_block_q, num_block_k).
+ - current_index (int): The current index in the sequence processing.
+ - threshold (float or None): A threshold value used to determine the minimum attention weight sum.
+ - num_to_choose (int or None): The number of blocks to be selected, ensuring sufficient information retrieval.
+ - decoding (bool): If True, operates in decoding mode; otherwise, it's in encoding mode.
+ - mode (str): Defines the processing mode, either 'both', 'prefill', or 'decode'.
+ - causal (bool): If True, applies causal masking to prevent future information leakage.
+
+ Returns:
+ - torch.Tensor: A boolean mask of shape (batch_size, head_num, num_block_q, num_block_k),
+ indicating which blocks should be attended to.
+ """
+ assert threshold is None or num_to_choose is None
+ batch_size, head_num, num_block_q, num_block_k = input_tensor.shape
+ input_tensor = input_tensor.to(float)
+
+ total_sum = input_tensor.sum(dim=-1, keepdim=True)
+ if isinstance(threshold, torch.Tensor):
+ threshold = threshold.to(float)
+ required_sum = total_sum * threshold.unsqueeze(0).unsqueeze(-1).unsqueeze(
+ -1
+ ).expand((batch_size, head_num, num_block_q, 1)).to(input_tensor.device)
+ else:
+ required_sum = total_sum * threshold
+
+
+ mask = torch.zeros_like(input_tensor, dtype=torch.bool)
+ mask[:, :, :, 0] = 1
+ mask[:, :, :, current_index : current_index + num_block_q] = (
+ torch.eye(num_block_q, device=mask.device)
+ .unsqueeze(0)
+ .unsqueeze(0)
+ .expand(1, head_num, num_block_q, num_block_q)
+ )
+ # Note that other_values only contains the values of the current block
+ # (the sink blocks and diagonal are filled with 0)
+ other_values = input_tensor.masked_fill(mask, 0)
+
+
+ # Get sorted values
+ sorted_values, _ = torch.sort(other_values, dim=-1, descending=True)
+ sorted_values = sorted_values.to(input_tensor.device)
+ sorted_values = torch.cat(
+ [
+ torch.zeros(
+ (batch_size, head_num, num_block_q, 1), device=input_tensor.device
+ ),
+ torch.where(mask, input_tensor, 0).sum(dim=-1, keepdim=True), # shape: (batch_size, head_num, num_block_q, 1)
+ sorted_values[:, :, :, :-2], # :-2 excludes the first and diagonal (which are marked 0 in other_values)
+ ],
+ dim=-1,
+ )
+
+ # Get sorted indices
+ # index will select the already-masked (sink and diagonal) at the beginning
+ _, index = torch.sort(
+ torch.where(mask, 100000 * (1 + input_tensor), input_tensor),
+ dim=-1,
+ descending=True,
+ )
+
+ # [batch_size, head_num, num_block_q, num_block_k]
+ cumulative_sum_without_self = torch.cat(
+ [
+ torch.zeros(
+ (batch_size, head_num, num_block_q, 1), device=input_tensor.device
+ ),
+ sorted_values[:, :, :, 0:-1],
+ ],
+ dim=-1,
+ ).cumsum(dim=-1)
+
+ # Mask for indices where cumulative sum is below the required threshold.
+ index_mask = cumulative_sum_without_self < required_sum
+ index = torch.where(index_mask, index, 0)
+
+ mask = mask.view(batch_size, head_num * num_block_q, num_block_k)
+ index = index.view(batch_size, head_num * num_block_q, num_block_k)
+ mask[:, torch.arange(mask.shape[1], device=mask.device).unsqueeze(dim=-1), index] = True
+ mask = mask.view(batch_size, head_num, num_block_q, num_block_k)
+
+
+ assert bool((torch.where(mask,input_tensor,0).sum(dim=-1, keepdim=True) >= required_sum * 0.99).all()), \
+ f"mask sum {torch.where(mask,input_tensor,0).sum(dim=-1, keepdim=True)} < required_sum {required_sum}"
+
+ try:
+ if causal:
+ assert (~mask[:, :, :, current_index + num_block_q :]).all()
+ except:
+ mask[:, :, :, current_index + num_block_q :] = False
+
+ if causal:
+ if decoding:
+ assert mask[:, :, :, 0].all() and mask[:, :, :, -1].all()
+ else:
+ lambda_mask = torch.zeros_like(input_tensor,dtype=bool,device=input_tensor.device)
+ lambda_mask[:,:,:,0] = 1
+ lambda_mask[:,:,:,current_index:current_index+num_block_q] = torch.eye(num_block_q, device=lambda_mask.device).unsqueeze(0).unsqueeze(0).expand(1,head_num,num_block_q,num_block_q)
+ assert(torch.where(lambda_mask,mask,True).all())
+
+ return mask
+
+
+def shuffle_zigzag_masks(
+ block_masks: torch.Tensor, # [batch_size, num_qo_heads, num_blocks_local, num_blocks]
+ process_group: dist.ProcessGroup = None
+ ):
+ dim = len(block_masks.shape) - 1
+ if not block_masks.is_contiguous():
+ block_masks = block_masks.contiguous()
+
+ # We must use outplace, otherwise it will raise error at backward due to inplace operations.
+ # We can not change to_send directly and create a new tensor to store the result.
+ to_send_f = torch.zeros_like(block_masks)
+
+ # assume the input sequence length is 8, and computation runs on 4 GPUs
+ # the seq is represented as [0 1 2 3 4 5 6 7], world size is 4
+ # the input status before `shuffle_zigzag_input` is
+ # - gpu A: [0 1]
+ # - gpu B: [2 3]
+ # - gpu C: [4 5]
+ # - gpu D: [6 7]
+ # the value of `to_send_slice` is
+ # - gpu A: [1]
+ # - gpu B: [3]
+ # - gpu C: [5]
+ # - gpu D: [7]
+ block_seq_len = block_masks.shape[dim] // 2
+ left_slicer = [slice(None)] * dim + [slice(None, block_seq_len)]
+ right_slicer = [slice(None)] * dim + [slice(block_seq_len, None)]
+ to_send_slice = block_masks[right_slicer].contiguous()
+
+ rank = dist.get_rank(process_group)
+ world_size = dist.get_world_size(process_group)
+
+ res = torch.zeros_like(to_send_slice)
+
+ _ops = []
+ offset = ((dist.get_rank() // world_size) * world_size)
+ # rank src_rank
+ # 0 3
+ # 1 2
+ # 2 1
+ # 3 0
+ src_rank = (world_size - rank - 1) % world_size + offset
+ send_op = dist.P2POp(
+ dist.isend, to_send_slice, src_rank, group=process_group
+ )
+ recv_op = dist.P2POp(
+ dist.irecv, res, src_rank, group=process_group)
+
+ _ops.append(send_op)
+ _ops.append(recv_op)
+
+ response = dist.batch_isend_irecv(_ops)
+ for resp in response:
+ resp.wait()
+
+ if rank >= world_size // 2: # D: 6 7, -> 1 6
+ to_send_f[right_slicer] = block_masks[left_slicer]
+ to_send_f[left_slicer] = res
+ else: # A: 0 1, -> 0 7
+ to_send_f[left_slicer] = block_masks[left_slicer]
+ to_send_f[right_slicer] = res
+ # after shuffle, the status of `to_send_f`
+ # GPU A: [0 7]
+ # GPU B: [2 5]
+ # GPU C: [3 4]
+ # GPU D: [1 6]
+
+ return to_send_f
+
+
+
+@triton.jit
+def softmax_fuse_block_sum_kernel_causal(
+ In,
+ Out,
+ scale,
+ input_stride_0,
+ input_stride_1,
+ input_stride_2,
+ output_stride_0,
+ output_stride_1,
+ output_stride_2,
+ real_q_len,
+ k_len, # we assume k_len is divisible by chunk size
+ chunk_start,
+ chunk_end,
+ segment_size: tl.constexpr,
+ block_size: tl.constexpr,
+):
+ block_id = tl.program_id(0)
+ head_id = tl.program_id(1)
+ batch_id = tl.program_id(2)
+
+ offs_q = tl.arange(0, block_size) + chunk_start + block_id * block_size
+ offs_k = tl.arange(0, segment_size)
+
+ num_iters = k_len // segment_size
+ num_iters_before_causal = (chunk_start + (block_id + 1) * block_size - 1) // segment_size
+
+ m_i = tl.zeros([block_size], dtype=tl.float32) - float("inf")
+ l_i = tl.zeros([block_size], dtype=tl.float32) + 1.0
+
+ input_ptr = In + batch_id * input_stride_0 + head_id * input_stride_1 + block_id * block_size * input_stride_2
+ input_ptr = input_ptr + tl.arange(0, segment_size) + tl.arange(0, block_size)[:, None] * input_stride_2
+
+ output_ptr = Out + batch_id * output_stride_0 + head_id * output_stride_1 + block_id * output_stride_2
+ output_ptr = output_ptr + tl.arange(0, segment_size // block_size)
+
+ for iter in range(0, num_iters_before_causal):
+ X = tl.load(input_ptr + iter * segment_size).to(tl.float32) * scale
+ m_local = tl.max(X, 1)
+ m_new = tl.maximum(m_i, m_local)
+ alpha = tl.math.exp2(m_i - m_new)
+
+ X = X - m_new[:, None]
+ l_local = tl.sum(tl.math.exp2(X), 1)
+ l_i = l_i * alpha + l_local
+
+ m_i = m_new
+
+ for iter in range(num_iters_before_causal, num_iters_before_causal + 1):
+ X = tl.load(input_ptr + iter * segment_size).to(tl.float32) * scale
+ mask = offs_q[:, None] >= (offs_k[None, :] + iter * segment_size)
+ X = tl.where(mask, X, -1.0e6)
+ m_local = tl.max(X, 1)
+ m_new = tl.maximum(m_i, m_local)
+ alpha = tl.math.exp2(m_i - m_new)
+
+ X = X - m_new[:, None]
+ l_local = tl.sum(tl.math.exp2(X), 1)
+ l_i = l_i * alpha + l_local
+
+ m_i = m_new
+
+ l_i_inv = 1.0 / l_i
+
+ sum_mask = offs_q[:, None] < real_q_len
+
+ for iter in range(0, num_iters_before_causal):
+ X = tl.load(input_ptr + iter * segment_size).to(tl.float32) * scale
+ X = tl.exp2(X - m_i[:, None]) * l_i_inv[:, None]
+ X = tl.where(sum_mask, X, 0)
+ X = tl.reshape(X, (block_size, segment_size // block_size, block_size))
+ X = tl.sum(X, 2)
+ X = tl.sum(X, 0)
+ tl.store(output_ptr + iter * segment_size // block_size, X.to(Out.type.element_ty))
+
+ for iter in range(num_iters_before_causal, num_iters_before_causal + 1):
+ X = tl.load(input_ptr + iter * segment_size).to(tl.float32) * scale
+ mask = offs_q[:, None] >= (offs_k[None, :] + iter * segment_size)
+ X = tl.where(mask, X, -1.0e6)
+ X = tl.exp2(X - m_i[:, None]) * l_i_inv[:, None]
+ X = tl.where(sum_mask, X, 0)
+ X = tl.reshape(X, (block_size, segment_size // block_size, block_size))
+ X = tl.sum(X, 2)
+ X = tl.sum(X, 0)
+ tl.store(output_ptr + iter * segment_size // block_size, X.to(Out.type.element_ty))
+
+ for iter in range(num_iters_before_causal + 1, num_iters):
+ X = tl.zeros([segment_size // block_size], dtype=tl.float32)
+ tl.store(output_ptr + iter * segment_size // block_size, X.to(Out.type.element_ty))
+
+
+@triton.jit
+def softmax_fuse_block_sum_kernel_non_causal(
+ In,
+ Out,
+ scale,
+ input_stride_0,
+ input_stride_1,
+ input_stride_2,
+ output_stride_0,
+ output_stride_1,
+ output_stride_2,
+ real_q_len,
+ k_len, # we assume k_len is divisible by chunk size
+ chunk_start,
+ chunk_end,
+ segment_size: tl.constexpr,
+ block_size: tl.constexpr,
+):
+ block_id = tl.program_id(0)
+ head_id = tl.program_id(1)
+ batch_id = tl.program_id(2)
+
+ offs_q = tl.arange(0, block_size) + chunk_start + block_id * block_size
+ offs_k = tl.arange(0, segment_size)
+
+ num_iters = k_len // segment_size
+
+ m_i = tl.zeros([block_size], dtype=tl.float32) - float("inf")
+ l_i = tl.zeros([block_size], dtype=tl.float32) + 1.0
+
+ input_ptr = In + batch_id * input_stride_0 + head_id * input_stride_1 + block_id * block_size * input_stride_2
+ input_ptr = input_ptr + tl.arange(0, segment_size) + tl.arange(0, block_size)[:, None] * input_stride_2
+
+ output_ptr = Out + batch_id * output_stride_0 + head_id * output_stride_1 + block_id * output_stride_2
+ output_ptr = output_ptr + tl.arange(0, segment_size // block_size)
+
+ for iter in range(0, num_iters):
+ X = tl.load(input_ptr + iter * segment_size).to(tl.float32) * scale
+ m_local = tl.max(X, 1)
+ m_new = tl.maximum(m_i, m_local)
+ alpha = tl.math.exp2(m_i - m_new)
+
+ X = X - m_new[:, None]
+ l_local = tl.sum(tl.math.exp2(X), 1)
+ l_i = l_i * alpha + l_local
+
+ m_i = m_new
+
+ l_i_inv = 1.0 / l_i
+
+ sum_mask = offs_q[:, None] < real_q_len
+
+ for iter in range(0, num_iters):
+ X = tl.load(input_ptr + iter * segment_size).to(tl.float32) * scale
+ X = tl.exp2(X - m_i[:, None]) * l_i_inv[:, None]
+ X = tl.where(sum_mask, X, 0)
+ X = tl.reshape(X, (block_size, segment_size // block_size, block_size))
+ X = tl.sum(X, 2)
+ X = tl.sum(X, 0)
+ tl.store(output_ptr + iter * segment_size // block_size, X.to(Out.type.element_ty))
+
+@triton.jit
+def flat_group_gemm_kernel(Q, K, Out,
+ stride_qz, stride_qh, stride_qn,
+ stride_kz, stride_kh, stride_kn,
+ stride_oz, stride_oh, stride_on,
+ chunk_start, chunk_end,
+ H: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ ):
+ block_m = tl.program_id(0).to(tl.int64)
+ block_n = tl.program_id(1).to(tl.int64)
+ batch_id = tl.program_id(2).to(tl.int64) // H
+ head_id = tl.program_id(2).to(tl.int64) % H
+
+ if chunk_start + (block_m + 1) * BLOCK_M <= block_n * BLOCK_N:
+ return
+
+ Q_ptrs = Q + batch_id * stride_qz + head_id * stride_qh + block_m * BLOCK_M * stride_qn
+ K_ptrs = K + batch_id * stride_kz + head_id * stride_kh + block_n * BLOCK_N * stride_kn
+
+ Q_ptrs = Q_ptrs + tl.arange(0, BLOCK_M)[:, None] * stride_qn + tl.arange(0, BLOCK_K)[None, :]
+ K_ptrs = K_ptrs + tl.arange(0, BLOCK_N)[None, :] * stride_kn + tl.arange(0, BLOCK_K)[:, None]
+
+ num_iters = HEAD_DIM // BLOCK_K
+ o = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+
+ for iter in range(num_iters):
+ q = tl.load(Q_ptrs + iter * BLOCK_K)
+ k = tl.load(K_ptrs + iter * BLOCK_K)
+ o += tl.dot(q, k)
+
+ O_ptrs = Out + batch_id * stride_oz + head_id * stride_oh + block_m * BLOCK_M * stride_on + block_n * BLOCK_N
+ O_ptrs = O_ptrs + tl.arange(0, BLOCK_M)[:, None] * stride_on + tl.arange(0, BLOCK_N)[None, :]
+
+ tl.store(O_ptrs, o.to(Out.type.element_ty))
+
+@triton.jit
+def flat_group_gemm_fuse_reshape_kernel(Q, K, Out,
+ stride_qz, stride_qh, stride_qn,
+ stride_kz, stride_kh, stride_kn,
+ stride_oz, stride_oh, stride_on,
+ chunk_start, chunk_end,
+ H: tl.constexpr,
+ STRIDE: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ is_caual: tl.constexpr,
+ ):
+ block_m = tl.program_id(0).to(tl.int64)
+ block_n = tl.program_id(1).to(tl.int64)
+ batch_id = tl.program_id(2).to(tl.int64) // H
+ head_id = tl.program_id(2).to(tl.int64) % H
+
+ if is_caual:
+ if chunk_start + (block_m + 1) * BLOCK_M <= block_n * BLOCK_N:
+ return
+
+ Q_ptrs = Q + batch_id * stride_qz + head_id * stride_qh + block_m * BLOCK_M * STRIDE * stride_qn
+ K_ptrs = K + batch_id * stride_kz + head_id * stride_kh + block_n * BLOCK_N * STRIDE * stride_kn
+
+ Q_ptrs = Q_ptrs + tl.arange(0, BLOCK_M)[:, None] * (stride_qn * STRIDE) + tl.arange(0, HEAD_DIM)[None, :] + stride_qn * (STRIDE - 1)
+ K_ptrs = K_ptrs + tl.arange(0, BLOCK_N)[None, :] * (stride_kn * STRIDE) + tl.arange(0, HEAD_DIM)[:, None]
+
+ o = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+
+ for iter in range(STRIDE):
+ q = tl.load(Q_ptrs - iter * stride_qn)
+ k = tl.load(K_ptrs + iter * stride_kn)
+ o += tl.dot(q, k)
+
+ O_ptrs = Out + batch_id * stride_oz + head_id * stride_oh + block_m * BLOCK_M * stride_on + block_n * BLOCK_N
+ O_ptrs = O_ptrs + tl.arange(0, BLOCK_M)[:, None] * stride_on + tl.arange(0, BLOCK_N)[None, :]
+
+ tl.store(O_ptrs, o.to(Out.type.element_ty))
+
+
+def softmax_fuse_block_sum(attn_weights_slice, reshaped_block_size, segment_size, chunk_start, chunk_end, real_q_len, scale, is_causal=True):
+ batch_size, num_heads, q_len, k_len = attn_weights_slice.shape
+ assert q_len % reshaped_block_size == 0
+ try:
+ assert k_len % segment_size == 0
+ except:
+ assert False, f"xAttention error, k_len: {k_len}, segment size: {segment_size}"
+ assert segment_size % reshaped_block_size == 0
+ assert attn_weights_slice.stride(-1) == 1
+
+ output = torch.empty((batch_size, num_heads, q_len // reshaped_block_size, k_len // reshaped_block_size), dtype=attn_weights_slice.dtype, device=attn_weights_slice.device)
+
+ grid = (q_len // reshaped_block_size, num_heads, batch_size)
+
+ if is_causal:
+ softmax_fuse_block_sum_kernel_causal[grid](
+ attn_weights_slice,
+ output,
+ scale,
+ attn_weights_slice.stride(0),
+ attn_weights_slice.stride(1),
+ attn_weights_slice.stride(2),
+ output.stride(0),
+ output.stride(1),
+ output.stride(2),
+ real_q_len,
+ k_len,
+ chunk_start,
+ chunk_end,
+ segment_size,
+ reshaped_block_size,
+ )
+ else:
+ softmax_fuse_block_sum_kernel_non_causal[grid](
+ attn_weights_slice,
+ output,
+ scale,
+ attn_weights_slice.stride(0),
+ attn_weights_slice.stride(1),
+ attn_weights_slice.stride(2),
+ output.stride(0),
+ output.stride(1),
+ output.stride(2),
+ real_q_len,
+ k_len,
+ chunk_start,
+ chunk_end,
+ segment_size,
+ reshaped_block_size,
+ )
+
+ return output
+
+def flat_group_gemm(query_states, key_states, chunk_start, chunk_end):
+ batch_size, num_heads, q_len, head_dim = query_states.shape
+ kv_len = key_states.shape[2]
+
+ output = torch.empty((batch_size, num_heads, q_len, kv_len), dtype=query_states.dtype, device=query_states.device)
+ BLOCK_M = 128
+ BLOCK_N = 128
+ BLOCK_K = 64
+
+ grid = (q_len // BLOCK_M, kv_len // BLOCK_N, batch_size * num_heads)
+ flat_group_gemm_kernel[grid](
+ query_states,
+ key_states,
+ output,
+ query_states.stride(0),
+ query_states.stride(1),
+ query_states.stride(2),
+ key_states.stride(0),
+ key_states.stride(1),
+ key_states.stride(2),
+ output.stride(0),
+ output.stride(1),
+ output.stride(2),
+ chunk_start,
+ chunk_end,
+ num_heads,
+ head_dim,
+ BLOCK_M,
+ BLOCK_N,
+ BLOCK_K,
+ )
+
+ return output
+
+def flat_group_gemm_fuse_reshape(query_states, key_states, stride, chunk_start, chunk_end, is_causal=True):
+ batch_size, num_heads, q_len, head_dim = query_states.shape
+ kv_len = key_states.shape[2]
+
+ assert (key_states.shape[0] == batch_size)
+ assert (key_states.shape[1] == num_heads)
+ assert (key_states.shape[3] == head_dim)
+
+ output = torch.empty((batch_size, num_heads, q_len // stride, kv_len // stride), dtype=query_states.dtype, device=query_states.device)
+ BLOCK_M = 128
+ BLOCK_N = 128
+ assert (q_len % (stride * BLOCK_M) == 0), f"q_len={q_len}, stride={stride}, BLOCK_M={BLOCK_M}"
+ assert (kv_len % (stride * BLOCK_N) == 0), f"kv_len={kv_len}, stride={stride}, BLOCK_N={BLOCK_N}"
+
+ grid = (q_len // stride // BLOCK_M, kv_len // stride // BLOCK_N, batch_size * num_heads)
+ flat_group_gemm_fuse_reshape_kernel[grid](
+ query_states,
+ key_states,
+ output,
+ query_states.stride(0),
+ query_states.stride(1),
+ query_states.stride(2),
+ key_states.stride(0),
+ key_states.stride(1),
+ key_states.stride(2),
+ output.stride(0),
+ output.stride(1),
+ output.stride(2),
+ chunk_start,
+ chunk_end,
+ num_heads,
+ stride,
+ head_dim,
+ BLOCK_M,
+ BLOCK_N,
+ is_causal,
+ )
+
+ return output
diff --git a/minference/ops/pit_sparse_flash_attention_v2.py b/minference/ops/pit_sparse_flash_attention_v2.py
index 825a6b0f..712716b0 100644
--- a/minference/ops/pit_sparse_flash_attention_v2.py
+++ b/minference/ops/pit_sparse_flash_attention_v2.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import math
@@ -9,6 +9,26 @@
from ..cuda import convert_vertical_slash_indexes
+try:
+ from sgl_kernel.sparse_flash_attn import sparse_attn_func
+except:
+ try:
+ from vllm_flash_attn import sparse_attn_func
+ except:
+ print("To benefit from fast kernel implementations, we recommend installing SGLang or vllm.")
+ sparse_attn_func = None
+
+try:
+ from sgl_kernel.sparse_flash_attn import (
+ convert_vertical_slash_indexes as convert_vertical_slash_indexes_opt,
+ )
+except:
+ try:
+ from vllm._custom_ops import (
+ convert_vertical_slash_indexes as convert_vertical_slash_indexes_opt,
+ )
+ except:
+ convert_vertical_slash_indexes_opt = None
# @triton.autotune(
# configs=[
@@ -181,8 +201,10 @@ def vertical_slash_sparse_attention(
block_size_M: int = 64,
block_size_N: int = 64,
):
+ if convert_vertical_slash_indexes_opt is not None:
+ return vertical_slash_sparse_attention_wo_pad(query, key, value, v_idx, s_idx)
batch_size, num_heads, context_size, head_dim = query.shape
- pad = block_size_M - (context_size & (block_size_M - 1))
+ pad = (block_size_M - context_size) & (block_size_M - 1)
query = torch.nn.functional.pad(query, [0, 0, 0, pad, 0, 0, 0, 0])
key = torch.nn.functional.pad(key, [0, 0, 0, pad, 0, 0, 0, 0])
value = torch.nn.functional.pad(value, [0, 0, 0, pad, 0, 0, 0, 0])
@@ -200,9 +222,53 @@ def vertical_slash_sparse_attention(
block_count, block_offset, column_count, column_index = convert_vertical_slash_indexes(
seqlens, v_idx, s_idx, context_size, block_size_M, block_size_N,
)
- out = _triton_mixed_sparse_attention(
- query, key, value, seqlens,
- block_count, block_offset, column_count, column_index,
- sm_scale, block_size_M, block_size_N,
- )
+
+ if sparse_attn_func is not None:
+ out = sparse_attn_func(
+ query.transpose(1, 2).contiguous(),
+ key.transpose(1, 2).contiguous(),
+ value.transpose(1, 2).contiguous(),
+ block_count, block_offset, column_count, column_index,
+ return_softmax_lse=False,
+ causal=True,
+ ).transpose(1, 2).contiguous()
+ else:
+ out = _triton_mixed_sparse_attention(
+ query, key, value, seqlens,
+ block_count, block_offset, column_count, column_index,
+ sm_scale, block_size_M, block_size_N,
+ )
+
return out[..., :context_size, :head_dim]
+
+def vertical_slash_sparse_attention_wo_pad(query, key, value, v_idx, s_idx, block_size_M: int = 64, block_size_N: int = 64):
+ batch_size, num_heads, context_size, head_dim = query.shape
+ seqlens = torch.tensor([context_size], dtype=torch.int32, device=query.device)
+
+ v_idx = v_idx.to(torch.int32).reshape((batch_size, num_heads, -1)).sort(dim=-1, descending=False)[0]
+ s_idx = s_idx.to(torch.int32).reshape((batch_size, num_heads, -1)).sort(dim=-1, descending=True)[0]
+
+ block_count, block_offset, column_count, column_index = (
+ convert_vertical_slash_indexes_opt(
+ seqlens,
+ seqlens,
+ v_idx.to(torch.int32),
+ s_idx.to(torch.int32),
+ context_size,
+ block_size_M,
+ block_size_N,
+ causal=True,
+ )
+ )
+ out = sparse_attn_func(
+ query.transpose(1, 2).contiguous(),
+ key.transpose(1, 2).contiguous(),
+ value.transpose(1, 2).contiguous(),
+ block_count,
+ block_offset,
+ column_count,
+ column_index,
+ causal=True,
+ return_softmax_lse=False,
+ )
+ return out.transpose(1, 2).contiguous()
diff --git a/minference/ops/pit_sparse_flash_attention_v3.py b/minference/ops/pit_sparse_flash_attention_v3.py
new file mode 100644
index 00000000..bf118d96
--- /dev/null
+++ b/minference/ops/pit_sparse_flash_attention_v3.py
@@ -0,0 +1,1498 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import math
+import os
+import sys
+from typing import List, Tuple
+
+import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+import triton
+import triton.language as tl
+
+# Save current flags
+if torch.version.hip is None:
+ original_flags = sys.getdlopenflags()
+ try:
+ sys.setdlopenflags(os.RTLD_LAZY | os.RTLD_GLOBAL)
+ import block_sparse_attn_cuda
+ from block_sparse_attn.block_sparse_attn_interface import (
+ convert_blockmask_col_reverse,
+ convert_blockmask_row_reverse,
+ )
+
+ # NOTE: Block-Sparse-Attention/csrc/block_sparse_attn/src/flash_blockmask.h: add head_idx to blockmask_ptr
+ except ModuleNotFoundError as e:
+ print(f"[Warning] Failed to import block_sparse_attn_cuda: {e}")
+ finally:
+ # Restore original flags for future imports
+ sys.setdlopenflags(original_flags)
+ # NOTE: Block-Sparse-Attention/csrc/block_sparse_attn/src/flash_blockmask.h: add head_idx to blockmask_ptr
+
+from .op_utils.vertical_slash_utils import build_index_local, convert_blockmask
+
+
+# ----------------------------------------------------------------------------
+# CUDA-based kernels (based on Block-Sparse-Attention)
+def block_attn_fwd(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, num_blocks]
+ granularity: int,
+ causal: bool,
+ step_idx: int=-1,
+):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ cu_seqlens = torch.arange(0, (batch_size + 1) * num_tokens, step=num_tokens, dtype=torch.int32, device=q.device)
+ head_mask_type = torch.ones((num_qo_heads, ), dtype=torch.int32, device=q.device) # Block-Sparse
+ streaming_info = torch.zeros((num_qo_heads * 2), dtype=torch.int32, device=q.device)
+ row_blockmask = convert_blockmask_row_reverse(block_mask, causal=True)
+
+ p_dropout = 0.0
+ out, q, k, v, out_padded, softmax_lse, S_dmask, rng_state = block_sparse_attn_cuda.fwd_block(
+ q.reshape((-1, num_qo_heads, head_dim)),
+ k.reshape((-1, num_kv_heads, head_dim)),
+ v.reshape((-1, num_kv_heads, head_dim)),
+ cu_seqlens, cu_seqlens,
+ granularity, granularity,
+ head_mask_type,
+ streaming_info,
+ row_blockmask,
+ num_tokens, num_tokens,
+ p_dropout,
+ softmax_scale,
+ causal, # is_causal
+ False, # exact_streaming
+ False, # return_softmax
+ -1, # window_size_left
+ -1, # window_size_right
+ None
+ )
+ out = out.reshape((batch_size, num_tokens, num_qo_heads, head_dim))
+ return out, softmax_lse
+
+
+def block_attn_bwd(
+ grad: torch.Tensor,
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ o: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ block_mask: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, num_blocks]
+ granularity: int,
+ deterministic: bool,
+ causal: bool,
+ converted: bool = False,
+):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ cu_seqlens = torch.arange(0, (batch_size + 1) * num_tokens, step=num_tokens, dtype=torch.int32, device=q.device)
+ head_mask_type = torch.ones((num_qo_heads, ), dtype=torch.int32, device=q.device) # Block-Sparse
+ streaming_info = torch.zeros((num_qo_heads * 2), dtype=torch.int32, device=q.device)
+ if converted:
+ col_blockmask = block_mask
+ else:
+ col_blockmask = convert_blockmask_col_reverse(block_mask, causal=True)
+ p_dropout = 0.0
+ dq, dk, dv = torch.empty_like(q), torch.empty_like(k), torch.empty_like(v)
+ dq, dk, dv, softmax_d = block_sparse_attn_cuda.bwd_block(
+ grad.reshape((-1, num_qo_heads, head_dim)),
+ q.reshape((-1, num_qo_heads, head_dim)),
+ k.reshape((-1, num_kv_heads, head_dim)),
+ v.reshape((-1, num_kv_heads, head_dim)),
+ o.reshape((-1, num_qo_heads, head_dim)),
+ softmax_lse,
+ dq.reshape((-1, num_qo_heads, head_dim)),
+ dk.reshape((-1, num_kv_heads, head_dim)),
+ dv.reshape((-1, num_kv_heads, head_dim)),
+ cu_seqlens, cu_seqlens,
+ granularity, granularity,
+ head_mask_type,
+ streaming_info,
+ col_blockmask,
+ num_tokens, num_tokens,
+ p_dropout,
+ softmax_scale,
+ True, # zero_tensors
+ causal, # is_causal
+ -1, # window_size_left
+ -1, # window_size_right
+ deterministic,
+ None, None
+ )
+ dq = dq.reshape((batch_size, num_tokens, num_qo_heads, head_dim))
+ dk = dk.reshape((batch_size, num_tokens, num_kv_heads, head_dim))
+ dv = dv.reshape((batch_size, num_tokens, num_kv_heads, head_dim))
+ return dq, dk, dv
+
+
+@triton.jit
+def _triton_bar_attn_fwd_kernel(
+ Q, K, V, sm_scale,
+ bar_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS, WORLD_SIZE + 1]
+ bar_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NNZ_V]
+ Out, # [BATCH, N_Q_HEADS, N_CTX, D_HEAD]
+ softmax_lse, # [BATCH, N_Q_HEADS, N_CTX]
+ stride_qz, stride_qh, stride_qm, stride_qd,
+ stride_kz, stride_kh, stride_kn, stride_kd,
+ stride_vz, stride_vh, stride_vn, stride_vd,
+ stride_oz, stride_oh, stride_om, stride_od,
+ stride_cz, stride_ch, stride_cm, stride_cr,
+ stride_iz, stride_ih, stride_im, stride_in,
+ stride_sz, stride_sh, stride_sm,
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_DMODEL: tl.constexpr,
+):
+ start_m = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ if start_m * BLOCK_M >= num_tokens:
+ return
+
+ # initialize offsets
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+ offs_d = tl.arange(0, BLOCK_DMODEL)
+ m_mask = offs_m < num_tokens
+
+ qo_offset = batch_idx * stride_qz + qo_head_idx * stride_qh
+ kv_offset = batch_idx * stride_kz + kv_head_idx * stride_kh
+
+ q_ptrs = Q + qo_offset + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qd
+ k_ptrs = K + kv_offset + offs_d[:, None] * stride_kd
+ v_ptrs = V + kv_offset + offs_d[None, :] * stride_vd
+ o_ptrs = Out + qo_offset + offs_m[:, None] * stride_om + offs_d[None, :] * stride_od
+
+ lse_ptrs = softmax_lse + batch_idx * stride_sz + qo_head_idx * stride_sh + offs_m * stride_sm
+
+ bar_l = tl.load(bar_cnt + batch_idx * stride_cz + qo_head_idx * stride_ch + start_m * stride_cm + step * stride_cr)
+ bar_r = tl.load(bar_cnt + batch_idx * stride_cz + qo_head_idx * stride_ch + start_m * stride_cm + (step + 1) * stride_cr)
+ bar_idx_ptr = bar_idx + batch_idx * stride_iz + qo_head_idx * stride_ih + start_m * stride_im
+
+ if bar_l >= bar_r:
+ return
+
+ # initialize pointer to m and l
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
+ l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
+
+ # scale sm_scale by log_2(e) and use
+ # 2^x instead of exp in the loop because CSE and LICM
+ # don't work as expected with `exp` in the loop
+ # 1/ln2 = lne/ln2 = log2(e) => 2^(x / ln2) = 2^(x * log2(e)) = (2^(log2(e)))^x = e^x
+ qk_scale = sm_scale * 1.44269504
+
+ # load q: it will stay in SRAM throughout
+ q = tl.load(q_ptrs, mask=m_mask[:, None], other=0)
+ q = (q * qk_scale).to(Q.type.element_ty)
+
+ # loop over k, v and update accumulator
+ for start_n in range(bar_l, bar_r, BLOCK_N):
+ n_mask = start_n + offs_n < bar_r
+ cols = tl.load(bar_idx_ptr + (start_n + offs_n) * stride_in, mask=n_mask, other=0)
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + cols[None, :] * stride_kn)
+ v = tl.load(v_ptrs + cols[:, None] * stride_vn)
+
+ # -- compute qk --
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = tl.where(m_mask[:, None] & n_mask[None, :], qk, float("-inf"))
+ qk = qk + tl.dot(q, k)
+
+ # -- compute scaling constant --
+ m_i_new = tl.maximum(m_i, tl.max(qk, 1))
+ alpha = tl.math.exp2(m_i - m_i_new)
+ p = tl.math.exp2(qk - m_i_new[:, None])
+
+ # -- scale and update acc --
+ acc_scale = l_i * 0 + alpha # workaround some compiler bug
+ acc = acc * acc_scale[:, None]
+ acc = acc + tl.dot(p.to(Q.type.element_ty), v)
+
+ # -- update m_i and l_i --
+ l_i = l_i * alpha + tl.sum(p, 1)
+ m_i = m_i_new
+
+ # write back O and LSE
+ acc_1 = acc / l_i[:, None]
+ s_1 = m_i * 0.69314718 + tl.math.log(l_i)
+ acc_0 = tl.load(o_ptrs, mask=m_mask[:, None], other=0.).to(tl.float32)
+ s_0 = tl.load(lse_ptrs, mask=m_mask, other=float("-inf"))
+
+ overflow_mask = (s_0 - s_1) < 88.0
+
+ theta = tl.math.exp(s_0 - s_1)
+ alpha_0 = 1 / (1 + 1 / theta)
+ alpha_1 = 1 / (1 + theta)
+ acc = alpha_0[:, None] * acc_0 + alpha_1[:, None] * acc_1
+ s = s_1 - tl.math.log(alpha_1)
+
+ tl.store(o_ptrs, acc.to(Out.type.element_ty), mask=m_mask[:, None])
+ tl.store(lse_ptrs, s, mask=(m_mask & overflow_mask))
+
+
+
+def bar_attn_fwd(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ o: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int,
+ step: int = 0,
+):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ num_blocks = bar_idx.shape[2]
+ _triton_bar_attn_fwd_kernel[(num_blocks, num_qo_heads, batch_size)](
+ q, k, v, softmax_scale, bar_cnt, bar_idx, o, lse,
+ q.stride(0), q.stride(2), q.stride(1), q.stride(3),
+ k.stride(0), k.stride(2), k.stride(1), k.stride(3),
+ v.stride(0), v.stride(2), v.stride(1), v.stride(3),
+ o.stride(0), o.stride(2), o.stride(1), o.stride(3),
+ bar_cnt.stride(0), bar_cnt.stride(1), bar_cnt.stride(2), bar_cnt.stride(3),
+ bar_idx.stride(0), bar_idx.stride(1), bar_idx.stride(2), bar_idx.stride(3),
+ lse.stride(0), lse.stride(1), lse.stride(2),
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M=granularity, BLOCK_N=64, BLOCK_DMODEL=head_dim,
+ num_warps=4, num_stages=2,
+ )
+ return o, lse
+
+
+@triton.jit
+def _triton_bar_attn_bwd_kernel(
+ Q, K, V, O,
+ DQ, DK, DV, DO,
+ sm_scale,
+ bar_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS, WORLD_SIZE + 1]
+ bar_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NNZ_V]
+ softmax_lse, # [BATCH, N_HEADS, N_CTX]
+ stride_qz, stride_qh, stride_qm, stride_qd,
+ stride_kz, stride_kh, stride_kn, stride_kd,
+ stride_vz, stride_vh, stride_vn, stride_vd,
+ stride_oz, stride_oh, stride_om, stride_od,
+ stride_dqz, stride_dqh, stride_dqm, stride_dqd,
+ stride_dkz, stride_dkh, stride_dkn, stride_dkd,
+ stride_dvz, stride_dvh, stride_dvn, stride_dvd,
+ stride_doz, stride_doh, stride_dom, stride_dod,
+ stride_cz, stride_ch, stride_cm, stride_cr,
+ stride_iz, stride_ih, stride_im, stride_in,
+ stride_sz, stride_sh, stride_sm,
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_DMODEL: tl.constexpr,
+):
+ start_m = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ if start_m * BLOCK_M >= num_tokens:
+ return
+
+ qk_scale = sm_scale * 1.44269504
+
+ # offset pointers for batch/head
+ Q += batch_idx * stride_qz + qo_head_idx * stride_qh
+ K += batch_idx * stride_kz + kv_head_idx * stride_kh
+ V += batch_idx * stride_vz + kv_head_idx * stride_vh
+ O += batch_idx * stride_oz + qo_head_idx * stride_oh
+ DQ += batch_idx * stride_dqz + qo_head_idx * stride_dqh
+ DK += batch_idx * stride_dkz + kv_head_idx * stride_dkh
+ DV += batch_idx * stride_dvz + kv_head_idx * stride_dvh
+ DO += batch_idx * stride_doz + qo_head_idx * stride_doh
+
+ # loop over rows
+ offs_d = tl.arange(0, BLOCK_DMODEL)
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+ m_mask = offs_m < num_tokens
+
+ # initialize pointers to value-like data
+ q_ptrs = Q + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qd
+ k_ptrs = K + offs_d[None, :] * stride_kd
+ v_ptrs = V + offs_d[None, :] * stride_vd
+ o_ptrs = O + offs_m[:, None] * stride_om + offs_d[None, :] * stride_od
+ dq_ptrs = DQ + offs_m[:, None] * stride_dqm + offs_d[None, :] * stride_dqd
+ dk_ptrs = DK + offs_d[None, :] * stride_dkd
+ dv_ptrs = DV + offs_d[None, :] * stride_dvd
+ do_ptrs = DO + offs_m[:, None] * stride_dom + offs_d[None, :] * stride_dod
+
+ l_ptrs = softmax_lse + batch_idx * stride_sz + qo_head_idx * stride_sh + offs_m * stride_sm
+
+ bar_l = tl.load(bar_cnt + batch_idx * stride_cz + qo_head_idx * stride_ch + start_m * stride_cm + step * stride_cr)
+ bar_r = tl.load(bar_cnt + batch_idx * stride_cz + qo_head_idx * stride_ch + start_m * stride_cm + (step + 1) * stride_cr)
+ bar_idx_ptr = bar_idx + batch_idx * stride_iz + qo_head_idx * stride_ih + start_m * stride_im
+
+ if bar_l >= bar_r:
+ return
+
+ o = tl.load(o_ptrs, mask=m_mask[:, None], other=0.).to(tl.float32)
+ do = tl.load(do_ptrs, mask=m_mask[:, None], other=0.).to(tl.float32)
+ d_i = tl.sum(o * do, axis=1)
+
+ q = tl.load(q_ptrs, mask=m_mask[:, None], other=0.)
+ do = do.to(DO.dtype.element_ty)
+ l_i = tl.load(l_ptrs, mask=m_mask, other=0.) * 1.44269504
+
+ dq = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
+
+ for start_n in range(bar_l, bar_r, BLOCK_N):
+ n_mask = start_n + offs_n < bar_r
+ cols = tl.load(bar_idx_ptr + (start_n + offs_n) * stride_in, mask=n_mask, other=0)
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + cols[:, None] * stride_kn)
+ v = tl.load(v_ptrs + cols[:, None] * stride_vn)
+
+ # Computer qk
+ qk = tl.where(m_mask[:, None] & n_mask[None, :], float(0.), float("-inf"))
+ qk = qk + tl.dot(q, tl.trans(k))
+ qk = qk * qk_scale
+ p = tl.math.exp2(qk - l_i[:, None])
+
+ # compute dv
+ dv_vals = tl.dot(tl.trans(p.to(Q.dtype.element_ty)), do).to(tl.float32)
+ tl.atomic_add(dv_ptrs + cols[:, None] * stride_dvn, dv_vals, mask=n_mask[:, None], sem="relaxed")
+
+ # compute dp = dot(v, do)
+ dp = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - d_i[:, None]
+ dp = dp + tl.dot(do, tl.trans(v))
+
+ # compute ds = p * (dp - delta[:, None])
+ ds = p * dp * sm_scale
+
+ # compute dk = dot(ds.T, q)
+ dk_vals = tl.dot(tl.trans(ds.to(Q.dtype.element_ty)), q).to(tl.float32)
+ tl.atomic_add(dk_ptrs + cols[:, None] * stride_dkn, dk_vals, mask=n_mask[:, None], sem="relaxed")
+
+ # compute dq
+ dq = dq + tl.dot(ds.to(Q.dtype.element_ty), k)
+
+ dq_old = tl.load(dq_ptrs, mask=m_mask[:, None], other=0.).to(tl.float32)
+ tl.store(dq_ptrs, (dq_old + dq).to(DQ.dtype.element_ty), mask=m_mask[:, None])
+
+
+def bar_attn_bwd(
+ grad: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ o: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ dq: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ dk: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ dv: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ granularity: int,
+ deterministic: bool,
+ step: int = 0,
+):
+ assert not deterministic
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ num_blocks = bar_idx.shape[2]
+ dq = torch.zeros_like(q, dtype=torch.float32) if dq is None else dq.to(torch.float32)
+ dk = torch.zeros_like(k, dtype=torch.float32) if dk is None else dk.to(torch.float32)
+ dv = torch.zeros_like(v, dtype=torch.float32) if dv is None else dv.to(torch.float32)
+ _triton_bar_attn_bwd_kernel[(num_blocks, num_qo_heads, batch_size)](
+ q, k, v, o, dq, dk, dv, grad, softmax_scale,
+ bar_cnt, bar_idx, softmax_lse,
+ q.stride(0), q.stride(2), q.stride(1), q.stride(3),
+ k.stride(0), k.stride(2), k.stride(1), k.stride(3),
+ v.stride(0), v.stride(2), v.stride(1), v.stride(3),
+ o.stride(0), o.stride(2), o.stride(1), o.stride(3),
+ dq.stride(0), dq.stride(2), dq.stride(1), dq.stride(3),
+ dk.stride(0), dk.stride(2), dk.stride(1), dk.stride(3),
+ dv.stride(0), dv.stride(2), dv.stride(1), dv.stride(3),
+ grad.stride(0), grad.stride(2), grad.stride(1), grad.stride(3),
+ bar_cnt.stride(0), bar_cnt.stride(1), bar_cnt.stride(2), bar_cnt.stride(3),
+ bar_idx.stride(0), bar_idx.stride(1), bar_idx.stride(2), bar_idx.stride(3),
+ softmax_lse.stride(0), softmax_lse.stride(1), softmax_lse.stride(2),
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M=granularity, BLOCK_N=64, BLOCK_DMODEL=head_dim,
+ num_warps=4, num_stages=2,
+ )
+ return dq, dk.to(dq.dtype), dv.to(dq.dtype)
+
+# ----------------------------------------------------------------------------
+# Purely Triton-based kernels
+@triton.jit
+def _triton_block_attn_fwd_kernel(
+ Q, K, V, sm_scale,
+ block_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS]
+ block_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NUM_COLS]
+ Out, # [BATCH, N_Q_HEADS, N_CTX, D_HEAD]
+ softmax_lse, # [BATCH, N_Q_HEADS, N_CTX]
+ stride_qz, stride_qh, stride_qm, stride_qd,
+ stride_kz, stride_kh, stride_kn, stride_kd,
+ stride_vz, stride_vh, stride_vn, stride_vd,
+ stride_oz, stride_oh, stride_om, stride_od,
+ stride_2cz, stride_2ch, stride_2cm,
+ stride_2iz, stride_2ih, stride_2im, stride_2in,
+ stride_sz, stride_sh, stride_sm,
+ num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_DMODEL: tl.constexpr,
+ CAUSAL: tl.constexpr,
+):
+ start_m = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ if start_m * BLOCK_M >= num_tokens:
+ return
+
+ # initialize offsets
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+ offs_d = tl.arange(0, BLOCK_DMODEL)
+
+ qo_offset = batch_idx * stride_qz + qo_head_idx * stride_qh
+ kv_offset = batch_idx * stride_kz + kv_head_idx * stride_kh
+
+ q_ptrs = Q + qo_offset + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qd
+ k_ptrs = K + kv_offset + offs_d[:, None] * stride_kd
+ v_ptrs = V + kv_offset + offs_d[None, :] * stride_vd
+ o_ptrs = Out + qo_offset + offs_m[:, None] * stride_om + offs_d[None, :] * stride_od
+ lse_ptrs = softmax_lse + batch_idx * stride_sz + qo_head_idx * stride_sh + offs_m * stride_sm
+
+ block_num = tl.load(block_cnt + batch_idx * stride_2cz + qo_head_idx * stride_2ch + start_m * stride_2cm)
+ if block_num <= 0:
+ return
+
+ block_idx_ptr = block_idx + batch_idx * stride_2iz + qo_head_idx * stride_2ih + start_m * stride_2im
+
+ # initialize pointer to m and l
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
+ l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
+
+ # scale sm_scale by log_2(e) and use
+ # 2^x instead of exp in the loop because CSE and LICM
+ # don't work as expected with `exp` in the loop
+ # 1/ln2 = lne/ln2 = log2(e) => 2^(x / ln2) = 2^(x * log2(e)) = (2^(log2(e)))^x = e^x
+ qk_scale = sm_scale * 1.44269504
+
+ # load q: it will stay in SRAM throughout
+ q = tl.load(q_ptrs)
+ q = (q * qk_scale).to(Q.type.element_ty)
+
+ if CAUSAL:
+ block_split = block_num - 2
+ else:
+ block_split = block_num
+
+ # Block
+ for start_n in range(0, block_split):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[None, :] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # -- compute qk --
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = qk + tl.dot(q, k)
+
+ # -- compute scaling constant --
+ m_i_new = tl.maximum(m_i, tl.max(qk, 1))
+ alpha = tl.math.exp2(m_i - m_i_new)
+ p = tl.math.exp2(qk - m_i_new[:, None])
+
+ # -- scale and update acc --
+ acc_scale = l_i * 0 + alpha # workaround some compiler bug
+ acc = acc * acc_scale[:, None]
+ acc = acc + tl.dot(p.to(Q.type.element_ty), v)
+
+ # -- update m_i and l_i --
+ l_i = l_i * alpha + tl.sum(p, 1)
+ m_i = m_i_new
+
+ # Block (Causal)
+ for start_n in range(max(block_split, 0), block_num):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[None, :] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # -- compute qk --
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = tl.where(offs_m[:, None] >= offs_n[None, :] + block_off, qk, float("-inf"))
+ qk = qk + tl.dot(q, k)
+
+ # -- compute scaling constant --
+ m_i_new = tl.maximum(m_i, tl.max(qk, 1))
+ alpha = tl.math.exp2(m_i - m_i_new)
+ p = tl.math.exp2(qk - m_i_new[:, None])
+
+ # -- scale and update acc --
+ acc_scale = l_i * 0 + alpha # workaround some compiler bug
+ acc = acc * acc_scale[:, None]
+ acc = acc + tl.dot(p.to(Q.type.element_ty), v)
+
+ # -- update m_i and l_i --
+ l_i = l_i * alpha + tl.sum(p, 1)
+ m_i = m_i_new
+
+ # write back O and LSE
+ acc_1 = acc / l_i[:, None]
+ s_1 = m_i * 0.69314718 + tl.math.log(l_i)
+ acc_0 = tl.load(o_ptrs).to(tl.float32)
+ s_0 = tl.load(lse_ptrs)
+
+ overflow_mask = (s_0 - s_1) < 88.0
+
+ theta = tl.math.exp(s_0 - s_1)
+ alpha_0 = 1 / (1 + 1 / theta)
+ alpha_1 = 1 / (1 + theta)
+ acc = alpha_0[:, None] * acc_0 + alpha_1[:, None] * acc_1
+ s = s_1 - tl.math.log(alpha_1)
+
+ tl.store(o_ptrs, acc.to(Out.type.element_ty))
+ tl.store(lse_ptrs, s, mask=overflow_mask)
+
+def triton_block_attn_fwd(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks]
+ granularity: int,
+ step: int = 0,
+ causal: bool = True,
+):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ num_blocks = block_idx.shape[2]
+
+ o = torch.zeros_like(q)
+ lse = torch.zeros((batch_size, num_qo_heads, num_tokens), dtype=torch.float32, device=q.device) - torch.inf
+
+ _triton_block_attn_fwd_kernel[(num_blocks, num_qo_heads, batch_size)](
+ q, k, v, softmax_scale,
+ block_cnt, block_idx,
+ o, lse,
+ q.stride(0), q.stride(2), q.stride(1), q.stride(3),
+ k.stride(0), k.stride(2), k.stride(1), k.stride(3),
+ v.stride(0), v.stride(2), v.stride(1), v.stride(3),
+ o.stride(0), o.stride(2), o.stride(1), o.stride(3),
+ block_cnt.stride(0), block_cnt.stride(1), block_cnt.stride(2),
+ block_idx.stride(0), block_idx.stride(1), block_idx.stride(2), block_idx.stride(3),
+ lse.stride(0), lse.stride(1), lse.stride(2),
+ num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M=granularity, BLOCK_N=64, BLOCK_DMODEL=head_dim, CAUSAL=causal,
+ num_warps=4, num_stages=2,
+ )
+ return o, lse
+
+@triton.jit
+def _triton_block_attn_bwd_kernel(
+ Q, K, V, O,
+ DQ, DK, DV, DO,
+ sm_scale,
+ block_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS]
+ block_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NUM_COLS]
+ softmax_lse, # [BATCH, N_HEADS, N_CTX]
+ stride_qz, stride_qh, stride_qm, stride_qd,
+ stride_kz, stride_kh, stride_kn, stride_kd,
+ stride_vz, stride_vh, stride_vn, stride_vd,
+ stride_oz, stride_oh, stride_om, stride_od,
+ stride_dqz, stride_dqh, stride_dqm, stride_dqd,
+ stride_dkz, stride_dkh, stride_dkn, stride_dkd,
+ stride_dvz, stride_dvh, stride_dvn, stride_dvd,
+ stride_doz, stride_doh, stride_dom, stride_dod,
+ stride_2cz, stride_2ch, stride_2cm,
+ stride_2iz, stride_2ih, stride_2im, stride_2in,
+ stride_sz, stride_sh, stride_sm,
+ num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_DMODEL: tl.constexpr,
+ CAUSAL: tl.constexpr,
+):
+ start_m = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ if start_m * BLOCK_M >= num_tokens:
+ return
+
+ qk_scale = sm_scale * 1.44269504
+
+ # offset pointers for batch/head
+ Q += batch_idx * stride_qz + qo_head_idx * stride_qh
+ K += batch_idx * stride_kz + kv_head_idx * stride_kh
+ V += batch_idx * stride_vz + kv_head_idx * stride_vh
+ O += batch_idx * stride_oz + qo_head_idx * stride_oh
+ DQ += batch_idx * stride_dqz + qo_head_idx * stride_dqh
+ DK += batch_idx * stride_dkz + kv_head_idx * stride_dkh
+ DV += batch_idx * stride_dvz + kv_head_idx * stride_dvh
+ DO += batch_idx * stride_doz + qo_head_idx * stride_doh
+
+ # loop over rows
+ offs_d = tl.arange(0, BLOCK_DMODEL)
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+
+ # initialize pointers to value-like data
+ q_ptrs = Q + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qd
+ k_ptrs = K + offs_d[None, :] * stride_kd
+ v_ptrs = V + offs_d[None, :] * stride_vd
+ o_ptrs = O + offs_m[:, None] * stride_om + offs_d[None, :] * stride_od
+ dq_ptrs = DQ + offs_m[:, None] * stride_dqm + offs_d[None, :] * stride_dqd
+ dk_ptrs = DK + offs_d[None, :] * stride_dkd
+ dv_ptrs = DV + offs_d[None, :] * stride_dvd
+ do_ptrs = DO + offs_m[:, None] * stride_dom + offs_d[None, :] * stride_dod
+ l_ptrs = softmax_lse + batch_idx * stride_sz + qo_head_idx * stride_sh + offs_m * stride_sm
+
+ block_num = tl.load(block_cnt + batch_idx * stride_2cz + qo_head_idx * stride_2ch + start_m * stride_2cm)
+ block_idx_ptr = block_idx + batch_idx * stride_2iz + qo_head_idx * stride_2ih + start_m * stride_2im
+
+ o = tl.load(o_ptrs).to(tl.float32)
+ do = tl.load(do_ptrs).to(tl.float32)
+ d_i = tl.sum(o * do, axis=1)
+
+ q = tl.load(q_ptrs)
+ do = do.to(DO.dtype.element_ty)
+ l_i = tl.load(l_ptrs) * 1.44269504
+
+ dq = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
+
+ if CAUSAL:
+ block_split = block_num - 2
+ else:
+ block_split = block_num
+
+ # Block
+ for start_n in range(0, block_split):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[:, None] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # Computer qk
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = qk + tl.dot(q, tl.trans(k))
+ qk = qk * qk_scale
+ p = tl.math.exp2(qk - l_i[:, None])
+
+ # compute dv
+ dv_vals = tl.dot(tl.trans(p.to(Q.dtype.element_ty)), do).to(tl.float32)
+ tl.atomic_add(dv_ptrs + block_off * stride_dvn + offs_n[:, None] * stride_dvn, dv_vals, sem="relaxed")
+
+ # compute dp = dot(v, do)
+ dp = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - d_i[:, None]
+ dp = dp + tl.dot(do, tl.trans(v))
+
+ # compute ds = p * (dp - delta[:, None])
+ ds = p * dp * sm_scale
+
+ # compute dk = dot(ds.T, q)
+ dk_vals = tl.dot(tl.trans(ds.to(Q.dtype.element_ty)), q).to(tl.float32)
+ tl.atomic_add(dk_ptrs + block_off * stride_dkn + offs_n[:, None] * stride_dkn, dk_vals, sem="relaxed")
+
+ # compute dq
+ dq = dq + tl.dot(ds.to(Q.dtype.element_ty), k)
+
+ # Block (Causal)
+ for start_n in range(max(block_split, 0), block_num):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[:, None] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # Computer qk
+ qk = tl.where(offs_m[:, None] >= offs_n[None, :] + block_off, float(0.), float("-inf"))
+ qk = qk + tl.dot(q, tl.trans(k))
+ qk = qk * qk_scale
+ p = tl.math.exp2(qk - l_i[:, None])
+
+ # compute dv
+ dv_vals = tl.dot(tl.trans(p.to(Q.dtype.element_ty)), do).to(tl.float32)
+ tl.atomic_add(dv_ptrs + block_off * stride_dvn + offs_n[:, None] * stride_dvn, dv_vals, sem="relaxed")
+
+ # compute dp = dot(v, do)
+ dp = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - d_i[:, None]
+ dp = dp + tl.dot(do, tl.trans(v))
+
+ # compute ds = p * (dp - delta[:, None])
+ ds = p * dp * sm_scale
+
+ # compute dk = dot(ds.T, q)
+ dk_vals = tl.dot(tl.trans(ds.to(Q.dtype.element_ty)), q).to(tl.float32)
+ tl.atomic_add(dk_ptrs + block_off * stride_dkn + offs_n[:, None] * stride_dkn, dk_vals, sem="relaxed")
+
+ # compute dq
+ dq = dq + tl.dot(ds.to(Q.dtype.element_ty), k)
+
+ dq_old = tl.load(dq_ptrs).to(tl.float32)
+ tl.store(dq_ptrs, (dq_old + dq).to(DQ.dtype.element_ty))
+
+
+def triton_block_attn_bwd(
+ grad: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ o: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ block_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks]
+ granularity: int,
+ deterministic: bool,
+ step: int = 0,
+ causal: bool = True,
+):
+ assert not deterministic
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ num_blocks = block_idx.shape[2]
+
+ dq = torch.zeros_like(q)
+ dk = torch.zeros_like(k, dtype=torch.float32)
+ dv = torch.zeros_like(v, dtype=torch.float32)
+
+ _triton_block_attn_bwd_kernel[(num_blocks, num_qo_heads, batch_size)](
+ q, k, v, o, dq, dk, dv, grad, softmax_scale,
+ block_cnt, block_idx, softmax_lse,
+ q.stride(0), q.stride(2), q.stride(1), q.stride(3),
+ k.stride(0), k.stride(2), k.stride(1), k.stride(3),
+ v.stride(0), v.stride(2), v.stride(1), v.stride(3),
+ o.stride(0), o.stride(2), o.stride(1), o.stride(3),
+ dq.stride(0), dq.stride(2), dq.stride(1), dq.stride(3),
+ dk.stride(0), dk.stride(2), dk.stride(1), dk.stride(3),
+ dv.stride(0), dv.stride(2), dv.stride(1), dv.stride(3),
+ grad.stride(0), grad.stride(2), grad.stride(1), grad.stride(3),
+ block_cnt.stride(0), block_cnt.stride(1), block_cnt.stride(2),
+ block_idx.stride(0), block_idx.stride(1), block_idx.stride(2), block_idx.stride(3),
+ softmax_lse.stride(0), softmax_lse.stride(1), softmax_lse.stride(2),
+ num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M=granularity, BLOCK_N=64, BLOCK_DMODEL=head_dim, CAUSAL=causal,
+ num_warps=4, num_stages=2,
+ )
+ return dq, dk.to(dq.dtype), dv.to(dq.dtype)
+
+
+@triton.jit
+def _triton_block_bar_attn_fwd_kernel(
+ Q, K, V, sm_scale,
+ bar_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS, WORLD_SIZE + 1]
+ bar_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NNZ_V]
+ block_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS]
+ block_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NUM_COLS]
+ Out, # [BATCH, N_Q_HEADS, N_CTX, D_HEAD]
+ softmax_lse, # [BATCH, N_Q_HEADS, N_CTX]
+ stride_qz, stride_qh, stride_qm, stride_qd,
+ stride_kz, stride_kh, stride_kn, stride_kd,
+ stride_vz, stride_vh, stride_vn, stride_vd,
+ stride_oz, stride_oh, stride_om, stride_od,
+ stride_1cz, stride_1ch, stride_1cm, stride_1cr,
+ stride_1iz, stride_1ih, stride_1im, stride_1in,
+ stride_2cz, stride_2ch, stride_2cm,
+ stride_2iz, stride_2ih, stride_2im, stride_2in,
+ stride_sz, stride_sh, stride_sm,
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_DMODEL: tl.constexpr,
+ CAUSAL: tl.constexpr,
+):
+ start_m = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ if start_m * BLOCK_M >= num_tokens:
+ return
+
+ # initialize offsets
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+ offs_d = tl.arange(0, BLOCK_DMODEL)
+
+ qo_offset = batch_idx * stride_qz + qo_head_idx * stride_qh
+ kv_offset = batch_idx * stride_kz + kv_head_idx * stride_kh
+
+ q_ptrs = Q + qo_offset + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qd
+ k_ptrs = K + kv_offset + offs_d[:, None] * stride_kd
+ v_ptrs = V + kv_offset + offs_d[None, :] * stride_vd
+ o_ptrs = Out + qo_offset + offs_m[:, None] * stride_om + offs_d[None, :] * stride_od
+
+ lse_ptrs = softmax_lse + batch_idx * stride_sz + qo_head_idx * stride_sh + offs_m * stride_sm
+
+ bar_l = tl.load(bar_cnt + batch_idx * stride_1cz + qo_head_idx * stride_1ch + start_m * stride_1cm + step * stride_1cr)
+ bar_r = tl.load(bar_cnt + batch_idx * stride_1cz + qo_head_idx * stride_1ch + start_m * stride_1cm + (step + 1) * stride_1cr)
+ bar_idx_ptr = bar_idx + batch_idx * stride_1iz + qo_head_idx * stride_1ih + start_m * stride_1im
+
+ block_num = tl.load(block_cnt + batch_idx * stride_2cz + qo_head_idx * stride_2ch + start_m * stride_2cm)
+ block_idx_ptr = block_idx + batch_idx * stride_2iz + qo_head_idx * stride_2ih + start_m * stride_2im
+
+ if (bar_l >= bar_r) and (block_num <= 0):
+ return
+
+ # initialize pointer to m and l
+ m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
+ l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
+ acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
+
+ # scale sm_scale by log_2(e) and use
+ # 2^x instead of exp in the loop because CSE and LICM
+ # don't work as expected with `exp` in the loop
+ # 1/ln2 = lne/ln2 = log2(e) => 2^(x / ln2) = 2^(x * log2(e)) = (2^(log2(e)))^x = e^x
+ qk_scale = sm_scale * 1.44269504
+
+ # load q: it will stay in SRAM throughout
+ q = tl.load(q_ptrs)
+ q = (q * qk_scale).to(Q.type.element_ty)
+
+ if CAUSAL:
+ block_split = block_num - 2
+ else:
+ block_split = block_num
+
+ # Block
+ for start_n in range(0, block_split):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[None, :] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # -- compute qk --
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = qk + tl.dot(q, k)
+
+ # -- compute scaling constant --
+ m_i_new = tl.maximum(m_i, tl.max(qk, 1))
+ alpha = tl.math.exp2(m_i - m_i_new)
+ p = tl.math.exp2(qk - m_i_new[:, None])
+
+ # -- scale and update acc --
+ acc_scale = l_i * 0 + alpha # workaround some compiler bug
+ acc = acc * acc_scale[:, None]
+ acc = acc + tl.dot(p.to(Q.type.element_ty), v)
+
+ # -- update m_i and l_i --
+ l_i = l_i * alpha + tl.sum(p, 1)
+ m_i = m_i_new
+
+ # Block (Causal)
+ for start_n in range(max(block_split, 0), block_num):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[None, :] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # -- compute qk --
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = tl.where(offs_m[:, None] >= offs_n[None, :] + block_off, qk, float("-inf"))
+ qk = qk + tl.dot(q, k)
+
+ # -- compute scaling constant --
+ m_i_new = tl.maximum(m_i, tl.max(qk, 1))
+ alpha = tl.math.exp2(m_i - m_i_new)
+ p = tl.math.exp2(qk - m_i_new[:, None])
+
+ # -- scale and update acc --
+ acc_scale = l_i * 0 + alpha # workaround some compiler bug
+ acc = acc * acc_scale[:, None]
+ acc = acc + tl.dot(p.to(Q.type.element_ty), v)
+
+ # -- update m_i and l_i --
+ l_i = l_i * alpha + tl.sum(p, 1)
+ m_i = m_i_new
+
+ # Bar
+ for start_n in range(bar_l, bar_r, BLOCK_N):
+ n_mask = start_n + offs_n < bar_r
+ cols = tl.load(bar_idx_ptr + (start_n + offs_n) * stride_1in, mask=n_mask, other=0)
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + cols[None, :] * stride_kn)
+ v = tl.load(v_ptrs + cols[:, None] * stride_vn)
+
+ # -- compute qk --
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = tl.where(n_mask[None, :], qk, float("-inf"))
+ qk = qk + tl.dot(q, k)
+
+ # -- compute scaling constant --
+ m_i_new = tl.maximum(m_i, tl.max(qk, 1))
+ alpha = tl.math.exp2(m_i - m_i_new)
+ p = tl.math.exp2(qk - m_i_new[:, None])
+
+ # -- scale and update acc --
+ acc_scale = l_i * 0 + alpha # workaround some compiler bug
+ acc = acc * acc_scale[:, None]
+ acc = acc + tl.dot(p.to(Q.type.element_ty), v)
+
+ # -- update m_i and l_i --
+ l_i = l_i * alpha + tl.sum(p, 1)
+ m_i = m_i_new
+
+ # write back O and LSE
+ acc_1 = acc / l_i[:, None]
+ s_1 = m_i * 0.69314718 + tl.math.log(l_i)
+ acc_0 = tl.load(o_ptrs).to(tl.float32)
+ s_0 = tl.load(lse_ptrs)
+
+ overflow_mask = (s_0 - s_1) < 88.0
+
+ theta = tl.math.exp(s_0 - s_1)
+ alpha_0 = 1 / (1 + 1 / theta)
+ alpha_1 = 1 / (1 + theta)
+ acc = alpha_0[:, None] * acc_0 + alpha_1[:, None] * acc_1
+ s = s_1 - tl.math.log(alpha_1)
+
+ tl.store(o_ptrs, acc.to(Out.type.element_ty))
+ tl.store(lse_ptrs, s, mask=overflow_mask)
+
+
+def block_bar_attn_fwd(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ o: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ block_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks]
+ granularity: int,
+ step: int = 0,
+ causal: bool = True,
+):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ num_blocks = bar_idx.shape[2]
+ if o is None:
+ o = torch.zeros_like(q)
+ lse = torch.zeros((batch_size, num_qo_heads, num_tokens), dtype=torch.float32, device=q.device) - torch.inf
+ _triton_block_bar_attn_fwd_kernel[(num_blocks, num_qo_heads, batch_size)](
+ q, k, v, softmax_scale, bar_cnt, bar_idx, block_cnt, block_idx, o, lse,
+ q.stride(0), q.stride(2), q.stride(1), q.stride(3),
+ k.stride(0), k.stride(2), k.stride(1), k.stride(3),
+ v.stride(0), v.stride(2), v.stride(1), v.stride(3),
+ o.stride(0), o.stride(2), o.stride(1), o.stride(3),
+ bar_cnt.stride(0), bar_cnt.stride(1), bar_cnt.stride(2), bar_cnt.stride(3),
+ bar_idx.stride(0), bar_idx.stride(1), bar_idx.stride(2), bar_idx.stride(3),
+ block_cnt.stride(0), block_cnt.stride(1), block_cnt.stride(2),
+ block_idx.stride(0), block_idx.stride(1), block_idx.stride(2), block_idx.stride(3),
+ lse.stride(0), lse.stride(1), lse.stride(2),
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M=granularity, BLOCK_N=64, BLOCK_DMODEL=head_dim, CAUSAL=causal,
+ num_warps=4, num_stages=2,
+ )
+ return o, lse
+
+
+@triton.jit
+def _triton_block_bar_attn_bwd_kernel(
+ Q, K, V, O,
+ DQ, DK, DV, DO,
+ sm_scale,
+ bar_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS, WORLD_SIZE + 1]
+ bar_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NNZ_V]
+ block_cnt, # [BATCH, N_Q_HEADS, NUM_ROWS]
+ block_idx, # [BATCH, N_Q_HEADS, NUM_ROWS, NUM_COLS]
+ softmax_lse, # [BATCH, N_HEADS, N_CTX]
+ stride_qz, stride_qh, stride_qm, stride_qd,
+ stride_kz, stride_kh, stride_kn, stride_kd,
+ stride_vz, stride_vh, stride_vn, stride_vd,
+ stride_oz, stride_oh, stride_om, stride_od,
+ stride_dqz, stride_dqh, stride_dqm, stride_dqd,
+ stride_dkz, stride_dkh, stride_dkn, stride_dkd,
+ stride_dvz, stride_dvh, stride_dvn, stride_dvd,
+ stride_doz, stride_doh, stride_dom, stride_dod,
+ stride_1cz, stride_1ch, stride_1cm, stride_1cr,
+ stride_1iz, stride_1ih, stride_1im, stride_1in,
+ stride_2cz, stride_2ch, stride_2cm,
+ stride_2iz, stride_2ih, stride_2im, stride_2in,
+ stride_sz, stride_sh, stride_sm,
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_DMODEL: tl.constexpr,
+ CAUSAL: tl.constexpr,
+):
+ start_m = tl.program_id(0)
+ qo_head_idx = tl.program_id(1)
+ batch_idx = tl.program_id(2)
+ kv_head_idx = qo_head_idx // (num_qo_heads // num_kv_heads)
+
+ if start_m * BLOCK_M >= num_tokens:
+ return
+
+ qk_scale = sm_scale * 1.44269504
+
+ # offset pointers for batch/head
+ Q += batch_idx * stride_qz + qo_head_idx * stride_qh
+ K += batch_idx * stride_kz + kv_head_idx * stride_kh
+ V += batch_idx * stride_vz + kv_head_idx * stride_vh
+ O += batch_idx * stride_oz + qo_head_idx * stride_oh
+ DQ += batch_idx * stride_dqz + qo_head_idx * stride_dqh
+ DK += batch_idx * stride_dkz + kv_head_idx * stride_dkh
+ DV += batch_idx * stride_dvz + kv_head_idx * stride_dvh
+ DO += batch_idx * stride_doz + qo_head_idx * stride_doh
+
+ # loop over rows
+ offs_d = tl.arange(0, BLOCK_DMODEL)
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+
+ # initialize pointers to value-like data
+ q_ptrs = Q + offs_m[:, None] * stride_qm + offs_d[None, :] * stride_qd
+ k_ptrs = K + offs_d[None, :] * stride_kd
+ v_ptrs = V + offs_d[None, :] * stride_vd
+ o_ptrs = O + offs_m[:, None] * stride_om + offs_d[None, :] * stride_od
+ dq_ptrs = DQ + offs_m[:, None] * stride_dqm + offs_d[None, :] * stride_dqd
+ dk_ptrs = DK + offs_d[None, :] * stride_dkd
+ dv_ptrs = DV + offs_d[None, :] * stride_dvd
+ do_ptrs = DO + offs_m[:, None] * stride_dom + offs_d[None, :] * stride_dod
+
+ l_ptrs = softmax_lse + batch_idx * stride_sz + qo_head_idx * stride_sh + offs_m * stride_sm
+
+ bar_l = tl.load(bar_cnt + batch_idx * stride_1cz + qo_head_idx * stride_1ch + start_m * stride_1cm + step * stride_1cr)
+ bar_r = tl.load(bar_cnt + batch_idx * stride_1cz + qo_head_idx * stride_1ch + start_m * stride_1cm + (step + 1) * stride_1cr)
+ bar_idx_ptr = bar_idx + batch_idx * stride_1iz + qo_head_idx * stride_1ih + start_m * stride_1im
+
+ block_num = tl.load(block_cnt + batch_idx * stride_2cz + qo_head_idx * stride_2ch + start_m * stride_2cm)
+ block_idx_ptr = block_idx + batch_idx * stride_2iz + qo_head_idx * stride_2ih + start_m * stride_2im
+
+ if (bar_l >= bar_r) and (block_num <= 0):
+ return
+
+ o = tl.load(o_ptrs).to(tl.float32)
+ do = tl.load(do_ptrs).to(tl.float32)
+ d_i = tl.sum(o * do, axis=1)
+
+ q = tl.load(q_ptrs)
+ do = do.to(DO.dtype.element_ty)
+ l_i = tl.load(l_ptrs) * 1.44269504
+
+ dq = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
+
+ if CAUSAL:
+ block_split = block_num - 2
+ else:
+ block_split = block_num
+
+ # Block
+ for start_n in range(0, block_split):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[:, None] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # Computer qk
+ qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
+ qk = qk + tl.dot(q, tl.trans(k))
+ qk = qk * qk_scale
+ p = tl.math.exp2(qk - l_i[:, None])
+
+ # compute dv
+ dv_vals = tl.dot(tl.trans(p.to(Q.dtype.element_ty)), do).to(tl.float32)
+ tl.atomic_add(dv_ptrs + block_off * stride_dvn + offs_n[:, None] * stride_dvn, dv_vals, sem="relaxed")
+
+ # compute dp = dot(v, do)
+ dp = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - d_i[:, None]
+ dp = dp + tl.dot(do, tl.trans(v))
+
+ # compute ds = p * (dp - delta[:, None])
+ ds = p * dp * sm_scale
+
+ # compute dk = dot(ds.T, q)
+ dk_vals = tl.dot(tl.trans(ds.to(Q.dtype.element_ty)), q).to(tl.float32)
+ tl.atomic_add(dk_ptrs + block_off * stride_dkn + offs_n[:, None] * stride_dkn, dk_vals, sem="relaxed")
+
+ # compute dq
+ dq = dq + tl.dot(ds.to(Q.dtype.element_ty), k)
+
+ # Block (Causal)
+ for start_n in range(max(block_split, 0), block_num):
+ block_off = tl.load(block_idx_ptr + start_n * stride_2in) * BLOCK_N
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + block_off * stride_kn + offs_n[:, None] * stride_kn)
+ v = tl.load(v_ptrs + block_off * stride_vn + offs_n[:, None] * stride_vn)
+
+ # Computer qk
+ qk = tl.where(offs_m[:, None] >= offs_n[None, :] + block_off, float(0.), float("-inf"))
+ qk = qk + tl.dot(q, tl.trans(k))
+ qk = qk * qk_scale
+ p = tl.math.exp2(qk - l_i[:, None])
+
+ # compute dv
+ dv_vals = tl.dot(tl.trans(p.to(Q.dtype.element_ty)), do).to(tl.float32)
+ tl.atomic_add(dv_ptrs + block_off * stride_dvn + offs_n[:, None] * stride_dvn, dv_vals, sem="relaxed")
+
+ # compute dp = dot(v, do)
+ dp = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - d_i[:, None]
+ dp = dp + tl.dot(do, tl.trans(v))
+
+ # compute ds = p * (dp - delta[:, None])
+ ds = p * dp * sm_scale
+
+ # compute dk = dot(ds.T, q)
+ dk_vals = tl.dot(tl.trans(ds.to(Q.dtype.element_ty)), q).to(tl.float32)
+ tl.atomic_add(dk_ptrs + block_off * stride_dkn + offs_n[:, None] * stride_dkn, dk_vals, sem="relaxed")
+
+ # compute dq
+ dq = dq + tl.dot(ds.to(Q.dtype.element_ty), k)
+
+ # Bar
+ for start_n in range(bar_l, bar_r, BLOCK_N):
+ n_mask = start_n + offs_n < bar_r
+ cols = tl.load(bar_idx_ptr + (start_n + offs_n) * stride_1in, mask=n_mask, other=0)
+
+ # -- load k, v --
+ k = tl.load(k_ptrs + cols[:, None] * stride_kn)
+ v = tl.load(v_ptrs + cols[:, None] * stride_vn)
+
+ # Computer qk
+ qk = tl.where(n_mask[None, :], float(0.), float("-inf"))
+ qk = qk + tl.dot(q, tl.trans(k))
+ qk = qk * qk_scale
+ p = tl.math.exp2(qk - l_i[:, None])
+
+ # compute dv
+ dv_vals = tl.dot(tl.trans(p.to(Q.dtype.element_ty)), do).to(tl.float32)
+ tl.atomic_add(dv_ptrs + cols[:, None] * stride_dvn, dv_vals, mask=n_mask[:, None], sem="relaxed")
+
+ # compute dp = dot(v, do)
+ dp = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) - d_i[:, None]
+ dp = dp + tl.dot(do, tl.trans(v))
+
+ # compute ds = p * (dp - delta[:, None])
+ ds = p * dp * sm_scale
+
+ # compute dk = dot(ds.T, q)
+ dk_vals = tl.dot(tl.trans(ds.to(Q.dtype.element_ty)), q).to(tl.float32)
+ tl.atomic_add(dk_ptrs + cols[:, None] * stride_dkn, dk_vals, mask=n_mask[:, None], sem="relaxed")
+
+ # compute dq
+ dq = dq + tl.dot(ds.to(Q.dtype.element_ty), k)
+
+ dq_old = tl.load(dq_ptrs).to(tl.float32)
+ tl.store(dq_ptrs, (dq_old + dq).to(DQ.dtype.element_ty))
+
+
+def block_bar_attn_bwd(
+ grad: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ o: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ dq: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ dk: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ dv: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ softmax_lse: torch.Tensor, # [batch_size, num_qo_heads, num_tokens]
+ softmax_scale: float,
+ bar_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, max_v_size]
+ bar_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, world_size + 1]
+ block_idx: torch.Tensor, # [batch_size, num_qo_heads, num_blocks, num_blocks]
+ block_cnt: torch.Tensor, # [batch_size, num_qo_heads, num_blocks]
+ granularity: int,
+ deterministic: bool,
+ step: int = 0,
+ causal: bool = True,
+):
+ assert not deterministic
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ num_kv_heads = k.shape[2]
+ num_blocks = bar_idx.shape[2]
+ dq = torch.zeros_like(q) if dq is None else dq
+ dk = torch.zeros_like(k, dtype=torch.float32) if dk is None else dk.to(torch.float32)
+ dv = torch.zeros_like(v, dtype=torch.float32) if dv is None else dv.to(torch.float32)
+ _triton_block_bar_attn_bwd_kernel[(num_blocks, num_qo_heads, batch_size)](
+ q, k, v, o, dq, dk, dv, grad, softmax_scale,
+ bar_cnt, bar_idx, block_cnt, block_idx, softmax_lse,
+ q.stride(0), q.stride(2), q.stride(1), q.stride(3),
+ k.stride(0), k.stride(2), k.stride(1), k.stride(3),
+ v.stride(0), v.stride(2), v.stride(1), v.stride(3),
+ o.stride(0), o.stride(2), o.stride(1), o.stride(3),
+ dq.stride(0), dq.stride(2), dq.stride(1), dq.stride(3),
+ dk.stride(0), dk.stride(2), dk.stride(1), dk.stride(3),
+ dv.stride(0), dv.stride(2), dv.stride(1), dv.stride(3),
+ grad.stride(0), grad.stride(2), grad.stride(1), grad.stride(3),
+ bar_cnt.stride(0), bar_cnt.stride(1), bar_cnt.stride(2), bar_cnt.stride(3),
+ bar_idx.stride(0), bar_idx.stride(1), bar_idx.stride(2), bar_idx.stride(3),
+ block_cnt.stride(0), block_cnt.stride(1), block_cnt.stride(2),
+ block_idx.stride(0), block_idx.stride(1), block_idx.stride(2), block_idx.stride(3),
+ softmax_lse.stride(0), softmax_lse.stride(1), softmax_lse.stride(2),
+ step, num_qo_heads, num_kv_heads, num_tokens,
+ BLOCK_M=granularity, BLOCK_N=64, BLOCK_DMODEL=head_dim, CAUSAL=causal,
+ num_warps=4, num_stages=2,
+ )
+ return dq, dk.to(dq.dtype), dv.to(dq.dtype)
+
+
+# ---------------------------------------------------------------------------------
+# Attention Classes
+class MInferenceAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ softmax_scale,
+ granularity,
+ return_softmax,
+ deterministic,
+ ):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ if softmax_scale is None:
+ softmax_scale = head_dim ** (-0.5)
+
+ block_mask, bar_idx, bar_cnt = build_index_local(
+ q, k, v_size, s_size, num_tokens, granularity
+ )
+
+ # Block Mask
+ out, softmax_lse = block_attn_fwd(
+ q, k, v, softmax_scale,
+ block_mask,
+ granularity=granularity,
+ causal=True,
+ )
+ # Bar Mask
+ out, softmax_lse = bar_attn_fwd(
+ q, k, v, out, softmax_lse, softmax_scale,
+ bar_idx, bar_cnt,
+ granularity=granularity,
+ step=0,
+ )
+
+ ctx.save_for_backward(q, k, v, out, softmax_lse, block_mask, bar_idx, bar_cnt)
+ ctx.granularity = granularity
+ ctx.deterministic = deterministic
+ ctx.softmax_scale = softmax_scale
+ return (out, softmax_lse, None) if return_softmax else out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ q, k, v, out, softmax_lse, block_mask, bar_idx, bar_cnt = ctx.saved_tensors
+ # Block Mask
+ dq, dk, dv = block_attn_bwd(
+ dout, q, k, v, out,
+ softmax_lse, ctx.softmax_scale,
+ block_mask,
+ granularity=ctx.granularity,
+ deterministic=ctx.deterministic,
+ causal=True,
+ )
+
+ # Bar Mask
+ dq, dk, dv = bar_attn_bwd(
+ dout, q, k, v, out, dq, dk, dv,
+ softmax_lse, ctx.softmax_scale,
+ bar_idx, bar_cnt,
+ granularity=ctx.granularity,
+ deterministic=ctx.deterministic,
+ step=0,
+ )
+ return dq, dk, dv, None, None, None, None, None, None
+
+
+
+class MInferenceAttnTritonFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ softmax_scale,
+ granularity,
+ return_softmax,
+ deterministic,
+ ):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ if softmax_scale is None:
+ softmax_scale = head_dim ** (-0.5)
+
+ block_mask, bar_idx, bar_cnt = build_index_local(q, k, v_size, s_size, num_tokens, granularity)
+ block_idx, block_cnt = convert_blockmask(block_mask, block_size_M=granularity, block_size_N=64)
+
+ out, softmax_lse = block_bar_attn_fwd(
+ q, k, v, None, None, softmax_scale,
+ bar_idx, bar_cnt, block_idx, block_cnt,
+ granularity=granularity,
+ step=0,
+ )
+
+ ctx.save_for_backward(q, k, v, out, softmax_lse, block_idx, block_cnt, bar_idx, bar_cnt)
+ ctx.granularity = granularity
+ ctx.deterministic = deterministic
+ ctx.softmax_scale = softmax_scale
+ return (out, softmax_lse, None) if return_softmax else out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ q, k, v, out, softmax_lse, block_idx, block_cnt, bar_idx, bar_cnt = ctx.saved_tensors
+
+ # Bar Mask
+ dq, dk, dv = block_bar_attn_bwd(
+ dout, q, k, v, out, None, None, None,
+ softmax_lse, ctx.softmax_scale,
+ bar_idx, bar_cnt, block_idx, block_cnt,
+ granularity=ctx.granularity,
+ deterministic=ctx.deterministic,
+ step=0,
+ )
+
+ return dq, dk, dv, None, None, None, None, None, None
+
+# ---------------------------------------------------------------------------------
+# Wrapped Attention Functions
+# --------------------------------------------
+# CUDA-Based
+def minference_flash_attn_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v_size: List[int], # [num_qo_heads]
+ s_size: List[int], # [num_qo_heads]
+ dropout_p: int = 0.0,
+ softmax_scale: float = None,
+ granularity: int = 128,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[float, float] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+):
+ assert dropout_p == 0
+ assert causal
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ return MInferenceAttnFunc.apply(
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ deterministic,
+ )
+
+
+def minference_flash_attn_qkvpacked_func(
+ qkv: torch.Tensor, # [batch_size, num_tokens, 3, num_heads, head_dim]
+ *args, **kwargs
+):
+ return minference_flash_attn_func(
+ qkv[:, :, 0], # q
+ qkv[:, :, 1], # k
+ qkv[:, :, 2], # v
+ *args, **kwargs
+ )
+
+
+def minference_flash_attn_kvpacked_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ kv: torch.Tensor, # [batch_size, num_tokens, 2, num_kv_heads, head_dim]\
+ *args, **kwargs
+):
+ return minference_flash_attn_func(
+ q,
+ kv[:, :, 0], # k
+ kv[:, :, 1], # v
+ *args, **kwargs
+ )
+
+# --------------------------------------------
+# Triton-Based
+def minference_flash_attn_triton_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v_size: List[int], # [num_qo_heads]
+ s_size: List[int], # [num_qo_heads]
+ dropout_p: int = 0.0,
+ softmax_scale: float = None,
+ granularity: int = 128,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[float, float] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+):
+ assert dropout_p == 0
+ assert causal
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+ return MInferenceAttnTritonFunc.apply(
+ q,
+ k,
+ v,
+ v_size,
+ s_size,
+ softmax_scale,
+ granularity,
+ return_attn_probs,
+ deterministic,
+ )
+
+def minference_flash_attn_triton_qkvpacked_func(
+ qkv: torch.Tensor, # [batch_size, num_tokens, 3, num_heads, head_dim]
+ *args, **kwargs
+):
+ return minference_flash_attn_triton_func(
+ qkv[:, :, 0], # q
+ qkv[:, :, 1], # k
+ qkv[:, :, 2], # v
+ *args, **kwargs
+ )
+
+
+def minference_flash_attn_triton_kvpacked_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ kv: torch.Tensor, # [batch_size, num_tokens, 2, num_kv_heads, head_dim]
+ *args, **kwargs
+):
+ return minference_flash_attn_triton_func(
+ q,
+ kv[:, :, 0], # k
+ kv[:, :, 1], # v
+ *args, **kwargs
+ )
diff --git a/minference/ops/streaming_kernel.py b/minference/ops/streaming_kernel.py
index 1eff0d7a..f5aee3ec 100644
--- a/minference/ops/streaming_kernel.py
+++ b/minference/ops/streaming_kernel.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
"""
@@ -113,6 +113,21 @@ def _attn_fwd_inner(acc, l_i, m_i, q,
return acc, l_i, m_i
+@triton.autotune(
+ configs=[
+ triton.Config({}, num_stages=1, num_warps=4),
+ triton.Config({}, num_stages=1, num_warps=8),
+ triton.Config({}, num_stages=2, num_warps=4),
+ triton.Config({}, num_stages=2, num_warps=8),
+ triton.Config({}, num_stages=3, num_warps=4),
+ triton.Config({}, num_stages=3, num_warps=8),
+ triton.Config({}, num_stages=4, num_warps=4),
+ triton.Config({}, num_stages=4, num_warps=8),
+ triton.Config({}, num_stages=5, num_warps=4),
+ triton.Config({}, num_stages=5, num_warps=8),
+ ],
+ key=['N_CTX'],
+)
@triton.heuristics(
{
"IS_EVEN_M": lambda args: args["N_CTX"] % args["BLOCK_M"] == 0,
@@ -411,7 +426,7 @@ def _forward(
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
assert Lq == Lk and Lk == Lv
- assert Lk in {16, 32, 64, 128}
+ assert Lk in {16, 32, 64, 128, 256}
q_round_len = math.ceil(q.shape[2] / 64) * 64
@@ -449,8 +464,6 @@ def _forward(
BLOCK_N=_BLOCK_N,
SLIDING_WINDOW=(sliding_window is not None),
COMPLEMENT_SLIDING_WINDOW=complement_sliding_window,
- num_warps=4,
- num_stages=4
)
except triton.OutOfResources as E:
_BLOCK_N = _BLOCK_N // 2
@@ -477,8 +490,6 @@ def _forward(
BLOCK_N=_BLOCK_N,
SLIDING_WINDOW=(sliding_window is not None),
COMPLEMENT_SLIDING_WINDOW=complement_sliding_window,
- num_warps=4,
- num_stages=4
)
@@ -766,3 +777,63 @@ def forward(self, query : torch.Tensor,
return score
return forward
+
+def a_shape_kernel(
+ q, k, v, config,
+):
+ # q,k,v should be tensors already equipped with RoPE
+ # k,v should already repeated to align with q.shape
+
+ n_init = config["attn_forward_config"].get("n_init", 128)
+ n_local = config["attn_forward_config"].get("n_local", 3968)
+
+ assert q.dim() == 4 # (bsz, num_heads, seqlen, head_dim)
+ assert q.shape == k.shape == v.shape
+
+ head_dim = q.shape[-1]
+ if head_dim not in [16, 32, 64, 128, 256]:
+ target_dim = 2 ** math.ceil(math.log2(head_dim)) - head_dim
+ q = torch.nn.functional.pad(q, [0, target_dim, 0, 0, 0, 0, 0, 0])
+ k = torch.nn.functional.pad(k, [0, target_dim, 0, 0, 0, 0, 0, 0])
+ v = torch.nn.functional.pad(v, [0, target_dim, 0, 0, 0, 0, 0, 0])
+
+ q_len = q.size(2)
+ k_len = k.size(2)
+
+ attn = TritonMultiStageDotProductionAttention(q.shape, q.dtype, q.device)
+
+ if k_len > n_local:
+ init_k = k[:, :, :n_init, :].contiguous()
+ init_v = v[:, :, :n_init, :].contiguous()
+
+ attn.append(q, k, v, sliding_window=n_local)
+ attn.append(
+ q, init_k, init_v, end=True,
+ sliding_window=(k_len - q_len, n_local), complement_sliding_window=True
+ )
+ else:
+ attn.append(q, k, v, sliding_window=n_local, end=True)
+
+ score, _ = attn.get_result()
+ return score[...,:head_dim]
+
+def tri_shape_kernel(q, k, v, config):
+ n_last = config["attn_forward_config"].get("n_last", 100)
+ n_last = min(n_last, q.size(2) - 1)
+
+ q1, q2 = q[:,:,:-n_last], q[:,:,-n_last:]
+ y1 = a_shape_kernel(q1, k[:,:,:-n_last], v[:,:,:-n_last], config)
+
+ qk = torch.einsum(f'bhmk, bhnk -> bhmn', q2, k) / math.sqrt(q.shape[-1])
+ arange = torch.arange(n_last, device="cuda")
+ mask = arange[None, None, :, None] >= arange[None, None, None, :]
+ qk[:, :, :, -n_last:] = torch.where(mask, qk[:, :, :, -n_last:], -torch.inf)
+ qk = torch.nn.functional.softmax(qk, dim=-1, dtype=torch.float32).to(q.dtype)
+ y2 = torch.einsum(f'bhmn, bhnk -> bhmk', qk, v)
+ return torch.cat([y1, y2], dim=2)
+
+
+if __name__ == "__main__":
+ b, h, m, k = 1, 1, 1024, 192
+ q, k, v = torch.rand(b, h, m, k).cuda(), torch.rand(b, h, m, k).cuda(), torch.rand(b, h, m, k).cuda()
+ output = a_shape_kernel(q, k, v, {"attn_forward_config": {"n_init": 128, "n_local": 512}})
diff --git a/minference/ops/utils.py b/minference/ops/utils.py
new file mode 100644
index 00000000..8050a75f
--- /dev/null
+++ b/minference/ops/utils.py
@@ -0,0 +1,80 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+
+import torch
+
+
+def set_seed(seed=42):
+ torch.manual_seed(seed)
+ torch.cuda.manual_seed(seed)
+ torch.cuda.manual_seed_all(seed)
+
+def use_triton():
+ return torch.version.hip is not None or os.getenv("FORCE_TRITON", "0") == "1"
+
+
+def check_correctness_by_row(
+ seq_len,
+ tensor_var,
+ ref_tensor_var,
+ tensor_name,
+ ATOL=1e-2,
+ RTOL=1e-2,
+):
+ if not torch.allclose(tensor_var, ref_tensor_var, atol=ATOL, rtol=RTOL):
+ for h in range(tensor_var.shape[2]):
+ for i in range(seq_len):
+ tensor_var_row = tensor_var[:, i, h]
+ ref_tensor_var_row = ref_tensor_var[:, i, h]
+
+ if not torch.allclose(tensor_var_row, ref_tensor_var_row, atol=ATOL, rtol=RTOL):
+ print('-' * 60 + '\n')
+ print(f"Mismatched {tensor_name} at Head {h}, Row {i}:\n")
+ print(f"Computed:\n{tensor_var_row}\n")
+ print(f"Ref:\n{ref_tensor_var_row}\n")
+
+ max_diff = torch.max(torch.abs(tensor_var_row - ref_tensor_var_row))
+ print(f"Maximal difference: {max_diff.item()}\n")
+ return False
+ else:
+ print(f"All {tensor_name} values are correct within the specified tolerance.")
+ return True
+
+def check_correct_rate(
+ tensor_var,
+ ref_tensor_var,
+ ATOL=1e-2,
+ RTOL=1e-2,
+):
+ assert len(tensor_var.shape) == 4, "Input tensor must be 3D (B, N, H, D)"
+ assert tensor_var.shape == ref_tensor_var.shape, (
+ "Input and reference tensors must have the same shape"
+ )
+ bsz, seq_len, num_heads, _ = tensor_var.shape
+
+ # Boolean mask of element-wise closeness
+ elem_close = torch.isclose(tensor_var, ref_tensor_var, atol=ATOL, rtol=RTOL)
+
+ # A row “matches” only if *all* its D elements are close
+ row_matches = elem_close.all(dim=-1) # shape (B, N, H)
+
+ # Count rows that do *not* match
+ num_mismatching = (~row_matches).sum().item()
+ num_mismatching_prop = num_mismatching / (bsz * seq_len * num_heads)
+ return 1 - num_mismatching_prop
+
+def check_by_correct_rate(
+ tensor_var,
+ ref_tensor_var,
+ ATOL=1e-2,
+ RTOL=1e-2,
+ threshold=0.99
+):
+ """
+ Check if the tensor_var is correct by comparing it with ref_tensor_var.
+ Returns True if the correctness rate is above 0.99, otherwise False.
+ """
+ correctness_rate = check_correct_rate(tensor_var, ref_tensor_var, ATOL, RTOL)
+ return correctness_rate >= threshold
diff --git a/minference/ops/xattention_fa.py b/minference/ops/xattention_fa.py
new file mode 100644
index 00000000..cd0bd89d
--- /dev/null
+++ b/minference/ops/xattention_fa.py
@@ -0,0 +1,376 @@
+# Copyright (c) 2025-2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://github.com/mit-han-lab/x-attention
+import math
+from typing import Any, Dict, List, Tuple
+
+import torch
+
+from minference.ops.op_utils.xattn_utils import (
+ LN2,
+ find_blocks_chunked,
+ flat_group_gemm_fuse_reshape,
+ softmax_fuse_block_sum,
+)
+from minference.ops.pit_sparse_flash_attention_v3 import block_attn_bwd, block_attn_fwd
+
+
+def xattn_estimate(
+ query_states: torch.Tensor, # (batch_size, num_q_head, q_len, head_dim)
+ key_states: torch.Tensor, # (batch_size, num_kv_head, k_len, head_dim)
+ block_size,
+ stride,
+ norm=1,
+ softmax=True,
+ threshold=0.9,
+ chunk_size=16384,
+ select_mode="inverse",
+ use_triton=True,
+ causal=True,
+ kdb: int = 1,
+ keep_sink=False,
+ keep_recent=False,
+) -> torch.Tensor:
+ batch_size, num_kv_head, k_len, head_dim = key_states.shape
+ batch_size, num_q_head, q_len, head_dim = query_states.shape
+ if num_q_head > num_kv_head:
+ key_states = torch.repeat_interleave(key_states.contiguous(), num_q_head // num_kv_head, dim=1)
+
+ assert q_len % chunk_size == 0
+ assert k_len % chunk_size == 0
+
+ q_chunk_num = q_len // chunk_size
+ q_block_num = q_len // block_size
+
+ # assert num_kv_head == num_q_head
+ attn_sum_list = []
+ simple_mask_list = []
+
+ if use_triton and (
+ "100" not in torch.cuda.get_device_properties(torch.cuda.current_device()).name
+ ):
+ use_triton = False
+ print(
+ "setting use triton to false. Triton kernel not surpported on this device"
+ )
+
+ num_strides_in_k = k_len // stride
+
+ num_strides_per_chunk = chunk_size // stride
+ num_strides_per_block = block_size // stride
+ num_blocks_per_chunk = num_strides_per_chunk // num_strides_per_block
+
+ for chunk_idx in range(q_chunk_num):
+ if kdb != 1:
+ raise ValueError("use_triton and kdb cannot be used together")
+
+ q_chunk_start = chunk_idx * num_strides_per_chunk * stride
+ q_chunk_end = (chunk_idx + 1) * num_strides_per_chunk * stride
+
+ q_chunk_start_stride = chunk_idx * num_strides_per_chunk
+ q_chunk_end_stride = (chunk_idx + 1) * num_strides_per_chunk
+
+ # attn_weights_slice: (batch_size, num_heads, chunk_size // stride, kv_len // stride)
+ # (i.e. the attention sum of each SxS stride block)
+ # This step is agnostic to block size and just computes the attention sum in each stride block
+ attn_weights_slice = flat_group_gemm_fuse_reshape(
+ # query_states, key_states, stride, chunk_start, chunk_end, is_causal=True
+ query_states[:, :, q_chunk_start : q_chunk_end, :,],
+ key_states,
+ stride,
+ q_chunk_start_stride,
+ q_chunk_end_stride,
+ is_causal=causal,
+ )
+
+ # (batch_size, num_heads, q_block_num, k_block_num),
+ attn_sum = softmax_fuse_block_sum(
+ attn_weights_slice, # (batch_size, num_heads, chunk_size // stride, kv_len // stride)
+ num_strides_per_block,
+ min(4096, num_strides_per_block),
+ q_chunk_start_stride, q_chunk_end_stride,
+ num_strides_in_k,
+ 1 / LN2 / math.sqrt(head_dim) / stride / norm,
+ is_causal=causal,
+ )
+
+
+ # (batch_size, head_num, num_blocks_per_chunk, block_num)
+ simple_mask = find_blocks_chunked(
+ attn_sum,
+ chunk_idx * num_blocks_per_chunk,
+ threshold,
+ None,
+ decoding=False,
+ mode="prefill",
+ causal=causal,
+ )
+
+ attn_sum_list.append(attn_sum)
+ simple_mask_list.append(simple_mask)
+
+ del attn_weights_slice
+
+ attn_sums = torch.cat(attn_sum_list, dim=-2)
+
+ # (batch_size, head_num, num_blocks_per_chunk * q_chunk_num, block_num)
+ # i.e. (batch_size, head_num, q_block_num, q_block_num)
+ simple_masks = torch.cat(simple_mask_list, dim=-2)
+
+ if causal:
+ simple_masks[:, :, -q_block_num:, -q_block_num:] = torch.where(
+ torch.tril(
+ torch.ones(
+ q_block_num, q_block_num, dtype=bool, device=key_states.device
+ ),
+ diagonal=0,
+ ),
+ simple_masks[:, :, -q_block_num:, -q_block_num:],
+ False,
+ )
+ # print(f"{__name__} | simple_masks[:, :, -q_block_num:, -q_block_num:].shape {simple_masks[:, :, -q_block_num:, -q_block_num:].shape} after torch.where")
+
+
+ if keep_sink:
+ simple_masks[:, :, 0, :] = True
+ if keep_recent:
+ eye_matrix = torch.eye(q_block_num, device=simple_masks.device, dtype=bool)
+ eye_matrix_expanded = (
+ eye_matrix.unsqueeze(0)
+ .unsqueeze(0)
+ .expand(1, num_kv_head, q_block_num, q_block_num)
+ )
+ simple_masks[:, :, -q_block_num:, -q_block_num:] = torch.where(
+ eye_matrix_expanded, True, simple_masks[:, :, -q_block_num:, -q_block_num:]
+ )
+
+ # simple_masks -> (batch_size, head_num, q_block_num, q_block_num)
+ return attn_sums, simple_masks
+
+class XAttnFunc(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ head_indices,
+ xattn_params, # Dict[str, Any]
+ granularity,
+ causal,
+ softmax_scale,
+ return_softmax,
+ deterministic,
+ ):
+ batch_size, num_tokens, num_qo_heads, head_dim = q.shape
+ if softmax_scale is None:
+ softmax_scale = head_dim ** (-0.5)
+
+ q_block_num = (q.shape[1] + granularity - 1) // granularity
+ # (batch_size, head_num, q_block_num, q_block_num)
+ _, block_mask = xattn_estimate(
+ q.transpose(1, 2), k.transpose(1, 2),
+ granularity,
+ **xattn_params
+ )
+ block_mask = block_mask[:, :, -q_block_num:, -q_block_num:].contiguous()
+
+ # Block Mask
+ out, softmax_lse = block_attn_fwd(
+ q, k, v, softmax_scale,
+ block_mask,
+ granularity=granularity,
+ causal=causal,
+ )
+
+ ctx.save_for_backward(q, k, v, out, softmax_lse, block_mask)
+ ctx.granularity = granularity
+ ctx.deterministic = deterministic
+ ctx.softmax_scale = softmax_scale
+ ctx.causal = causal
+ ctx.head_indices = head_indices
+
+ # print(f"{__name__} | out shape: {out.shape}")
+ return (out, softmax_lse, None) if return_softmax else out
+
+ @staticmethod
+ def backward(ctx, dout, *args):
+ q, k, v, out, softmax_lse, block_mask = ctx.saved_tensors
+ causal = ctx.causal
+
+ # Block Mask
+ dq, dk, dv = block_attn_bwd(
+ dout, q, k, v, out,
+ softmax_lse, ctx.softmax_scale,
+ block_mask,
+ granularity=ctx.granularity,
+ deterministic=ctx.deterministic,
+ causal=causal,
+ )
+ return dq, dk, dv, None, None, None, None, None, None, None
+
+def xattn_flash_attn_func(
+ q: torch.Tensor, # [batch_size, num_tokens, num_qo_heads, head_dim]
+ k: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ v: torch.Tensor, # [batch_size, num_tokens, num_kv_heads, head_dim]
+ head_indices: List[int], # [num_qo_heads]
+ xattn_params: Dict[str, Any],
+ granularity: int = 128,
+ dropout_p: int = 0.0,
+ softmax_scale: float = None,
+ causal: bool = True,
+ window_size: Tuple[int, int] = (-1, -1), # -1 means infinite context window
+ alibi_slopes: Tuple[float, float] = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+):
+ assert dropout_p == 0
+ assert causal
+ assert window_size == (-1, -1)
+ assert alibi_slopes is None
+
+ return XAttnFunc.apply(
+ q, k, v,
+ head_indices,
+ xattn_params,
+ granularity,
+ causal,
+ softmax_scale,
+ return_attn_probs,
+ deterministic,
+ )
+
+
+
+if __name__ == "__main__":
+ import argparse
+
+ from flash_attn import flash_attn_func
+
+ from minference.ops.utils import set_seed
+
+
+ parser = argparse.ArgumentParser(description="XAttn Test")
+ parser.add_argument("--use_ones", action="store_true", help="Use ones for q, k, v")
+ parser.add_argument("--enable_sparse", action="store_true", help="Enable Sparse XAttenion")
+ parser.add_argument("--test_backward", action="store_true", help="Test backward pass")
+ parser.add_argument("--seq_len", type=int, default=16384, help="Sequence length")
+ args = parser.parse_args()
+
+ ATOL, RTOL = 1e-2, 1e-2
+ # dtype = torch.bfloat16
+ dtype = torch.float16
+ device = torch.device(f"cuda:0")
+ torch.cuda.set_device(device)
+ set_seed(2025)
+
+ batch_size, seq_len, num_q_heads, head_dim = 1, args.seq_len, 8, 128
+ num_kv_heads = 4
+ head_indices = list(range(num_q_heads))
+
+ granularity = 128
+ xattn_params = {
+ "stride": 16,
+ "norm": 1,
+ "softmax": True,
+ "threshold": 0.9 if args.enable_sparse else 1,
+ "chunk_size": 16384,
+ "select_mode": "inverse",
+ "use_triton": True,
+ "causal": True,
+ "kdb": 1,
+ "keep_sink": False,
+ "keep_recent": False
+ }
+
+ if args.use_ones:
+ q = torch.ones((batch_size, seq_len, num_q_heads, head_dim), device=device, dtype=dtype, requires_grad=args.test_backward)
+ k = torch.ones((batch_size, seq_len, num_kv_heads, head_dim), device=device, dtype=dtype, requires_grad=args.test_backward)
+ v = torch.ones((batch_size, seq_len, num_kv_heads, head_dim), device=device, dtype=dtype, requires_grad=args.test_backward)
+ else:
+ q = torch.randn((batch_size, seq_len, num_q_heads, head_dim), device=device, dtype=dtype, requires_grad=args.test_backward)
+ k = torch.randn((batch_size, seq_len, num_kv_heads, head_dim), device=device, dtype=dtype, requires_grad=args.test_backward)
+ v = torch.randn((batch_size, seq_len, num_kv_heads, head_dim), device=device, dtype=dtype, requires_grad=args.test_backward)
+
+ # Clone inputs for reference implementation to ensure separate gradient computation
+ if args.test_backward:
+ q_ref = q.clone().detach().requires_grad_(True)
+ k_ref = k.clone().detach().requires_grad_(True)
+ v_ref = v.clone().detach().requires_grad_(True)
+ else:
+ q_ref, k_ref, v_ref = q, k, v
+
+ out = xattn_flash_attn_func(
+ q, k, v,
+ head_indices,
+ xattn_params,
+ granularity=granularity,
+ )
+ print(f"out shape: {out.shape}")
+
+ ref_out = flash_attn_func(
+ q_ref, k_ref, v_ref,
+ causal=True,
+ softmax_scale=head_dim ** (-0.5)
+ )
+
+
+ # Compare out and ref_out
+ if not torch.allclose(out, ref_out, atol=ATOL, rtol=RTOL):
+ num_blocks = seq_len // granularity
+ for i in range(num_blocks):
+ start = i * granularity
+ end = (i + 1) * granularity
+ out_chunk = out[:, start:end, :, :]
+ ref_out_chunk = ref_out[:, start:end, :, :]
+
+ print('-' * 60)
+ if not torch.allclose(out_chunk, ref_out_chunk, atol=ATOL, rtol=RTOL):
+ print(f"Forward Output mismatch at chunk {i}:")
+ print(f"Forward out_chunk: {out_chunk}")
+ print(f"Forward ref_out_chunk: {ref_out_chunk}")
+ else:
+ print(f"Forward Output match at chunk {i}")
+ else:
+ print("Forward Output match")
+
+
+ # Backward pass testing
+ if args.test_backward:
+ print("\nTesting backward pass...")
+
+ # Create gradient for backward pass
+ grad_output = torch.randn_like(out)
+ grad_output_ref = grad_output.clone()
+
+ # Backward pass for custom implementation
+ out.backward(grad_output)
+
+ # Backward pass for reference implementation
+ ref_out.backward(grad_output_ref)
+
+ # Compare gradients
+ print("\nGradient comparison:")
+
+ # Compare q gradients
+ q_grad_match = torch.allclose(q.grad, q_ref.grad, atol=ATOL, rtol=RTOL)
+ print(f"q grad match: {q_grad_match}")
+ if not q_grad_match:
+ q_diff = (q.grad - q_ref.grad).abs()
+ print(f"q grad max diff: {q_diff.max().item()}, mean diff: {q_diff.mean().item()}")
+
+ # Compare k gradients
+ k_grad_match = torch.allclose(k.grad, k_ref.grad, atol=ATOL, rtol=RTOL)
+ print(f"k grad match: {k_grad_match}")
+ if not k_grad_match:
+ k_diff = (k.grad - k_ref.grad).abs()
+ print(f"k grad max diff: {k_diff.max().item()}, mean diff: {k_diff.mean().item()}")
+
+ # Compare v gradients
+ v_grad_match = torch.allclose(v.grad, v_ref.grad, atol=ATOL, rtol=RTOL)
+ print(f"v grad match: {v_grad_match}")
+ if not v_grad_match:
+ v_diff = (v.grad - v_ref.grad).abs()
+ print(f"v grad max diff: {v_diff.max().item()}, mean diff: {v_diff.mean().item()}")
+
+ print(f"\nOverall gradient match: {q_grad_match and k_grad_match and v_grad_match}")
diff --git a/minference/patch.py b/minference/patch.py
index 10fb24eb..f3094705 100644
--- a/minference/patch.py
+++ b/minference/patch.py
@@ -1,27 +1,42 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
-import json
+import os
+import types
+from functools import partial
+from typing import Any, Dict, List, Optional, Tuple, Union
import torch
import transformers
-from transformers.cache_utils import *
-from transformers.models.llama.modeling_llama import *
+from torch.nn import CrossEntropyLoss
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.models.llama.modeling_llama import (
+ BaseModelOutputWithPast,
+ CausalLMOutputWithPast,
+ LlamaAttention,
+ logger,
+)
+
+try:
+ from transformers.models.llama.modeling_llama import LlamaFlashAttention2
+except:
+ LlamaFlashAttention2 = None
+from .modules.forward import attn_forward, decoding_forwards, prefill_forwards
from .modules.inf_llm import InfLLMGenerator, inf_llm_forward
+from .modules.kvcompression import (
+ method_to_cache_obj,
+ prepare_inputs_for_generation_kvcompression,
+)
from .modules.minference_forward import (
gather_last_q_vertical_slash_topk_v4,
- gather_last_q_vertical_slash_topk_vllm,
init_minference_parameters,
+ kvcompress_forward,
minference_forward,
minference_kv_cache_cpu_forward,
- minference_vllm_forward,
- minference_with_snapkv_forward,
- search_pattern,
- sum_all_diagonal_matrix,
)
from .ops.streaming_kernel import stream_llm_forward
-from .utils import patch_glm_4_1m
+from .utils import causal_model_forward, glm_forward, prepare_input, update_kwargs
KV_CACHE_CPU_DEVICE = "cpu"
@@ -177,7 +192,6 @@ def apply_rotary_pos_emb_glm4(
rot_dim = rope_cache.shape[-2] * 2
x, x_pass = x[..., :rot_dim], x[..., rot_dim:]
# truncate to support variable sizes
- # import ipdb;ipdb.set_trace()
rope_cache = rope_cache[:sq]
xshaped = x.reshape(b, np, sq, rot_dim // 2, 2)
rope_cache = rope_cache.view(-1, 1, sq, xshaped.size(3), 2)
@@ -193,7 +207,7 @@ def apply_rotary_pos_emb_glm4(
ATTN_FORWRAD = {
- "streaming": stream_llm_forward,
+ "a_shape": stream_llm_forward,
"minference": minference_forward,
"inf_llm": inf_llm_forward,
}
@@ -443,70 +457,20 @@ def prepare_inputs_for_generation(
return model_inputs
-def prepare_inputs_for_generation_snapkv(
- self,
- input_ids,
- past_key_values=None,
- attention_mask=None,
- inputs_embeds=None,
- **kwargs,
-):
- if past_key_values is None: # [SnapKV]
- for layer in self.model.layers:
- layer.self_attn.kv_seq_len = 0
- if past_key_values is not None:
- if isinstance(past_key_values, Cache):
- cache_length = past_key_values.get_seq_length()
- past_length = past_key_values.seen_tokens
- max_cache_length = past_key_values.get_max_length()
- else:
- # cache_length = past_length = past_key_values[0][0].shape[2]
- # max_cache_length = None
- cache_length = past_length = self.model.layers[0].self_attn.kv_seq_len
- max_cache_length = None
- # Keep only the unprocessed tokens:
- # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
- # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
- # input)
- if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
- input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
- # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
- # input_ids based on the past_length.
- elif past_length < input_ids.shape[1]:
- input_ids = input_ids[:, past_length:]
- # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+def prepare_cache(method: str, config):
+ cache_obj: Cache = method_to_cache_obj[method]
- # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
- if (
- max_cache_length is not None
- and attention_mask is not None
- and cache_length + input_ids.shape[1] > max_cache_length
- ):
- attention_mask = attention_mask[:, -max_cache_length:]
+ def _prepare_cache_for_generation(
+ self, generation_config, model_kwargs: Dict, *args, **kwargs
+ ) -> bool:
+ """
+ Prepares the cache for generation (if applicable), given `generate`'s paramaterization. If a cache is
+ instantiated, writes it to `model_kwargs`, under the name expected by the model.
+ """
+ config.num_layers = self.config.num_hidden_layers
+ model_kwargs["past_key_values"] = cache_obj(config)
- position_ids = kwargs.get("position_ids", None)
- if attention_mask is not None and position_ids is None:
- # create position_ids on the fly for batch generation
- position_ids = attention_mask.long().cumsum(-1) - 1
- position_ids.masked_fill_(attention_mask == 0, 1)
- if past_key_values:
- position_ids = position_ids[:, -input_ids.shape[1] :]
-
- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
- if inputs_embeds is not None and past_key_values is None:
- model_inputs = {"inputs_embeds": inputs_embeds}
- else:
- model_inputs = {"input_ids": input_ids}
-
- model_inputs.update(
- {
- "position_ids": position_ids,
- "past_key_values": past_key_values,
- "use_cache": kwargs.get("use_cache"),
- "attention_mask": attention_mask,
- }
- )
- return model_inputs
+ return _prepare_cache_for_generation
def _prepare_decoder_attention_mask_inference(
@@ -538,63 +502,58 @@ def forward_llama_decoder_layer(
hidden_states: torch.Tensor,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.LongTensor] = None,
- past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ past_key_value: Optional[Cache] = None,
output_attentions: Optional[bool] = False,
use_cache: Optional[bool] = False,
- padding_mask: Optional[torch.LongTensor] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[
+ Tuple[torch.Tensor, torch.Tensor]
+ ] = None, # necessary, but kept here for BC
+ chunk_size: int = 96_000,
+ **kwargs,
) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
- """
- Args:
- hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
- attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
- `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
- output_attentions (`bool`, *optional*):
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under
- returned tensors for more detail.
- use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
- (see `past_key_values`).
- past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
- """
-
residual = hidden_states.clone()
batch, seq_len, embed_dim = hidden_states.shape
+ if chunk_size == -1:
+ chunk_size = seq_len
- for start_idx in range(0, seq_len, 32000):
- end_idx = min(seq_len, start_idx + 32000)
+ for start_idx in range(0, seq_len, chunk_size):
+ end_idx = min(seq_len, start_idx + chunk_size)
hidden_states[:, start_idx:end_idx, :] = self.input_layernorm(
hidden_states[:, start_idx:end_idx, :]
)
# Self Attention
- hidden_states, self_attn_weights, present_key_value = self.self_attn(
+ attention_outputs = self.self_attn(
hidden_states=hidden_states,
attention_mask=attention_mask,
position_ids=position_ids,
past_key_value=past_key_value,
output_attentions=output_attentions,
use_cache=use_cache,
- padding_mask=padding_mask,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ **kwargs,
)
+ hidden_states, self_attn_weights = attention_outputs[:2]
if residual.device != hidden_states.device:
residual = residual.to(hidden_states.device)
hidden_states = residual + hidden_states
# Fully Connected
- for start_idx in range(0, seq_len, 32000):
- end_idx = min(seq_len, start_idx + 32000)
+ for start_idx in range(0, seq_len, chunk_size):
+ end_idx = min(seq_len, start_idx + chunk_size)
part_hidden_states = hidden_states[:, start_idx:end_idx, :].clone()
part_hidden_states = self.post_attention_layernorm(part_hidden_states)
part_hidden_states = self.mlp(part_hidden_states)
hidden_states[:, start_idx:end_idx, :] += part_hidden_states
outputs = (hidden_states,)
-
if output_attentions:
outputs += (self_attn_weights,)
- if use_cache:
- outputs += (present_key_value,)
+ if use_cache and len(attention_outputs) == 3:
+ outputs += (attention_outputs[-1],)
return outputs
@@ -770,6 +729,7 @@ def forward_llama_for_causal_lm(
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
+ logits_to_keep: int = 1,
) -> Union[Tuple, CausalLMOutputWithPast]:
# assert labels is not None
output_attentions = (
@@ -837,16 +797,109 @@ def forward_llama_for_causal_lm(
)
-def minference_patch(model, config):
- from transformers import LlamaForCausalLM
+def patch_glm_4_1m(model, config):
+ Attention = model.transformer.encoder.layers[0].self_attention.__class__
+ Transformer = model.transformer.encoder.__class__
+
+ prefill_forward = prefill_forwards[config.attn_type]
+ decoding_forward = decoding_forwards[config.kv_type]
+
+ attn_forward = glm_forward(
+ prefill_forward=prefill_forward,
+ decoding_forward=decoding_forward,
+ attn_forward_config=config.attn_kwargs,
+ class_name="attn_forward",
+ )
+ transformer_forward = glm_forward(
+ prefill_forward=None,
+ decoding_forward=None,
+ attn_forward_config=None,
+ class_name="transformer_forward",
+ )
+
+ def update_module(m):
+ if isinstance(m, Attention):
+ m.forward = (
+ lambda self, *args, **kwargs: attn_forward(self, *args, **kwargs)
+ ).__get__(m, Attention)
+ if isinstance(m, Transformer):
+ m.forward = (
+ lambda self, *args, **kwargs: transformer_forward(self, *args, **kwargs)
+ ).__get__(m, Transformer)
+
+ model.apply(update_module)
+ prepare_cache_func = prepare_cache(config.kv_type, config)
+ model._prepare_cache_for_generation = prepare_cache_func.__get__(
+ model, model.__class__
+ )
+ model.prepare_inputs_for_generation = prepare_input.__get__(model, model.__class__)
+ model._update_model_kwargs_for_generation = update_kwargs.__get__(
+ model, model.__class__
+ )
+ model.forward = causal_model_forward(model.forward).__get__(model, model.__class__)
+ prepare_inputs_func = prepare_inputs_for_generation_kvcompression(
+ config.kv_type, config, model.prepare_inputs_for_generation
+ )
+ model.prepare_inputs_for_generation = prepare_inputs_func.__get__(
+ model, model.__class__
+ )
+ print(f"Patched model for {config.attn_type} with {config.kv_type} ..")
+ return model
+
+
+def new_patch(model, config):
+ if model.__class__.__name__ == "ChatGLMForConditionalGeneration":
+ model = patch_glm_4_1m(model, config)
+ return model
+ Attention = model.model.layers[0].self_attn.__class__
+ Model = model.model.__class__
+ DecoderLayer = model.model.layers[0].__class__
+
+ prefill_forward = prefill_forwards[config.attn_type]
+ decoding_forward = decoding_forwards[config.kv_type]
+
+ custom_rope_func = None # apply custom rope func if needed
+ forward = partial(
+ attn_forward,
+ prefill_forward=prefill_forward,
+ decoding_forward=decoding_forward,
+ attn_forward_config=config.attn_kwargs,
+ customized_rope_func=custom_rope_func,
+ )
+
+ def update_module(m):
+ if isinstance(m, Attention):
+ m.forward = (
+ lambda self, *args, **kwargs: forward(self, *args, **kwargs)
+ ).__get__(m, Attention)
+ if isinstance(m, DecoderLayer):
+ m.forward = forward_llama_decoder_layer.__get__(m, DecoderLayer)
+
+ model.apply(update_module)
+ prepare_cache_func = prepare_cache(config.kv_type, config)
+ model._prepare_cache_for_generation = prepare_cache_func.__get__(
+ model, model.__class__
+ )
+
+ prepare_inputs_func = prepare_inputs_for_generation_kvcompression(
+ config.kv_type, config, model.prepare_inputs_for_generation
+ )
+ model.prepare_inputs_for_generation = prepare_inputs_func.__get__(
+ model, model.__class__
+ )
+
+ return model
+
+
+def minference_patch(model, config):
if config.kv_cache_cpu:
global KV_CACHE_CPU_DEVICE
KV_CACHE_CPU_DEVICE = config.kv_cache_cpu_device
model.config.kv_cache_cpu_device = config.kv_cache_cpu_device
return minference_patch_kv_cache_cpu(model)
- if config.use_snapkv:
- return minference_patch_with_snapkv(model)
+ if config.kv_type:
+ return minference_patch_with_kvcompress(model, config)
model = patch_glm_4_1m(model)
@@ -889,8 +942,6 @@ def update_module(m):
def minference_patch_kv_cache_cpu(model):
- from transformers import LlamaForCausalLM
-
transformers.cache_utils.DynamicCache.update = cpu_cache_update
transformers.cache_utils.DynamicCache.get = cpu_cache_get
@@ -934,168 +985,284 @@ def update_module(m):
return model
-def minference_patch_with_snapkv(model):
- from transformers import LlamaForCausalLM
-
+def minference_patch_with_kvcompress(model, config):
model = patch_glm_4_1m(model)
Attention = model.model.layers[0].self_attn.__class__
Model = model.model.__class__
DecoderLayer = model.model.layers[0].__class__
- forward = minference_with_snapkv_forward()
+ forward = kvcompress_forward(
+ Attention.forward, method=config.kv_type, config=config
+ )
def update_module(m):
if isinstance(m, Attention):
- m.init_minference_parameters = init_minference_parameters.__get__(
- m, Attention
- )
- m.gather_last_q_vertical_slash_topk_v4 = (
- gather_last_q_vertical_slash_topk_v4.__get__(m, Attention)
- )
+ # if use minference with kvcompress, then patch with minference kernels
+ if config.attn_type in ["minference"]:
+ m.init_minference_parameters = init_minference_parameters.__get__(
+ m, Attention
+ )
+ m.gather_last_q_vertical_slash_topk_v4 = (
+ gather_last_q_vertical_slash_topk_v4.__get__(m, Attention)
+ )
+ if config.kv_type == "quest":
+ if LlamaFlashAttention2 is not None:
+ m.flash_forward = types.MethodType(LlamaFlashAttention2.forward, m)
+ else:
+ m.flash_forward = types.MethodType(LlamaAttention.forward, m)
+ m.token_budget = (
+ 1024 if not hasattr(m, "token_budget") else m.token_budget
+ )
+ m.chunk_size = 16 if not hasattr(m, "chunk_size") else m.chunk_size
m.forward = forward.__get__(m, Attention)
- if isinstance(m, DecoderLayer):
- m.forward = forward_llama_decoder_layer.__get__(m, DecoderLayer)
model.apply(update_module)
- model.prepare_inputs_for_generation = prepare_inputs_for_generation_snapkv.__get__(
+ prepare_cache_func = prepare_cache(config.kv_type, config)
+ model._prepare_cache_for_generation = prepare_cache_func.__get__(
model, model.__class__
)
- model.model._use_sdpa = False
- model.model._prepare_decoder_attention_mask = (
- _prepare_decoder_attention_mask_inference.__get__(
- model.model, model.model.__class__
- )
+ prepare_inputs_func = prepare_inputs_for_generation_kvcompression(
+ config.kv_type, config, model.prepare_inputs_for_generation
)
- model.model.forward = forward_llama_model.__get__(
- model.model, model.model.__class__
+ model.prepare_inputs_for_generation = prepare_inputs_func.__get__(
+ model, model.__class__
)
- model.forward = forward_llama_for_causal_lm.__get__(model, model.__class__)
- print("Patched model for minference with SanpKV..")
+ # model.model._use_sdpa = False
+ # model.model._prepare_decoder_attention_mask = (
+ # _prepare_decoder_attention_mask_inference.__get__(
+ # model.model, model.model.__class__
+ # )
+ # )
+ print(f"Patched model for minference with {config.kv_type} ..")
return model
-def llama_model_forward_vllm(
- self,
- input_ids: Optional[torch.Tensor],
- positions: torch.Tensor,
- kv_caches: List[torch.Tensor],
- attn_metadata,
- inputs_embeds: Optional[torch.Tensor] = None,
-) -> torch.Tensor:
- if inputs_embeds is not None:
- hidden_states = inputs_embeds
- else:
- hidden_states = self.get_input_embeddings(input_ids)
- residual = None
- for i in range(len(self.layers)):
- layer = self.layers[i]
- hidden_states, residual = layer(
- positions,
- hidden_states,
- kv_caches[i],
- attn_metadata,
- residual,
- layer_idx=i,
- )
- hidden_states, _ = self.norm(hidden_states, residual)
- return hidden_states
+def minference_patch_vllm_tp(self, config_file, patch_config):
+ self.model_runner.model.apply(
+ minference_patch_vllm_executor(config_file, patch_config)
+ )
-def llama_layer_forward_vllm(
- self,
- positions: torch.Tensor,
- hidden_states: torch.Tensor,
- kv_cache: torch.Tensor,
- attn_metadata,
- residual: Optional[torch.Tensor],
- layer_idx: int,
-) -> Tuple[torch.Tensor, torch.Tensor]:
- # Self Attention
- if residual is None:
- residual = hidden_states
- hidden_states = self.input_layernorm(hidden_states)
- else:
- hidden_states, residual = self.input_layernorm(hidden_states, residual)
- hidden_states = self.self_attn(
- positions=positions,
- hidden_states=hidden_states,
- kv_cache=kv_cache,
- attn_metadata=attn_metadata,
- layer_idx=layer_idx,
+def minference_patch_vllm_executor(config_file: str, patch_config={}):
+ import json
+ from collections import defaultdict
+
+ import vllm
+ from vllm.attention import Attention
+ from vllm.forward_context import get_forward_context
+ from vllm.model_executor.models.chatglm import (
+ GLMAttention,
+ GLMBlock,
+ GLMTransformer,
+ )
+ from vllm.model_executor.models.llama import (
+ LlamaAttention,
+ LlamaDecoderLayer,
+ LlamaModel,
)
- # Fully Connected
- hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
- hidden_states = self.mlp(hidden_states)
- return hidden_states, residual
+ from minference.modules.minference_forward import (
+ gather_last_q_vertical_slash_topk_vllm,
+ minference_vllm_forward,
+ )
+ vllm_version = vllm.__version__
-def llama_attn_forward_vllm(
- vllm_version: str = "0.4.2",
-):
- def llama_attn_forward_vllm(
+ config = defaultdict(dict)
+ if os.path.exists(config_file):
+ config = json.load(open(config_file))
+ attn_forward = minference_vllm_forward(
+ config, vllm_version=vllm_version, patch_config=patch_config
+ )
+
+ def vllm_attn_forward(
self,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ # kv_cache: Optional[torch.Tensor],
+ # attn_metadata,
+ # kv_scale: float = 1.0,
+ layer_idx: int = 0,
+ ) -> torch.Tensor:
+ output_shape = query.shape
+ output = torch.zeros(output_shape, dtype=query.dtype, device=query.device)
+ hidden_size = output_shape[-1]
+ forward_context = get_forward_context()
+ attn_metadata = forward_context.attn_metadata
+ if isinstance(attn_metadata, dict):
+ attn_metadata = attn_metadata[self.layer_name]
+ self_kv_cache = self.kv_cache[forward_context.virtual_engine]
+ return self.impl.forward(
+ self,
+ query,
+ key,
+ value,
+ self_kv_cache,
+ attn_metadata,
+ output=output,
+ layer_idx=layer_idx,
+ )
+ # check self._kv_scale
+ # kv_scale = getattr(self, "_kv_scale", getattr(self, "_k_scale", kv_scale))
+ return self.impl.forward(query, key, value, layer_idx)
+
+ def llama_model_forward_vllm(
+ self,
+ input_ids: Optional[torch.Tensor],
positions: torch.Tensor,
+ intermediate_tensors,
+ inputs_embeds: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ if inputs_embeds is not None:
+ hidden_states = inputs_embeds
+ else:
+ hidden_states = self.get_input_embeddings(input_ids)
+ residual = None
+ for i in range(len(self.layers)):
+ layer = self.layers[i]
+ hidden_states, residual = layer(positions, hidden_states, residual, i)
+ hidden_states, _ = self.norm(hidden_states, residual)
+ return hidden_states
+
+ def chatglm_model_forward_vllm(
+ self,
hidden_states: torch.Tensor,
- kv_cache: torch.Tensor,
+ position_ids: torch.Tensor,
+ kv_caches: List[torch.Tensor],
attn_metadata,
- layer_idx: int,
) -> torch.Tensor:
- qkv, _ = self.qkv_proj(hidden_states)
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
- q, k = self.rotary_emb(positions, q, k)
- if "0.4.1" <= vllm_version <= "0.4.2":
- attn_output = self.attn(
- q, k, v, kv_cache, attn_metadata, self.kv_scale, layer_idx
+ for i in range(self.num_layers):
+ layer = self.layers[i]
+ hidden_states = layer(
+ hidden_states=hidden_states,
+ position_ids=position_ids,
+ kv_cache=kv_caches[i],
+ attn_metadata=attn_metadata,
+ layer_idx=i,
)
- elif vllm_version >= "0.4.3":
- attn_output = self.attn(q, k, v, kv_cache, attn_metadata, layer_idx)
- else:
- assert False, "Only support 'vllm>=0.4.1'. Please update your vllm version."
+ # Final layer norm.
+ if self.post_layer_norm:
+ hidden_states = self.final_layernorm(hidden_states)
- output, _ = self.o_proj(attn_output)
- return output
+ return hidden_states
- return llama_attn_forward_vllm
+ def llama_layer_forward_vllm(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ residual: Optional[torch.Tensor],
+ layer_idx: int,
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ # Self Attention
+ if residual is None:
+ residual = hidden_states
+ hidden_states = self.input_layernorm(hidden_states)
+ else:
+ hidden_states, residual = self.input_layernorm(hidden_states, residual)
+ hidden_states = self.self_attn(
+ positions=positions,
+ hidden_states=hidden_states,
+ layer_idx=layer_idx,
+ )
+ # Fully Connected
+ hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
+ hidden_states = self.mlp(hidden_states)
+ return hidden_states, residual
-def vllm_attn_forward(
- self,
- query: torch.Tensor,
- key: torch.Tensor,
- value: torch.Tensor,
- kv_cache: Optional[torch.Tensor],
- attn_metadata,
- kv_scale: float = 1.0,
- layer_idx: int = 0,
-) -> torch.Tensor:
- # check self._kv_scale
- kv_scale = getattr(self, "_kv_scale", kv_scale)
- return self.impl.forward(
- query, key, value, kv_cache, attn_metadata, kv_scale, layer_idx
- )
+ def chatglm_layer_forward_vllm(
+ self,
+ hidden_states: torch.Tensor,
+ position_ids: torch.Tensor,
+ kv_cache: torch.Tensor,
+ attn_metadata,
+ layer_idx=0,
+ ) -> torch.Tensor:
+ # hidden_states: [num_tokens, h]
+ # Layer norm at the beginning of the transformer layer.
+ layernorm_output = self.input_layernorm(hidden_states)
+ # Self attention.
+ attention_output = self.self_attention(
+ hidden_states=layernorm_output,
+ position_ids=position_ids,
+ kv_cache=kv_cache,
+ attn_metadata=attn_metadata,
+ layer_idx=layer_idx,
+ )
+ # Residual connection.
+ if self.apply_residual_connection_post_layernorm:
+ residual = layernorm_output
+ else:
+ residual = hidden_states
+ layernorm_input = residual + attention_output
+ # Layer norm post the self attention.
+ layernorm_output = self.post_attention_layernorm(layernorm_input)
+ # Second residual connection.
+ if self.apply_residual_connection_post_layernorm:
+ residual = layernorm_output
+ else:
+ residual = layernorm_input
+ output = self.mlp(layernorm_output) + residual
+ return output
-def minference_patch_vllm(
- llm,
- config_file,
-):
- import vllm
- from vllm.attention import Attention
- from vllm.model_executor.models.llama import (
- LlamaAttention,
- LlamaDecoderLayer,
- LlamaForCausalLM,
- LlamaModel,
- )
+ def llama_attn_forward_vllm(
+ vllm_version: str = "0.4.2",
+ ):
+ def llama_attn_forward_vllm(
+ self,
+ positions: torch.Tensor,
+ hidden_states: torch.Tensor,
+ layer_idx: int,
+ ) -> torch.Tensor:
+ qkv, _ = self.qkv_proj(hidden_states)
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+ q, k = self.rotary_emb(positions, q, k)
+ if "0.4.1" <= vllm_version <= "0.4.2":
+ attn_output = self.attn(
+ q, k, v, kv_cache, attn_metadata, self.kv_scale, layer_idx
+ )
+ elif vllm_version >= "0.8.0":
+ attn_output = self.attn(q, k, v, layer_idx=layer_idx)
+ elif vllm_version >= "0.4.3":
+ attn_output = self.attn(
+ q, k, v, kv_cache, attn_metadata, layer_idx=layer_idx
+ )
+ else:
+ assert (
+ False
+ ), "Only support 'vllm>=0.4.1'. Please update your vllm version."
- vllm_version = vllm.__version__
+ output, _ = self.o_proj(attn_output)
+ return output
- config = json.load(open(config_file))
- attn_forward = minference_vllm_forward(config, vllm_version=vllm_version)
+ return llama_attn_forward_vllm
+
+ def chatglm_attn_forward_vllm(
+ self,
+ hidden_states: torch.Tensor,
+ position_ids: torch.Tensor,
+ kv_cache: torch.Tensor,
+ attn_metadata,
+ layer_idx: int = 0,
+ ) -> torch.Tensor:
+ qkv, _ = self.query_key_value(hidden_states)
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
+ q, k = self.rotary_emb(position_ids, q, k)
+ context_layer = self.attn(
+ q,
+ k,
+ v,
+ kv_cache,
+ attn_metadata,
+ layer_idx=layer_idx,
+ )
+ attn_output, _ = self.dense(context_layer)
+ return attn_output
def update_module(m):
if isinstance(m, Attention):
@@ -1113,8 +1280,31 @@ def update_module(m):
m.forward = llama_model_forward_vllm.__get__(m, LlamaModel)
if isinstance(m, LlamaAttention):
m.forward = llama_attn_forward_vllm(vllm_version).__get__(m, LlamaAttention)
+ if isinstance(m, GLMBlock):
+ m.forward = chatglm_layer_forward_vllm.__get__(m, GLMBlock)
+ if isinstance(m, GLMTransformer):
+ m.forward = chatglm_model_forward_vllm.__get__(m, GLMTransformer)
+ if isinstance(m, GLMAttention):
+ m.forward = chatglm_attn_forward_vllm.__get__(m, GLMAttention)
+
+ return update_module
- llm.llm_engine.model_executor.driver_worker.model_runner.model.apply(update_module)
+
+def minference_patch_vllm(
+ llm,
+ config_file,
+ patch_config: dict = {},
+):
+ if "workers" in llm.llm_engine.model_executor.__dict__:
+ llm.llm_engine.model_executor._run_workers(
+ "minference_patch_vllm_tp",
+ config_file=config_file,
+ patch_config=patch_config,
+ )
+ else:
+ llm.llm_engine.model_executor.driver_worker.model_runner.model.apply(
+ minference_patch_vllm_executor(config_file, patch_config)
+ )
print("Patched model for minference with vLLM..")
return llm
@@ -1131,16 +1321,7 @@ def patch_hf(
attn_kwargs.update(kwargs)
# This approach lacks scalability and will be refactored.
from transformers import LlamaForCausalLM, MistralForCausalLM, Qwen2ForCausalLM
- from transformers.models.llama.modeling_llama import (
- BaseModelOutputWithPast,
- LlamaAttention,
- LlamaModel,
- )
- from transformers.models.mistral.modeling_mistral import (
- MistralAttention,
- MistralModel,
- )
- from transformers.models.qwen2.modeling_qwen2 import Qwen2Attention, Qwen2Model
+ from transformers.models.llama.modeling_llama import BaseModelOutputWithPast
def model_forward(
self,
diff --git a/minference/utils.py b/minference/utils.py
index e71c2521..c03e10d0 100644
--- a/minference/utils.py
+++ b/minference/utils.py
@@ -1,38 +1,553 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
+import functools
+import inspect
+import types
+from typing import Optional
+
import torch
+from transformers.cache_utils import Cache, StaticCache
+from transformers.modeling_flash_attention_utils import _flash_attention_forward
+from transformers.models.glm.modeling_glm import GlmMLP, GlmRotaryEmbedding
+from transformers.models.llama.modeling_llama import (
+ ACT2FN,
+ LlamaDecoderLayer,
+ LlamaForCausalLM,
+ LlamaModel,
+ LlamaPreTrainedModel,
+ logger,
+)
+
+
+def update_kwargs(
+ self,
+ outputs,
+ model_kwargs,
+ is_encoder_decoder: bool = False,
+ num_new_tokens: int = 1,
+):
+ # update past_key_values keeping its naming used in model code
+ cache_name, cache = self._extract_past_from_model_output(outputs)
+ model_kwargs[cache_name] = cache
+ if getattr(outputs, "state", None) is not None:
+ model_kwargs["state"] = outputs.state
+
+ # update token_type_ids with last value
+ if "token_type_ids" in model_kwargs:
+ token_type_ids = model_kwargs["token_type_ids"]
+ model_kwargs["token_type_ids"] = torch.cat(
+ [token_type_ids, token_type_ids[:, -1].unsqueeze(-1)], dim=-1
+ )
+
+ if not is_encoder_decoder:
+ # update attention mask
+ if "attention_mask" in model_kwargs:
+ attention_mask = model_kwargs["attention_mask"]
+ model_kwargs["attention_mask"] = torch.cat(
+ [attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))],
+ dim=-1,
+ )
+ else:
+ # update decoder attention mask
+ if "decoder_attention_mask" in model_kwargs:
+ decoder_attention_mask = model_kwargs["decoder_attention_mask"]
+ model_kwargs["decoder_attention_mask"] = torch.cat(
+ [
+ decoder_attention_mask,
+ decoder_attention_mask.new_ones(
+ (decoder_attention_mask.shape[0], 1)
+ ),
+ ],
+ dim=-1,
+ )
+
+ if model_kwargs.get("use_cache", True):
+ model_kwargs["cache_position"] = (
+ model_kwargs["cache_position"][-1:] + num_new_tokens
+ )
+ else:
+ past_positions = model_kwargs.pop("cache_position")
+ new_positions = torch.arange(
+ past_positions[-1] + 1,
+ past_positions[-1] + num_new_tokens + 1,
+ dtype=past_positions.dtype,
+ ).to(past_positions.device)
+ model_kwargs["cache_position"] = torch.cat((past_positions, new_positions))
+ return model_kwargs
+
+
+def causal_model_forward(original_forward):
+ @functools.wraps(original_forward)
+ def new_forward(self, *args, **kwargs):
+ if kwargs.get("logits_to_keep", None) == 1:
+ kwargs["return_last_logit"] = True
+ kwargs.pop("logits_to_keep")
+ if kwargs.get("position_ids", None) is None:
+ kv_cache = kwargs.get("past_key_values")
+ input_ids = kwargs.get("input_ids")
+ past_seen_tokens = kv_cache.get_seq_length() if kv_cache is not None else 0
+ pos_ids = torch.arange(
+ past_seen_tokens,
+ past_seen_tokens + input_ids.shape[1],
+ dtype=torch.long,
+ device=input_ids.device,
+ )
+ pos_ids = pos_ids.unsqueeze(0)
+ kwargs["position_ids"] = pos_ids
+ kv_cache.pos_ids = pos_ids
+ return original_forward(*args, **kwargs)
+
+ return new_forward
+
+
+def prepare_input(
+ self,
+ input_ids: torch.LongTensor,
+ past_key_values: Optional[Cache] = None,
+ attention_mask: Optional[torch.LongTensor] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs,
+):
+ model_inputs = {}
+ if self._supports_cache_class:
+ model_inputs["cache_position"] = cache_position
+ elif cache_position is None:
+ past_length = (
+ past_key_values[0][0].shape[2] if past_key_values is not None else 0
+ )
+ cache_position = torch.arange(
+ past_length, input_ids.shape[1], dtype=torch.long, device=input_ids.device
+ )
+
+ if past_key_values is not None:
+ model_inputs["past_key_values"] = past_key_values
+ if (
+ inputs_embeds is not None or cache_position[-1] >= input_ids.shape[1]
+ ): # Exception 1 or Exception 3
+ input_ids = input_ids[:, -cache_position.shape[0] :]
+ elif (
+ input_ids.shape[1] != cache_position.shape[0]
+ ): # Default case (the "else", a no op, is Exception 2)
+ input_ids = input_ids[:, cache_position]
+
+ input_ids_key = (
+ "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids"
+ )
+ if not self.config.is_encoder_decoder:
+ if inputs_embeds is not None and cache_position[0] == 0:
+ model_inputs[input_ids_key] = None
+ model_inputs["inputs_embeds"] = inputs_embeds
+ else:
+ model_inputs[input_ids_key] = input_ids.clone(
+ memory_format=torch.contiguous_format
+ )
+ model_inputs["inputs_embeds"] = None
+ else:
+ model_inputs[input_ids_key] = input_ids.clone(
+ memory_format=torch.contiguous_format
+ )
+
+ if (
+ attention_mask is not None
+ and kwargs.get("position_ids") is None
+ and "position_ids" in set(inspect.signature(self.forward).parameters.keys())
+ ):
+ position_ids = attention_mask.long().cumsum(-1) - 1
+ position_ids.masked_fill_(attention_mask == 0, 1)
+ kwargs[
+ "position_ids"
+ ] = position_ids # placed in kwargs for further processing (see below)
+
+ for model_input_name in ["position_ids", "token_type_ids"]:
+ model_input = kwargs.get(model_input_name)
+ if model_input is not None:
+ if past_key_values:
+ model_input = model_input[:, -input_ids.shape[1] :]
+ model_input = model_input.clone(memory_format=torch.contiguous_format)
+ model_inputs[model_input_name] = model_input
+
+ if isinstance(past_key_values, StaticCache) and attention_mask.ndim == 2:
+ if model_inputs["inputs_embeds"] is not None:
+ batch_size, sequence_length, _ = model_inputs["inputs_embeds"].shape
+ device = model_inputs["inputs_embeds"].device
+ else:
+ batch_size, sequence_length = model_inputs[input_ids_key].shape
+ device = model_inputs[input_ids_key].device
+
+ base_model = getattr(self, self.base_model_prefix, None)
+ if base_model is None:
+ causal_mask_creation_function = getattr(
+ self, "_prepare_4d_causal_attention_mask_with_cache_position", None
+ )
+ else:
+ causal_mask_creation_function = getattr(
+ base_model,
+ "_prepare_4d_causal_attention_mask_with_cache_position",
+ None,
+ )
+ if causal_mask_creation_function is None:
+ logger.warning_once(
+ f"{self.__class__.__name__} has no `_prepare_4d_causal_attention_mask_with_cache_position` method "
+ "defined in its base modeling class. Compiled forward passes will be sub-optimal. If you're "
+ "writing code, see Llama for an example implementation. If you're a user, please report this "
+ "issue on GitHub."
+ )
+ else:
+ attention_mask = causal_mask_creation_function(
+ attention_mask,
+ sequence_length=sequence_length,
+ target_length=past_key_values.get_max_cache_shape(),
+ dtype=self.dtype,
+ device=device,
+ cache_position=cache_position,
+ batch_size=batch_size,
+ config=self.config,
+ past_key_values=past_key_values,
+ )
+ if attention_mask is not None:
+ model_inputs["attention_mask"] = attention_mask
+
+ for key, value in kwargs.items():
+ if key not in model_inputs:
+ model_inputs[key] = value
+
+ model_inputs.pop("labels", None)
+ model_inputs.pop("cache_position", None)
+ model_inputs["return_last_logit"] = True
+
+ return model_inputs
+
+
+@torch.jit.script
+def apply_rotary_pos_emb_glm_legacy(
+ x: torch.Tensor, rope_cache: torch.Tensor
+) -> torch.Tensor:
+ # x: [b, np, sq, hn]
+ b, np, sq, hn = x.size(0), x.size(1), x.size(2), x.size(3)
+ rot_dim = rope_cache.shape[-2] * 2
+ x, x_pass = x[..., :rot_dim], x[..., rot_dim:]
+ # truncate to support variable sizes
+ rope_cache = rope_cache[:, :sq]
+ xshaped = x.reshape(b, np, sq, rot_dim // 2, 2)
+ rope_cache = rope_cache.view(-1, 1, sq, xshaped.size(3), 2)
+ x_out2 = torch.stack(
+ [
+ xshaped[..., 0] * rope_cache[..., 0] - xshaped[..., 1] * rope_cache[..., 1],
+ xshaped[..., 1] * rope_cache[..., 0] + xshaped[..., 0] * rope_cache[..., 1],
+ ],
+ -1,
+ )
+ x_out2 = x_out2.flatten(3)
+ return torch.cat((x_out2, x_pass), dim=-1)
+
+
+def glm_forward(
+ prefill_forward, decoding_forward, attn_forward_config, class_name: str
+):
+ def attn_forward(
+ self,
+ hidden_states,
+ attention_mask,
+ rotary_pos_emb,
+ kv_cache=None,
+ use_cache=True,
+ ):
+ mixed_x_layer = self.query_key_value(hidden_states)
+ q_len = mixed_x_layer.size(1)
+ bsz = mixed_x_layer.size(0)
+ if self.multi_query_attention:
+ (query_layer, key_layer, value_layer) = mixed_x_layer.split(
+ [
+ self.num_attention_heads_per_partition
+ * self.hidden_size_per_attention_head,
+ self.num_multi_query_groups_per_partition
+ * self.hidden_size_per_attention_head,
+ self.num_multi_query_groups_per_partition
+ * self.hidden_size_per_attention_head,
+ ],
+ dim=-1,
+ )
+ query_layer = query_layer.view(
+ query_layer.size()[:-1]
+ + (
+ self.num_attention_heads_per_partition,
+ self.hidden_size_per_attention_head,
+ )
+ )
+ key_layer = key_layer.view(
+ key_layer.size()[:-1]
+ + (
+ self.num_multi_query_groups_per_partition,
+ self.hidden_size_per_attention_head,
+ )
+ )
+ value_layer = value_layer.view(
+ value_layer.size()[:-1]
+ + (
+ self.num_multi_query_groups_per_partition,
+ self.hidden_size_per_attention_head,
+ )
+ )
+ num_kv_groups = (
+ self.num_attention_heads_per_partition
+ // self.num_multi_query_groups_per_partition
+ )
+ else:
+ new_tensor_shape = mixed_x_layer.size()[:-1] + (
+ self.num_attention_heads_per_partition,
+ 3 * self.hidden_size_per_attention_head,
+ )
+ mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
+
+ # [b, sq, np, 3 * hn] --> 3 [b, sq, np, hn]
+ (query_layer, key_layer, value_layer) = torch.split(
+ mixed_x_layer,
+ [
+ self.hidden_size_per_attention_head,
+ self.hidden_size_per_attention_head,
+ self.hidden_size_per_attention_head,
+ ],
+ dim=-1,
+ )
+
+ # [b, sq, np, hn] -> [b, np, sq, hn]
+ query_layer, key_layer, value_layer = [
+ k.transpose(1, 2) for k in [query_layer, key_layer, value_layer]
+ ]
+ # apply relative positional encoding (rotary embedding)
+ if rotary_pos_emb is not None:
+ query_layer = apply_rotary_pos_emb_glm_legacy(query_layer, rotary_pos_emb)
+ key_layer = apply_rotary_pos_emb_glm_legacy(key_layer, rotary_pos_emb)
-def patch_glm_4_1m(model):
+ if kv_cache is not None:
+ cache_kwargs = {
+ "attn_forward_config": attn_forward_config,
+ "attention_mask": attention_mask,
+ "num_key_value_groups": num_kv_groups,
+ "query_states": query_layer,
+ "update_global_past_kv": getattr(self, "update_global_past_kv", True),
+ }
+ (
+ key_layer,
+ value_layer,
+ ) = kv_cache.update( # DynamicCache/KvcompressCache
+ key_layer,
+ value_layer,
+ self.layer_number - 1,
+ cache_kwargs,
+ )
+
+ if q_len == kv_cache.get_seq_length(self.layer_number - 1): # prefilling
+ if prefill_forward is not None: # eg, a-shape/tri-shape/minference
+ prefill_kwargs = {
+ "attention_mask": attention_mask,
+ "layer_idx": self.layer_number - 1,
+ "attn_forward_config": attn_forward_config,
+ }
+ attn_output = prefill_forward( # [bsz, num_heads, q_len, head_dim]
+ query_layer,
+ key_layer,
+ value_layer,
+ prefill_kwargs,
+ )
+ attn_output = attn_output.transpose(1, 2).contiguous()
+
+ else: # if not specified, use flash attention
+ attn_output = (
+ _flash_attention_forward( # [bsz, q_len, num_heads, head_dim]
+ query_layer.transpose(1, 2),
+ key_layer.transpose(1, 2),
+ value_layer.transpose(1, 2),
+ attention_mask,
+ q_len,
+ sliding_window=getattr(self, "sliding_window", None),
+ is_causal=True,
+ )
+ )
+
+ else: # decoding
+ # assert q_len == 1
+ if decoding_forward is not None: # eg, retr_attn
+ decoding_kwargs = {
+ "layer_idx": self.layer_number - 1,
+ "attn_forward_config": attn_forward_config,
+ "position_ids": kv_cache.pos_ids,
+ "num_key_value_groups": num_kv_groups,
+ }
+ attn_output = decoding_forward( # [bsz, num_heads, q_len, head_dim]
+ query_layer,
+ key_layer,
+ value_layer,
+ decoding_kwargs,
+ )
+ attn_output = attn_output.transpose(
+ 1, 2
+ ) # [bsz, q_len, num_heads, head_dim]
+ else:
+ attn_output = _flash_attention_forward(
+ query_layer.transpose(1, 2),
+ key_layer.transpose(1, 2),
+ value_layer.transpose(1, 2),
+ attention_mask,
+ q_len,
+ sliding_window=getattr(self, "sliding_window", None),
+ is_causal=True,
+ )
+
+ assert attn_output.size(1) == q_len
+ attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
+ output = self.dense(attn_output)
+
+ return output, kv_cache
+
+ def transformer_forward(
+ self,
+ hidden_states,
+ attention_mask,
+ rotary_pos_emb,
+ kv_caches=None,
+ use_cache: Optional[bool] = True,
+ output_hidden_states: Optional[bool] = False,
+ ):
+ if kv_caches is None:
+ # if not kv_caches:
+ kv_caches = [None for _ in range(self.num_layers)]
+ if self.gradient_checkpointing and self.training:
+ if use_cache:
+ logger.warning_once(
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+ )
+ use_cache = False
+
+ all_self_attentions = None
+ all_hidden_states = () if output_hidden_states else None
+ for index in range(self.num_layers):
+ if output_hidden_states:
+ all_hidden_states = all_hidden_states + (hidden_states,)
+
+ layer = self._get_layer(index)
+ if self.gradient_checkpointing and self.training:
+ layer_ret = torch.utils.checkpoint.checkpoint(
+ layer,
+ hidden_states,
+ attention_mask,
+ rotary_pos_emb,
+ kv_caches,
+ use_cache,
+ use_reentrant=False,
+ )
+ else:
+ layer_ret = layer(
+ hidden_states,
+ attention_mask,
+ rotary_pos_emb,
+ kv_caches,
+ use_cache=use_cache,
+ )
+ hidden_states, kv_caches = layer_ret
+
+ if output_hidden_states:
+ all_hidden_states = all_hidden_states + (hidden_states,)
+
+ # Final layer norm.
+ if self.post_layer_norm:
+ hidden_states = self.final_layernorm(hidden_states)
+
+ return hidden_states, kv_caches, all_hidden_states, all_self_attentions
+
+ forwards = {
+ "attn_forward": attn_forward,
+ "transformer_forward": transformer_forward,
+ }
+ return forwards[class_name]
+
+
+def convert_glm_4_1m(model):
# Support THUDM/glm-4-9b-chat-1m
+ from transformers.models.llama.modeling_llama import LlamaAttention
+
+ try:
+ from transformers.models.llama.modeling_llama import LlamaFlashAttention2
+ except ImportError:
+ pass
+
+ def patch_forward(model):
+ if model.__class__.__name__ == "ChatGLMForConditionalGeneration":
+ model.forward = types.MethodType(LlamaForCausalLM.forward, model)
+ model.prepare_inputs_for_generation = types.MethodType(
+ LlamaForCausalLM.prepare_inputs_for_generation, model
+ )
+ model._update_model_kwargs_for_generation = types.MethodType(
+ LlamaForCausalLM._update_model_kwargs_for_generation, model
+ )
+
+ for name, module in reversed(model._modules.items()):
+ if len(list(module.children())) > 0:
+ patch_forward(module)
+ if module.__class__.__name__ == "ChatGLMModel":
+ model._modules[name].forward = types.MethodType(
+ LlamaModel.forward, model._modules[name]
+ )
+ rope_emb = GlmRotaryEmbedding(
+ dim=model._modules[name].rotary_pos_emb.dim,
+ max_position_embeddings=model._modules[name].config.seq_length,
+ base=model._modules[name].rotary_pos_emb.rope_ratio,
+ )
+ rope_emb.to(model._modules[name].device)
+ model._modules[name].rotary_emb = rope_emb
+ model._modules[name]._update_causal_mask = types.MethodType(
+ LlamaModel._update_causal_mask, model._modules[name]
+ )
+ if module.__class__.__name__ == "ChatGLMPreTrainedModel":
+ model._modules[name].forward = types.MethodType(
+ LlamaPreTrainedModel.forward, model._modules[name]
+ )
+ if module.__class__.__name__ == "GLMBlock":
+ model._modules[name].forward = types.MethodType(
+ LlamaDecoderLayer.forward, model._modules[name]
+ )
+ if module.__class__.__name__ == "SelfAttention":
+ model._modules[name].forward = types.MethodType(
+ LlamaAttention.forward, model._modules[name]
+ )
+ if module.__class__.__name__ == "FlashAttention2":
+ model._modules[name].forward = types.MethodType(
+ LlamaFlashAttention2.forward, model._modules[name]
+ )
+ if module.__class__.__name__ == "MLP":
+ model._modules[name].forward = types.MethodType(
+ GlmMLP.forward, model._modules[name]
+ )
+
if model.__class__.__name__ == "ChatGLMForConditionalGeneration":
model.model = model.transformer
- del model.transformer
model.model.embed_tokens = model.model.embedding.word_embeddings
- del model.model.embedding
model.lm_head = model.model.output_layer
- del model.model.output_layer
model.model.norm = model.model.encoder.final_layernorm
- del model.model.encoder.final_layernorm
model.model.layers = model.model.encoder.layers
+
+ del model.model.embedding
+ del model.model.output_layer
+ del model.model.encoder.final_layernorm
+ del model.transformer
del model.model.encoder
+
for layer_idx in range(0, len(model.model.layers)):
model.model.layers[layer_idx].self_attn = model.model.layers[
layer_idx
].self_attention
- del model.model.layers[layer_idx].self_attention
model.model.layers[layer_idx].self_attn.qkv_proj = model.model.layers[
layer_idx
].self_attn.query_key_value
- del model.model.layers[layer_idx].self_attn.query_key_value
model.model.layers[layer_idx].self_attn.o_proj = model.model.layers[
layer_idx
].self_attn.dense
del model.model.layers[layer_idx].self_attn.dense
- model.model.layers[
- layer_idx
- ].self_attn.rotary_emb = model.model.rotary_pos_emb
+ del model.model.layers[layer_idx].self_attention
+ del model.model.layers[layer_idx].self_attn.query_key_value
model.model.layers[layer_idx].self_attn.config = model.config
model.model.layers[layer_idx].self_attn.layer_idx = layer_idx
config = model.config
@@ -57,6 +572,19 @@ def patch_glm_4_1m(model):
].self_attn.max_position_embeddings = config.seq_length
model.model.layers[layer_idx].self_attn.rope_theta = config.rope_ratio
model.model.layers[layer_idx].self_attn.is_causal = True
+ model.model.layers[layer_idx].mlp.gate_up_proj = model.model.layers[
+ layer_idx
+ ].mlp.dense_h_to_4h
+ model.model.layers[layer_idx].mlp.down_proj = model.model.layers[
+ layer_idx
+ ].mlp.dense_4h_to_h
+ model.model.layers[layer_idx].mlp.activation_fn = ACT2FN["silu"]
+
+ del model.model.layers[layer_idx].mlp.dense_h_to_4h
+ del model.model.layers[layer_idx].mlp.dense_4h_to_h
model.model.gradient_checkpointing = False
+ model.config.pretraining_tp = 1
+ patch_forward(model)
torch.cuda.empty_cache()
+
return model
diff --git a/minference/version.py b/minference/version.py
index ff46d656..a1ac9270 100644
--- a/minference/version.py
+++ b/minference/version.py
@@ -1,14 +1,14 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
_MAJOR = "0"
_MINOR = "1"
# On master and in a nightly release the patch should be one ahead of the last
# released build.
-_PATCH = "5"
+_PATCH = "6"
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
# https://semver.org/#is-v123-a-semantic-version for the semantics.
-_SUFFIX = ".post1"
+_SUFFIX = ".0"
VERSION_SHORT = "{0}.{1}".format(_MAJOR, _MINOR)
VERSION = "{0}.{1}.{2}{3}".format(_MAJOR, _MINOR, _PATCH, _SUFFIX)
diff --git a/mtraining/.gitignore b/mtraining/.gitignore
new file mode 100644
index 00000000..7a129c2e
--- /dev/null
+++ b/mtraining/.gitignore
@@ -0,0 +1,13 @@
+**/__pycache__/
+MTraining.egg-info/
+**.ipynb
+**/prof_logs/
+.vscode/
+.pytest_cache/
+*.log
+**/draft/
+output/
+**/ring_attn_comp_data/
+**/ring_attn_comp_data/
+**/ring_attn_pt_logs/
+expr_data_store/
diff --git a/mtraining/README.md b/mtraining/README.md
new file mode 100644
index 00000000..38e374d6
--- /dev/null
+++ b/mtraining/README.md
@@ -0,0 +1,99 @@
+# MTraining
+
+[[Paper]](https://arxiv.org/abs/2510.18830)
+
+
+
+
+MTraining is the dynamic-sparse-attention-based long-context training strategy. Its implementation builds on top of nnScaler and provides:
+
+- training-time sparse attention integration for ultra-long context LLMs,
+- distributed sparse attention operators (implemented in `minference/dist_ops`),
+- example scripts for data preparation and training in `mtraining/experiments/scripts`.
+
+
+
+## Installation
+
+Use the setup script to install dependencies and build from source:
+
+```bash
+cd mtraining
+bash setup.sh
+```
+
+`setup.sh` will:
+
+- install pinned training dependencies (including nnScaler and FlashAttention variants),
+- install `MInference` from source (`pip install -e .`) and then `mtraining` in editable mode.
+
+
+
+## Operator Correctness Tests
+
+From repository root:
+
+```bash
+# Run ring sparse attention tests
+bash minference/dist_ops/test/run_ring_pytests.sh
+```
+
+Or run tests individually:
+
+```bash
+pytest -s minference/dist_ops/test/minfer_ring_test.py
+pytest -s minference/dist_ops/test/moba_ring_test.py
+pytest -s minference/dist_ops/test/xattn_ring_test.py
+```
+
+
+
+## Quick Start: Data Preparation + Training
+
+### 1) Prepare ProLong 512K data
+
+```bash
+cd mtraining
+bash experiments/scripts/prolong_data_prepare.sh
+```
+
+This script downloads `princeton-nlp/prolong-data-512K` in `RAW_DATASET_DIR/long-context-524288` (~228G) and pre-processes it to `PROCESSED_DATA_DIR` (~19G) by data sampling (default interval: 4) and re-tokenization.
+
+
+
+### 2) Launch training
+
+We have provided sample training script in `mtraining/experiments/scripts` for training Qwen-2.5 models (0.5B and 3B) with or without sparse attention. For example, `mtraining/experiments/scripts/train_qwen2_3B_ProLong512K.sh` is to train Qwen-2.5-3B with MTraining under Striped Ring Attention.
+
+You can adjust the type of the attention operator to be used during the training, where the supported `--attn_type` values include:
+
+- `dense`
+- `zigzag_ring`
+- `stripe_ring`
+- `minfer`
+- `moba`
+- `xattn`
+
+where `minfer` refer to our dynamic sparse attention operators. You may need to further configure the selected attention by giving a `yaml`-based configuration. The directory `mtraining/train_attn_configs` has provided a set of such configurations. For example, `mtraining/train_attn_configs/qwen_flex_090.yaml` is like below:
+
+```yaml
+pattern_config_name: Qwen2.5_3B_flex_0.90
+implementation: stripe
+```
+
+which specifies the sparse pattern file under `minference/configs` and the usage of striped Ring Attention. The field `implementation` can be `zigzag`, `stripe` and `dr_stripe`, corresponding to Zigzag, Striped and Hierarchical Striped Ring Attention respectively.
+
+
+
+## Citation
+
+If you use MTraining, please cite:
+
+```bibtex
+@article{li2025mtraining,
+ title={MTraining: Distributed Dynamic Sparse Attention for Efficient Ultra-Long Context Training},
+ author={Li, Wenxuan and Zhang, Chengruidong and Jiang, Huiqiang and Li, Yucheng and Yang, Yuqing and Qiu, Lili},
+ journal={arXiv preprint arXiv:2510.18830},
+ year={2025}
+}
+```
diff --git a/mtraining/__init__.py b/mtraining/__init__.py
new file mode 100644
index 00000000..0d2017b4
--- /dev/null
+++ b/mtraining/__init__.py
@@ -0,0 +1,2 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
diff --git a/mtraining/attn_funcs/__init__.py b/mtraining/attn_funcs/__init__.py
new file mode 100644
index 00000000..49808885
--- /dev/null
+++ b/mtraining/attn_funcs/__init__.py
@@ -0,0 +1,44 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from typing import Callable, Dict
+
+from .dense_func import (
+ fa_attn_forward,
+ stripe_ring_attention_forward,
+ zigzag_ring_attention_forward,
+)
+from .minfer_func import minfer_attention_forward
+from .moba_func import moba_attention_forward
+from .xattn_func import xattn_attention_forward
+
+
+class AttnType:
+ DENSE: str = "dense"
+ ZIGZAG_RING: str = "zigzag_ring"
+ STRIPE_RING: str = "stripe_ring"
+
+ MINFER: str = "minfer"
+ MOBA: str = "moba"
+ XATTN: str = "xattn"
+
+
+ATTN_TO_FUNC = {
+ AttnType.DENSE: fa_attn_forward,
+ AttnType.ZIGZAG_RING: zigzag_ring_attention_forward,
+ AttnType.STRIPE_RING: stripe_ring_attention_forward,
+ AttnType.MINFER: minfer_attention_forward,
+ AttnType.MOBA: moba_attention_forward,
+ AttnType.XATTN: xattn_attention_forward,
+}
+
+
+def overwrite_attn_implementation(
+ attn_dict: Dict[str, Callable],
+ attn_type: AttnType,
+):
+ attn_func: Callable = ATTN_TO_FUNC[attn_type]
+ print(f"Overwriting attention implementation to {attn_type} ({attn_func.__name__})")
+
+ for attn_name in attn_dict:
+ attn_dict[attn_name] = attn_func
diff --git a/mtraining/attn_funcs/dense_func.py b/mtraining/attn_funcs/dense_func.py
new file mode 100644
index 00000000..be3ca630
--- /dev/null
+++ b/mtraining/attn_funcs/dense_func.py
@@ -0,0 +1,329 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+# This file modifies the official modeling_llama.py file at runtime to
+# 1. register the flash attention function to nnscaler and update related code
+# 2. replace the un-fused RMSNorm with apex's fused version
+import torch
+from torch import Tensor
+from transformers.utils import is_flash_attn_2_available
+
+if is_flash_attn_2_available():
+ from flash_attn.bert_padding import pad_input
+ from flash_attn import flash_attn_func, flash_attn_varlen_func
+
+from nnscaler.graph.parser.register import register_op
+from nnscaler.ir import IRTensor
+from nnscaler.ir.operator import IRFwOperation
+from nnscaler.runtime.device import DeviceGroup
+
+from minference.dist_ops.striped_attention import stripe_flash_attn_func
+from minference.dist_ops.zigzag_attention import zigzag_ring_flash_attn_func
+
+from .utils import nnscaler_upad_input
+
+
+def fa_attn_forward(
+ module: torch.nn.Module,
+ query_states, # [B, H, N, D]
+ key_states,
+ value_states,
+ attention_mask,
+ dropout=0.0,
+ scaling=None,
+ **kwargs,
+):
+ # Contains at least one padding token in the sequence
+ if attention_mask is not None:
+ batch_size = query_states.shape[0]
+ query_length = kwargs.get("query_length", query_states.shape[2])
+ (
+ query_states,
+ key_states,
+ value_states,
+ indices_q,
+ cu_seq_lens,
+ max_seq_lens,
+ ) = nnscaler_upad_input(
+ query_states, key_states, value_states, attention_mask, query_length
+ )
+
+ cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+ max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+ attn_output_unpad = flash_attn_varlen_func(
+ query_states,
+ key_states,
+ value_states,
+ cu_seqlens_q=cu_seqlens_q,
+ cu_seqlens_k=cu_seqlens_k,
+ max_seqlen_q=max_seqlen_in_batch_q,
+ max_seqlen_k=max_seqlen_in_batch_k,
+ dropout_p=dropout,
+ softmax_scale=scaling,
+ causal=True,
+ )
+
+ attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+ else:
+ attn_output = flash_attn_func(
+ query_states.transpose(1, 2),
+ key_states.transpose(1, 2),
+ value_states.transpose(1, 2),
+ dropout,
+ softmax_scale=scaling,
+ causal=True,
+ )
+
+ return attn_output, None
+
+
+# ---------------------------------------------------------------------------
+def zigzag_ring_attention_forward(
+ module: torch.nn.Module,
+ query: Tensor, # [B, H, N, D]
+ key: Tensor,
+ value: Tensor,
+ attention_mask: Optional[Tensor],
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ softcap: Optional[float] = None,
+ **kwargs,
+) -> Tuple[Tensor, None]:
+ return (
+ wrap_zigzag_attn_func(
+ query.transpose(1, 2),
+ key.transpose(1, 2),
+ value.transpose(1, 2),
+ layer_idx=module.layer_idx,
+ softmax_scale=scaling,
+ dropout_p=dropout,
+ causal=True,
+ ),
+ None,
+ )
+
+
+def wrap_zigzag_attn_func(
+ q: Tensor,
+ k: Tensor,
+ v: Tensor,
+ layer_idx: int,
+ softmax_scale: Tensor = None,
+ dropout_p: float = 0.0,
+ causal: bool = True,
+ window_size: Tuple[int] = (-1, -1),
+ alibi_slopes: Tensor = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ process_group: Tuple[int] = None,
+) -> Tensor:
+ if process_group is None or len(process_group) == 1:
+ # there is an additional checker for the `softmax_scale`, which is equivalent
+ # to the behavior of the original flash_attn_func.
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+ output = flash_attn_func(q, k, v, 0.0, softmax_scale, causal)
+ return output
+
+ assert causal == True, "zigzag_ring is meaningless for causal=False"
+ assert len(q.shape) == 4, "q must have shape [bs, ql, qh, dim]"
+ assert len(k.shape) == 4, "k must have shape [bs, kl, kh, dim]"
+ assert len(v.shape) == 4, "v must have shape [bs, vl, vh, dim]"
+ qbsz, qlen, qheads, qdim = q.shape
+ kbsz, klen, kheads, kdim = k.shape
+ vbsz, vlen, vheads, vdim = v.shape
+ assert qbsz == kbsz == vbsz, "batch size must be the same"
+ assert qlen == klen == vlen, "sequence length must be the same"
+ assert kheads == vheads, "number of k and v heads must be the same"
+ assert qheads % kheads == 0, "number of q heads must be a multiple of k heads"
+ assert qdim == kdim == vdim, "dimension must be the same"
+
+ local_process_group = DeviceGroup().get_group(process_group)
+ output = zigzag_ring_flash_attn_func(
+ q,
+ k,
+ v,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ local_process_group,
+ ).contiguous()
+ return output
+
+
+# ---------------------------------------------------------------------------
+def stripe_ring_attention_forward(
+ module: torch.nn.Module,
+ query: Tensor, # [B, H, N, D]
+ key: Tensor,
+ value: Tensor,
+ attention_mask: Optional[Tensor],
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ softcap: Optional[float] = None,
+ **kwargs,
+) -> Tuple[Tensor, None]:
+ return (
+ wrap_striped_attn_func(
+ query.transpose(1, 2),
+ key.transpose(1, 2),
+ value.transpose(1, 2),
+ layer_idx=module.layer_idx,
+ softmax_scale=scaling,
+ dropout_p=dropout,
+ causal=True,
+ ),
+ None,
+ )
+
+
+def wrap_striped_attn_func(
+ q: Tensor,
+ k: Tensor,
+ v: Tensor,
+ layer_idx: int,
+ granularity: int = 1,
+ softmax_scale: Tensor = None,
+ dropout_p: float = 0.0,
+ causal: bool = True,
+ window_size: Tuple[int] = (-1, -1),
+ alibi_slopes: Tensor = None,
+ deterministic: bool = False,
+ return_attn_probs: bool = False,
+ process_group: Tuple[int] = None,
+) -> Tensor:
+ if process_group is None or len(process_group) == 1:
+ # there is an additional checker for the `softmax_scale`, which is equivalent
+ # to the behavior of the original flash_attn_func.
+ if softmax_scale is None:
+ softmax_scale = q.shape[-1] ** (-0.5)
+ output = flash_attn_func(q, k, v, 0.0, softmax_scale, causal)
+ return output
+
+ assert len(q.shape) == 4, "q must have shape [bs, ql, qh, dim]"
+ assert len(k.shape) == 4, "k must have shape [bs, kl, kh, dim]"
+ assert len(v.shape) == 4, "v must have shape [bs, vl, vh, dim]"
+ qbsz, qlen, qheads, qdim = q.shape
+ kbsz, klen, kheads, kdim = k.shape
+ vbsz, vlen, vheads, vdim = v.shape
+ assert qbsz == kbsz == vbsz, "batch size must be the same"
+ assert qlen == klen == vlen, "sequence length must be the same"
+ assert kheads == vheads, "number of k and v heads must be the same"
+ assert qheads % kheads == 0, "number of q heads must be a multiple of k heads"
+ assert qdim == kdim == vdim, "dimension must be the same"
+
+ local_process_group = DeviceGroup().get_group(process_group)
+ output = stripe_flash_attn_func(
+ q,
+ k,
+ v,
+ layer_idx,
+ dropout_p,
+ softmax_scale,
+ granularity,
+ causal,
+ window_size,
+ alibi_slopes,
+ deterministic,
+ return_attn_probs,
+ local_process_group,
+ ).contiguous()
+ return output
+
+
+# ---------------------------------------------------------------------------
+def flash_attention_anno(
+ query_states, key_states, value_states, attention_mask, *args, **kwargs
+) -> str:
+ if query_states.shape[2] != key_states.shape[2]:
+ assert query_states.shape[2] % key_states.shape[2] == 0
+ group_size = query_states.shape[2] // key_states.shape[2]
+ assert query_states.shape[2] == value_states.shape[2] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ if isinstance(attention_mask, IRTensor):
+ return f"b l^ {q_anno} hd^, b s^ {kv_anno} hd^, b s^ {kv_anno} vd^, b l^ -> b l^ {q_anno} vd^"
+ else:
+ return f"b l^ {q_anno} hd^, b s^ {kv_anno} hd^, b s^ {kv_anno} vd^ -> b l^ {q_anno} vd^"
+
+
+def emit_ring(
+ node: IRFwOperation,
+ args: List[str],
+ kwargs: Dict[str, str],
+ runtime_devid: int,
+ plan_ndevs: int,
+ runtime_ndevs: int,
+) -> str:
+ """Special rule to generate zigzag_attn node"""
+
+ signature = node.signature
+
+ offset = (runtime_devid // plan_ndevs) * plan_ndevs
+ scale_unit_dev_ids = [local_rank + offset for local_rank in range(plan_ndevs)]
+
+ kw_pairs = list()
+ for key, val in kwargs.items():
+ code = f"{key}={val}"
+ kw_pairs.append(code)
+
+ sub_input = node.inputs()[0]
+ full_input = sub_input.parent
+ partition_dims = [
+ i for i, (s, f) in enumerate(zip(sub_input.shape, full_input.shape)) if s != f
+ ]
+ assert (
+ len(partition_dims) <= 1
+ ), f"support no more than one partition dim, but got {partition_dims}"
+ if not partition_dims:
+ kw_pairs.append("process_group=None")
+ else:
+ # if the 'process_group' is None, we will use the local attention (flash_attn_func)
+ if partition_dims[0] == 0: # partition on batch dim
+ # partition the bsz dim, use local flash_attn_func
+ kw_pairs.append("process_group=None")
+ elif partition_dims[0] == 1: # partition on sequence dim
+ # the synchronization should occur across scaleunits
+ kw_pairs.append(f"process_group={scale_unit_dev_ids}")
+ elif partition_dims[0] == 2:
+ # partition on num_head dim
+ kw_pairs.append("process_group=None")
+ else:
+ raise ValueError(f"unsupported partition dim: {partition_dims[0]}")
+
+ args = ", ".join(list(args) + kw_pairs)
+ return f"{signature}({args})"
+
+
+def ring_attn_anno(query_states, key_states, value_states, *args, **kwargs) -> str:
+ if query_states.shape[2] != key_states.shape[2]:
+ assert query_states.shape[2] % key_states.shape[2] == 0
+ group_size = query_states.shape[2] // key_states.shape[2]
+ assert query_states.shape[2] == value_states.shape[2] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ return f"b l {q_anno} hd^, b l {kv_anno} hd^, b l {kv_anno} vd^ -> b l {q_anno} vd^"
+
+
+register_op(flash_attention_anno)(fa_attn_forward)
+register_op(ring_attn_anno, emit_fn=emit_ring)(wrap_zigzag_attn_func)
+register_op(ring_attn_anno, emit_fn=emit_ring)(wrap_striped_attn_func)
diff --git a/mtraining/attn_funcs/minfer_func.py b/mtraining/attn_funcs/minfer_func.py
new file mode 100644
index 00000000..22452750
--- /dev/null
+++ b/mtraining/attn_funcs/minfer_func.py
@@ -0,0 +1,456 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# This file modifies the official modeling_llama.py file at runtime to
+# 1. register the flash attention function to nnscaler and update related code
+# 2. replace the un-fused RMSNorm with apex's fused version
+import json
+import logging
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+from typing import Callable, Dict, List, Optional, Tuple
+
+from transformers.utils import is_flash_attn_2_available, logging
+
+if is_flash_attn_2_available():
+ from flash_attn import flash_attn_func
+
+from nnscaler.graph.parser.register import register_op
+from nnscaler.ir import IRTensor
+from nnscaler.ir.operator import IRFwOperation
+from nnscaler.runtime.device import DeviceGroup
+
+from minference.dist_ops import (
+ minfer_dr_stripe_func,
+ minfer_stripe_func,
+ minfer_zigzag_func,
+)
+from minference.ops.pit_sparse_flash_attention_v3 import (
+ minference_flash_attn_func,
+ minference_flash_attn_triton_func,
+)
+from minference.ops.utils import use_triton
+
+
+# =======================================================
+def minfer_op(
+ query_states: torch.Tensor,
+ key_states: torch.Tensor,
+ value_states: torch.Tensor,
+ head_indices: torch.Tensor,
+ bsz: int,
+ q_len: int,
+ head_dim: int,
+ layer_idx: int,
+ pattern_dict: Dict[int, Tuple[str, int, int, int]],
+ attn_dropout: float = 0.0,
+ granularity: int = 128,
+ group: Optional[torch.distributed.ProcessGroup] = None,
+):
+ v_sizes = [
+ pattern_dict[head_indices[idx].item()][1] for idx in range(query_states.size(1))
+ ]
+ s_sizes = [
+ pattern_dict[head_indices[idx].item()][2] for idx in range(query_states.size(1))
+ ]
+ if not use_triton():
+ attn_output = minference_flash_attn_func(
+ query_states.transpose(1, 2).contiguous(),
+ key_states.transpose(1, 2).contiguous(),
+ value_states.transpose(1, 2).contiguous(),
+ v_sizes,
+ s_sizes,
+ attn_dropout,
+ softmax_scale=None,
+ granularity=granularity,
+ causal=True,
+ window_size=(-1, -1),
+ deterministic=False,
+ return_attn_probs=False,
+ group=group,
+ )
+ else:
+ attn_output = minference_flash_attn_triton_func(
+ query_states.transpose(1, 2).contiguous(),
+ key_states.transpose(1, 2).contiguous(),
+ value_states.transpose(1, 2).contiguous(),
+ v_sizes,
+ s_sizes,
+ attn_dropout,
+ softmax_scale=None,
+ granularity=granularity,
+ causal=True,
+ window_size=(-1, -1),
+ deterministic=False,
+ return_attn_probs=False,
+ group=group,
+ )
+ return attn_output.contiguous()
+
+
+def minfer_stripe_op(
+ query_states: torch.Tensor, # [batch_size, num_heads, num_tokens, head_dim]
+ key_states: torch.Tensor,
+ value_states: torch.Tensor,
+ head_indices: torch.Tensor,
+ bsz: int,
+ q_len: int,
+ head_dim: int,
+ layer_idx: int,
+ pattern_dict: Dict[int, Tuple[str, int, int, int]],
+ attn_dropout: float = 0.0,
+ granularity: int = 128,
+ process_group: Optional[torch.distributed.ProcessGroup] = None,
+):
+ if process_group is None or len(process_group) == 1:
+ softmax_scale = query_states.shape[-1] ** (-0.5)
+
+ output = flash_attn_func(
+ query_states.transpose(1, 2),
+ key_states.transpose(1, 2),
+ value_states.transpose(1, 2),
+ attn_dropout,
+ softmax_scale,
+ causal=True,
+ )
+ return output
+ group = DeviceGroup().get_group(process_group)
+
+ v_sizes = [
+ pattern_dict[head_indices[idx].item()][1] for idx in range(query_states.size(1))
+ ]
+ s_sizes = [
+ pattern_dict[head_indices[idx].item()][2] for idx in range(query_states.size(1))
+ ]
+ attn_output = minfer_stripe_func(
+ query_states.transpose(1, 2).contiguous(),
+ key_states.transpose(1, 2).contiguous(),
+ value_states.transpose(1, 2).contiguous(),
+ v_sizes,
+ s_sizes,
+ layer_idx,
+ attn_dropout,
+ softmax_scale=None,
+ granularity=granularity,
+ causal=True,
+ window_size=(-1, -1),
+ deterministic=False,
+ return_attn_probs=False,
+ group=group,
+ ) # expect: b {q_anno} l^ vd^'
+
+ return attn_output.contiguous()
+
+
+def minfer_zigzag_op(
+ query_states: torch.Tensor, # [batch_size, num_heads, num_tokens, head_dim]
+ key_states: torch.Tensor,
+ value_states: torch.Tensor,
+ head_indices: torch.Tensor,
+ bsz: int,
+ q_len: int,
+ head_dim: int,
+ layer_idx: int,
+ pattern_dict: Dict[int, Tuple[str, int, int, int]],
+ attn_dropout: float = 0.0,
+ granularity: int = 128,
+ process_group: Optional[torch.distributed.ProcessGroup] = None,
+):
+ if process_group is None or len(process_group) == 1:
+ # there is an additional checker for the `softmax_scale`, which is equivalent
+ # to the behavior of the original flash_attn_func.
+ softmax_scale = query_states.shape[-1] ** (-0.5)
+ output = flash_attn_func(
+ query_states.transpose(1, 2),
+ key_states.transpose(1, 2),
+ value_states.transpose(1, 2),
+ attn_dropout,
+ softmax_scale,
+ causal=True,
+ )
+ return output
+ group = DeviceGroup().get_group(process_group)
+
+ v_sizes = [
+ pattern_dict[head_indices[idx].item()][1] for idx in range(query_states.size(1))
+ ]
+ s_sizes = [
+ pattern_dict[head_indices[idx].item()][2] for idx in range(query_states.size(1))
+ ]
+ if not use_triton():
+ attn_output = minfer_zigzag_func(
+ query_states.transpose(1, 2).contiguous(),
+ key_states.transpose(1, 2).contiguous(),
+ value_states.transpose(1, 2).contiguous(),
+ v_sizes,
+ s_sizes,
+ layer_idx,
+ attn_dropout,
+ softmax_scale=None,
+ granularity=granularity,
+ causal=True,
+ window_size=(-1, -1),
+ deterministic=False,
+ return_attn_probs=False,
+ group=group,
+ ) # expect: b {q_anno} l^ vd^'
+ else:
+ raise NotImplementedError(
+ "Triton-only version is not implemented for MInfer w. zigzag"
+ )
+ return attn_output.contiguous()
+
+
+def minfer_dr_stripe_op(
+ query_states: torch.Tensor, # [batch_size, num_heads, num_tokens, head_dim]
+ key_states: torch.Tensor,
+ value_states: torch.Tensor,
+ head_indices: torch.Tensor,
+ bsz: int,
+ q_len: int,
+ head_dim: int,
+ layer_idx: int,
+ pattern_dict: Dict[int, Tuple[str, int, int, int]],
+ attn_dropout: float = 0.0,
+ granularity: int = 128,
+ process_group: Optional[torch.distributed.ProcessGroup] = None,
+):
+ if process_group is None or len(process_group) == 1:
+ # there is an additional checker for the `softmax_scale`, which is equivalent
+ # to the behavior of the original flash_attn_func.
+ softmax_scale = query_states.shape[-1] ** (-0.5)
+
+ output = flash_attn_func(
+ query_states.transpose(1, 2),
+ key_states.transpose(1, 2),
+ value_states.transpose(1, 2),
+ attn_dropout,
+ softmax_scale,
+ causal=True,
+ )
+ return output
+
+ group = DeviceGroup().get_group(process_group)
+ v_sizes = [
+ pattern_dict[head_indices[idx].item()][1] for idx in range(query_states.size(1))
+ ]
+ s_sizes = [
+ pattern_dict[head_indices[idx].item()][2] for idx in range(query_states.size(1))
+ ]
+
+ attn_output = minfer_dr_stripe_func(
+ query_states.transpose(1, 2).contiguous(),
+ key_states.transpose(1, 2).contiguous(),
+ value_states.transpose(1, 2).contiguous(),
+ v_sizes,
+ s_sizes,
+ layer_idx,
+ attn_dropout,
+ softmax_scale=None,
+ granularity=granularity,
+ causal=True,
+ window_size=(-1, -1),
+ deterministic=False,
+ return_attn_probs=False,
+ group=group,
+ ) # expect: b {q_anno} l^ vd^'
+ return attn_output.contiguous()
+
+
+MINFER_IMPLEMENTATIONS: Dict[str, Callable] = {
+ "default": minfer_op,
+ "stripe": minfer_stripe_op,
+ "zigzag": minfer_zigzag_op,
+ "dr_stripe": minfer_dr_stripe_op,
+}
+
+
+def emit_minfer_ring(
+ node: IRFwOperation,
+ args: List[str],
+ kwargs: Dict[str, str],
+ runtime_devid: int,
+ plan_ndevs: int,
+ runtime_ndevs: int,
+) -> str:
+ """Special rule to generate zigzag_attn node"""
+
+ signature = node.signature
+
+ offset = (runtime_devid // plan_ndevs) * plan_ndevs
+ scale_unit_dev_ids = [local_rank + offset for local_rank in range(plan_ndevs)]
+
+ kw_pairs = list()
+ for key, val in kwargs.items():
+ code = f"{key}={val}"
+ kw_pairs.append(code)
+
+ sub_input = node.inputs()[0]
+ full_input = sub_input.parent
+ partition_dims = [
+ i for i, (s, f) in enumerate(zip(sub_input.shape, full_input.shape)) if s != f
+ ]
+ assert (
+ len(partition_dims) <= 1
+ ), f"support no more than one partition dim, but got {partition_dims}"
+ if not partition_dims:
+ kw_pairs.append("process_group=None")
+ else:
+ # if the 'process_group' is None, we will use the local attention (flash_attn_func)
+ if partition_dims[0] == 0: # partition on batch dim
+ # partition the bsz dim, use local flash_attn_func
+ kw_pairs.append("process_group=None")
+ elif partition_dims[0] == 1:
+ # partition on num_head dim
+ kw_pairs.append("process_group=None")
+ elif partition_dims[0] == 2: # partition on sequence dim
+ # the synchronization should occur across scaleunits
+ kw_pairs.append(f"process_group={scale_unit_dev_ids}")
+ else:
+ raise ValueError(f"unsupported partition dim: {partition_dims[0]}")
+
+ args = ", ".join(list(args) + kw_pairs)
+ return f"{signature}({args})"
+
+
+def minfer_attn_anno(query_states, key_states, value_states, *args, **kwargs) -> str:
+ if query_states.shape[1] != key_states.shape[1]:
+ assert query_states.shape[1] % key_states.shape[1] == 0
+ group_size = query_states.shape[1] // key_states.shape[1]
+ assert query_states.shape[1] == value_states.shape[1] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ return f"b {q_anno} l^ hd^, b {kv_anno} s^ hd^, b {kv_anno} s^ vd^, {q_anno} -> b l^ {q_anno} vd^"
+
+
+def minfer_attn_ring_anno(
+ query_states, key_states, value_states, *args, **kwargs
+) -> str:
+ if query_states.shape[1] != key_states.shape[1]:
+ assert query_states.shape[1] % key_states.shape[1] == 0
+ group_size = query_states.shape[1] // key_states.shape[1]
+ assert query_states.shape[1] == value_states.shape[1] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ return f"b {q_anno} l hd^, b {kv_anno} l hd^, b {kv_anno} l vd^, {q_anno} -> b l {q_anno} vd^"
+
+
+if __name__ != "__main__":
+ register_op(minfer_attn_anno)(minfer_op)
+ register_op(minfer_attn_ring_anno, emit_fn=emit_minfer_ring)(minfer_stripe_op)
+ register_op(minfer_attn_ring_anno, emit_fn=emit_minfer_ring)(minfer_zigzag_op)
+ register_op(minfer_attn_ring_anno, emit_fn=emit_minfer_ring)(minfer_dr_stripe_op)
+
+
+class MInferAttnFunc:
+ def __init__(self):
+ self.initialized = False
+
+ def init_minfer_params(
+ self,
+ config_path: str,
+ minfer_implementation: str, # "fa", "stripe", "zigzag"
+ granularity: int = 128,
+ ):
+ assert (
+ minfer_implementation in MINFER_IMPLEMENTATIONS
+ ), f"minfer_implementation should be one of {MINFER_IMPLEMENTATIONS}, but got {self.minfer_implementation}"
+ self.minfer_implementation: str = minfer_implementation
+
+ self.config_path = config_path
+ self.all_pattern_dict = json.load(open(self.config_path))
+ self.granularity = granularity
+
+ self.initialized = True
+
+ def get_pattern_dict(self, layer_idx):
+ return {int(ii): jj for ii, jj in self.all_pattern_dict[layer_idx].items()}
+
+ def forward(
+ self,
+ query_states: torch.Tensor,
+ key_states: torch.Tensor,
+ value_states: torch.Tensor,
+ head_indices: torch.Tensor,
+ attn_module_config: Dict[str, int],
+ attn_dropout: float = 0.0,
+ ):
+ bsz, q_len = query_states.shape[0], query_states.shape[2]
+ head_dim, layer_idx = (
+ attn_module_config["head_dim"],
+ attn_module_config["layer_idx"],
+ )
+
+ pattern_dict = self.get_pattern_dict(layer_idx)
+ minfer_args = (
+ query_states,
+ key_states,
+ value_states,
+ head_indices,
+ bsz,
+ q_len,
+ head_dim,
+ layer_idx,
+ pattern_dict,
+ attn_dropout,
+ self.granularity,
+ )
+
+ if self.minfer_implementation == "default":
+ return minfer_op(*minfer_args)
+ elif self.minfer_implementation == "stripe":
+ return minfer_stripe_op(*minfer_args)
+ elif self.minfer_implementation == "zigzag":
+ return minfer_zigzag_op(*minfer_args)
+ elif self.minfer_implementation == "dr_stripe":
+ return minfer_dr_stripe_op(*minfer_args)
+ else:
+ raise ValueError(
+ f"Unsupported minfer_implementation: {self.minfer_implementation}"
+ )
+
+
+def minfer_attention_forward(
+ module: torch.nn.Module,
+ query: torch.Tensor, # [B, H, N, D]
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attention_mask: Optional[torch.Tensor],
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ softcap: Optional[float] = None,
+ **kwargs,
+) -> Tuple[torch.Tensor, None]:
+ attn_module_config = {
+ "num_heads": module.config.num_attention_heads,
+ "head_dim": module.head_dim,
+ "layer_idx": module.layer_idx,
+ }
+ head_indices = torch.arange(
+ attn_module_config["num_heads"], device=query.device, dtype=torch.int32
+ )
+
+ return (
+ module.minfer_attn_func.forward(
+ query,
+ key,
+ value,
+ head_indices,
+ attn_module_config,
+ dropout,
+ ),
+ None,
+ )
diff --git a/mtraining/attn_funcs/moba_func.py b/mtraining/attn_funcs/moba_func.py
new file mode 100644
index 00000000..86e41778
--- /dev/null
+++ b/mtraining/attn_funcs/moba_func.py
@@ -0,0 +1,232 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+from typing import Any, Dict, List, Optional, Tuple
+
+import torch
+import torch.distributed as dist
+import yaml
+from nnscaler.graph.parser.register import register_op
+from nnscaler.ir.operator import IRFwOperation
+from nnscaler.runtime.device import DeviceGroup
+from torch import Tensor
+
+from minference.dist_ops.moba_zigzag import moba_zigzag_func
+from minference.ops.moba import moba_attn_func
+from minference.ops.op_utils.moba_utils import MoBAConfig
+
+
+def load_moba_config(moba_config_dict: Dict[str, Any]):
+ moba_config = MoBAConfig(**moba_config_dict)
+ return moba_config
+
+
+def moba_attention_forward(
+ module: torch.nn.Module,
+ query: torch.Tensor, # [B, H, N, D]
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attention_mask: Optional[torch.Tensor],
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ softcap: Optional[float] = None,
+ **kwargs,
+) -> Tuple[torch.Tensor, None]:
+ seq_len = query.shape[2]
+ moba_topk, moba_chunk_size = module.moba_topk, module.moba_chunk_size
+ implementation = module.implementation
+
+ if implementation == "default":
+ return (
+ wrapped_moba_func(
+ query.transpose(1, 2),
+ key.transpose(1, 2),
+ value.transpose(1, 2),
+ seq_len,
+ moba_topk,
+ moba_chunk_size,
+ ),
+ None,
+ )
+ elif implementation == "zigzag":
+ layer_idx = module.layer_idx
+ return (
+ wrapped_moba_zigzag_func(
+ query.transpose(1, 2),
+ key.transpose(1, 2),
+ value.transpose(1, 2),
+ seq_len,
+ moba_topk,
+ moba_chunk_size,
+ layer_idx,
+ attention_mask,
+ dropout,
+ scaling,
+ sliding_window,
+ softcap,
+ ),
+ None,
+ )
+ else:
+ raise ValueError(
+ f"Unsupported MoBA implementation: {implementation}. "
+ f"Supported implementations are 'default' and 'zigzag'."
+ )
+
+
+# ------------------------------------------
+def wrapped_moba_func(
+ q: Tensor,
+ k: Tensor,
+ v: Tensor,
+ seq_len: int,
+ moba_topk: int,
+ moba_chunk_size: int,
+):
+ return moba_attn_func(
+ q,
+ k,
+ v,
+ seq_len,
+ moba_chunk_size,
+ moba_topk,
+ )
+
+
+def wrapped_moba_zigzag_func(
+ query: Tensor, # [B, N, H, D]
+ key: Tensor,
+ value: Tensor,
+ seq_len: int,
+ moba_topk: int,
+ moba_chunk_size: int,
+ layer_idx: int,
+ attention_mask: Optional[torch.Tensor],
+ dropout: float = 0.0,
+ softmax_scale: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ softcap: Optional[float] = None,
+ process_group: Tuple[int] = None,
+):
+ if process_group is None or len(process_group) == 1:
+ # there is an additional checker for the `softmax_scale`, which is equivalent
+ # to the behavior of the original flash_attn_func.
+ from flash_attn import flash_attn_func
+
+ if softmax_scale is None:
+ softmax_scale = query.shape[-1] ** (-0.5)
+ output = flash_attn_func(query, key, value, 0.0, softmax_scale, True)
+ return output
+
+ batch_size, block_seq_len, q_heads, head_dim = query.shape
+ assert batch_size == 1, "Current implementation only supports batch size = 1"
+
+ local_process_group = DeviceGroup().get_group(process_group)
+ output = moba_zigzag_func(
+ query,
+ key,
+ value,
+ layer_idx,
+ seq_len,
+ moba_chunk_size,
+ moba_topk,
+ dropout,
+ softmax_scale,
+ True, # causal,
+ (-1, -1), # window_size,
+ None, # alibi_slopes,
+ False, # deterministic,
+ False, # return_softmax,
+ local_process_group, # group
+ ).contiguous()
+ return output.view(batch_size, block_seq_len, q_heads, head_dim)
+
+
+# --------------------------------------------------
+def moba_attn_anno(query_states, key_states, value_states, *args, **kwargs) -> str:
+ if query_states.shape[2] != key_states.shape[2]:
+ assert query_states.shape[2] % key_states.shape[2] == 0
+ group_size = query_states.shape[2] // key_states.shape[2]
+ assert query_states.shape[2] == value_states.shape[2] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ return f"b l^ {q_anno} hd^, b s^ {kv_anno} hd^, b s^ {kv_anno} vd^ -> b l^ {q_anno} vd^"
+
+
+def moba_zigzag_attn_anno(
+ query_states, key_states, value_states, *args, **kwargs
+) -> str:
+ num_q_heads, num_kv_heads = query_states.shape[2], key_states.shape[2]
+ if num_q_heads != num_kv_heads:
+ assert num_q_heads % num_kv_heads == 0
+ group_size = num_q_heads // num_kv_heads
+ assert num_q_heads == value_states.shape[2] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ attn_anno = (
+ f"b l {q_anno} hd^, b l {kv_anno} hd^, b l {kv_anno} vd^ -> b l {q_anno} vd^"
+ )
+ return attn_anno
+
+
+def emit_moba_zigzag(
+ node: IRFwOperation,
+ args: List[str],
+ kwargs: Dict[str, str],
+ runtime_devid: int,
+ plan_ndevs: int,
+ runtime_ndevs: int,
+) -> str:
+ """Special rule to generate zigzag_attn node"""
+
+ signature = node.signature
+
+ offset = (runtime_devid // plan_ndevs) * plan_ndevs
+ scale_unit_dev_ids = [local_rank + offset for local_rank in range(plan_ndevs)]
+
+ kw_pairs = list()
+ for key, val in kwargs.items():
+ code = f"{key}={val}"
+ kw_pairs.append(code)
+
+ sub_input = node.inputs()[0]
+ full_input = sub_input.parent
+ partition_dims = [
+ i for i, (s, f) in enumerate(zip(sub_input.shape, full_input.shape)) if s != f
+ ]
+ assert (
+ len(partition_dims) <= 1
+ ), f"support no more than one partition dim, but got {partition_dims}"
+ if not partition_dims:
+ kw_pairs.append("process_group=None")
+ else:
+ # if the 'process_group' is None, we will use the local attention (flash_attn_func)
+ if partition_dims[0] == 0: # partition on batch dim
+ # partition the bsz dim, use local flash_attn_func
+ kw_pairs.append("process_group=None")
+ elif partition_dims[0] == 1: # partition on sequence dim
+ # the synchronization should occur across scaleunits
+ kw_pairs.append(f"process_group={scale_unit_dev_ids}")
+ elif partition_dims[0] == 2:
+ # partition on num_head dim
+ kw_pairs.append("process_group=None")
+ else:
+ raise ValueError(f"unsupported partition dim: {partition_dims[0]}")
+
+ args = ", ".join(list(args) + kw_pairs)
+ return f"{signature}({args})"
+
+
+if __name__ != "__main__":
+ register_op(moba_attn_anno)(wrapped_moba_func)
+ register_op(moba_zigzag_attn_anno, emit_fn=emit_moba_zigzag)(
+ wrapped_moba_zigzag_func
+ )
diff --git a/mtraining/attn_funcs/utils.py b/mtraining/attn_funcs/utils.py
new file mode 100644
index 00000000..e00bdbca
--- /dev/null
+++ b/mtraining/attn_funcs/utils.py
@@ -0,0 +1,60 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import logging
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+from transformers.modeling_flash_attention_utils import _get_unpad_data
+from transformers.utils import is_flash_attn_2_available, logging
+
+if is_flash_attn_2_available():
+ from flash_attn.bert_padding import index_first_axis, unpad_input # noqa
+
+
+def nnscaler_upad_input(
+ query_layer, key_layer, value_layer, attention_mask, query_length
+):
+ indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+ _, _, num_heads, _ = query_layer.shape
+ batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+ key_layer = index_first_axis(
+ key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+ indices_k,
+ )
+ value_layer = index_first_axis(
+ value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim),
+ indices_k,
+ )
+ if query_length == kv_seq_len:
+ query_layer = index_first_axis(
+ query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k
+ )
+ cu_seqlens_q = cu_seqlens_k
+ max_seqlen_in_batch_q = max_seqlen_in_batch_k
+ indices_q = indices_k
+ elif query_length == 1:
+ max_seqlen_in_batch_q = 1
+ cu_seqlens_q = torch.arange(
+ batch_size + 1, dtype=torch.int32, device=query_layer.device
+ ) # There is a memcpy here, that is very bad.
+ indices_q = cu_seqlens_q[:-1]
+ query_layer = query_layer.squeeze(1)
+ else:
+ # The -q_len: slice assumes left padding.
+ attention_mask = attention_mask[:, -query_length:]
+ query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(
+ query_layer, attention_mask
+ )
+
+ return (
+ query_layer,
+ key_layer,
+ value_layer,
+ indices_q,
+ (cu_seqlens_q, cu_seqlens_k),
+ (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+ )
diff --git a/mtraining/attn_funcs/xattn_func.py b/mtraining/attn_funcs/xattn_func.py
new file mode 100644
index 00000000..6bf0ec5a
--- /dev/null
+++ b/mtraining/attn_funcs/xattn_func.py
@@ -0,0 +1,277 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import copy
+import os
+from functools import partial
+from typing import Any, Dict, List, Optional, Tuple
+
+import torch
+import torch.distributed as dist
+import yaml
+from flash_attn import flash_attn_func
+from nnscaler.graph.parser.register import register_op
+from nnscaler.ir import IRTensor
+from nnscaler.ir.operator import IRFwOperation
+from nnscaler.runtime.device import DeviceGroup
+from torch import Tensor
+
+from minference.dist_ops.xattn_zigzag import xattn_zigzag_func
+from minference.ops.xattention_fa import xattn_flash_attn_func
+
+
+def xattn_attention_forward(
+ module: torch.nn.Module,
+ query: torch.Tensor, # [B, H, N, D]
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attention_mask: Optional[torch.Tensor],
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ softcap: Optional[float] = None,
+ **kwargs,
+) -> Tuple[torch.Tensor, None]:
+ granularity, xattn_params = module.granularity, module.xattn_params
+ implementation = module.implementation
+ layer_idx = module.layer_idx
+
+ if implementation == "default":
+ head_indices = torch.arange(
+ module.config.num_attention_heads, device=query.device
+ )
+ return wrapped_xattn_func_(
+ query.transpose(1, 2),
+ key.transpose(1, 2),
+ value.transpose(1, 2),
+ head_indices,
+ granularity,
+ xattn_params,
+ dropout,
+ scaling,
+ sliding_window,
+ )
+ elif implementation == "zigzag":
+ return wrapped_xattn_zigzag_func_(
+ query.transpose(1, 2),
+ key.transpose(1, 2),
+ value.transpose(1, 2),
+ layer_idx,
+ granularity,
+ xattn_params,
+ dropout=dropout,
+ scaling=scaling,
+ sliding_window=sliding_window,
+ )
+ else:
+ raise NotImplementedError(
+ f"Unsupported implementation for xattn_attention_forward: {implementation}"
+ )
+
+
+# ------------------------------------------
+# Non-CP version
+def wrapped_xattn_func_(
+ q: Tensor,
+ k: Tensor,
+ v: Tensor, # [B, N, H, D]
+ head_indices: torch.Tensor,
+ granularity: int,
+ xattn_params: Dict[str, Any],
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+):
+ return (
+ wrapped_xattn_func(
+ q,
+ k,
+ v,
+ head_indices,
+ granularity,
+ xattn_params,
+ dropout,
+ scaling,
+ sliding_window,
+ ),
+ None,
+ )
+
+
+def wrapped_xattn_func(
+ q: Tensor,
+ k: Tensor,
+ v: Tensor,
+ head_indices: torch.Tensor,
+ granularity: int,
+ xattn_params: Dict[str, Any],
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+):
+ sliding_window = -1 if sliding_window is None else sliding_window
+ return xattn_flash_attn_func(
+ q,
+ k,
+ v,
+ head_indices.detach().cpu().numpy().tolist(),
+ xattn_params,
+ granularity,
+ dropout_p=dropout,
+ softmax_scale=scaling,
+ causal=True,
+ window_size=(sliding_window, sliding_window),
+ alibi_slopes=None,
+ deterministic=False,
+ )
+
+
+# ------------------------------------------
+# Zigzag Version
+def wrapped_xattn_zigzag_func_(
+ q: Tensor,
+ k: Tensor,
+ v: Tensor, # [B, N, H, D]
+ layer_idx: int,
+ granularity: int,
+ xattn_params: Dict[str, Any],
+ causal: bool = True,
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ process_group: Optional[dist.ProcessGroup] = None,
+):
+ return (
+ wrapped_xattn_zigzag_func(
+ q,
+ k,
+ v,
+ layer_idx,
+ granularity,
+ xattn_params,
+ causal=causal,
+ dropout=dropout,
+ scaling=scaling,
+ sliding_window=sliding_window,
+ ),
+ None,
+ )
+
+
+def wrapped_xattn_zigzag_func(
+ q: Tensor,
+ k: Tensor,
+ v: Tensor, # [B, N, H, D]
+ layer_idx: int,
+ granularity: int,
+ xattn_params: Dict[str, Any],
+ causal: bool = True,
+ dropout: float = 0.0,
+ scaling: Optional[float] = None,
+ sliding_window: Optional[int] = None,
+ process_group: Tuple[int] = None,
+):
+ if process_group is None or len(process_group) == 1:
+ # there is an additional checker for the `scaling`, which is equivalent
+ # to the behavior of the original flash_attn_func.
+ if scaling is None:
+ scaling = q.shape[-1] ** (-0.5)
+ output = flash_attn_func(q, k, v, 0.0, scaling, causal)
+ return output
+
+ group = DeviceGroup().get_group(process_group)
+
+ xattn_params = copy.copy(xattn_params)
+ xattn_params.pop("chunk_size", None)
+ return xattn_zigzag_func(
+ q,
+ k,
+ v,
+ layer_idx,
+ xattn_params,
+ granularity,
+ dropout_p=dropout,
+ softmax_scale=scaling,
+ causal=causal,
+ group=group,
+ ).contiguous()
+
+
+def xattn_attn_anno(query_states, key_states, value_states, *args, **kwargs) -> str:
+ if query_states.shape[2] != key_states.shape[2]:
+ assert query_states.shape[2] % key_states.shape[2] == 0
+ group_size = query_states.shape[2] // key_states.shape[2]
+ assert query_states.shape[2] == value_states.shape[2] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ return f"b l^ {q_anno} hd^, b s^ {kv_anno} hd^, b s^ {kv_anno} vd^, {q_anno} -> b l^ {q_anno} vd^"
+
+
+def emit_xattn_zigzag(
+ node: IRFwOperation,
+ args: List[str],
+ kwargs: Dict[str, str],
+ runtime_devid: int,
+ plan_ndevs: int,
+ runtime_ndevs: int,
+) -> str:
+ """Special rule to generate zigzag_attn node"""
+
+ signature = node.signature
+
+ offset = (runtime_devid // plan_ndevs) * plan_ndevs
+ scale_unit_dev_ids = [local_rank + offset for local_rank in range(plan_ndevs)]
+
+ kw_pairs = list()
+ for key, val in kwargs.items():
+ code = f"{key}={val}"
+ kw_pairs.append(code)
+
+ sub_input = node.inputs()[0]
+ full_input = sub_input.parent
+ partition_dims = [
+ i for i, (s, f) in enumerate(zip(sub_input.shape, full_input.shape)) if s != f
+ ]
+ assert (
+ len(partition_dims) <= 1
+ ), f"support no more than one partition dim, but got {partition_dims}"
+ if not partition_dims:
+ kw_pairs.append("process_group=None")
+ else:
+ # if the 'process_group' is None, we will use the local attention (flash_attn_func)
+ if partition_dims[0] == 0: # partition on batch dim
+ # partition the bsz dim, use local flash_attn_func
+ kw_pairs.append("process_group=None")
+ elif partition_dims[0] == 1: # partition on sequence dim
+ # the synchronization should occur across scaleunits
+ kw_pairs.append(f"process_group={scale_unit_dev_ids}")
+ elif partition_dims[0] == 2:
+ # partition on num_head dim
+ kw_pairs.append("process_group=None")
+ else:
+ raise ValueError(f"unsupported partition dim: {partition_dims[0]}")
+
+ args = ", ".join(list(args) + kw_pairs)
+ return f"{signature}({args})"
+
+
+def xattn_zigzag_anno(query_states, key_states, value_states, *args, **kwargs) -> str:
+ if query_states.shape[2] != key_states.shape[2]:
+ assert query_states.shape[2] % key_states.shape[2] == 0
+ group_size = query_states.shape[2] // key_states.shape[2]
+
+ assert query_states.shape[2] == value_states.shape[2] * group_size
+ q_anno = f"(group_num {group_size})"
+ kv_anno = "group_num"
+ else:
+ q_anno = kv_anno = "num_heads"
+
+ return f"b l {q_anno} hd^, b l {kv_anno} hd^, b l {kv_anno} vd^ -> b l {q_anno} vd^"
+
+
+if __name__ != "__main__":
+ register_op(xattn_attn_anno)(wrapped_xattn_func)
+ register_op(xattn_zigzag_anno, emit_fn=emit_xattn_zigzag)(wrapped_xattn_zigzag_func)
diff --git a/mtraining/custom_parallel.py b/mtraining/custom_parallel.py
new file mode 100644
index 00000000..3b006b7d
--- /dev/null
+++ b/mtraining/custom_parallel.py
@@ -0,0 +1,643 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import inspect
+import os
+import shutil
+from pathlib import Path
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ List,
+ Literal,
+ Optional,
+ Set,
+ Tuple,
+ Type,
+ TypeVar,
+ Union,
+)
+
+import torch
+import torch.distributed as dist
+from nnscaler.autodist.apis import parallelize_graph
+from nnscaler.autodist.autodist_config import AutoDistConfig
+from nnscaler.autodist.util import get_default_profile_path
+from nnscaler.graph import IRGraph
+from nnscaler.graph.parser import FxModuleParser
+from nnscaler.parallel import (
+ _FORWARD_ARGS_DUMP_FILE,
+ _GENCODE_FILE_TEMPLATE,
+ _GRAPH_DUMP_FILE,
+ _PREDEFINED_POLICIES,
+ BroadcastGenFilesStrategy,
+ ComputeConfig,
+ RegenStatus,
+ ReuseType,
+ _broadcast_gen_files,
+ _clean_files,
+ _compile_flags,
+ _gencode,
+ _is_any_gencode_loaded,
+ _load_parallel_module_class,
+ _prepare_namespace,
+)
+from nnscaler.runtime.device import DeviceGroup
+from nnscaler.runtime.module import (
+ AttrMeta,
+ CubeModule,
+ ExtraState,
+ OriginModuleMetadata,
+ ParallelModule,
+)
+
+_CUSTOM_PREDEFINED_POLICIES: Dict[
+ str, Callable[[IRGraph, "ComputeConfig"], IRGraph]
+] = {}
+for k, v in _PREDEFINED_POLICIES.items():
+ if k != "autodist":
+ _CUSTOM_PREDEFINED_POLICIES[k] = v
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def pas_autodist(graph: IRGraph, cfg: "ComputeConfig") -> IRGraph:
+ print(f"{__name__} | Using custom autodist policy defined in {__file__}")
+ pas_cfg = cfg.pas_config
+
+ update_freq = pas_cfg.get("update_freq", 1)
+ if isinstance(update_freq, (tuple, list)):
+ update_freq = update_freq[0]
+
+ # optional parameters
+ explore_pipeline = pas_cfg.get("explore_pipeline", False)
+ if explore_pipeline and not cfg.use_end2end:
+ raise ValueError("explore_pipeline cannot be enabled if use_end2end is False")
+ if explore_pipeline and cfg.use_async_reducer:
+ raise ValueError(
+ "explore_pipeline cannot be enabled if use_async_reducer is True"
+ )
+
+ pipeline_scheduler = pas_cfg.get("pipeline_scheduler", "1f1b")
+ if pipeline_scheduler != "1f1b":
+ raise ValueError(f"Only 1f1b scheduler is supported in autodist.")
+
+ mesh_col = pas_cfg.get("max_partition_degree", cfg.plan_ngpus)
+ if cfg.plan_ngpus % mesh_col != 0:
+ raise ValueError(
+ f"plan_ngpus {cfg.plan_ngpus} should be divisible by max_partition_degree {mesh_col}"
+ )
+ mesh_row = cfg.plan_ngpus // mesh_col
+ if not explore_pipeline and mesh_row != 1:
+ raise ValueError("mesh_row should be 1 if pipeline is not enabled")
+ memory_constraint = pas_cfg.get("mem_constraint", -1)
+ task_name = pas_cfg.get("task_name", "_")
+ use_memory_efficient_fp16 = pas_cfg.get("use_memory_efficient_fp16", False)
+ use_memory_efficient_bf16 = pas_cfg.get("use_memory_efficient_bf16", False)
+ use_fp16 = pas_cfg.get("use_fp16", use_memory_efficient_fp16)
+ use_bf16 = pas_cfg.get("use_bf16", use_memory_efficient_bf16)
+ re_profile = pas_cfg.get("re_profile", False)
+ verbose = pas_cfg.get("verbose", False)
+ load_plan_path = pas_cfg.get("load_plan_path", None)
+ save_plan_path = pas_cfg.get("save_plan_path", None)
+ partition_constraints_path = pas_cfg.get("partition_constraints_path", "")
+ recompute_modules = pas_cfg.get("recompute_modules", "")
+ pipeline_pivots = pas_cfg.get("pipeline_pivots", "")
+ use_apex_fused_adam_v2 = pas_cfg.get("use_apex_fused_adam_v2", False)
+ parallel_profile = pas_cfg.get("parallel_profile", True)
+ transient_mem_coef = pas_cfg.get("transient_mem_coef", 2)
+ profile_dir = pas_cfg.get("profile_dir", get_default_profile_path())
+ solver = pas_cfg.get("solver", "dp")
+
+ task_name = f"{task_name}_{cfg.plan_ngpus}gpus_{update_freq}update_freq"
+ if memory_constraint == -1:
+ # consider memory fragmentation and other buffers, use 80% of the memory
+ memory_constraint = int(0.8 * torch.cuda.mem_get_info()[1] / 1024 / 1024 / 1024)
+ if cfg.use_zero:
+ zero_stage = 1
+ zero_ngroups = cfg.zero_ngroups
+ else:
+ zero_stage = 0
+ zero_ngroups = 1
+ if use_fp16 or use_bf16:
+ support_inkernel_cast = use_apex_fused_adam_v2
+ if use_memory_efficient_fp16 or use_memory_efficient_bf16:
+ # Check fairseq/optim/fused_adam.py
+ # If memory efficient:
+ # Considered in opt_resident_mem: fp32 moment1, fp32 moment2.
+ # Considered in opt_transient_mem: fp32 weight, fp32 gradient,
+ # because fp16 weight and gradient are casted to fp32.
+ # Here weight_mem is in fp16, so multiply by (2+2).
+ opt_resident_coef = 4
+ opt_transient_coef = 0 if support_inkernel_cast else 4
+ else:
+ # If not memory efficient:
+ # Considered in opt_resident_mem: fp32 moment1, fp32 moment2, fp32 weight.
+ # Considered in opt_transient_mem: fp32 gradient,
+ # because fp16 gradient are casted to fp32.
+ # Here weight_mem is in fp16, so multiply by (2+2+2).
+ opt_resident_coef = 6
+ # inkernel cast between fp32 weight and fp16 grad has not support
+ opt_transient_coef = 2 if support_inkernel_cast else 2
+ else:
+ # Considered in opt_resident_mem: fp32 moment1, fp32 moment2
+ # Considered in opt_transient_mem: 0
+ # Here weight_mem is in fp32, so multiply by (1+1).
+ opt_resident_coef = 2
+ opt_transient_coef = 0
+
+ autodist_cfg = AutoDistConfig(
+ mesh_row=mesh_row,
+ mesh_col=mesh_col,
+ update_freq=update_freq,
+ task_name=task_name,
+ profile_dir=profile_dir,
+ is_train=not cfg.inference_only,
+ ignore_small_tensor_threshold=524288, # 0.5 MB is a good threshold to reduce search time and make the result correct, will refine later
+ memory_granularity=524288, # 0.5 MB is a good threshold to reduce search time and make the result correct, will refine later
+ consider_mem=True,
+ partition_constraints_path=partition_constraints_path,
+ memory_constraint=memory_constraint,
+ opt_resident_coef=opt_resident_coef,
+ opt_transient_coef=opt_transient_coef,
+ verbose=verbose,
+ re_profile=re_profile,
+ world_size=cfg.runtime_ngpus,
+ recompute_modules=recompute_modules,
+ zero_stage=zero_stage,
+ zero_ngroups=zero_ngroups,
+ load_plan_path=load_plan_path,
+ save_plan_path=save_plan_path,
+ pipeline=explore_pipeline,
+ pipeline_pivots=pipeline_pivots,
+ parallel_profile=parallel_profile,
+ transient_mem_coef=transient_mem_coef,
+ solver=solver,
+ )
+ logger.info(f"{__name__} | Using autodist config: {autodist_cfg}")
+
+ return parallelize_graph(graph, autodist_cfg)
+
+
+_CUSTOM_PREDEFINED_POLICIES["autodist"] = pas_autodist
+
+
+def compute_config_safe_equals(
+ a: Optional["ComputeConfig"], b: Optional["ComputeConfig"]
+) -> bool:
+ """
+ Return False if a and b are from incompatible version of ComputeConfig
+ This is only for backward compatibility, and will be removed in future
+ and can use `==` when we save dict version of ComputeConfig to file.
+ """
+ res = True
+ try:
+ for key in a.__dataclass_fields__:
+ if getattr(a, key) != getattr(b, key):
+ print(
+ f"{key} not equal: {getattr(a, key)} (old_config) != {getattr(b, key)} (current_config)"
+ )
+
+ if key == "user_config":
+ continue
+ else:
+ print(
+ f"compute_config_safe_equals | {key} not equal: {getattr(a, key)} (old_config) != {getattr(b, key)} (current_config)"
+ )
+ res = False
+ return res
+ except AttributeError:
+ logger.warning(
+ f"compute_config_safe_equals | Failed to compare ComputeConfig. They are incompatible."
+ f"Old config: {a}\n"
+ f"New config: {b}\n"
+ )
+ return False
+
+
+GRAPH_CONFIG_FIELDS = [
+ "constant_folding",
+ "user_config",
+ "inference_only",
+ "end2end_mode",
+ "trace_strategy",
+]
+
+
+def graph_config_equals(a: Dict[str, Any], b: Dict[str, Any]) -> bool:
+ """
+ Return False if a and b are from incompatible version of ComputeConfig
+ This is only for backward compatibility, and will be removed in future
+ and can use `==` when we save dict version of ComputeConfig to file.
+ """
+ res = True
+ try:
+ for key in GRAPH_CONFIG_FIELDS:
+ if a[key] != b[key]:
+ print(
+ f"graph_config_equals | {key} not equal: {getattr(a, key)} (old_config) != {getattr(b, key)} (current_config)"
+ )
+ if key != "user_config":
+ res = False
+ return res
+ except KeyError as e:
+ import traceback
+
+ logger.warning(
+ "graph_config_equals | Failed to compare GraphConfig with exception.\n"
+ f"Exception: {traceback.format_exc()}\n"
+ f"Old config: {a}\n"
+ f"New config: {b}\n"
+ )
+ return False
+
+
+TRACE_FILE_EXTENSIONS = [
+ FxModuleParser.ATTR_CONTENT_FILE_0, # init weights file(fullmodel.pt.*),
+ FxModuleParser.ATTR_MAP_FILE, # param name mapping (dist_param_map.pt)\
+ _GRAPH_DUMP_FILE, # graph dump (graph.ckp),
+ _FORWARD_ARGS_DUMP_FILE, # forward args dump(forward_args.pkl),
+ ParallelModule.ORIGIN_MODULE_METADATA_FILE, # origin module metadata (origin_module_metadata.pt),
+]
+
+
+def transfer_metadata(out_dir, transfer_config: Dict[str, Any]):
+ transfer_config_dir, transfer_force = (
+ transfer_config["transfer_config_dir"],
+ transfer_config["transfer_force"],
+ )
+ if not os.path.exists(transfer_config_dir):
+ # if transfer_config_dir is not set, use the default directory
+ transfer_config_dir = transfer_config_dir.replace(
+ "compile_config/", "compile_config/rank_0/"
+ )
+ assert os.path.exists(
+ transfer_config_dir
+ ), f"Source directory {transfer_config_dir} for transferring does not exist"
+
+ for file in os.listdir(transfer_config_dir):
+ if file in TRACE_FILE_EXTENSIONS or file.startswith(
+ FxModuleParser.ATTR_CONTENT_FILE_STEM
+ ):
+ src_file = os.path.join(transfer_config_dir, file)
+ dst_file = os.path.join(out_dir, file)
+
+ print(
+ f"{__name__} | Copying {src_file} to {dst_file} (local_rank={os.getenv('LOCAL_RANK')})"
+ )
+ if not os.path.exists(dst_file) or transfer_force:
+ shutil.copyfile(src_file, dst_file)
+
+ if not os.path.exists(dst_file):
+ raise FileNotFoundError(
+ f"{__name__} | Copy failed ({dst_file} does not exist after copying)"
+ )
+
+ # Create a file 'transferred.sign' to indicate that the transfer is done
+ with open(os.path.join(out_dir, "transferred.sign"), "w") as f:
+ f.write("Transferred from " + transfer_config_dir)
+
+
+def _prepare_and_check_reusable(
+ gen_savedir: str,
+ module_or_module_class: Union[Type[torch.nn.Module], torch.nn.Module],
+ compute_config: ComputeConfig,
+ instance_name: Optional[str] = None,
+ reuse: ReuseType = ReuseType.MATCH,
+ transfer_config: Dict[str, Any] = None,
+) -> Tuple[str, bool, bool]:
+ """
+ Prepare the output directory for code generation, and also check if the existing code is reusable.
+
+ Args:
+ gen_savedir (str): the directory to save generated code
+ module_or_module_class (Union[Type[torch.nn.Module], torch.nn.Module]): the original module or module class
+ compute_config (ComputeConfig): the environment resource
+ instance_name (Optional[str]): the instance name of the generated module. If it is None, will use the default name.
+ reuse (ReuseType): specify which part can be reused.
+
+ Returns:
+ Tuple[str, bool]: the output directory and whether the existing code is reusable.
+
+ Raises:
+ RuntimeError: if the existing code is not reusable,
+ will raise RuntimeError if the code is not reusable but the module is already loaded.
+ """
+ namespace, outdir = _prepare_namespace(
+ gen_savedir, module_or_module_class, instance_name
+ )
+ reusable = False
+ transferred = False
+
+ config_file = outdir / ParallelModule.COMPUTE_CONFIG_FILE
+
+ # Empty + Transfer -> config match, graph match, tracing file present -> generate code by MATCH or MOO
+ # Empty w.o. Transfer -> Empty -> generate code by MATCH or MOO
+ has_transferred = os.path.exists(os.path.join(outdir, "transferred.sign"))
+ if (
+ transfer_config is not None
+ and transfer_config.get("transfer_config_dir", None) is not None
+ and (not has_transferred or transfer_config["transfer_force"])
+ ):
+ # transfer_config_dir: Optional[str] = None,
+ transfer_metadata(outdir, transfer_config)
+ ComputeConfig.safe_dump_to_file(compute_config, config_file)
+ transferred = True
+
+ # decision matrix for code generation
+ # reuse flag | dir condition(imported, empty, match, unmatched) | action
+ # ---------------------------------------------------------
+ # OVERRIDE | empty | generate
+ # OVERRIDE | imported | raise error
+ # OVERRIDE | whatever match | generate
+ # OVERRIDE | unmatch | generate
+ # GRAPH | empty | generate
+ # GRAPH | imported | raise error
+ # GRAPH | graph match | reuse graph, and regenerate code
+ # GRAPH | all match | reuse graph, and regenerate code
+ # GRAPH | unmatch | generate
+ # MATCH | empty | generate
+ # MATCH | match | reuse(do nothing)
+ # MATCH* | whatever unmatch| raise error (except when there's no python source code, see below)
+ # MATCH | imported | doesn't matter
+ # MOO | empty | generate
+ # MOO | match | reuse(do nothing)
+ # MOO | match graph | reuse graph, and regenerate code
+ # MOO | imported | raise error if whatever unmatch
+ # *: The precondition for `except` part is the compute config should match.
+ # you can take it as a continous operation after a failed generation.
+ old_config: Optional[ComputeConfig] = ComputeConfig.safe_load_from_file(config_file)
+ is_config_match = compute_config_safe_equals(old_config, compute_config)
+ is_graph_config_match = old_config is not None and graph_config_equals(
+ old_config.graph_config, compute_config.graph_config
+ )
+ trace_meta_files = [
+ outdir
+ / FxModuleParser.ATTR_CONTENT_FILE_0, # init weights file(fullmodel.pt.*),
+ outdir / FxModuleParser.ATTR_MAP_FILE, # param name mapping (dist_param_map.pt)
+ ]
+
+ if reuse == ReuseType.MATCH or reuse == ReuseType.MOO:
+ # check if the module is already generated
+ expected_output_files = [
+ outdir / _GENCODE_FILE_TEMPLATE.format(rank)
+ for rank in range(compute_config.runtime_ngpus)
+ ]
+ expected_output_files.extend(trace_meta_files)
+ expected_output_files.append(config_file)
+ expected_output_files.append(
+ outdir / _GRAPH_DUMP_FILE
+ ) # graph dump (graph.ckp),
+ expected_output_files.append(
+ outdir / _FORWARD_ARGS_DUMP_FILE
+ ) # forward args dump(forward_args.pkl),
+ expected_output_files.append(
+ outdir / ParallelModule.ORIGIN_MODULE_METADATA_FILE
+ ) # origin module metadata (origin_module_metadata.pt),
+ existing_output_files = [
+ f
+ for f in outdir.glob("*")
+ if f.is_file()
+ and ( # just take fullmodel.pt.0 to compare
+ not f.name.startswith(FxModuleParser.ATTR_CONTENT_FILE_STEM)
+ or f.name == FxModuleParser.ATTR_CONTENT_FILE_0
+ )
+ and not f.name.endswith(".sign")
+ ]
+
+ print(f"{__name__} | compute config match: {is_config_match}")
+ print(f"{__name__} | graph config match: {is_graph_config_match}")
+ print(f"{__name__} | existing output files: {existing_output_files}")
+ print(f"{__name__} | expected output files: {expected_output_files}")
+
+ if existing_output_files: # if the directory is not empty
+ if (
+ is_config_match
+ and all([output_file.exists() for output_file in expected_output_files])
+ and len(existing_output_files) == len(expected_output_files)
+ ):
+ print(f"{__name__} | Reuse existing files in {outdir}")
+ reusable = True # everything is matched.
+ elif is_config_match and all(
+ f.suffix != ".py" for f in existing_output_files
+ ):
+ # No python source code is generated.
+ # which means its last generation failed.
+ # in this case, we can reuse the same directory safely.
+ logger.info(
+ f"Output directory {outdir} is not empty. "
+ f"But no python source code is present. "
+ f"Will reuse the directory and the graph dump if present."
+ )
+ # we have to trace the graph again if not all meta files are present.
+ print(
+ f"{__name__} | compute config match but no python code exists in {outdir}"
+ )
+ if not all([meta_file.exists() for meta_file in trace_meta_files]):
+ print(
+ f"{__name__} | compute config match but no python code exists in {outdir} and not all meta files are present"
+ )
+ _clean_files(outdir)
+ elif reuse == ReuseType.MATCH:
+ raise RuntimeError(
+ f"Output directory {outdir} is not empty. "
+ f"And the existing files do not match with current config. "
+ f"You can remove the directory and try again, "
+ f"or set reuse to ReuseType.NONE/ReuseType.OVERRIDE to regenerate the code."
+ )
+ else:
+ assert reuse == ReuseType.MOO
+ if _is_any_gencode_loaded(namespace):
+ raise RuntimeError(
+ f"Output directory {outdir} is already loaded. "
+ f"You can not override a loaded module."
+ )
+ elif is_graph_config_match:
+ # reuse the graph dump
+ print(
+ f"{__name__} | MOO | graph match -> reuse graph but clean the current code"
+ )
+ _clean_files(outdir, "*.py")
+ else:
+ _clean_files(outdir)
+ else:
+ # check if the module is already loaded
+ if _is_any_gencode_loaded(namespace):
+ raise RuntimeError(
+ f"Output directory {outdir} is already loaded. "
+ f"You can not override a loaded module."
+ )
+ # clear existing generated files
+ if (
+ reuse == ReuseType.OVERRIDE
+ or not is_graph_config_match
+ or not all([meta_file.exists() for meta_file in trace_meta_files])
+ ):
+ # we have to trace the graph again if not all meta files are present even when reuse=graph.
+ print(f"{__name__} | OVERRIDE | Override existing files in {outdir}")
+ glob_pattern = "*"
+ else:
+ print(
+ f"{__name__} | GRAPH | keep the graph dump in {outdir} and regenerate the code"
+ )
+ glob_pattern = "*.py" # so we can keep graph dumps.
+ _clean_files(outdir, glob_pattern)
+
+ return outdir, reusable, transferred
+
+
+def parallelize(
+ module_or_module_class: Union[torch.nn.Module, Type[torch.nn.Module]],
+ dummy_forward_args: Dict[str, Any],
+ pas_policy: Union[str, Callable[[IRGraph, ComputeConfig], IRGraph]],
+ compute_config: ComputeConfig,
+ *,
+ gen_savedir: Union[str, Path] = "./.nnscaler",
+ reuse: Union[ReuseType, str] = ReuseType.MATCH,
+ instance_name: Optional[str] = None,
+ load_module: bool = True,
+ module_dtype: Optional[torch.dtype] = None,
+ module_fn: Optional[Callable[[], torch.nn.Module]] = None,
+ init_module_params: bool = True,
+ broadcast_strategy: Union[str, BroadcastGenFilesStrategy] = "none",
+ transfer_config: Optional[Dict[str, Any]] = None,
+ force_broadcast_all: bool = False,
+) -> Union[None, ParallelModule, Type[ParallelModule]]:
+ if isinstance(module_or_module_class, ParallelModule) or (
+ inspect.isclass(module_or_module_class)
+ and issubclass(module_or_module_class, ParallelModule)
+ ):
+ # already done
+ return module_or_module_class if load_module else None
+
+ if isinstance(module_or_module_class, CubeModule) or (
+ inspect.isclass(module_or_module_class)
+ and issubclass(module_or_module_class, CubeModule)
+ ):
+ raise RuntimeError("Old style CubeModule is not supported")
+
+ if isinstance(pas_policy, str):
+ if not pas_policy in _CUSTOM_PREDEFINED_POLICIES:
+ raise ValueError(f"Invalid pas_policy: {pas_policy}")
+ pas_policy = _CUSTOM_PREDEFINED_POLICIES[pas_policy]
+
+ is_module_class = inspect.isclass(module_or_module_class)
+ module_class = (
+ module_or_module_class if is_module_class else module_or_module_class.__class__
+ )
+ reuse = ReuseType(reuse) if isinstance(reuse, str) else reuse
+ broadcast_strategy = (
+ BroadcastGenFilesStrategy(broadcast_strategy)
+ if isinstance(broadcast_strategy, str)
+ else broadcast_strategy
+ )
+
+ # Call it here just to ensure the device group is initialized.
+ # If the user initializes dist
+ # and doesn't call `nnscaler.init()` before calling this function, this is necessary.
+ if dist.is_initialized():
+ _ = DeviceGroup()
+
+ # generate code only in node0
+ # if it is not in a torchrun environment, just generate.
+ if not dist.is_initialized() or dist.get_rank() == 0:
+ outdir, reusable, transferred = _prepare_and_check_reusable(
+ gen_savedir,
+ module_class,
+ compute_config,
+ instance_name,
+ reuse,
+ transfer_config,
+ )
+ if not reusable:
+ config_file = outdir / ParallelModule.COMPUTE_CONFIG_FILE
+ ComputeConfig.safe_dump_to_file(
+ compute_config, config_file
+ ) # always refresh compute config
+ with _compile_flags(compute_config):
+ regen_status = _gencode(
+ module_or_module_class,
+ dummy_forward_args,
+ pas_policy,
+ compute_config,
+ outdir,
+ module_dtype=module_dtype,
+ module_fn=module_fn,
+ )
+ else:
+ regen_status = RegenStatus.NONE
+ logger.info(f"Reuse generated code in {outdir}")
+
+ if regen_status == RegenStatus.CODE and transferred:
+ regen_status = RegenStatus.ALL
+
+ if dist.is_initialized():
+ # code generation can take very long time (for example, over 1 hour)
+ # It is not always OK to use dist.barrier() directly.
+ # because the default timeout for nccl is 30 minutes
+ # (we can't control the timeout setting if dist is not initialized by us)
+ DeviceGroup().long_barrier()
+
+ if broadcast_strategy != BroadcastGenFilesStrategy.NONE or force_broadcast_all:
+ if not dist.is_initialized(): # we only support loading in torchrun environment
+ raise RuntimeError(
+ "Broadcast generated files failed: dist is not initialized."
+ )
+ dist.barrier()
+ # sync regen_status
+ curr_rank = dist.get_rank()
+ if curr_rank == 0:
+ sent_obj = [regen_status]
+ else:
+ sent_obj = [None]
+ dist.broadcast_object_list(
+ sent_obj,
+ src=0,
+ )
+ if curr_rank != 0:
+ regen_status = sent_obj[0]
+
+ # narrow down broadcast_strategy according to regen_status
+ if force_broadcast_all:
+ logger.info(f"Force broadcast all generated files in {gen_savedir}")
+ broadcast_strategy = BroadcastGenFilesStrategy.ALL
+ elif regen_status == RegenStatus.NONE:
+ # we don't need to broadcast anything
+ broadcast_strategy = BroadcastGenFilesStrategy.NONE
+ elif regen_status == RegenStatus.CODE:
+ # narrow ALL/NO_WEIGHTS down to code
+ broadcast_strategy = BroadcastGenFilesStrategy.CODE
+ else:
+ # we don't need to narrow broadcast_strategy in this case
+ # keep the original broadcast_strategy
+ assert regen_status == RegenStatus.ALL
+
+ # broadcast generated files according to regen_status
+ if broadcast_strategy != BroadcastGenFilesStrategy.NONE:
+ _broadcast_gen_files(
+ module_class,
+ gen_savedir=gen_savedir,
+ instance_name=instance_name,
+ broadcast_strategy=broadcast_strategy,
+ )
+
+ if load_module:
+ if not dist.is_initialized(): # we only support loading in torchrun environment
+ raise RuntimeError("Load ParallelModule failed: dist is not initialized.")
+ dist.barrier()
+ parallel_module_class = _load_parallel_module_class(
+ module_class,
+ gen_savedir=gen_savedir,
+ instance_name=instance_name,
+ )
+ if is_module_class:
+ return parallel_module_class
+ else:
+ parallel_module = parallel_module_class(init_module_params)
+ parallel_module.train(
+ module_or_module_class.training
+ ) # set training state to the same as original module
+ return parallel_module
diff --git a/mtraining/experiments/scripts/launch_auto_merge.sh b/mtraining/experiments/scripts/launch_auto_merge.sh
new file mode 100755
index 00000000..91eb0784
--- /dev/null
+++ b/mtraining/experiments/scripts/launch_auto_merge.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/bash
+
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+i=$(hostname | awk -F'-' '{print $2}')
+NODE_RANK=$i
+export NUM_NODES=4
+export REUSE_TYPE="match"
+export FORCE_TRITON=1
+
+export HF_HOME=/scratch/hf_cache/huggingface
+mkdir -p $HF_HOME
+export HF_TRUST_REMOTE_CODE=true
+export HF_DATASETS_TRUST_REMOTE_CODE=true
+
+export MASTER_ADDR="node-0"
+export MASTER_PORT="12345"
+
+export NNSCALER_HOME="${HOME}/.conda/envs/ptca/lib/python3.10/site-packages/nnscaler/"
+export PYTHONPATH="${NNSCALER_HOME}:${PYTHONPATH}"
+
+# -----------------------------------------------
+# TODO: Basic Environment Settings
+SEQUENCE_LENGTH=524288
+export GPU_NAME=A100
+export GPU_PER_NODE=8
+export WORLD_SIZE=32
+export GPU_SET="${GPU_NAME}_${WORLD_SIZE}"
+
+export SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+export EXPR_HOME="$(cd "${SCRIPT_DIR}/../.." && pwd)" # .../mtraining
+export EXPR_DATA_STORE="/blob/mtrain_expr_data_store/${GPU_SET}"
+mkdir -p $EXPR_DATA_STORE
+cd $EXPR_HOME
+
+export EXPR_DIR="mtrain_qwen" # Name for the experiment set
+export EXPR_NAME="qwen_3B_fp090_512K" # Name for the single experiment run
+export MODEL_ID="Qwen/Qwen2.5-3B"
+
+# -----------------------------------------------
+export MERGE_CKPT_DIR="${EXPR_DATA_STORE}/${EXPR_DIR}/${EXPR_NAME}/merged_ckpts"
+mkdir -p $MERGE_CKPT_DIR
+
+export LOG_PATH="${MERGE_CKPT_DIR}/auto_merge.log"
+echo "log path: $LOG_PATH"
+
+python -m utils.auto_merge_ckpt \
+ --gpu_set ${GPU_NAME}_${WORLD_SIZE} \
+ --expr_dir $EXPR_DIR \
+ --expr_name $EXPR_NAME \
+ --model_id $MODEL_ID \
+ --num_gpus $WORLD_SIZE \
+ > $LOG_PATH 2>&1 &
diff --git a/mtraining/experiments/scripts/prolong_data_prepare.sh b/mtraining/experiments/scripts/prolong_data_prepare.sh
new file mode 100755
index 00000000..fb965d2f
--- /dev/null
+++ b/mtraining/experiments/scripts/prolong_data_prepare.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/bash
+
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+export HF_TRUST_REMOTE_CODE=1
+export HF_HOME=/scratch/hf_cache/huggingface
+export HF_TOKEN_PATH="/scratch/.hf_access_token"
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+BASE_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+PROJECT_ROOT="$(cd "${BASE_DIR}/.." && pwd)"
+
+# ------------------------------------------
+# Download data
+# Prerequisite: sudo apt-get install git-lfs && git lfs install
+RAW_DATASET_DIR="/scratch/datasets" # "/path/to/datasets"
+mkdir -p $RAW_DATASET_DIR
+
+
+# Check whether the data is already downloaded
+if [ -d $RAW_DATASET_DIR/long-context-524288 ]; then
+ echo "Data already downloaded"
+else
+ echo "Downloading data..."
+ git clone https://huggingface.co/datasets/princeton-nlp/prolong-data-512K $RAW_DATASET_DIR/long-context-524288
+ cd $RAW_DATASET_DIR/long-context-524288
+ git lfs fetch
+ git lfs checkout
+fi
+
+# ------------------------------------------
+# Data Processing
+cd $BASE_DIR
+MODEL_ID="Qwen/Qwen2.5-7B"
+PROCESSED_DATA_DIR="/scratch/datasets/processed_datasets"
+mkdir -p $PROCESSED_DATA_DIR
+
+torchrun --nproc_per_node=1 \
+ utils/data_utils/prolong.py \
+ --model_id $MODEL_ID \
+ --dataset_mix fixed_524288 \
+ --dataset_path $RAW_DATASET_DIR/long-context-524288 \
+ --save_path $PROCESSED_DATA_DIR/long-context-524288 \
+ --sample_interval 4
diff --git a/mtraining/experiments/scripts/train_dense_qwen2_3B_ProLong512K.sh b/mtraining/experiments/scripts/train_dense_qwen2_3B_ProLong512K.sh
new file mode 100755
index 00000000..429347a0
--- /dev/null
+++ b/mtraining/experiments/scripts/train_dense_qwen2_3B_ProLong512K.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/bash
+
+
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+i=$(hostname | awk -F'-' '{print $2}')
+NODE_RANK=$i
+export NUM_NODES=4
+export REUSE_TYPE="match"
+export FORCE_TRITON=1
+
+export HF_HOME=/scratch/hf_cache/huggingface
+mkdir -p $HF_HOME
+export HF_TRUST_REMOTE_CODE=true
+export HF_DATASETS_TRUST_REMOTE_CODE=true
+
+export MASTER_ADDR="node-0"
+export MASTER_PORT="12345"
+
+export NNSCALER_HOME="${HOME}/.conda/envs/ptca/lib/python3.10/site-packages/nnscaler/"
+export PYTHONPATH="${NNSCALER_HOME}:${PYTHONPATH}"
+
+# -----------------------------------------------
+# TODO: Basic Environment Settings
+SEQUENCE_LENGTH=524288
+export GPU_NAME=A100
+export GPU_PER_NODE=8
+export WORLD_SIZE=32
+export GPU_SET="${GPU_NAME}_${WORLD_SIZE}"
+
+export SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+export EXPR_HOME="$(cd "${SCRIPT_DIR}/../.." && pwd)" # .../mtraining
+export EXPR_DATA_STORE="/blob/mtrain_expr_data_store/${GPU_SET}"
+mkdir -p $EXPR_DATA_STORE
+cd $EXPR_HOME
+
+# ------------------------------------------
+export EXPR_DIR="dense_qwen" # Name for the experiment set
+export EXPR_NAME="qwen_3B_dense" # Name for the single experiment run
+export MODEL_ID="Qwen/Qwen2.5-3B"
+export DATASET_PATH="/scratch/datasets/processed_datasets/long-context-524288"
+export MODEL_CONFIG_PATH="${EXPR_HOME}/model_configs/qwen2/lc_config_3B"
+echo "Using model config path: $MODEL_CONFIG_PATH"
+TRANSFER_CONFIG_DIR="none"
+export TRAIN_ATTN_CONFIG_PATH="${EXPR_HOME}/train_attn_configs/qwen_flex_090.yaml"
+export ATTN_TYPE="zigzag_ring"
+
+# ------------------------------------------
+# Training Path settings
+export TF_LOG_PATH="$EXPR_DATA_STORE/$EXPR_DIR/tf_logs"
+export CKPT_PATH="$EXPR_DATA_STORE/$EXPR_DIR/$EXPR_NAME/checkpoints"
+export COMPILE_PATH="$EXPR_DATA_STORE/compile_config/rank_${NODE_RANK}"
+export PAS_PROFILE_DIR="$EXPR_DATA_STORE/$EXPR_DIR/$EXPR_NAME/pas_profile"
+mkdir -p $TF_LOG_PATH
+mkdir -p $CKPT_PATH
+mkdir -p $COMPILE_PATH
+mkdir -p $PAS_PROFILE_DIR
+
+# -------------------------------------------
+# Training Settings
+export SOLVER="dp"
+export TRACE_STRATEGY="reuse_cache"
+export FORCE_BROADCAST_ALL=0
+if [ "$FORCE_BROADCAST_ALL" -eq 1 ]; then
+ FORCE_BROADCAST_ALL_FLAG="--force_broadcast_all"
+else
+ FORCE_BROADCAST_ALL_FLAG=""
+fi
+
+export GLOBAL_BATCH_SIZE=64
+export MICRO_BATCH_SIZE=1
+export MEM_CONSTRAINT=40
+
+export NUM_ITER=0
+export NUM_EPOCH=1
+
+export CKPT_SAVE_STEP=1
+export CKPT_SAVE_EPOCH=0
+
+export CHECK_RESUME=0
+if [ "$CHECK_RESUME" -eq 1 ]; then
+ CHECK_RESUME="--check_resume"
+else
+ CHECK_RESUME=""
+fi
+
+# -------------------------------------------
+# Logging Path
+export LOG_PATH="${EXPR_DATA_STORE}/${EXPR_DIR}/${EXPR_NAME}/rank_${NODE_RANK}"
+mkdir -p $LOG_PATH
+max=0
+# Loop through matching files (if no match, the glob expands to itself, so check existence)
+for file in "$LOG_PATH"/train*.log; do
+ [ -e "$file" ] || continue # Skip if no file exists
+
+ base=$(basename "$file")
+ if [ "$base" = "train.log" ]; then
+ num=0
+ elif [[ "$base" =~ train_([0-9]+)\.log ]]; then
+ num="${BASH_REMATCH[1]}"
+ else
+ continue
+ fi
+
+ if (( num > max )); then
+ max=$num
+ fi
+done
+next=$((max + 1))
+echo "Logging directed to $LOG_PATH/train_${next}.log"
+
+torchrun --nproc_per_node=$GPU_PER_NODE \
+ --nnodes=$NUM_NODES \
+ --node_rank=$NODE_RANK \
+ --master_addr=$MASTER_ADDR \
+ --master_port=$MASTER_PORT \
+ train.py --plan_ngpus $WORLD_SIZE \
+ --runtime_ngpus $WORLD_SIZE \
+ --name $EXPR_NAME \
+ --seq_len $SEQUENCE_LENGTH \
+ --attn_type $ATTN_TYPE \
+ --solver $SOLVER \
+ --train_attn_config_path $TRAIN_ATTN_CONFIG_PATH \
+ --reuse_type $REUSE_TYPE \
+ --model_id $MODEL_ID \
+ --n_iter $NUM_ITER \
+ --n_epochs $NUM_EPOCH \
+ --global_batch_size $GLOBAL_BATCH_SIZE \
+ --micro_batch_size $MICRO_BATCH_SIZE \
+ --pas_profile_dir $PAS_PROFILE_DIR \
+ --dataset_path $DATASET_PATH \
+ --compile_save_path $COMPILE_PATH \
+ --tf_log_dir $TF_LOG_PATH \
+ --model_config_path $MODEL_CONFIG_PATH \
+ --ckpt_save_dir $CKPT_PATH \
+ --ckpt_n_step $CKPT_SAVE_STEP \
+ --ckpt_n_epoch $CKPT_SAVE_EPOCH \
+ --trace_strategy $TRACE_STRATEGY \
+ --transfer_config_dir $TRANSFER_CONFIG_DIR \
+ --mem_constraint $MEM_CONSTRAINT \
+ $FORCE_BROADCAST_ALL_FLAG \
+ $CHECK_RESUME > $LOG_PATH/train_${next}.log 2>&1
+echo "Log saved to $LOG_PATH/train_${next}.log"
diff --git a/mtraining/experiments/scripts/train_qwen2_3B_ProLong512K.sh b/mtraining/experiments/scripts/train_qwen2_3B_ProLong512K.sh
new file mode 100755
index 00000000..d2223664
--- /dev/null
+++ b/mtraining/experiments/scripts/train_qwen2_3B_ProLong512K.sh
@@ -0,0 +1,144 @@
+#!/usr/bin/bash
+
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+i=$(hostname | awk -F'-' '{print $2}')
+NODE_RANK=$i
+export NUM_NODES=4
+export REUSE_TYPE="match"
+export FORCE_TRITON=1
+
+export HF_HOME=/scratch/hf_cache/huggingface
+mkdir -p $HF_HOME
+export HF_TRUST_REMOTE_CODE=true
+export HF_DATASETS_TRUST_REMOTE_CODE=true
+
+export MASTER_ADDR="node-0"
+export MASTER_PORT="12345"
+
+export NNSCALER_HOME="${HOME}/.conda/envs/ptca/lib/python3.10/site-packages/nnscaler/"
+export PYTHONPATH="${NNSCALER_HOME}:${PYTHONPATH}"
+
+# -----------------------------------------------
+# TODO: Basic Environment Settings
+SEQUENCE_LENGTH=524288
+export GPU_NAME=A100
+export GPU_PER_NODE=8
+export WORLD_SIZE=32
+export GPU_SET="${GPU_NAME}_${WORLD_SIZE}"
+
+export SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+export EXPR_HOME="$(cd "${SCRIPT_DIR}/../.." && pwd)" # .../mtraining
+export EXPR_DATA_STORE="/blob/mtrain_expr_data_store/${GPU_SET}"
+mkdir -p $EXPR_DATA_STORE
+cd $EXPR_HOME
+
+# ------------------------------------------
+export EXPR_DIR="mtrain_qwen" # Name for the experiment set
+export EXPR_NAME="qwen_3B_fp090_512K" # Name for the single experiment run
+export MODEL_ID="Qwen/Qwen2.5-3B"
+export DATASET_PATH="/scratch/datasets/processed_datasets/long-context-524288"
+export MODEL_CONFIG_PATH="${EXPR_HOME}/model_configs/qwen2/lc_config_3B"
+echo "Using model config path: $MODEL_CONFIG_PATH"
+TRANSFER_CONFIG_DIR="none"
+export TRAIN_ATTN_CONFIG_PATH="${EXPR_HOME}/train_attn_configs/qwen_flex_090.yaml"
+export ATTN_TYPE="minfer"
+
+# ------------------------------------------
+# Training Path settings
+export TF_LOG_PATH="$EXPR_DATA_STORE/$EXPR_DIR/tf_logs"
+export CKPT_PATH="$EXPR_DATA_STORE/$EXPR_DIR/$EXPR_NAME/checkpoints"
+export COMPILE_PATH="$EXPR_DATA_STORE/compile_config/rank_${NODE_RANK}"
+export PAS_PROFILE_DIR="$EXPR_DATA_STORE/$EXPR_DIR/$EXPR_NAME/pas_profile"
+mkdir -p $TF_LOG_PATH
+mkdir -p $CKPT_PATH
+mkdir -p $COMPILE_PATH
+mkdir -p $PAS_PROFILE_DIR
+
+# -------------------------------------------
+# Training Settings
+export SOLVER="dp"
+export TRACE_STRATEGY="reuse_cache"
+export FORCE_BROADCAST_ALL=0
+if [ "$FORCE_BROADCAST_ALL" -eq 1 ]; then
+ FORCE_BROADCAST_ALL_FLAG="--force_broadcast_all"
+else
+ FORCE_BROADCAST_ALL_FLAG=""
+fi
+
+export GLOBAL_BATCH_SIZE=64
+export MICRO_BATCH_SIZE=1
+export MEM_CONSTRAINT=40
+
+export NUM_ITER=0
+export NUM_EPOCH=1
+
+export CKPT_SAVE_STEP=1
+export CKPT_SAVE_EPOCH=0
+
+export CHECK_RESUME=0
+if [ "$CHECK_RESUME" -eq 1 ]; then
+ CHECK_RESUME="--check_resume"
+else
+ CHECK_RESUME=""
+fi
+
+# -------------------------------------------
+# Logging Path
+
+export LOG_PATH="${EXPR_DATA_STORE}/${EXPR_DIR}/${EXPR_NAME}/rank_${NODE_RANK}"
+mkdir -p $LOG_PATH
+max=0
+# Loop through matching files (if no match, the glob expands to itself, so check existence)
+for file in "$LOG_PATH"/train*.log; do
+ [ -e "$file" ] || continue # Skip if no file exists
+
+ base=$(basename "$file")
+ if [ "$base" = "train.log" ]; then
+ num=0
+ elif [[ "$base" =~ train_([0-9]+)\.log ]]; then
+ num="${BASH_REMATCH[1]}"
+ else
+ continue
+ fi
+
+ if (( num > max )); then
+ max=$num
+ fi
+done
+next=$((max + 1))
+echo "Logging directed to $LOG_PATH/train_${next}.log"
+
+torchrun --nproc_per_node=$GPU_PER_NODE \
+ --nnodes=$NUM_NODES \
+ --node_rank=$NODE_RANK \
+ --master_addr=$MASTER_ADDR \
+ --master_port=$MASTER_PORT \
+ train.py --plan_ngpus $WORLD_SIZE \
+ --runtime_ngpus $WORLD_SIZE \
+ --name $EXPR_NAME \
+ --seq_len $SEQUENCE_LENGTH \
+ --attn_type $ATTN_TYPE \
+ --solver $SOLVER \
+ --train_attn_config_path $TRAIN_ATTN_CONFIG_PATH \
+ --reuse_type $REUSE_TYPE \
+ --model_id $MODEL_ID \
+ --n_iter $NUM_ITER \
+ --n_epochs $NUM_EPOCH \
+ --global_batch_size $GLOBAL_BATCH_SIZE \
+ --micro_batch_size $MICRO_BATCH_SIZE \
+ --pas_profile_dir $PAS_PROFILE_DIR \
+ --dataset_path $DATASET_PATH \
+ --compile_save_path $COMPILE_PATH \
+ --tf_log_dir $TF_LOG_PATH \
+ --model_config_path $MODEL_CONFIG_PATH \
+ --ckpt_save_dir $CKPT_PATH \
+ --ckpt_n_step $CKPT_SAVE_STEP \
+ --ckpt_n_epoch $CKPT_SAVE_EPOCH \
+ --trace_strategy $TRACE_STRATEGY \
+ --transfer_config_dir $TRANSFER_CONFIG_DIR \
+ --mem_constraint $MEM_CONSTRAINT \
+ $FORCE_BROADCAST_ALL_FLAG \
+ $CHECK_RESUME > $LOG_PATH/train_${next}.log 2>&1
+echo "Log saved to $LOG_PATH/train_${next}.log"
diff --git a/mtraining/experiments/scripts/train_qwen_0_5B_ProLong512K.sh b/mtraining/experiments/scripts/train_qwen_0_5B_ProLong512K.sh
new file mode 100755
index 00000000..0cb714da
--- /dev/null
+++ b/mtraining/experiments/scripts/train_qwen_0_5B_ProLong512K.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/bash
+
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+
+i=$(hostname | awk -F'-' '{print $2}')
+NODE_RANK=$i
+export NUM_NODES=1
+export REUSE_TYPE="match"
+export FORCE_TRITON=1
+
+export HF_HOME=/scratch/hf_cache/huggingface
+mkdir -p $HF_HOME
+export HF_TRUST_REMOTE_CODE=true
+export HF_DATASETS_TRUST_REMOTE_CODE=true
+
+export MASTER_ADDR="node-0"
+export MASTER_PORT="12345"
+
+export NNSCALER_HOME="${HOME}/.conda/envs/mtrain/lib/python3.10/site-packages/nnscaler/"
+export PYTHONPATH="${NNSCALER_HOME}:${PYTHONPATH}"
+
+# -----------------------------------------------
+# TODO: Basic Environment Settings
+SEQUENCE_LENGTH=524288
+export GPU_NAME=A100
+export GPU_PER_NODE=8
+export WORLD_SIZE=8
+export GPU_SET="${GPU_NAME}_${WORLD_SIZE}"
+
+export SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+export EXPR_HOME="$(cd "${SCRIPT_DIR}/../.." && pwd)" # .../mtraining
+export EXPR_DATA_STORE="${EXPR_HOME}/experiments/expr_data_store/${GPU_SET}"
+mkdir -p $EXPR_DATA_STORE
+cd $EXPR_HOME
+
+# ------------------------------------------
+export EXPR_DIR="mtrain_qwen" # Name for the experiment set
+export EXPR_NAME="qwen_05B_fp090_512K" # Name for the single experiment run
+export MODEL_ID="Qwen/Qwen2.5-0.5B"
+export DATASET_PATH="${EXPR_HOME}/experiments/processed_datasets/long-context-524288"
+export MODEL_CONFIG_PATH="${EXPR_HOME}/model_configs/qwen2/lc_config_0_5B"
+echo "Using model config path: $MODEL_CONFIG_PATH"
+TRANSFER_CONFIG_DIR="none"
+export TRAIN_ATTN_CONFIG_PATH="${EXPR_HOME}/train_attn_configs/qwen_05B_flex_090.yaml"
+export ATTN_TYPE="minfer"
+
+# ------------------------------------------
+# Training Path settings
+export TF_LOG_PATH="$EXPR_DATA_STORE/$EXPR_DIR/tf_logs"
+export CKPT_PATH="$EXPR_DATA_STORE/$EXPR_DIR/$EXPR_NAME/checkpoints"
+export COMPILE_PATH="$EXPR_DATA_STORE/compile_config/rank_${NODE_RANK}"
+mkdir -p $TF_LOG_PATH
+mkdir -p $CKPT_PATH
+mkdir -p $COMPILE_PATH
+
+# -------------------------------------------
+# Training Settings
+export TRACE_STRATEGY="reuse_cache"
+
+export GLOBAL_BATCH_SIZE=4 # TODO
+export MICRO_BATCH_SIZE=1
+export MEM_CONSTRAINT=37
+
+export NUM_ITER=10
+export NUM_EPOCH=0
+
+export CKPT_SAVE_STEP=0
+export CKPT_SAVE_EPOCH=1
+
+export CHECK_RESUME=0
+if [ "$CHECK_RESUME" -eq 1 ]; then
+ CHECK_RESUME="--check_resume"
+else
+ CHECK_RESUME=""
+fi
+
+# -------------------------------------------
+# Logging Path
+export LOG_PATH="${EXPR_DATA_STORE}/${EXPR_DIR}/${EXPR_NAME}/rank_${NODE_RANK}"
+mkdir -p $LOG_PATH
+echo "Logging directed to $LOG_PATH/train.log"
+
+torchrun --nproc_per_node=$GPU_PER_NODE \
+ --nnodes=$NUM_NODES \
+ --node_rank=$NODE_RANK \
+ --master_addr=$MASTER_ADDR \
+ --master_port=$MASTER_PORT \
+ train.py --plan_ngpus $WORLD_SIZE \
+ --runtime_ngpus $WORLD_SIZE \
+ --name $EXPR_NAME \
+ --seq_len $SEQUENCE_LENGTH \
+ --attn_type $ATTN_TYPE \
+ --train_attn_config_path $TRAIN_ATTN_CONFIG_PATH \
+ --reuse_type $REUSE_TYPE \
+ --model_id $MODEL_ID \
+ --n_iter $NUM_ITER \
+ --n_epochs $NUM_EPOCH \
+ --global_batch_size $GLOBAL_BATCH_SIZE \
+ --micro_batch_size $MICRO_BATCH_SIZE \
+ --dataset_path $DATASET_PATH \
+ --compile_save_path $COMPILE_PATH \
+ --tf_log_dir $TF_LOG_PATH \
+ --model_config_path $MODEL_CONFIG_PATH \
+ --ckpt_save_dir $CKPT_PATH \
+ --ckpt_n_step $CKPT_SAVE_STEP \
+ --ckpt_n_epoch $CKPT_SAVE_EPOCH \
+ --trace_strategy $TRACE_STRATEGY \
+ --transfer_config_dir $TRANSFER_CONFIG_DIR \
+ --mem_constraint $MEM_CONSTRAINT \
+ $CHECK_RESUME > $LOG_PATH/train.log 2>&1
+
+echo "Log saved to $LOG_PATH/train.log"
diff --git a/mtraining/model_configs/__init__.py b/mtraining/model_configs/__init__.py
new file mode 100644
index 00000000..75d94ed5
--- /dev/null
+++ b/mtraining/model_configs/__init__.py
@@ -0,0 +1,45 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from .phi3 import PHI_ATTN_FUNCS, Phi3ForCausalLM
+from .qwen2 import QWEN_ATTN_FUNCS, Qwen2ForCausalLM
+
+SUPPORTED_MODEL_SERIRS = {"Phi-3", "Qwen2.5"}
+
+MODEL_TO_ATTN_FUNC = {
+ "Phi-3": PHI_ATTN_FUNCS,
+ "Qwen2.5": QWEN_ATTN_FUNCS,
+}
+
+
+MODEL_ID_TO_MODEL_CLS = {
+ "Phi-3": Phi3ForCausalLM,
+ "Qwen2.5": Qwen2ForCausalLM,
+}
+
+MODEL_ID_TO_PREFIX = {
+ "Phi-3": "Phi3",
+ "Qwen2.5": "Qwen2",
+}
+
+
+def _get_model_series(model_id: str):
+ for model_series in SUPPORTED_MODEL_SERIRS:
+ if model_series in model_id:
+ return model_series
+ raise ValueError(f"Model series not found in {model_id}")
+
+
+def get_model_attn_funcs(model_id: str):
+ model_series = _get_model_series(model_id)
+ return MODEL_TO_ATTN_FUNC[model_series]
+
+
+def get_model_cls(model_id: str):
+ model_series = _get_model_series(model_id)
+ return MODEL_ID_TO_MODEL_CLS[model_series]
+
+
+def get_model_prefix(model_id: str):
+ model_series = _get_model_series(model_id)
+ return MODEL_ID_TO_PREFIX[model_series]
diff --git a/mtraining/model_configs/phi3/__init__.py b/mtraining/model_configs/phi3/__init__.py
new file mode 100644
index 00000000..a311fa45
--- /dev/null
+++ b/mtraining/model_configs/phi3/__init__.py
@@ -0,0 +1,11 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from .modelling_phi import (
+ PHI_ATTN_FUNCS,
+ Phi3Attention,
+ Phi3Config,
+ Phi3ForCausalLM,
+ apply_rotary_pos_emb,
+ repeat_kv,
+)
diff --git a/mtraining/model_configs/phi3/configuration_phi3.py b/mtraining/model_configs/phi3/configuration_phi3.py
new file mode 100644
index 00000000..82ec029b
--- /dev/null
+++ b/mtraining/model_configs/phi3/configuration_phi3.py
@@ -0,0 +1,237 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Phi-3 model configuration"""
+
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+PHI3_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+ "microsoft/Phi-3-mini-4k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/config.json",
+ "microsoft/Phi-3-mini-128k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/config.json",
+}
+
+
+class Phi3Config(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the
+ [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 32064):
+ Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Phi3Model`].
+ hidden_size (`int`, *optional*, defaults to 3072):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 8192):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer decoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer decoder.
+ num_key_value_heads (`int`, *optional*):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+ `num_attention_heads`.
+ resid_pdrop (`float`, *optional*, defaults to 0.0):
+ Dropout probability for mlp outputs.
+ embd_pdrop (`int`, *optional*, defaults to 0.0):
+ The dropout ratio for the embeddings.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio after computing the attention scores.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model might ever be used with.
+ original_max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model was trained with. This is used to determine the size of the
+ original RoPE embeddings when using long scaling.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+ The epsilon value used for the RMSNorm.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether to tie weight embeddings
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`dict`, *optional*):
+ The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must
+ contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be `longrope` and
+ the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size
+ divided by the number of attention heads divided by 2.
+ bos_token_id (`int`, *optional*, defaults to 1):
+ The id of the "beginning-of-sequence" token.
+ eos_token_id (`int`, *optional*, defaults to 32000):
+ The id of the "end-of-sequence" token.
+ pad_token_id (`int`, *optional*, defaults to 32000):
+ The id of the padding token.
+ sliding_window (`int`, *optional*):
+ Sliding window attention window size. If `None`, no sliding window is applied.
+
+ Example:
+
+ ```python
+ >>> from transformers import Phi3Model, Phi3Config
+
+ >>> # Initializing a Phi-3 style configuration
+ >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+
+ >>> # Initializing a model from the configuration
+ >>> model = Phi3Model(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "phi3"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ def __init__(
+ self,
+ vocab_size=32064,
+ hidden_size=3072,
+ intermediate_size=8192,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=None,
+ resid_pdrop=0.0,
+ embd_pdrop=0.0,
+ attention_dropout=0.0,
+ hidden_act="silu",
+ max_position_embeddings=4096,
+ original_max_position_embeddings=4096,
+ initializer_range=0.02,
+ rms_norm_eps=1e-5,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ bos_token_id=1,
+ eos_token_id=32000,
+ pad_token_id=32000,
+ sliding_window=None,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.resid_pdrop = resid_pdrop
+ self.embd_pdrop = embd_pdrop
+ self.attention_dropout = attention_dropout
+ self.hidden_act = hidden_act
+ self.max_position_embeddings = max_position_embeddings
+ self.original_max_position_embeddings = original_max_position_embeddings
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self._rope_scaling_adjustment()
+ self._rope_scaling_validation()
+ self.sliding_window = sliding_window
+
+ super().__init__(
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ pad_token_id=pad_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+
+ def _rope_scaling_adjustment(self):
+ """
+ Adjust the `type` of the `rope_scaling` configuration for backward compatibility.
+ """
+ if self.rope_scaling is None:
+ return
+
+ rope_scaling_type = self.rope_scaling.get("type", None)
+
+ # For backward compatibility if previous version used "su" or "yarn"
+ if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]:
+ self.rope_scaling["type"] = "longrope"
+
+ def _rope_scaling_validation(self):
+ """
+ Validate the `rope_scaling` configuration.
+ """
+ if self.rope_scaling is None:
+ return
+
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:
+ raise ValueError(
+ "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, "
+ f"got {self.rope_scaling}"
+ )
+ rope_scaling_type = self.rope_scaling.get("type", None)
+ rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)
+ rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
+ if rope_scaling_type is None or rope_scaling_type not in ["longrope"]:
+ raise ValueError(
+ f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}"
+ )
+ if not (
+ isinstance(rope_scaling_short_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
+ )
+ if (
+ not len(rope_scaling_short_factor)
+ == self.hidden_size // self.num_attention_heads // 2
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"
+ )
+ if not (
+ isinstance(rope_scaling_long_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
+ )
+ if (
+ not len(rope_scaling_long_factor)
+ == self.hidden_size // self.num_attention_heads // 2
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"
+ )
diff --git a/mtraining/model_configs/phi3/lc_config/configuration_phi3.py b/mtraining/model_configs/phi3/lc_config/configuration_phi3.py
new file mode 100644
index 00000000..82ec029b
--- /dev/null
+++ b/mtraining/model_configs/phi3/lc_config/configuration_phi3.py
@@ -0,0 +1,237 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Phi-3 model configuration"""
+
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+PHI3_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+ "microsoft/Phi-3-mini-4k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/config.json",
+ "microsoft/Phi-3-mini-128k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/config.json",
+}
+
+
+class Phi3Config(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the
+ [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 32064):
+ Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Phi3Model`].
+ hidden_size (`int`, *optional*, defaults to 3072):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 8192):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer decoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer decoder.
+ num_key_value_heads (`int`, *optional*):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+ `num_attention_heads`.
+ resid_pdrop (`float`, *optional*, defaults to 0.0):
+ Dropout probability for mlp outputs.
+ embd_pdrop (`int`, *optional*, defaults to 0.0):
+ The dropout ratio for the embeddings.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio after computing the attention scores.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model might ever be used with.
+ original_max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model was trained with. This is used to determine the size of the
+ original RoPE embeddings when using long scaling.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+ The epsilon value used for the RMSNorm.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether to tie weight embeddings
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`dict`, *optional*):
+ The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must
+ contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be `longrope` and
+ the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size
+ divided by the number of attention heads divided by 2.
+ bos_token_id (`int`, *optional*, defaults to 1):
+ The id of the "beginning-of-sequence" token.
+ eos_token_id (`int`, *optional*, defaults to 32000):
+ The id of the "end-of-sequence" token.
+ pad_token_id (`int`, *optional*, defaults to 32000):
+ The id of the padding token.
+ sliding_window (`int`, *optional*):
+ Sliding window attention window size. If `None`, no sliding window is applied.
+
+ Example:
+
+ ```python
+ >>> from transformers import Phi3Model, Phi3Config
+
+ >>> # Initializing a Phi-3 style configuration
+ >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+
+ >>> # Initializing a model from the configuration
+ >>> model = Phi3Model(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "phi3"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ def __init__(
+ self,
+ vocab_size=32064,
+ hidden_size=3072,
+ intermediate_size=8192,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=None,
+ resid_pdrop=0.0,
+ embd_pdrop=0.0,
+ attention_dropout=0.0,
+ hidden_act="silu",
+ max_position_embeddings=4096,
+ original_max_position_embeddings=4096,
+ initializer_range=0.02,
+ rms_norm_eps=1e-5,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ bos_token_id=1,
+ eos_token_id=32000,
+ pad_token_id=32000,
+ sliding_window=None,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.resid_pdrop = resid_pdrop
+ self.embd_pdrop = embd_pdrop
+ self.attention_dropout = attention_dropout
+ self.hidden_act = hidden_act
+ self.max_position_embeddings = max_position_embeddings
+ self.original_max_position_embeddings = original_max_position_embeddings
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self._rope_scaling_adjustment()
+ self._rope_scaling_validation()
+ self.sliding_window = sliding_window
+
+ super().__init__(
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ pad_token_id=pad_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+
+ def _rope_scaling_adjustment(self):
+ """
+ Adjust the `type` of the `rope_scaling` configuration for backward compatibility.
+ """
+ if self.rope_scaling is None:
+ return
+
+ rope_scaling_type = self.rope_scaling.get("type", None)
+
+ # For backward compatibility if previous version used "su" or "yarn"
+ if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]:
+ self.rope_scaling["type"] = "longrope"
+
+ def _rope_scaling_validation(self):
+ """
+ Validate the `rope_scaling` configuration.
+ """
+ if self.rope_scaling is None:
+ return
+
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:
+ raise ValueError(
+ "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, "
+ f"got {self.rope_scaling}"
+ )
+ rope_scaling_type = self.rope_scaling.get("type", None)
+ rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)
+ rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
+ if rope_scaling_type is None or rope_scaling_type not in ["longrope"]:
+ raise ValueError(
+ f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}"
+ )
+ if not (
+ isinstance(rope_scaling_short_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
+ )
+ if (
+ not len(rope_scaling_short_factor)
+ == self.hidden_size // self.num_attention_heads // 2
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"
+ )
+ if not (
+ isinstance(rope_scaling_long_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
+ )
+ if (
+ not len(rope_scaling_long_factor)
+ == self.hidden_size // self.num_attention_heads // 2
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"
+ )
diff --git a/mtraining/model_configs/phi3/lc_config_mini/configuration_phi3.py b/mtraining/model_configs/phi3/lc_config_mini/configuration_phi3.py
new file mode 100644
index 00000000..82ec029b
--- /dev/null
+++ b/mtraining/model_configs/phi3/lc_config_mini/configuration_phi3.py
@@ -0,0 +1,237 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Phi-3 model configuration"""
+
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+PHI3_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+ "microsoft/Phi-3-mini-4k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/config.json",
+ "microsoft/Phi-3-mini-128k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/config.json",
+}
+
+
+class Phi3Config(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the
+ [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 32064):
+ Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Phi3Model`].
+ hidden_size (`int`, *optional*, defaults to 3072):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 8192):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer decoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer decoder.
+ num_key_value_heads (`int`, *optional*):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+ `num_attention_heads`.
+ resid_pdrop (`float`, *optional*, defaults to 0.0):
+ Dropout probability for mlp outputs.
+ embd_pdrop (`int`, *optional*, defaults to 0.0):
+ The dropout ratio for the embeddings.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio after computing the attention scores.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model might ever be used with.
+ original_max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model was trained with. This is used to determine the size of the
+ original RoPE embeddings when using long scaling.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+ The epsilon value used for the RMSNorm.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether to tie weight embeddings
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`dict`, *optional*):
+ The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must
+ contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be `longrope` and
+ the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size
+ divided by the number of attention heads divided by 2.
+ bos_token_id (`int`, *optional*, defaults to 1):
+ The id of the "beginning-of-sequence" token.
+ eos_token_id (`int`, *optional*, defaults to 32000):
+ The id of the "end-of-sequence" token.
+ pad_token_id (`int`, *optional*, defaults to 32000):
+ The id of the padding token.
+ sliding_window (`int`, *optional*):
+ Sliding window attention window size. If `None`, no sliding window is applied.
+
+ Example:
+
+ ```python
+ >>> from transformers import Phi3Model, Phi3Config
+
+ >>> # Initializing a Phi-3 style configuration
+ >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+
+ >>> # Initializing a model from the configuration
+ >>> model = Phi3Model(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "phi3"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ def __init__(
+ self,
+ vocab_size=32064,
+ hidden_size=3072,
+ intermediate_size=8192,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=None,
+ resid_pdrop=0.0,
+ embd_pdrop=0.0,
+ attention_dropout=0.0,
+ hidden_act="silu",
+ max_position_embeddings=4096,
+ original_max_position_embeddings=4096,
+ initializer_range=0.02,
+ rms_norm_eps=1e-5,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ bos_token_id=1,
+ eos_token_id=32000,
+ pad_token_id=32000,
+ sliding_window=None,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.resid_pdrop = resid_pdrop
+ self.embd_pdrop = embd_pdrop
+ self.attention_dropout = attention_dropout
+ self.hidden_act = hidden_act
+ self.max_position_embeddings = max_position_embeddings
+ self.original_max_position_embeddings = original_max_position_embeddings
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self._rope_scaling_adjustment()
+ self._rope_scaling_validation()
+ self.sliding_window = sliding_window
+
+ super().__init__(
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ pad_token_id=pad_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+
+ def _rope_scaling_adjustment(self):
+ """
+ Adjust the `type` of the `rope_scaling` configuration for backward compatibility.
+ """
+ if self.rope_scaling is None:
+ return
+
+ rope_scaling_type = self.rope_scaling.get("type", None)
+
+ # For backward compatibility if previous version used "su" or "yarn"
+ if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]:
+ self.rope_scaling["type"] = "longrope"
+
+ def _rope_scaling_validation(self):
+ """
+ Validate the `rope_scaling` configuration.
+ """
+ if self.rope_scaling is None:
+ return
+
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:
+ raise ValueError(
+ "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, "
+ f"got {self.rope_scaling}"
+ )
+ rope_scaling_type = self.rope_scaling.get("type", None)
+ rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)
+ rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
+ if rope_scaling_type is None or rope_scaling_type not in ["longrope"]:
+ raise ValueError(
+ f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}"
+ )
+ if not (
+ isinstance(rope_scaling_short_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
+ )
+ if (
+ not len(rope_scaling_short_factor)
+ == self.hidden_size // self.num_attention_heads // 2
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"
+ )
+ if not (
+ isinstance(rope_scaling_long_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
+ )
+ if (
+ not len(rope_scaling_long_factor)
+ == self.hidden_size // self.num_attention_heads // 2
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"
+ )
diff --git a/mtraining/model_configs/phi3/modelling_phi.py b/mtraining/model_configs/phi3/modelling_phi.py
new file mode 100644
index 00000000..543bcbb8
--- /dev/null
+++ b/mtraining/model_configs/phi3/modelling_phi.py
@@ -0,0 +1,1324 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# This file was automatically generated from src/transformers/models/phi3/modular_phi3.py.
+# Do NOT edit this file manually as any edits will be overwritten by the generation of
+# the file from the modular. If any change should be done, please apply the change to the
+# modular_phi3.py file directly. One of our CI enforces this.
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Callable, List, Optional, Tuple, Union
+
+import torch
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.cache_utils import (
+ Cache,
+ DynamicCache,
+ SlidingWindowCache,
+ StaticCache,
+)
+from transformers.generation import GenerationMixin
+from transformers.modeling_attn_mask_utils import AttentionMaskConverter
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.modeling_outputs import (
+ BaseModelOutputWithPast,
+ CausalLMOutputWithPast,
+ SequenceClassifierOutputWithPast,
+ TokenClassifierOutput,
+)
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import (
+ LossKwargs,
+ add_code_sample_docstrings,
+ add_start_docstrings,
+ add_start_docstrings_to_model_forward,
+ logging,
+ replace_return_docstrings,
+)
+from transformers.utils.deprecation import deprecate_kwarg
+
+from .configuration_phi3 import Phi3Config
+
+logger = logging.get_logger(__name__)
+
+_CHECKPOINT_FOR_DOC = "microsoft/Phi-3-mini-4k-instruct"
+_CONFIG_FOR_DOC = "Phi3Config"
+
+PHI_ATTN_FUNCS = ALL_ATTENTION_FUNCTIONS.copy()
+
+
+class Phi3MLP(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+
+ self.config = config
+ self.gate_up_proj = nn.Linear(
+ config.hidden_size, 2 * config.intermediate_size, bias=False
+ )
+ self.down_proj = nn.Linear(
+ config.intermediate_size, config.hidden_size, bias=False
+ )
+ self.activation_fn = ACT2FN[config.hidden_act]
+
+ def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+ up_states = self.gate_up_proj(hidden_states)
+
+ gate, up_states = up_states.chunk(2, dim=-1)
+ up_states = up_states * self.activation_fn(gate)
+
+ return self.down_proj(up_states)
+
+
+def rotate_half(x):
+ """Rotates half the hidden dims of the input."""
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2 :]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+ """Applies Rotary Position Embedding to the query and key tensors.
+
+ Args:
+ q (`torch.Tensor`): The query tensor.
+ k (`torch.Tensor`): The key tensor.
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
+ position_ids (`torch.Tensor`, *optional*):
+ Deprecated and unused.
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+ Returns:
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+ """
+ cos = cos.unsqueeze(unsqueeze_dim)
+ sin = sin.unsqueeze(unsqueeze_dim)
+ q_embed = (q * cos) + (rotate_half(q) * sin)
+ k_embed = (k * cos) + (rotate_half(k) * sin)
+ return q_embed, k_embed
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+ """
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+ """
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+ if n_rep == 1:
+ return hidden_states
+ hidden_states = hidden_states[:, :, None, :, :].expand(
+ batch, num_key_value_heads, n_rep, slen, head_dim
+ )
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+ module: nn.Module,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attention_mask: Optional[torch.Tensor],
+ scaling: float,
+ dropout: float = 0.0,
+ **kwargs,
+):
+ key_states = repeat_kv(key, module.num_key_value_groups)
+ value_states = repeat_kv(value, module.num_key_value_groups)
+
+ attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+ if attention_mask is not None:
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+ attn_weights = attn_weights + causal_mask
+
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ query.dtype
+ )
+ attn_weights = nn.functional.dropout(
+ attn_weights, p=dropout, training=module.training
+ )
+ attn_output = torch.matmul(attn_weights, value_states)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+
+ return attn_output, attn_weights
+
+
+class Phi3Attention(nn.Module):
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+ def __init__(self, config: Phi3Config, layer_idx: Optional[int] = None):
+ super().__init__()
+ self.config = config
+ self.layer_idx = layer_idx
+ self.head_dim = getattr(
+ config, "head_dim", config.hidden_size // config.num_attention_heads
+ )
+ self.num_key_value_groups = (
+ config.num_attention_heads // config.num_key_value_heads
+ )
+ self.num_key_value_heads = config.num_key_value_heads
+ self.scaling = self.head_dim**-0.5
+ self.attention_dropout = config.attention_dropout
+ self.is_causal = True
+
+ op_size = config.num_attention_heads * self.head_dim + 2 * (
+ config.num_key_value_heads * self.head_dim
+ )
+ self.o_proj = nn.Linear(
+ config.num_attention_heads * self.head_dim, config.hidden_size, bias=False
+ )
+ self.qkv_proj = nn.Linear(config.hidden_size, op_size, bias=False)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+ attention_mask: Optional[torch.Tensor],
+ past_key_value: Optional[Cache] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ input_shape = hidden_states.shape[:-1]
+ hidden_shape = (*input_shape, -1, self.head_dim)
+
+ qkv = self.qkv_proj(hidden_states)
+ query_pos = self.config.num_attention_heads * self.head_dim
+ query_states = qkv[..., :query_pos]
+ key_states = qkv[
+ ..., query_pos : query_pos + self.num_key_value_heads * self.head_dim
+ ]
+ value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]
+
+ query_states = query_states.view(hidden_shape).transpose(1, 2)
+ key_states = key_states.view(hidden_shape).transpose(1, 2)
+ value_states = value_states.view(hidden_shape).transpose(1, 2)
+
+ cos, sin = position_embeddings
+ query_states, key_states = apply_rotary_pos_emb(
+ query_states, key_states, cos, sin
+ )
+
+ if past_key_value is not None:
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+ key_states, value_states = past_key_value.update(
+ key_states, value_states, self.layer_idx, cache_kwargs
+ )
+
+ attention_interface: Callable = eager_attention_forward
+ if self.config._attn_implementation != "eager":
+ if self.config._attn_implementation == "sdpa" and kwargs.get(
+ "output_attentions", False
+ ):
+ logger.warning_once(
+ "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to "
+ 'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+ )
+ else:
+ attention_interface = PHI_ATTN_FUNCS[self.config._attn_implementation]
+
+ attn_output, attn_weights = attention_interface(
+ self,
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ dropout=0.0 if not self.training else self.attention_dropout,
+ scaling=self.scaling,
+ sliding_window=getattr(self.config, "sliding_window", None),
+ **kwargs,
+ )
+
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+ attn_output = self.o_proj(attn_output)
+ return attn_output, attn_weights
+
+
+class Phi3RMSNorm(nn.Module):
+ def __init__(self, hidden_size, eps=1e-6):
+ """
+ Phi3RMSNorm is equivalent to T5LayerNorm
+ """
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.variance_epsilon = eps
+
+ def forward(self, hidden_states):
+ input_dtype = hidden_states.dtype
+ hidden_states = hidden_states.to(torch.float32)
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+ return self.weight * hidden_states.to(input_dtype)
+
+ def extra_repr(self):
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class Phi3DecoderLayer(nn.Module):
+ def __init__(self, config: Phi3Config, layer_idx: int):
+ super().__init__()
+ self.hidden_size = config.hidden_size
+ self.self_attn = Phi3Attention(config=config, layer_idx=layer_idx)
+ self.mlp = Phi3MLP(config)
+ self.input_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.post_attention_layernorm = Phi3RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.config = config
+ self.resid_attn_dropout = nn.Dropout(config.resid_pdrop)
+ self.resid_mlp_dropout = nn.Dropout(config.resid_pdrop)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: Optional[bool] = False,
+ use_cache: Optional[bool] = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[
+ Tuple[torch.Tensor, torch.Tensor]
+ ] = None, # necessary, but kept here for BC
+ **kwargs: Unpack[FlashAttentionKwargs],
+ ) -> Tuple[
+ torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
+ ]:
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`):
+ input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
+ `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
+ past_key_value (`Cache`, *optional*): cached past key and value projection states
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ use_cache (`bool`, *optional*):
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
+ Indices depicting the position of the input sequence tokens in the sequence
+ kwargs (`dict`, *optional*):
+ Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code
+ into the model
+ """
+ residual = hidden_states
+
+ hidden_states = self.input_layernorm(hidden_states)
+
+ # Self Attention
+ hidden_states, self_attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ **kwargs,
+ )
+ hidden_states = residual + self.resid_attn_dropout(
+ hidden_states
+ ) # main diff with Llama
+
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + self.resid_mlp_dropout(
+ hidden_states
+ ) # main diff with Llama
+
+ outputs = (hidden_states,)
+ if output_attentions:
+ outputs += (self_attn_weights,)
+
+ return outputs
+
+
+class Phi3RotaryEmbedding(nn.Module):
+ def __init__(self, config: Phi3Config, device=None):
+ super().__init__()
+ # BC: "rope_type" was originally "type"
+ if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
+ self.rope_type = config.rope_scaling.get(
+ "rope_type", config.rope_scaling.get("type")
+ )
+ else:
+ self.rope_type = "default"
+ self.max_seq_len_cached = config.max_position_embeddings
+ self.original_max_seq_len = config.max_position_embeddings
+
+ self.config = config
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+ self.original_inv_freq = self.inv_freq
+
+ def _dynamic_frequency_update(self, position_ids, device):
+ """
+ dynamic RoPE layers should recompute `inv_freq` in the following situations:
+ 1 - growing beyond the cached sequence length (allow scaling)
+ 2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
+ """
+ seq_len = torch.max(position_ids) + 1
+ if seq_len > self.max_seq_len_cached: # growth
+ inv_freq, self.attention_scaling = self.rope_init_fn(
+ self.config, device, seq_len=seq_len
+ )
+ self.register_buffer(
+ "inv_freq", inv_freq, persistent=False
+ ) # TODO joao: may break with compilation
+ self.max_seq_len_cached = seq_len
+
+ if (
+ seq_len < self.original_max_seq_len
+ and self.max_seq_len_cached > self.original_max_seq_len
+ ): # reset
+ # This .to() is needed if the model has been moved to a device after being initialized (because
+ # the buffer is automatically moved, but not the original copy)
+ self.original_inv_freq = self.original_inv_freq.to(device)
+ self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
+ self.max_seq_len_cached = self.original_max_seq_len
+
+ @torch.no_grad()
+ def forward(self, x, position_ids):
+ if "dynamic" in self.rope_type:
+ self._dynamic_frequency_update(position_ids, device=x.device)
+ elif self.rope_type == "longrope":
+ self._longrope_frequency_update(position_ids, device=x.device)
+
+ # Core RoPE block
+ inv_freq_expanded = (
+ self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
+ )
+ position_ids_expanded = position_ids[:, None, :].float()
+ # Force float32 (see https://github.com/huggingface/transformers/pull/29285)
+ device_type = x.device.type
+ device_type = (
+ device_type
+ if isinstance(device_type, str) and device_type != "mps"
+ else "cpu"
+ )
+ with torch.autocast(device_type=device_type, enabled=False):
+ freqs = (
+ inv_freq_expanded.float() @ position_ids_expanded.float()
+ ).transpose(1, 2)
+ emb = torch.cat((freqs, freqs), dim=-1)
+ cos = emb.cos()
+ sin = emb.sin()
+
+ # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention
+ cos = cos * self.attention_scaling
+ sin = sin * self.attention_scaling
+
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+ def _longrope_frequency_update(self, position_ids, device):
+ """Longrope uses long factor if sequence is larger than original pretraining length, short otherwise."""
+ seq_len = torch.max(position_ids) + 1
+ if hasattr(self.config, "original_max_position_embeddings"):
+ original_max_position_embeddings = (
+ self.config.original_max_position_embeddings
+ )
+ else:
+ original_max_position_embeddings = self.config.max_position_embeddings
+ if seq_len > original_max_position_embeddings:
+ if not hasattr(self, "long_inv_freq"):
+ self.long_inv_freq, _ = self.rope_init_fn(
+ self.config, device, seq_len=original_max_position_embeddings + 1
+ )
+ self.register_buffer("inv_freq", self.long_inv_freq, persistent=False)
+ else:
+ # This .to() is needed if the model has been moved to a device after being initialized (because
+ # the buffer is automatically moved, but not the original copy)
+ self.original_inv_freq = self.original_inv_freq.to(device)
+ self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
+
+
+PHI3_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`Phi3Config`]):
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
+ load the weights associated with the model, only the configuration. Check out the
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+ "The bare Phi3 Model outputting raw hidden-states without any specific head on top.",
+ PHI3_START_DOCSTRING,
+)
+class Phi3PreTrainedModel(PreTrainedModel):
+ config_class = Phi3Config
+ base_model_prefix = "model"
+ supports_gradient_checkpointing = True
+ _no_split_modules = ["Phi3DecoderLayer"]
+ _skip_keys_device_placement = ["past_key_values"]
+ _supports_flash_attn_2 = True
+ _supports_sdpa = True
+ _supports_flex_attn = True
+ _supports_cache_class = True
+ _supports_quantized_cache = True
+ _supports_static_cache = True
+ _supports_attention_backend = True
+ _version = "0.0.5"
+
+ def _init_weights(self, module):
+ std = self.config.initializer_range
+ if isinstance(module, nn.Linear):
+ module.weight.data.normal_(mean=0.0, std=std)
+ if module.bias is not None:
+ module.bias.data.zero_()
+ elif isinstance(module, nn.Embedding):
+ module.weight.data.normal_(mean=0.0, std=std)
+ if module.padding_idx is not None:
+ module.weight.data[module.padding_idx].zero_()
+
+
+PHI3_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
+ `past_key_values`).
+
+ If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+ information on the default strategy.
+
+ - 1 indicates the head is **not masked**,
+ - 0 indicates the head is **masked**.
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.n_positions - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+ Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+ blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
+ returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
+
+ Two formats are allowed:
+ - a [`~cache_utils.Cache`] instance, see our
+ [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
+ - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
+ cache format.
+
+ The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
+ legacy cache format will be returned.
+
+ If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+ have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+ of shape `(batch_size, sequence_length)`.
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
+ use_cache (`bool`, *optional*):
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
+ Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
+ this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
+ the complete sequence length.
+"""
+
+
+@add_start_docstrings(
+ "The bare Phi3 Model outputting raw hidden-states without any specific head on top.",
+ PHI3_START_DOCSTRING,
+)
+class Phi3Model(Phi3PreTrainedModel):
+ """
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`]
+
+ Args:
+ config: Phi3Config
+ """
+
+ def __init__(self, config: Phi3Config):
+ super().__init__(config)
+ self.padding_idx = config.pad_token_id
+ self.vocab_size = config.vocab_size
+
+ self.embed_tokens = nn.Embedding(
+ config.vocab_size, config.hidden_size, self.padding_idx
+ )
+ self.layers = nn.ModuleList(
+ [
+ Phi3DecoderLayer(config, layer_idx)
+ for layer_idx in range(config.num_hidden_layers)
+ ]
+ )
+ self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.rotary_emb = Phi3RotaryEmbedding(config=config)
+ self.gradient_checkpointing = False
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Cache] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **flash_attn_kwargs: Unpack[FlashAttentionKwargs],
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
+ output_attentions = (
+ output_attentions
+ if output_attentions is not None
+ else self.config.output_attentions
+ )
+ output_hidden_states = (
+ output_hidden_states
+ if output_hidden_states is not None
+ else self.config.output_hidden_states
+ )
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ if (input_ids is None) ^ (inputs_embeds is not None):
+ raise ValueError(
+ "You must specify exactly one of input_ids or inputs_embeds"
+ )
+
+ if self.gradient_checkpointing and self.training and use_cache:
+ logger.warning_once(
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
+ )
+ use_cache = False
+
+ if inputs_embeds is None:
+ inputs_embeds = self.embed_tokens(input_ids)
+
+ if use_cache and past_key_values is None:
+ past_key_values = DynamicCache()
+
+ if cache_position is None:
+ past_seen_tokens = (
+ past_key_values.get_seq_length() if past_key_values is not None else 0
+ )
+ cache_position = torch.arange(
+ past_seen_tokens,
+ past_seen_tokens + inputs_embeds.shape[1],
+ device=inputs_embeds.device,
+ )
+
+ if position_ids is None:
+ position_ids = cache_position.unsqueeze(0)
+
+ causal_mask = self._update_causal_mask(
+ attention_mask,
+ inputs_embeds,
+ cache_position,
+ past_key_values,
+ output_attentions,
+ )
+
+ hidden_states = inputs_embeds
+
+ # create position embeddings to be shared across the decoder layers
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+ # decoder layers
+ all_hidden_states = () if output_hidden_states else None
+ all_self_attns = () if output_attentions else None
+
+ for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ if self.gradient_checkpointing and self.training:
+ layer_outputs = self._gradient_checkpointing_func(
+ decoder_layer.__call__,
+ hidden_states,
+ causal_mask,
+ position_ids,
+ past_key_values,
+ output_attentions,
+ use_cache,
+ cache_position,
+ position_embeddings,
+ )
+ else:
+ layer_outputs = decoder_layer(
+ hidden_states,
+ attention_mask=causal_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_values,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ **flash_attn_kwargs,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_self_attns += (layer_outputs[1],)
+
+ hidden_states = self.norm(hidden_states)
+
+ # add hidden states from the last decoder layer
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ output = BaseModelOutputWithPast(
+ last_hidden_state=hidden_states,
+ past_key_values=past_key_values if use_cache else None,
+ hidden_states=all_hidden_states,
+ attentions=all_self_attns,
+ )
+ return output if return_dict else output.to_tuple()
+
+ def _update_causal_mask(
+ self,
+ attention_mask: torch.Tensor,
+ input_tensor: torch.Tensor,
+ cache_position: torch.Tensor,
+ past_key_values: Cache,
+ output_attentions: bool,
+ ):
+ if self.config._attn_implementation == "flash_attention_2":
+ if attention_mask is not None and past_key_values is not None:
+ is_padding_right = (
+ attention_mask[:, -1].sum().item() != input_tensor.size()[0]
+ )
+ if is_padding_right:
+ raise ValueError(
+ "You are attempting to perform batched generation with padding_side='right'"
+ " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to "
+ " call `tokenizer.padding_side = 'left'` before tokenizing the input. "
+ )
+ if attention_mask is not None and 0.0 in attention_mask:
+ return attention_mask
+ return None
+
+ # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
+ # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
+ # to infer the attention mask.
+ past_seen_tokens = (
+ past_key_values.get_seq_length() if past_key_values is not None else 0
+ )
+ using_static_cache = isinstance(past_key_values, StaticCache)
+ using_sliding_window_cache = isinstance(past_key_values, SlidingWindowCache)
+
+ # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
+ if (
+ self.config._attn_implementation == "sdpa"
+ and not (using_static_cache or using_sliding_window_cache)
+ and not output_attentions
+ ):
+ if AttentionMaskConverter._ignore_causal_mask_sdpa(
+ attention_mask,
+ inputs_embeds=input_tensor,
+ past_key_values_length=past_seen_tokens,
+ sliding_window=self.config.sliding_window,
+ is_training=self.training,
+ ):
+ return None
+
+ dtype, device = input_tensor.dtype, input_tensor.device
+ min_dtype = torch.finfo(dtype).min
+ sequence_length = input_tensor.shape[1]
+ # SlidingWindowCache or StaticCache
+ if using_sliding_window_cache or using_static_cache:
+ target_length = past_key_values.get_max_cache_shape()
+ # DynamicCache or no cache
+ else:
+ target_length = (
+ attention_mask.shape[-1]
+ if isinstance(attention_mask, torch.Tensor)
+ else past_seen_tokens + sequence_length + 1
+ )
+
+ # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
+ causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
+ attention_mask,
+ sequence_length=sequence_length,
+ target_length=target_length,
+ dtype=dtype,
+ device=device,
+ cache_position=cache_position,
+ batch_size=input_tensor.shape[0],
+ config=self.config,
+ past_key_values=past_key_values,
+ )
+
+ if (
+ self.config._attn_implementation == "sdpa"
+ and attention_mask is not None
+ and attention_mask.device.type == "cuda"
+ and not output_attentions
+ ):
+ # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
+ # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
+ # Details: https://github.com/pytorch/pytorch/issues/110213
+ causal_mask = AttentionMaskConverter._unmask_unattended(
+ causal_mask, min_dtype
+ )
+
+ return causal_mask
+
+ @staticmethod
+ def _prepare_4d_causal_attention_mask_with_cache_position(
+ attention_mask: torch.Tensor,
+ sequence_length: int,
+ target_length: int,
+ dtype: torch.dtype,
+ device: torch.device,
+ cache_position: torch.Tensor,
+ batch_size: int,
+ config: Phi3Config,
+ past_key_values: Cache,
+ ):
+ """
+ Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
+ `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
+
+ Args:
+ attention_mask (`torch.Tensor`):
+ A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
+ sequence_length (`int`):
+ The sequence length being processed.
+ target_length (`int`):
+ The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
+ dtype (`torch.dtype`):
+ The dtype to use for the 4D attention mask.
+ device (`torch.device`):
+ The device to plcae the 4D attention mask on.
+ cache_position (`torch.Tensor`):
+ Indices depicting the position of the input sequence tokens in the sequence.
+ batch_size (`torch.Tensor`):
+ Batch size.
+ config (`Phi3Config`):
+ The model's configuration class
+ past_key_values (`Cache`):
+ The cache class that is being used currently to generate
+ """
+ if attention_mask is not None and attention_mask.dim() == 4:
+ # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
+ causal_mask = attention_mask
+ else:
+ min_dtype = torch.finfo(dtype).min
+ causal_mask = torch.full(
+ (sequence_length, target_length),
+ fill_value=min_dtype,
+ dtype=dtype,
+ device=device,
+ )
+ diagonal_attend_mask = torch.arange(
+ target_length, device=device
+ ) > cache_position.reshape(-1, 1)
+ if config.sliding_window is not None:
+ # if we have sliding window, we should not attend to tokens beyond sliding window length, so we mask them out also
+ # the check is needed to verify is current checkpoint was trained with sliding window or not
+ if (
+ not isinstance(past_key_values, SlidingWindowCache)
+ or sequence_length > target_length
+ ):
+ sliding_attend_mask = torch.arange(
+ target_length, device=device
+ ) <= (cache_position.reshape(-1, 1) - config.sliding_window)
+ diagonal_attend_mask.bitwise_or_(sliding_attend_mask)
+ causal_mask *= diagonal_attend_mask
+ causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
+ if attention_mask is not None:
+ causal_mask = (
+ causal_mask.clone()
+ ) # copy to contiguous memory for in-place edit
+ if attention_mask.shape[-1] > target_length:
+ attention_mask = attention_mask[:, :target_length]
+ mask_length = attention_mask.shape[-1]
+ padding_mask = (
+ causal_mask[:, :, :, :mask_length]
+ + attention_mask[:, None, None, :]
+ )
+ padding_mask = padding_mask == 0
+ causal_mask[:, :, :, :mask_length] = causal_mask[
+ :, :, :, :mask_length
+ ].masked_fill(padding_mask, min_dtype)
+ return causal_mask
+
+
+class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs):
+ ...
+
+
+class Phi3ForCausalLM(Phi3PreTrainedModel, GenerationMixin):
+ _tied_weights_keys = ["lm_head.weight"]
+ _tp_plan = {"lm_head": "colwise_rep"}
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.model = Phi3Model(config)
+ self.vocab_size = config.vocab_size
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ def set_output_embeddings(self, new_embeddings):
+ self.lm_head = new_embeddings
+
+ def set_decoder(self, decoder):
+ self.model = decoder
+
+ def get_decoder(self):
+ return self.model
+
+ @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")
+ @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
+ @replace_return_docstrings(
+ output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC
+ )
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ logits_to_keep: Union[int, torch.Tensor] = 0,
+ **kwargs: Unpack[KwargsForCausalLM],
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
+ r"""
+ Args:
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+ logits_to_keep (`int` or `torch.Tensor`, *optional*):
+ If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
+ `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
+ token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
+ If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
+ This is useful when using packed tensor format (single dimension for batch and sequence length).
+
+ Returns:
+
+ Example:
+
+ ```python
+ >>> from transformers import AutoTokenizer, Phi3ForCausalLM
+
+ >>> model = Phi3ForCausalLM.from_pretrained("meta-phi3/Phi3-2-7b-hf")
+ >>> tokenizer = AutoTokenizer.from_pretrained("meta-phi3/Phi3-2-7b-hf")
+
+ >>> prompt = "Hey, are you conscious? Can you talk to me?"
+ >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+ >>> # Generate
+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+ "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
+ ```"""
+ output_attentions = (
+ output_attentions
+ if output_attentions is not None
+ else self.config.output_attentions
+ )
+ output_hidden_states = (
+ output_hidden_states
+ if output_hidden_states is not None
+ else self.config.output_hidden_states
+ )
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+ outputs = self.model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ cache_position=cache_position,
+ **kwargs,
+ )
+
+ hidden_states = outputs[0]
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+ slice_indices = (
+ slice(-logits_to_keep, None)
+ if isinstance(logits_to_keep, int)
+ else logits_to_keep
+ )
+ logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+ loss = None
+ if labels is not None:
+ loss = self.loss_function(
+ logits=logits,
+ labels=labels,
+ vocab_size=self.config.vocab_size,
+ **kwargs,
+ )
+
+ if not return_dict:
+ output = (logits,) + outputs[1:]
+ return (loss,) + output if loss is not None else output
+
+ return CausalLMOutputWithPast(
+ loss=loss,
+ logits=logits,
+ past_key_values=outputs.past_key_values,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ )
+
+ def prepare_inputs_for_generation(
+ self,
+ input_ids,
+ past_key_values=None,
+ attention_mask=None,
+ inputs_embeds=None,
+ cache_position=None,
+ position_ids=None,
+ use_cache=True,
+ logits_to_keep=None,
+ **kwargs,
+ ):
+ # Overwritten -- this model may need to switch between short and long rope, invalidating the cache in the
+ # process
+
+ # When the first time input length reached long and short factor switching point, enforce re-compute cache
+ # It will cause downside of slower at this single token position, however, better than current failure.
+ if (
+ past_key_values
+ and self.config.rope_scaling
+ and input_ids.shape[1] >= self.config.original_max_position_embeddings + 1
+ ):
+ past_length = cache_position[0]
+ if past_length <= self.config.original_max_position_embeddings:
+ past_key_values = None
+
+ model_inputs = super().prepare_inputs_for_generation(
+ input_ids=input_ids,
+ past_key_values=past_key_values,
+ attention_mask=attention_mask,
+ inputs_embeds=inputs_embeds,
+ cache_position=cache_position,
+ position_ids=position_ids,
+ use_cache=use_cache,
+ logits_to_keep=logits_to_keep,
+ **kwargs,
+ )
+ return model_inputs
+
+
+@add_start_docstrings(
+ """
+ The Phi3 Model transformer with a sequence classification head on top (linear layer).
+
+ [`Phi3ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-2) do.
+
+ Since it does classification on the last token, it requires to know the position of the last token. If a
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
+ """,
+ PHI3_START_DOCSTRING,
+)
+class Phi3ForSequenceClassification(Phi3PreTrainedModel):
+ def __init__(self, config):
+ super().__init__(config)
+ self.num_labels = config.num_labels
+ self.model = Phi3Model(config)
+ self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+ r"""
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ """
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ transformer_outputs = self.model(
+ input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+ hidden_states = transformer_outputs[0]
+ logits = self.score(hidden_states)
+
+ if input_ids is not None:
+ batch_size = input_ids.shape[0]
+ else:
+ batch_size = inputs_embeds.shape[0]
+
+ if self.config.pad_token_id is None and batch_size != 1:
+ raise ValueError(
+ "Cannot handle batch sizes > 1 if no padding token is defined."
+ )
+ if self.config.pad_token_id is None:
+ sequence_lengths = -1
+ else:
+ if input_ids is not None:
+ # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+ sequence_lengths = (
+ torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+ )
+ sequence_lengths = sequence_lengths % input_ids.shape[-1]
+ sequence_lengths = sequence_lengths.to(logits.device)
+ else:
+ sequence_lengths = -1
+
+ pooled_logits = logits[
+ torch.arange(batch_size, device=logits.device), sequence_lengths
+ ]
+
+ loss = None
+ if labels is not None:
+ loss = self.loss_function(
+ logits=logits,
+ labels=labels,
+ pooled_logits=pooled_logits,
+ config=self.config,
+ )
+
+ if not return_dict:
+ output = (pooled_logits,) + transformer_outputs[1:]
+ return ((loss,) + output) if loss is not None else output
+
+ return SequenceClassifierOutputWithPast(
+ loss=loss,
+ logits=pooled_logits,
+ past_key_values=transformer_outputs.past_key_values,
+ hidden_states=transformer_outputs.hidden_states,
+ attentions=transformer_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """
+ The Phi3 Model transformer with a token classification head on top (a linear layer on top of the hidden-states
+ output) e.g. for Named-Entity-Recognition (NER) tasks.
+ """,
+ PHI3_START_DOCSTRING,
+)
+class Phi3ForTokenClassification(Phi3PreTrainedModel):
+ def __init__(self, config):
+ super().__init__(config)
+ self.num_labels = config.num_labels
+ self.model = Phi3Model(config)
+ if getattr(config, "classifier_dropout", None) is not None:
+ classifier_dropout = config.classifier_dropout
+ elif getattr(config, "hidden_dropout", None) is not None:
+ classifier_dropout = config.hidden_dropout
+ else:
+ classifier_dropout = 0.1
+ self.dropout = nn.Dropout(classifier_dropout)
+ self.score = nn.Linear(config.hidden_size, config.num_labels)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
+ @add_code_sample_docstrings(
+ checkpoint=_CHECKPOINT_FOR_DOC,
+ output_type=TokenClassifierOutput,
+ config_class=_CONFIG_FOR_DOC,
+ )
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, TokenClassifierOutput]:
+ r"""
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ """
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ outputs = self.model(
+ input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+ sequence_output = outputs[0]
+ sequence_output = self.dropout(sequence_output)
+ logits = self.score(sequence_output)
+
+ loss = None
+ if labels is not None:
+ loss = self.loss_function(logits, labels, self.config)
+
+ if not return_dict:
+ output = (logits,) + outputs[2:]
+ return ((loss,) + output) if loss is not None else output
+
+ return TokenClassifierOutput(
+ loss=loss,
+ logits=logits,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ )
+
+
+__all__ = [
+ "Phi3PreTrainedModel",
+ "Phi3Model",
+ "Phi3ForCausalLM",
+ "Phi3ForSequenceClassification",
+ "Phi3ForTokenClassification",
+]
diff --git a/mtraining/model_configs/qwen2/__init__.py b/mtraining/model_configs/qwen2/__init__.py
new file mode 100644
index 00000000..dc6b73dc
--- /dev/null
+++ b/mtraining/model_configs/qwen2/__init__.py
@@ -0,0 +1,11 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from .configuration_qwen2 import Qwen2Config
+from .modeling_qwen2 import (
+ QWEN_ATTN_FUNCS,
+ Qwen2Attention,
+ Qwen2ForCausalLM,
+ apply_rotary_pos_emb,
+ repeat_kv,
+)
diff --git a/mtraining/model_configs/qwen2/configuration_qwen2.py b/mtraining/model_configs/qwen2/configuration_qwen2.py
new file mode 100644
index 00000000..f2e0f4bf
--- /dev/null
+++ b/mtraining/model_configs/qwen2/configuration_qwen2.py
@@ -0,0 +1,198 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Qwen2 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class Qwen2Config(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Qwen2Model`]. It is used to instantiate a
+ Qwen2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of
+ Qwen2-7B-beta [Qwen/Qwen2-7B-beta](https://huggingface.co/Qwen/Qwen2-7B-beta).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 151936):
+ Vocabulary size of the Qwen2 model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Qwen2Model`]
+ hidden_size (`int`, *optional*, defaults to 4096):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 22016):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ num_key_value_heads (`int`, *optional*, defaults to 32):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 32768):
+ The maximum sequence length that this model might ever be used with.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+ The epsilon used by the rms normalization layers.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether the model's input and output word embeddings should be tied.
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`Dict`, *optional*):
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+ accordingly.
+ Expected contents:
+ `rope_type` (`str`):
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+ 'llama3'], with 'default' being the original RoPE implementation.
+ `factor` (`float`, *optional*):
+ Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+ most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+ original maximum pre-trained length.
+ `original_max_position_embeddings` (`int`, *optional*):
+ Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+ pretraining.
+ `attention_factor` (`float`, *optional*):
+ Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+ computation. If unspecified, it defaults to value recommended by the implementation, using the
+ `factor` field to infer the suggested value.
+ `beta_fast` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+ ramp function. If unspecified, it defaults to 32.
+ `beta_slow` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+ ramp function. If unspecified, it defaults to 1.
+ `short_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `long_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `low_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+ `high_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+ use_sliding_window (`bool`, *optional*, defaults to `False`):
+ Whether to use sliding window attention.
+ sliding_window (`int`, *optional*, defaults to 4096):
+ Sliding window attention (SWA) window size. If not specified, will default to `4096`.
+ max_window_layers (`int`, *optional*, defaults to 28):
+ The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+
+ ```python
+ >>> from transformers import Qwen2Model, Qwen2Config
+
+ >>> # Initializing a Qwen2 style configuration
+ >>> configuration = Qwen2Config()
+
+ >>> # Initializing a model from the Qwen2-7B style configuration
+ >>> model = Qwen2Model(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "qwen2"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ # Default tensor parallel plan for base model `Qwen2`
+ base_model_tp_plan = {
+ "layers.*.self_attn.q_proj": "colwise",
+ "layers.*.self_attn.k_proj": "colwise",
+ "layers.*.self_attn.v_proj": "colwise",
+ "layers.*.self_attn.o_proj": "rowwise",
+ "layers.*.mlp.gate_proj": "colwise",
+ "layers.*.mlp.up_proj": "colwise",
+ "layers.*.mlp.down_proj": "rowwise",
+ }
+
+ def __init__(
+ self,
+ vocab_size=151936,
+ hidden_size=4096,
+ intermediate_size=22016,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=32,
+ hidden_act="silu",
+ max_position_embeddings=32768,
+ initializer_range=0.02,
+ rms_norm_eps=1e-6,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ use_sliding_window=False,
+ sliding_window=4096,
+ max_window_layers=28,
+ attention_dropout=0.0,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.max_position_embeddings = max_position_embeddings
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.use_sliding_window = use_sliding_window
+ self.sliding_window = sliding_window if use_sliding_window else None
+ self.max_window_layers = max_window_layers
+
+ # for backward compatibility
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self.attention_dropout = attention_dropout
+ # Validate the correctness of rotary position embeddings parameters
+ # BC: if there is a 'type' field, move it to 'rope_type'.
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+ rope_config_validation(self)
+
+ super().__init__(
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
diff --git a/mtraining/model_configs/qwen2/lc_config_0_5B/config.json b/mtraining/model_configs/qwen2/lc_config_0_5B/config.json
new file mode 100644
index 00000000..8779bfda
--- /dev/null
+++ b/mtraining/model_configs/qwen2/lc_config_0_5B/config.json
@@ -0,0 +1,28 @@
+{
+ "architectures": [
+ "Qwen2ForCausalLM"
+ ],
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "eos_token_id": 151643,
+ "hidden_act": "silu",
+ "hidden_size": 896,
+ "initializer_range": 0.02,
+ "intermediate_size": 4864,
+ "max_position_embeddings": 32768,
+ "max_window_layers": 24,
+ "model_type": "qwen2",
+ "num_attention_heads": 14,
+ "num_hidden_layers": 24,
+ "num_key_value_heads": 2,
+ "rms_norm_eps": 1e-06,
+ "rope_theta": 1000000.0,
+ "sliding_window": 32768,
+ "tie_word_embeddings": true,
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.40.1",
+ "use_cache": true,
+ "use_mrope": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+ }
diff --git a/mtraining/model_configs/qwen2/lc_config_0_5B/configuration_qwen2.py b/mtraining/model_configs/qwen2/lc_config_0_5B/configuration_qwen2.py
new file mode 100644
index 00000000..f2e0f4bf
--- /dev/null
+++ b/mtraining/model_configs/qwen2/lc_config_0_5B/configuration_qwen2.py
@@ -0,0 +1,198 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Qwen2 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class Qwen2Config(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Qwen2Model`]. It is used to instantiate a
+ Qwen2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of
+ Qwen2-7B-beta [Qwen/Qwen2-7B-beta](https://huggingface.co/Qwen/Qwen2-7B-beta).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 151936):
+ Vocabulary size of the Qwen2 model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Qwen2Model`]
+ hidden_size (`int`, *optional*, defaults to 4096):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 22016):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ num_key_value_heads (`int`, *optional*, defaults to 32):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 32768):
+ The maximum sequence length that this model might ever be used with.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+ The epsilon used by the rms normalization layers.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether the model's input and output word embeddings should be tied.
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`Dict`, *optional*):
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+ accordingly.
+ Expected contents:
+ `rope_type` (`str`):
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+ 'llama3'], with 'default' being the original RoPE implementation.
+ `factor` (`float`, *optional*):
+ Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+ most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+ original maximum pre-trained length.
+ `original_max_position_embeddings` (`int`, *optional*):
+ Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+ pretraining.
+ `attention_factor` (`float`, *optional*):
+ Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+ computation. If unspecified, it defaults to value recommended by the implementation, using the
+ `factor` field to infer the suggested value.
+ `beta_fast` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+ ramp function. If unspecified, it defaults to 32.
+ `beta_slow` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+ ramp function. If unspecified, it defaults to 1.
+ `short_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `long_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `low_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+ `high_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+ use_sliding_window (`bool`, *optional*, defaults to `False`):
+ Whether to use sliding window attention.
+ sliding_window (`int`, *optional*, defaults to 4096):
+ Sliding window attention (SWA) window size. If not specified, will default to `4096`.
+ max_window_layers (`int`, *optional*, defaults to 28):
+ The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+
+ ```python
+ >>> from transformers import Qwen2Model, Qwen2Config
+
+ >>> # Initializing a Qwen2 style configuration
+ >>> configuration = Qwen2Config()
+
+ >>> # Initializing a model from the Qwen2-7B style configuration
+ >>> model = Qwen2Model(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "qwen2"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ # Default tensor parallel plan for base model `Qwen2`
+ base_model_tp_plan = {
+ "layers.*.self_attn.q_proj": "colwise",
+ "layers.*.self_attn.k_proj": "colwise",
+ "layers.*.self_attn.v_proj": "colwise",
+ "layers.*.self_attn.o_proj": "rowwise",
+ "layers.*.mlp.gate_proj": "colwise",
+ "layers.*.mlp.up_proj": "colwise",
+ "layers.*.mlp.down_proj": "rowwise",
+ }
+
+ def __init__(
+ self,
+ vocab_size=151936,
+ hidden_size=4096,
+ intermediate_size=22016,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=32,
+ hidden_act="silu",
+ max_position_embeddings=32768,
+ initializer_range=0.02,
+ rms_norm_eps=1e-6,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ use_sliding_window=False,
+ sliding_window=4096,
+ max_window_layers=28,
+ attention_dropout=0.0,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.max_position_embeddings = max_position_embeddings
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.use_sliding_window = use_sliding_window
+ self.sliding_window = sliding_window if use_sliding_window else None
+ self.max_window_layers = max_window_layers
+
+ # for backward compatibility
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self.attention_dropout = attention_dropout
+ # Validate the correctness of rotary position embeddings parameters
+ # BC: if there is a 'type' field, move it to 'rope_type'.
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+ rope_config_validation(self)
+
+ super().__init__(
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
diff --git a/mtraining/model_configs/qwen2/lc_config_3B/config.json b/mtraining/model_configs/qwen2/lc_config_3B/config.json
new file mode 100644
index 00000000..92562389
--- /dev/null
+++ b/mtraining/model_configs/qwen2/lc_config_3B/config.json
@@ -0,0 +1,33 @@
+{
+ "architectures": [
+ "Qwen2ForCausalLM"
+ ],
+ "attention_dropout": 0.0,
+ "bos_token_id": 151643,
+ "eos_token_id": 151643,
+ "hidden_act": "silu",
+ "hidden_size": 2048,
+ "initializer_range": 0.02,
+ "intermediate_size": 11008,
+ "max_position_embeddings": 524288,
+ "max_window_layers": 36,
+ "model_type": "qwen2",
+ "num_attention_heads": 16,
+ "num_hidden_layers": 36,
+ "num_key_value_heads": 2,
+ "rms_norm_eps": 1e-06,
+ "rope_theta": 1000000.0,
+ "rope_scaling": {
+ "factor": 16.0,
+ "original_max_position_embeddings": 32768,
+ "type": "yarn"
+ },
+ "sliding_window": 32768,
+ "tie_word_embeddings": true,
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.40.1",
+ "use_cache": true,
+ "use_mrope": false,
+ "use_sliding_window": false,
+ "vocab_size": 151936
+ }
diff --git a/mtraining/model_configs/qwen2/lc_config_3B/configuration_qwen2.py b/mtraining/model_configs/qwen2/lc_config_3B/configuration_qwen2.py
new file mode 100644
index 00000000..f2e0f4bf
--- /dev/null
+++ b/mtraining/model_configs/qwen2/lc_config_3B/configuration_qwen2.py
@@ -0,0 +1,198 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Qwen2 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class Qwen2Config(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Qwen2Model`]. It is used to instantiate a
+ Qwen2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of
+ Qwen2-7B-beta [Qwen/Qwen2-7B-beta](https://huggingface.co/Qwen/Qwen2-7B-beta).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 151936):
+ Vocabulary size of the Qwen2 model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Qwen2Model`]
+ hidden_size (`int`, *optional*, defaults to 4096):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 22016):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ num_key_value_heads (`int`, *optional*, defaults to 32):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 32768):
+ The maximum sequence length that this model might ever be used with.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+ The epsilon used by the rms normalization layers.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether the model's input and output word embeddings should be tied.
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`Dict`, *optional*):
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+ accordingly.
+ Expected contents:
+ `rope_type` (`str`):
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+ 'llama3'], with 'default' being the original RoPE implementation.
+ `factor` (`float`, *optional*):
+ Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+ most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+ original maximum pre-trained length.
+ `original_max_position_embeddings` (`int`, *optional*):
+ Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+ pretraining.
+ `attention_factor` (`float`, *optional*):
+ Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+ computation. If unspecified, it defaults to value recommended by the implementation, using the
+ `factor` field to infer the suggested value.
+ `beta_fast` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+ ramp function. If unspecified, it defaults to 32.
+ `beta_slow` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+ ramp function. If unspecified, it defaults to 1.
+ `short_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `long_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `low_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+ `high_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+ use_sliding_window (`bool`, *optional*, defaults to `False`):
+ Whether to use sliding window attention.
+ sliding_window (`int`, *optional*, defaults to 4096):
+ Sliding window attention (SWA) window size. If not specified, will default to `4096`.
+ max_window_layers (`int`, *optional*, defaults to 28):
+ The number of layers that use SWA (Sliding Window Attention). The bottom layers use SWA while the top use full attention.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+
+ ```python
+ >>> from transformers import Qwen2Model, Qwen2Config
+
+ >>> # Initializing a Qwen2 style configuration
+ >>> configuration = Qwen2Config()
+
+ >>> # Initializing a model from the Qwen2-7B style configuration
+ >>> model = Qwen2Model(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "qwen2"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ # Default tensor parallel plan for base model `Qwen2`
+ base_model_tp_plan = {
+ "layers.*.self_attn.q_proj": "colwise",
+ "layers.*.self_attn.k_proj": "colwise",
+ "layers.*.self_attn.v_proj": "colwise",
+ "layers.*.self_attn.o_proj": "rowwise",
+ "layers.*.mlp.gate_proj": "colwise",
+ "layers.*.mlp.up_proj": "colwise",
+ "layers.*.mlp.down_proj": "rowwise",
+ }
+
+ def __init__(
+ self,
+ vocab_size=151936,
+ hidden_size=4096,
+ intermediate_size=22016,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=32,
+ hidden_act="silu",
+ max_position_embeddings=32768,
+ initializer_range=0.02,
+ rms_norm_eps=1e-6,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ use_sliding_window=False,
+ sliding_window=4096,
+ max_window_layers=28,
+ attention_dropout=0.0,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.max_position_embeddings = max_position_embeddings
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.use_sliding_window = use_sliding_window
+ self.sliding_window = sliding_window if use_sliding_window else None
+ self.max_window_layers = max_window_layers
+
+ # for backward compatibility
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self.attention_dropout = attention_dropout
+ # Validate the correctness of rotary position embeddings parameters
+ # BC: if there is a 'type' field, move it to 'rope_type'.
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
+ rope_config_validation(self)
+
+ super().__init__(
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
diff --git a/mtraining/model_configs/qwen2/modeling_qwen2.py b/mtraining/model_configs/qwen2/modeling_qwen2.py
new file mode 100644
index 00000000..14aa148b
--- /dev/null
+++ b/mtraining/model_configs/qwen2/modeling_qwen2.py
@@ -0,0 +1,1275 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# This file was automatically generated from src/transformers/models/qwen2/modular_qwen2.py.
+# Do NOT edit this file manually as any edits will be overwritten by the generation of
+# the file from the modular. If any change should be done, please apply the change to the
+# modular_qwen2.py file directly. One of our CI enforces this.
+# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+from typing import Callable, List, Optional, Tuple, Union
+
+import torch
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.cache_utils import (
+ Cache,
+ DynamicCache,
+ SlidingWindowCache,
+ StaticCache,
+)
+from transformers.generation import GenerationMixin
+from transformers.modeling_attn_mask_utils import AttentionMaskConverter
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.modeling_outputs import (
+ BaseModelOutputWithPast,
+ CausalLMOutputWithPast,
+ QuestionAnsweringModelOutput,
+ SequenceClassifierOutputWithPast,
+ TokenClassifierOutput,
+)
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import (
+ LossKwargs,
+ add_code_sample_docstrings,
+ add_start_docstrings,
+ add_start_docstrings_to_model_forward,
+ logging,
+ replace_return_docstrings,
+)
+from transformers.utils.deprecation import deprecate_kwarg
+
+from .configuration_qwen2 import Qwen2Config
+
+logger = logging.get_logger(__name__)
+
+_CHECKPOINT_FOR_DOC = "meta-qwen2/Qwen2-2-7b-hf"
+_CONFIG_FOR_DOC = "Qwen2Config"
+
+QWEN_ATTN_FUNCS = ALL_ATTENTION_FUNCTIONS.copy()
+
+
+class Qwen2MLP(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+ self.config = config
+ self.hidden_size = config.hidden_size
+ self.intermediate_size = config.intermediate_size
+ self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+ self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+ self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+ self.act_fn = ACT2FN[config.hidden_act]
+
+ def forward(self, x):
+ down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+ return down_proj
+
+
+def rotate_half(x):
+ """Rotates half the hidden dims of the input."""
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2 :]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+ """Applies Rotary Position Embedding to the query and key tensors.
+
+ Args:
+ q (`torch.Tensor`): The query tensor. # (B, H, S, D)
+ k (`torch.Tensor`): The key tensor.
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
+ position_ids (`torch.Tensor`, *optional*):
+ Deprecated and unused.
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+ Returns:
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+ """
+ cos = cos.unsqueeze(unsqueeze_dim) # (B, S, D) -> (B, 1, S, D)
+ sin = sin.unsqueeze(unsqueeze_dim) # (B, S, D) -> (B, 1, S, D)
+ q_embed = (q * cos) + (rotate_half(q) * sin)
+ k_embed = (k * cos) + (rotate_half(k) * sin)
+ return q_embed, k_embed
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+ """
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+ """
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+ if n_rep == 1:
+ return hidden_states
+ hidden_states = hidden_states[:, :, None, :, :].expand(
+ batch, num_key_value_heads, n_rep, slen, head_dim
+ )
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+def eager_attention_forward(
+ module: nn.Module,
+ query: torch.Tensor,
+ key: torch.Tensor,
+ value: torch.Tensor,
+ attention_mask: Optional[torch.Tensor],
+ scaling: float,
+ dropout: float = 0.0,
+ **kwargs,
+):
+ key_states = repeat_kv(key, module.num_key_value_groups)
+ value_states = repeat_kv(value, module.num_key_value_groups)
+
+ attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+ if attention_mask is not None:
+ causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+ attn_weights = attn_weights + causal_mask
+
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(
+ query.dtype
+ )
+ attn_weights = nn.functional.dropout(
+ attn_weights, p=dropout, training=module.training
+ )
+ attn_output = torch.matmul(attn_weights, value_states)
+ attn_output = attn_output.transpose(1, 2).contiguous()
+
+ return attn_output, attn_weights
+
+
+class Qwen2Attention(nn.Module):
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+ def __init__(self, config: Qwen2Config, layer_idx: int):
+ super().__init__()
+ self.config = config
+ self.layer_idx = layer_idx
+ self.head_dim = getattr(
+ config, "head_dim", config.hidden_size // config.num_attention_heads
+ )
+ self.num_key_value_groups = (
+ config.num_attention_heads // config.num_key_value_heads
+ )
+ self.scaling = self.head_dim**-0.5
+ self.attention_dropout = config.attention_dropout
+ self.is_causal = True
+ self.q_proj = nn.Linear(
+ config.hidden_size, config.num_attention_heads * self.head_dim, bias=True
+ )
+ self.k_proj = nn.Linear(
+ config.hidden_size, config.num_key_value_heads * self.head_dim, bias=True
+ )
+ self.v_proj = nn.Linear(
+ config.hidden_size, config.num_key_value_heads * self.head_dim, bias=True
+ )
+ self.o_proj = nn.Linear(
+ config.num_attention_heads * self.head_dim, config.hidden_size, bias=False
+ )
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+ attention_mask: Optional[torch.Tensor],
+ past_key_value: Optional[Cache] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **kwargs: Unpack[FlashAttentionKwargs],
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ input_shape = hidden_states.shape[:-1]
+ hidden_shape = (*input_shape, -1, self.head_dim)
+
+ query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+ key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+ value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+ cos, sin = position_embeddings
+ query_states, key_states = apply_rotary_pos_emb(
+ query_states, key_states, cos, sin
+ )
+
+ if past_key_value is not None:
+ # sin and cos are specific to RoPE models; cache_position needed for the static cache
+ cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+ key_states, value_states = past_key_value.update(
+ key_states, value_states, self.layer_idx, cache_kwargs
+ )
+
+ sliding_window = None
+ if (
+ self.config.use_sliding_window
+ and getattr(self.config, "sliding_window", None) is not None
+ and self.layer_idx >= self.config.max_window_layers
+ ):
+ sliding_window = self.config.sliding_window
+
+ attention_interface: Callable = eager_attention_forward
+ if self.config._attn_implementation != "eager":
+ if self.config._attn_implementation == "sdpa" and kwargs.get(
+ "output_attentions", False
+ ):
+ logger.warning_once(
+ "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to "
+ 'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+ )
+ else:
+ attention_interface = QWEN_ATTN_FUNCS[self.config._attn_implementation]
+
+ attn_output, attn_weights = attention_interface(
+ self,
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ dropout=0.0 if not self.training else self.attention_dropout,
+ scaling=self.scaling,
+ sliding_window=sliding_window, # main diff with Llama
+ **kwargs,
+ )
+
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+ attn_output = self.o_proj(attn_output)
+ return attn_output, attn_weights
+
+
+class Qwen2RMSNorm(nn.Module):
+ def __init__(self, hidden_size, eps=1e-6):
+ """
+ Qwen2RMSNorm is equivalent to T5LayerNorm
+ """
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.variance_epsilon = eps
+
+ def forward(self, hidden_states):
+ input_dtype = hidden_states.dtype
+ hidden_states = hidden_states.to(torch.float32)
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+ return self.weight * hidden_states.to(input_dtype)
+
+ def extra_repr(self):
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class Qwen2DecoderLayer(nn.Module):
+ def __init__(self, config: Qwen2Config, layer_idx: int):
+ super().__init__()
+ self.hidden_size = config.hidden_size
+ self.self_attn = Qwen2Attention(config=config, layer_idx=layer_idx)
+ self.mlp = Qwen2MLP(config)
+ self.input_layernorm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.post_attention_layernorm = Qwen2RMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps
+ )
+ if config.sliding_window and config._attn_implementation != "flash_attention_2":
+ logger.warning_once(
+ f"Sliding Window Attention is enabled but not implemented for `{config._attn_implementation}`; "
+ "unexpected results may be encountered."
+ )
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: Optional[bool] = False,
+ use_cache: Optional[bool] = False,
+ cache_position: Optional[torch.LongTensor] = None,
+ position_embeddings: Optional[
+ Tuple[torch.Tensor, torch.Tensor]
+ ] = None, # necessary, but kept here for BC
+ **kwargs: Unpack[FlashAttentionKwargs],
+ ) -> Tuple[
+ torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
+ ]:
+ residual = hidden_states
+
+ hidden_states = self.input_layernorm(hidden_states)
+
+ # Self Attention
+ hidden_states, self_attn_weights = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ **kwargs,
+ )
+ hidden_states = residual + hidden_states
+
+ # Fully Connected
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+
+ outputs = (hidden_states,)
+ if output_attentions:
+ outputs += (self_attn_weights,)
+
+ return outputs
+
+
+class Qwen2RotaryEmbedding(nn.Module):
+ def __init__(self, config: Qwen2Config, device=None):
+ super().__init__()
+ # BC: "rope_type" was originally "type"
+ if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
+ self.rope_type = config.rope_scaling.get(
+ "rope_type", config.rope_scaling.get("type")
+ )
+ else:
+ self.rope_type = "default"
+ self.max_seq_len_cached = config.max_position_embeddings
+ self.original_max_seq_len = config.max_position_embeddings
+
+ self.config = config
+ self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+
+ inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+ self.original_inv_freq = self.inv_freq
+
+ def _dynamic_frequency_update(self, position_ids, device):
+ """
+ dynamic RoPE layers should recompute `inv_freq` in the following situations:
+ 1 - growing beyond the cached sequence length (allow scaling)
+ 2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
+ """
+ seq_len = torch.max(position_ids) + 1
+ if seq_len > self.max_seq_len_cached: # growth
+ inv_freq, self.attention_scaling = self.rope_init_fn(
+ self.config, device, seq_len=seq_len
+ )
+ self.register_buffer(
+ "inv_freq", inv_freq, persistent=False
+ ) # TODO joao: may break with compilation
+ self.max_seq_len_cached = seq_len
+
+ if (
+ seq_len < self.original_max_seq_len
+ and self.max_seq_len_cached > self.original_max_seq_len
+ ): # reset
+ # This .to() is needed if the model has been moved to a device after being initialized (because
+ # the buffer is automatically moved, but not the original copy)
+ self.original_inv_freq = self.original_inv_freq.to(device)
+ self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
+ self.max_seq_len_cached = self.original_max_seq_len
+
+ @torch.no_grad()
+ def forward(self, x, position_ids):
+ if "dynamic" in self.rope_type:
+ self._dynamic_frequency_update(position_ids, device=x.device)
+
+ # Core RoPE block
+ inv_freq_expanded = (
+ self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
+ ) # (B, D // 2, 1)
+ position_ids_expanded = position_ids[:, None, :].float() # (B, 1, S)
+ # Force float32 (see https://github.com/huggingface/transformers/pull/29285)
+ device_type = x.device.type
+ device_type = (
+ device_type
+ if isinstance(device_type, str) and device_type != "mps"
+ else "cpu"
+ )
+ with torch.autocast(device_type=device_type, enabled=False):
+ freqs = (
+ inv_freq_expanded.float() @ position_ids_expanded.float()
+ ).transpose(
+ 1, 2
+ ) # (B, S, D // 2)
+ emb = torch.cat((freqs, freqs), dim=-1) # (B, S, D)
+ cos = emb.cos()
+ sin = emb.sin()
+
+ # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention
+ cos = cos * self.attention_scaling
+ sin = sin * self.attention_scaling
+
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+QWEN2_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`Qwen2Config`]):
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
+ load the weights associated with the model, only the configuration. Check out the
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+ "The bare Qwen2 Model outputting raw hidden-states without any specific head on top.",
+ QWEN2_START_DOCSTRING,
+)
+class Qwen2PreTrainedModel(PreTrainedModel):
+ config_class = Qwen2Config
+ base_model_prefix = "model"
+ supports_gradient_checkpointing = True
+ _no_split_modules = ["Qwen2DecoderLayer"]
+ _skip_keys_device_placement = ["past_key_values"]
+ _supports_flash_attn_2 = True
+ _supports_sdpa = True
+ _supports_flex_attn = True
+ _supports_cache_class = True
+ _supports_quantized_cache = True
+ _supports_static_cache = True
+ _supports_attention_backend = True
+
+ def _init_weights(self, module):
+ std = self.config.initializer_range
+ if isinstance(module, nn.Linear):
+ module.weight.data.normal_(mean=0.0, std=std)
+ if module.bias is not None:
+ module.bias.data.zero_()
+ elif isinstance(module, nn.Embedding):
+ module.weight.data.normal_(mean=0.0, std=std)
+ if module.padding_idx is not None:
+ module.weight.data[module.padding_idx].zero_()
+
+
+QWEN2_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
+ `past_key_values`).
+
+ If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+ information on the default strategy.
+
+ - 1 indicates the head is **not masked**,
+ - 0 indicates the head is **masked**.
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.n_positions - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+ Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+ blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
+ returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
+
+ Two formats are allowed:
+ - a [`~cache_utils.Cache`] instance, see our
+ [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
+ - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
+ cache format.
+
+ The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
+ legacy cache format will be returned.
+
+ If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+ have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+ of shape `(batch_size, sequence_length)`.
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
+ use_cache (`bool`, *optional*):
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+ cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
+ Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
+ this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
+ the complete sequence length.
+"""
+
+
+@add_start_docstrings(
+ "The bare Qwen2 Model outputting raw hidden-states without any specific head on top.",
+ QWEN2_START_DOCSTRING,
+)
+class Qwen2Model(Qwen2PreTrainedModel):
+ """
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Qwen2DecoderLayer`]
+
+ Args:
+ config: Qwen2Config
+ """
+
+ def __init__(self, config: Qwen2Config):
+ super().__init__(config)
+ self.padding_idx = config.pad_token_id
+ self.vocab_size = config.vocab_size
+
+ self.embed_tokens = nn.Embedding(
+ config.vocab_size, config.hidden_size, self.padding_idx
+ )
+ self.layers = nn.ModuleList(
+ [
+ Qwen2DecoderLayer(config, layer_idx)
+ for layer_idx in range(config.num_hidden_layers)
+ ]
+ )
+ self.norm = Qwen2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.rotary_emb = Qwen2RotaryEmbedding(config=config)
+ self.gradient_checkpointing = False
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[
+ torch.LongTensor
+ ] = None, # example: torch.arange(0, 1024).unsqueeze(0) (shape: [B, S])
+ past_key_values: Optional[Cache] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ **flash_attn_kwargs: Unpack[FlashAttentionKwargs],
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
+ output_attentions = (
+ output_attentions
+ if output_attentions is not None
+ else self.config.output_attentions
+ )
+ output_hidden_states = (
+ output_hidden_states
+ if output_hidden_states is not None
+ else self.config.output_hidden_states
+ )
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ if (input_ids is None) ^ (inputs_embeds is not None):
+ raise ValueError(
+ "You must specify exactly one of input_ids or inputs_embeds"
+ )
+
+ if self.gradient_checkpointing and self.training and use_cache:
+ logger.warning_once(
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
+ )
+ use_cache = False
+
+ if inputs_embeds is None:
+ inputs_embeds = self.embed_tokens(input_ids)
+
+ if use_cache and past_key_values is None:
+ past_key_values = DynamicCache()
+
+ if cache_position is None:
+ past_seen_tokens = (
+ past_key_values.get_seq_length() if past_key_values is not None else 0
+ )
+ cache_position = torch.arange(
+ past_seen_tokens,
+ past_seen_tokens + inputs_embeds.shape[1],
+ device=inputs_embeds.device,
+ )
+
+ if position_ids is None:
+ position_ids = cache_position.unsqueeze(0)
+
+ causal_mask = self._update_causal_mask(
+ attention_mask,
+ inputs_embeds,
+ cache_position,
+ past_key_values,
+ output_attentions,
+ )
+
+ hidden_states = inputs_embeds
+
+ # create position embeddings to be shared across the decoder layers
+ position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+ # decoder layers
+ all_hidden_states = () if output_hidden_states else None
+ all_self_attns = () if output_attentions else None
+
+ for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ if self.gradient_checkpointing and self.training:
+ layer_outputs = self._gradient_checkpointing_func(
+ decoder_layer.__call__,
+ hidden_states,
+ causal_mask,
+ position_ids,
+ past_key_values,
+ output_attentions,
+ use_cache,
+ cache_position,
+ position_embeddings,
+ )
+ else:
+ layer_outputs = decoder_layer(
+ hidden_states,
+ attention_mask=causal_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_values,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ cache_position=cache_position,
+ position_embeddings=position_embeddings,
+ **flash_attn_kwargs,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if output_attentions:
+ all_self_attns += (layer_outputs[1],)
+
+ hidden_states = self.norm(hidden_states)
+
+ # add hidden states from the last decoder layer
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ output = BaseModelOutputWithPast(
+ last_hidden_state=hidden_states,
+ past_key_values=past_key_values if use_cache else None,
+ hidden_states=all_hidden_states,
+ attentions=all_self_attns,
+ )
+ return output if return_dict else output.to_tuple()
+
+ def _update_causal_mask(
+ self,
+ attention_mask: torch.Tensor,
+ input_tensor: torch.Tensor,
+ cache_position: torch.Tensor,
+ past_key_values: Cache,
+ output_attentions: bool,
+ ):
+ if self.config._attn_implementation == "flash_attention_2":
+ if attention_mask is not None and (attention_mask == 0.0).any():
+ return attention_mask
+ return None
+
+ # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in
+ # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail
+ # to infer the attention mask.
+ past_seen_tokens = (
+ past_key_values.get_seq_length() if past_key_values is not None else 0
+ )
+ using_static_cache = isinstance(past_key_values, StaticCache)
+
+ # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward
+ if (
+ self.config._attn_implementation == "sdpa"
+ and not using_static_cache
+ and not output_attentions
+ ):
+ if AttentionMaskConverter._ignore_causal_mask_sdpa(
+ attention_mask,
+ inputs_embeds=input_tensor,
+ past_key_values_length=past_seen_tokens,
+ is_training=self.training,
+ ):
+ return None
+
+ dtype, device = input_tensor.dtype, input_tensor.device
+ sequence_length = input_tensor.shape[1]
+ if using_static_cache:
+ target_length = past_key_values.get_max_cache_shape()
+ else:
+ target_length = (
+ attention_mask.shape[-1]
+ if isinstance(attention_mask, torch.Tensor)
+ else past_seen_tokens + sequence_length + 1
+ )
+
+ # In case the provided `attention` mask is 2D, we generate a causal mask here (4D).
+ causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position(
+ attention_mask,
+ sequence_length=sequence_length,
+ target_length=target_length,
+ dtype=dtype,
+ device=device,
+ cache_position=cache_position,
+ batch_size=input_tensor.shape[0],
+ )
+
+ if (
+ self.config._attn_implementation == "sdpa"
+ and attention_mask is not None
+ and attention_mask.device.type == "cuda"
+ and not output_attentions
+ ):
+ # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when
+ # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path.
+ # Details: https://github.com/pytorch/pytorch/issues/110213
+ min_dtype = torch.finfo(dtype).min
+ causal_mask = AttentionMaskConverter._unmask_unattended(
+ causal_mask, min_dtype
+ )
+
+ return causal_mask
+
+ @staticmethod
+ def _prepare_4d_causal_attention_mask_with_cache_position(
+ attention_mask: torch.Tensor,
+ sequence_length: int,
+ target_length: int,
+ dtype: torch.dtype,
+ device: torch.device,
+ cache_position: torch.Tensor,
+ batch_size: int,
+ **kwargs,
+ ):
+ """
+ Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
+ `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
+
+ Args:
+ attention_mask (`torch.Tensor`):
+ A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
+ `(batch_size, 1, query_length, key_value_length)`.
+ sequence_length (`int`):
+ The sequence length being processed.
+ target_length (`int`):
+ The target length: when generating with static cache, the mask should be as long as the static cache,
+ to account for the 0 padding, the part of the cache that is not filled yet.
+ dtype (`torch.dtype`):
+ The dtype to use for the 4D attention mask.
+ device (`torch.device`):
+ The device to plcae the 4D attention mask on.
+ cache_position (`torch.Tensor`):
+ Indices depicting the position of the input sequence tokens in the sequence.
+ batch_size (`torch.Tensor`):
+ Batch size.
+ """
+ if attention_mask is not None and attention_mask.dim() == 4:
+ # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
+ causal_mask = attention_mask
+ else:
+ min_dtype = torch.finfo(dtype).min
+ causal_mask = torch.full(
+ (sequence_length, target_length),
+ fill_value=min_dtype,
+ dtype=dtype,
+ device=device,
+ )
+ if sequence_length != 1:
+ causal_mask = torch.triu(causal_mask, diagonal=1)
+ causal_mask *= torch.arange(
+ target_length, device=device
+ ) > cache_position.reshape(-1, 1)
+ causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
+ if attention_mask is not None:
+ causal_mask = (
+ causal_mask.clone()
+ ) # copy to contiguous memory for in-place edit
+ mask_length = attention_mask.shape[-1]
+ padding_mask = (
+ causal_mask[:, :, :, :mask_length]
+ + attention_mask[:, None, None, :]
+ )
+ padding_mask = padding_mask == 0
+ causal_mask[:, :, :, :mask_length] = causal_mask[
+ :, :, :, :mask_length
+ ].masked_fill(padding_mask, min_dtype)
+
+ return causal_mask
+
+
+class KwargsForCausalLM(FlashAttentionKwargs, LossKwargs):
+ ...
+
+
+class Qwen2ForCausalLM(Qwen2PreTrainedModel, GenerationMixin):
+ _tied_weights_keys = ["lm_head.weight"]
+ _tp_plan = {"lm_head": "colwise_rep"}
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.model = Qwen2Model(config)
+ self.vocab_size = config.vocab_size
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ def set_output_embeddings(self, new_embeddings):
+ self.lm_head = new_embeddings
+
+ def set_decoder(self, decoder):
+ self.model = decoder
+
+ def get_decoder(self):
+ return self.model
+
+ @deprecate_kwarg("num_logits_to_keep", version="4.50", new_name="logits_to_keep")
+ @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+ @replace_return_docstrings(
+ output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC
+ )
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ cache_position: Optional[torch.LongTensor] = None,
+ logits_to_keep: Union[int, torch.Tensor] = 0,
+ **kwargs: Unpack[KwargsForCausalLM],
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
+ r"""
+ Args:
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+ logits_to_keep (`int` or `torch.Tensor`, *optional*):
+ If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
+ `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
+ token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
+ If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
+ This is useful when using packed tensor format (single dimension for batch and sequence length).
+
+ Returns:
+
+ Example:
+
+ ```python
+ >>> from transformers import AutoTokenizer, Qwen2ForCausalLM
+
+ >>> model = Qwen2ForCausalLM.from_pretrained("meta-qwen2/Qwen2-2-7b-hf")
+ >>> tokenizer = AutoTokenizer.from_pretrained("meta-qwen2/Qwen2-2-7b-hf")
+
+ >>> prompt = "Hey, are you conscious? Can you talk to me?"
+ >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+ >>> # Generate
+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+ "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
+ ```"""
+ output_attentions = (
+ output_attentions
+ if output_attentions is not None
+ else self.config.output_attentions
+ )
+ output_hidden_states = (
+ output_hidden_states
+ if output_hidden_states is not None
+ else self.config.output_hidden_states
+ )
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+ outputs = self.model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ cache_position=cache_position,
+ **kwargs,
+ )
+
+ hidden_states = outputs[0]
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+ slice_indices = (
+ slice(-logits_to_keep, None)
+ if isinstance(logits_to_keep, int)
+ else logits_to_keep
+ )
+ logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+ loss = None
+ if labels is not None:
+ loss = self.loss_function(
+ logits=logits,
+ labels=labels,
+ vocab_size=self.config.vocab_size,
+ **kwargs,
+ )
+
+ if not return_dict:
+ output = (logits,) + outputs[1:]
+ return (loss,) + output if loss is not None else output
+
+ return CausalLMOutputWithPast(
+ loss=loss,
+ logits=logits,
+ past_key_values=outputs.past_key_values,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """
+ The Qwen2 Model transformer with a sequence classification head on top (linear layer).
+
+ [`Qwen2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-2) do.
+
+ Since it does classification on the last token, it requires to know the position of the last token. If a
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
+ """,
+ QWEN2_START_DOCSTRING,
+)
+class Qwen2ForSequenceClassification(Qwen2PreTrainedModel):
+ def __init__(self, config):
+ super().__init__(config)
+ self.num_labels = config.num_labels
+ self.model = Qwen2Model(config)
+ self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+ r"""
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ """
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ transformer_outputs = self.model(
+ input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+ hidden_states = transformer_outputs[0]
+ logits = self.score(hidden_states)
+
+ if input_ids is not None:
+ batch_size = input_ids.shape[0]
+ else:
+ batch_size = inputs_embeds.shape[0]
+
+ if self.config.pad_token_id is None and batch_size != 1:
+ raise ValueError(
+ "Cannot handle batch sizes > 1 if no padding token is defined."
+ )
+ if self.config.pad_token_id is None:
+ sequence_lengths = -1
+ else:
+ if input_ids is not None:
+ # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+ sequence_lengths = (
+ torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+ )
+ sequence_lengths = sequence_lengths % input_ids.shape[-1]
+ sequence_lengths = sequence_lengths.to(logits.device)
+ else:
+ sequence_lengths = -1
+
+ pooled_logits = logits[
+ torch.arange(batch_size, device=logits.device), sequence_lengths
+ ]
+
+ loss = None
+ if labels is not None:
+ loss = self.loss_function(
+ logits=logits,
+ labels=labels,
+ pooled_logits=pooled_logits,
+ config=self.config,
+ )
+
+ if not return_dict:
+ output = (pooled_logits,) + transformer_outputs[1:]
+ return ((loss,) + output) if loss is not None else output
+
+ return SequenceClassifierOutputWithPast(
+ loss=loss,
+ logits=pooled_logits,
+ past_key_values=transformer_outputs.past_key_values,
+ hidden_states=transformer_outputs.hidden_states,
+ attentions=transformer_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """
+ The Qwen2 Model transformer with a token classification head on top (a linear layer on top of the hidden-states
+ output) e.g. for Named-Entity-Recognition (NER) tasks.
+ """,
+ QWEN2_START_DOCSTRING,
+)
+class Qwen2ForTokenClassification(Qwen2PreTrainedModel):
+ def __init__(self, config):
+ super().__init__(config)
+ self.num_labels = config.num_labels
+ self.model = Qwen2Model(config)
+ if getattr(config, "classifier_dropout", None) is not None:
+ classifier_dropout = config.classifier_dropout
+ elif getattr(config, "hidden_dropout", None) is not None:
+ classifier_dropout = config.hidden_dropout
+ else:
+ classifier_dropout = 0.1
+ self.dropout = nn.Dropout(classifier_dropout)
+ self.score = nn.Linear(config.hidden_size, config.num_labels)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+ @add_code_sample_docstrings(
+ checkpoint=_CHECKPOINT_FOR_DOC,
+ output_type=TokenClassifierOutput,
+ config_class=_CONFIG_FOR_DOC,
+ )
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, TokenClassifierOutput]:
+ r"""
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ """
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ outputs = self.model(
+ input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+ sequence_output = outputs[0]
+ sequence_output = self.dropout(sequence_output)
+ logits = self.score(sequence_output)
+
+ loss = None
+ if labels is not None:
+ loss = self.loss_function(logits, labels, self.config)
+
+ if not return_dict:
+ output = (logits,) + outputs[2:]
+ return ((loss,) + output) if loss is not None else output
+
+ return TokenClassifierOutput(
+ loss=loss,
+ logits=logits,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """
+The Qwen2 Model transformer with a span classification head on top for extractive question-answering tasks like
+SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`).
+ """,
+ QWEN2_START_DOCSTRING,
+)
+class Qwen2ForQuestionAnswering(Qwen2PreTrainedModel):
+ base_model_prefix = "transformer"
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.transformer = Qwen2Model(config)
+ self.qa_outputs = nn.Linear(config.hidden_size, 2)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.transformer.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.transformer.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(QWEN2_INPUTS_DOCSTRING)
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ attention_mask: Optional[torch.FloatTensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ start_positions: Optional[torch.LongTensor] = None,
+ end_positions: Optional[torch.LongTensor] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ **kwargs,
+ ) -> Union[Tuple, QuestionAnsweringModelOutput]:
+ r"""
+ start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for position (index) of the start of the labelled span for computing the token classification loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
+ end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for position (index) of the end of the labelled span for computing the token classification loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
+ """
+ return_dict = (
+ return_dict if return_dict is not None else self.config.use_return_dict
+ )
+
+ outputs = self.transformer(
+ input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ sequence_output = outputs[0]
+
+ logits = self.qa_outputs(sequence_output)
+ start_logits, end_logits = logits.split(1, dim=-1)
+ start_logits = start_logits.squeeze(-1).contiguous()
+ end_logits = end_logits.squeeze(-1).contiguous()
+
+ loss = None
+ if start_positions is not None and end_positions is not None:
+ loss = self.loss_function(
+ start_logits, end_logits, start_positions, end_positions, **kwargs
+ )
+
+ if not return_dict:
+ output = (start_logits, end_logits) + outputs[2:]
+ return ((loss,) + output) if loss is not None else output
+
+ return QuestionAnsweringModelOutput(
+ loss=loss,
+ start_logits=start_logits,
+ end_logits=end_logits,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ )
diff --git a/mtraining/requirements.txt b/mtraining/requirements.txt
new file mode 100644
index 00000000..f7b8e409
--- /dev/null
+++ b/mtraining/requirements.txt
@@ -0,0 +1,6 @@
+transformers==4.48.0
+datasets==2.20.0
+tensorboard
+
+# For Data Preparation
+mosaicml-streaming==0.8.1
diff --git a/mtraining/setup.py b/mtraining/setup.py
new file mode 100644
index 00000000..dddd7718
--- /dev/null
+++ b/mtraining/setup.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from setuptools import find_packages, setup
+
+setup(
+ name="mtraining", # Name of your project
+ version="0.1.0",
+ packages=find_packages(), # Automatically discover all packages
+ install_requires=[], # List dependencies if any (or use requirements.txt)
+ classifiers=[
+ "Programming Language :: Python :: 3",
+ "License :: OSI Approved :: MIT License",
+ "Operating System :: OS Independent",
+ ],
+ python_requires=">=3.10", # Specify the Python version
+)
diff --git a/mtraining/setup.sh b/mtraining/setup.sh
new file mode 100755
index 00000000..aa80d987
--- /dev/null
+++ b/mtraining/setup.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/bash
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+set -e
+
+BASE_DIR="$(cd "$(dirname "$0")" && pwd)" # path/to/MInference/mtraining
+PROJECT_ROOT="$(cd "${BASE_DIR}/.." && pwd)" # path/to/MInference
+PIP="$(which pip)"
+
+sudo $PIP install -U pip setuptools wheel
+sudo $PIP install ninja cmake pybind11 packaging psutil pytest
+sudo $PIP install -r "${BASE_DIR}/requirements.txt"
+
+sudo $PIP install git+https://github.com/microsoft/nnscaler.git@2368540417bc3b77b7e714d3f1a0de8a51bb66e8
+sudo $PIP install "rotary-emb @ git+https://github.com/Dao-AILab/flash-attention.git@9356a1c0389660d7e231ff3163c1ac17d9e3824a#subdirectory=csrc/rotary" --no-build-isolation
+sudo $PIP install "block_sparse_attn @ git+https://github.com/HalberdOfPineapple/flash-attention.git@block-sparse" --no-build-isolation
+sudo $PIP install git+https://github.com/Dao-AILab/flash-attention.git@v2.7.4.post1 --no-build-isolation
+sudo $PIP install torch==2.3.1 torchvision==0.18.1
+sudo $PIP install triton==3.0.0
+
+# Get the path to nnscaler and write its path to PYTHONPATH in ~/.profile
+NNSCALER_HOME=$(python -c "import nnscaler; print(nnscaler.__path__[0])")
+echo "export NNSCALER_HOME=${NNSCALER_HOME}" >> ~/.profile
+echo "export PYTHONPATH=${NNSCALER_HOME}:${PROJECT_ROOT}:\${PYTHONPATH}" >> ~/.profile
+source ~/.profile
+
+cd $PROJECT_ROOT
+sudo MINFERENCE_FORCE_BUILD=TRUE $PIP install -e . --no-build-isolation
+
+cd $BASE_DIR
+sudo $PIP install -e $BASE_DIR
+
+cp -r $PROJECT_ROOT/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/* $NNSCALER_HOME/resources/profile/mi200/comm/
diff --git a/mtraining/train.py b/mtraining/train.py
new file mode 100644
index 00000000..8d4d4072
--- /dev/null
+++ b/mtraining/train.py
@@ -0,0 +1,697 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+import logging
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+import os
+from typing import Dict, List, Optional
+
+import numpy as np
+import torch
+import yaml
+from datasets import load_from_disk
+from nnscaler.cli.loggers.tensorboard import TensorBoardLogger
+from nnscaler.cli.trainer_args import (
+ CheckpointConfig,
+ DataloaderConfig,
+ DatasetConfig,
+ DatasetSamplerConfig,
+ HookMapConfig,
+ LogConfig,
+ ModelConfig,
+ OptimizerConfig,
+)
+from nnscaler.parallel import ComputeConfig
+from nnscaler.runtime.f16_optimizer import MixedPrecisionAdamW
+from nnscaler.utils import set_default_logger_level
+from transformers import AutoConfig, DataCollatorForLanguageModeling
+from transformers.modeling_utils import PreTrainedModel
+
+from minference.configs.model2path import BASE_DIR as SPARSE_PATTERN_CONFIG_DIR
+from minference.minference_configuration import MInferenceConfig
+from minference.models_patch import MInference
+from mtraining.attn_funcs import AttnType, overwrite_attn_implementation
+from mtraining.model_configs import (
+ get_model_attn_funcs,
+ get_model_cls,
+ get_model_prefix,
+)
+from mtraining.trainer import CustomTrainer as Trainer
+from mtraining.trainer import CustomTrainerArgs as TrainerArgs
+from mtraining.utils import (
+ aggregate_outputs_fn,
+ chunk_linear_cross_entropy,
+ get_resume_path,
+ get_tokenizer,
+)
+from mtraining.utils.general import freeze_model_params, load_comm_profile_data
+from mtraining.utils.paths import update_expr_data_save_path
+
+IGNORE_IDX = -100
+logger = logging.getLogger(__name__)
+set_default_logger_level("INFO")
+
+
+def init_by_attn_type(model_id: str, attn_type: AttnType):
+ attn_dict = get_model_attn_funcs(model_id)
+ logger.info(f"Using {attn_type} attention implementation for {model_id}")
+ overwrite_attn_implementation(attn_dict, attn_type)
+
+
+class BaselineModel(torch.nn.Module):
+ def __init__(
+ self,
+ model_id,
+ config_path: str = None,
+ # merged_ckpt_path: str=None,
+ active_param_config_path: str = None,
+ ):
+ super().__init__()
+ model_cls: PreTrainedModel = get_model_cls(model_id)
+
+ if not config_path:
+ self.model = model_cls.from_pretrained(
+ model_id, attn_implementation="flash_attention_2"
+ )
+ else:
+ model_config = AutoConfig.from_pretrained(
+ config_path, trust_remote_code=True
+ )
+ model_config._attn_implementation = "flash_attention_2"
+ self.model = model_cls.from_pretrained(
+ model_id,
+ config=model_config,
+ )
+
+ if active_param_config_path:
+ freeze_model_params(self.model, active_param_config_path)
+
+ print(
+ f"{__class__.__name__} Self-Attention Class: {self.model.model.layers[0].self_attn.__class__.__name__}"
+ )
+
+ def forward(self, samples):
+ with torch.autocast(device_type="cuda", dtype=self.model.config.torch_dtype):
+ outputs = self.model.model(
+ input_ids=samples["net_input"]["src_tokens"],
+ use_cache=False,
+ return_dict=False,
+ )
+ hidden_states = outputs[0]
+ losses = chunk_linear_cross_entropy(
+ hidden_states,
+ self.model.lm_head.weight,
+ samples["target"],
+ IGNORE_IDX,
+ 1024,
+ )
+ loss = torch.sum(losses)
+
+ return loss, loss.data, samples["ntokens"], samples["nsentences"]
+
+
+class MInferModel(BaselineModel):
+ def __init__(
+ self,
+ model_id,
+ config_path: str = None,
+ minfer_config: Dict = {},
+ **kwargs,
+ ):
+ super().__init__(
+ model_id=model_id,
+ config_path=config_path,
+ **kwargs,
+ )
+
+ # ----------------------------------------------
+ # Ring Attention specific
+ granularity: int = minfer_config.pop("granularity", 128)
+
+ # --------------------------------------------
+ # MInference Setup
+ minfer_implementation: str = minfer_config.pop("implementation", "default")
+ minfer_attn_type = minfer_config.pop("attn_type", "minference")
+ minfer_config["config_path"] = os.path.join(
+ SPARSE_PATTERN_CONFIG_DIR,
+ f'{minfer_config.pop("pattern_config_name")}.json',
+ )
+ print(
+ f"{__name__} | MInference Pattern Config Path: {minfer_config['config_path']}"
+ )
+ minfer = MInference(
+ attn_type=minfer_attn_type,
+ model_name=model_id,
+ **minfer_config,
+ )
+ minfer_config: MInferenceConfig = minfer.config
+
+ # --------------------------------------------
+ # We still need to attach the function object to the model
+ # otherwise the states of the function will be lost as nnscaler will only load the model from file
+ # but not call this procedure again
+ from mtraining.attn_funcs.minfer_func import MInferAttnFunc
+
+ Attention = self.model.model.layers[0].self_attn.__class__
+
+ def update_module(m):
+ if isinstance(m, Attention):
+ m.minfer_attn_func = MInferAttnFunc()
+ m.minfer_attn_func.init_minfer_params(
+ config_path=minfer_config.config_path,
+ minfer_implementation=minfer_implementation,
+ granularity=granularity,
+ )
+
+ self.model.apply(update_module)
+
+
+class XAttnModel(BaselineModel):
+ def __init__(
+ self,
+ model_id,
+ config_path: str = None,
+ xattn_params: Dict = {},
+ **kwargs,
+ ):
+ super().__init__(
+ model_id=model_id,
+ config_path=config_path,
+ **kwargs,
+ )
+
+ # --------------------------------------------
+ implementation: str = xattn_params.pop("implementation", "fa")
+ granularity: int = xattn_params.pop("granularity", 128)
+
+ # --------------------------------------------
+ Attention = self.model.model.layers[0].self_attn.__class__
+
+ def update_module(m):
+ if isinstance(m, Attention):
+ m.granularity = granularity
+ m.xattn_params = xattn_params
+ m.implementation = implementation
+
+ self.model.apply(update_module)
+
+
+class MoBAModel(BaselineModel):
+ def __init__(
+ self,
+ model_id,
+ config_path: str = None,
+ moba_config_dict: Dict = {},
+ **kwargs,
+ ):
+ super().__init__(
+ model_id=model_id,
+ config_path=config_path,
+ **kwargs,
+ )
+ # --------------------------------------------
+ moba_topk, moba_chunk_size = (
+ moba_config_dict["moba_topk"],
+ moba_config_dict["moba_chunk_size"],
+ )
+ moba_implementation = moba_config_dict.get("implementation", "default")
+
+ # --------------------------------------------
+ # We still need to attach the function object to the model
+ # otherwise the states of the function will be lost as nnscaler will only load the model from file
+ # but not call this procedure again
+ Attention = self.model.model.layers[0].self_attn.__class__
+
+ def update_module(m):
+ if isinstance(m, Attention):
+ m.moba_topk = moba_topk
+ m.moba_chunk_size = moba_chunk_size
+ m.implementation = moba_implementation
+
+ self.model.apply(update_module)
+
+
+ATTN_TO_MODEL = {
+ AttnType.DENSE: BaselineModel,
+ AttnType.STRIPE_RING: BaselineModel,
+ AttnType.ZIGZAG_RING: BaselineModel,
+ AttnType.MINFER: MInferModel,
+ AttnType.MOBA: MoBAModel,
+ AttnType.XATTN: XAttnModel,
+}
+
+
+def load_train_attn_config(train_attn_config_path: str) -> MInferenceConfig:
+ if train_attn_config_path is None or train_attn_config_path.lower() == "none":
+ train_attn_config_path = None
+
+ if train_attn_config_path is None:
+ print(f"{__name__} | Use empty Training Attention config")
+ train_attn_config = {}
+ elif os.path.exists(train_attn_config_path):
+ print(
+ f"{__name__} | Training Attention config found in {train_attn_config_path}."
+ )
+ with open(train_attn_config_path, "r") as f:
+ train_attn_config = yaml.safe_load(f)
+ print("-" * 20)
+ print("Training Attention Config:")
+ print(train_attn_config)
+ print("-" * 20)
+ else:
+ raise FileNotFoundError(
+ f"Training Attention config {train_attn_config_path} not found. Exit."
+ )
+ return train_attn_config
+
+
+def build_model_args(args, train_attn_config: MInferenceConfig) -> Dict:
+ model_args = {
+ "model_id": args.model_id,
+ "config_path": args.model_config_path,
+ "active_param_config_path": args.active_param_config_path,
+ }
+ if args.attn_type == AttnType.MINFER:
+ model_args["minfer_config"] = train_attn_config
+ elif args.attn_type == AttnType.XATTN:
+ model_args["xattn_params"] = train_attn_config
+ elif args.attn_type == AttnType.MOBA:
+ model_args["moba_config_dict"] = train_attn_config
+
+ return model_args
+
+
+def main(args):
+ update_expr_data_save_path(args.ckpt_save_dir, args.compile_save_path)
+
+ local_rank = int(os.environ["LOCAL_RANK"])
+ if local_rank == 0:
+ load_comm_profile_data(args)
+
+ init_by_attn_type(args.model_id, args.attn_type)
+ train_attn_config = load_train_attn_config(args.train_attn_config_path)
+
+ # ---------------------------------
+ # Compute config
+ if args.run_mode == "compile":
+ if args.runtime_ngpus is None:
+ raise ValueError("runtime_ngpus must be specified in compile mode")
+ runtime_ngpus = args.runtime_ngpus
+ elif args.run_mode == "run":
+ world_size = int(os.getenv("WORLD_SIZE"))
+ if args.runtime_ngpus is None:
+ runtime_ngpus = world_size
+ else:
+ if args.runtime_ngpus != world_size:
+ raise ValueError(
+ f"runtime_ngpus ({args.runtime_ngpus}) must match the number of GPUs in run mode ({world_size})"
+ )
+ runtime_ngpus = args.runtime_ngpus
+
+ if runtime_ngpus % args.plan_ngpus != 0:
+ raise ValueError("runtime_ngpus must be a multiple of plan_ngpus")
+
+ scaling_factor: int = runtime_ngpus // args.plan_ngpus
+ grad_accu_step: int = args.global_batch_size // (
+ args.micro_batch_size * scaling_factor
+ )
+
+ model_prefix = get_model_prefix(args.model_id)
+ pas_config = {
+ "parallel_profile": False,
+ "recompute_modules": f"{model_prefix}DecoderLayer",
+ }
+ if args.mem_constraint > 0:
+ pas_config["mem_constraint"] = args.mem_constraint
+ if args.pas_profile_dir:
+ print(
+ f"Setting profile_dir of PAS config to {args.pas_profile_dir} in pas_config"
+ )
+ pas_config["profile_dir"] = args.pas_profile_dir
+ if args.solver:
+ pas_config["solver"] = args.solver
+ compute_config = ComputeConfig(
+ plan_ngpus=args.plan_ngpus,
+ trace_strategy=args.trace_strategy,
+ runtime_ngpus=runtime_ngpus,
+ constant_folding=True,
+ use_zero=True,
+ use_end2end=True,
+ pas_config=pas_config,
+ )
+
+ # ---------------------------------
+ ## Setup Dataset ##
+ dataset = load_from_disk(args.dataset_path)
+ tokenizer = get_tokenizer(args.model_id)
+ data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+ def collate_fn(samples):
+ if len(samples) == 0:
+ return {}
+
+ mini_batch = data_collator(samples)
+ _mini_batch = {}
+
+ src_tokens = mini_batch.pop("input_ids")
+ seq_len = src_tokens.size(-1)
+ _mini_batch["src_tokens"] = src_tokens
+
+ shift_labels = mini_batch["labels"][..., 1:]
+ _mini_batch["labels"] = torch.nn.functional.pad(
+ shift_labels, (0, 1), "constant", IGNORE_IDX
+ ).contiguous()
+
+ return {
+ "nsentences": len(samples),
+ "ntokens": len(samples) * seq_len,
+ "net_input": _mini_batch,
+ "target": _mini_batch.pop("labels"),
+ }
+
+ dataset_config = DatasetConfig(
+ type=(lambda split: dataset),
+ train_args={"split": "train"},
+ )
+ dataloader_config = DataloaderConfig(
+ train_args={
+ "collate_fn": collate_fn,
+ "drop_last": True,
+ },
+ )
+ sampler_config = DatasetSamplerConfig(
+ train_args={
+ "shuffle": True,
+ "seed": args.seed,
+ },
+ )
+
+ # ---------------------------------
+ # Model Config
+ model_args = build_model_args(args, train_attn_config)
+ model_config = ModelConfig(
+ type=ATTN_TO_MODEL[args.attn_type],
+ args=model_args,
+ )
+
+ # ---------------------------------
+ # optimizer hyperparameters are from YaRN
+ optimizer_config = OptimizerConfig(
+ type=MixedPrecisionAdamW,
+ args={"lr": 2e-5, "betas": (0.9, 0.95), "weight_decay": 0.0, "fused": True},
+ clip_gnorm=1.0,
+ loss_reduction="sum",
+ grad_reduction="per-token-mean",
+ aggregate_outputs_fn=aggregate_outputs_fn,
+ )
+
+ # ---------------------------------
+ # Checkpoint Config
+ checkpoint_config = CheckpointConfig(
+ save_dir=args.ckpt_save_dir
+ if args.ckpt_save_dir
+ else f"./checkpoints_{args.name}",
+ every_n_epochs=args.ckpt_n_epoch,
+ every_n_train_steps=args.ckpt_n_step,
+ save_type="deduped",
+ resume_from=args.resume_from,
+ )
+
+ # ---------------------------------
+ # Log Config
+ log_config = LogConfig(
+ type=TensorBoardLogger,
+ args={
+ "name": args.name,
+ "root_dir": args.tf_log_dir or f"./runs_{args.name}",
+ },
+ )
+
+ # ---------------------------------
+ trainer_args = TrainerArgs(
+ global_batch_size=args.global_batch_size,
+ micro_batch_size=args.micro_batch_size,
+ grad_accumulation_steps=grad_accu_step,
+ pas_policy="autodist",
+ precision="bf16",
+ seed=args.seed,
+ gen_reuse=args.reuse_type,
+ gen_savedir=args.compile_save_path,
+ instance_name=args.name,
+ run_mode=args.run_mode,
+ max_epochs=args.n_epochs,
+ max_train_steps=args.n_iter,
+ enable_progress_bar=not args.disable_progressbar,
+ compute_config=compute_config,
+ model=model_config,
+ optimizer=optimizer_config,
+ dataset=dataset_config,
+ dataloader=dataloader_config,
+ checkpoint=checkpoint_config,
+ log=[log_config],
+ broadcast_strategy="all",
+ dataset_sampler=sampler_config,
+ transfer_config={
+ "transfer_config_dir": args.transfer_config_dir,
+ "transfer_force": args.transfer_force,
+ },
+ merged_ckpt_path=args.resume_merged_ckpt,
+ force_broadcast_all=args.force_broadcast_all,
+ )
+
+ trainer = Trainer(train_args=trainer_args)
+ trainer.run()
+
+
+def print_args(args: argparse.Namespace):
+ print("=" * 80)
+ print(f"Start Experiment:\t{args.name}")
+ print(f"Seed:\t{args.seed}")
+ print(f"Reuse Type:\t{args.reuse_type}")
+ print(f"Run Mode:\t{args.run_mode}")
+ print(f"Total number of GPUs:\t{args.runtime_ngpus}")
+ print(f"GPU unit size:\t{args.plan_ngpus}")
+ print(f"Model ID:\t{args.model_id}")
+
+ print("-" * 40)
+ if args.n_iter:
+ print(
+ f"Number of Iterations:\t{args.n_iter} (number of tokens: {args.n_iter * args.global_batch_size * args.seq_len})"
+ )
+ else:
+ print(f"Number of Epochs:\t{args.n_epochs}")
+
+ print(f"Global Batch Size:\t{args.global_batch_size}")
+ print(f"Micro Batch Size:\t{args.micro_batch_size}")
+
+ scaling_factor = args.runtime_ngpus // args.plan_ngpus
+ grad_accu_step = args.global_batch_size // (args.micro_batch_size * scaling_factor)
+ print(f"Scaling Factor (INFERRED):\t{scaling_factor}")
+ print(f"Gradient Accumulation Steps (INFERRED):\t{grad_accu_step}")
+
+ print("-" * 40)
+ print(f"Model Config Path:\t{args.model_config_path}")
+ print(f"Dataset path:\t{args.dataset_path}")
+ print(f"Training Attention Config Path:\t{args.train_attn_config_path}")
+ print(f"Compile Save Path:\t{args.compile_save_path}")
+ print(f"Tensorboard Log Path:\t{args.tf_log_dir}")
+ print(f"Checkpoint Save Path:\t{args.ckpt_save_dir}")
+ print(f"Resume from Checkpoint:\t{args.check_resume}")
+ print(f"Path to the checkpoint to resume from:\t{args.resume_from}")
+ print(f"Path to the merged checkpoint to resume from:\t{args.resume_merged_ckpt}")
+
+ print(f"Trace Strategy:\t{args.trace_strategy}")
+ if args.transfer_config_dir:
+ print(f"Transfer Configs from another experiment:\t{args.transfer_config_dir}")
+ print(f"Force Transfer Configs:\t{args.transfer_force}")
+
+ if args.active_param_config_path:
+ print(f"Active Param Config Path:\t{args.active_param_config_path}")
+
+ if args.ckpt_n_step:
+ print(f"Checkpoint Save Every {args.ckpt_n_step} Steps")
+ else:
+ print(f"Checkpoint Save Every {args.ckpt_n_epoch} Epochs")
+ print("=" * 80, flush=True)
+
+
+if __name__ == "__main__":
+ ## Parse Args ##
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--seed", type=int, default=0, help="random seed")
+ parser.add_argument(
+ "--name", type=str, default="phi-grad", help="name of the experiment"
+ )
+ parser.add_argument("--seq_len", type=int, default=131072, help="sequence length")
+ parser.add_argument(
+ "--attn_type",
+ type=str,
+ default=AttnType.DENSE,
+ choices=AttnType.__dict__.values(),
+ help="minference type",
+ )
+ parser.add_argument(
+ "--reuse_type",
+ type=str,
+ default="match",
+ choices=["match", "override", "moo", "graph"],
+ help="reuse type",
+ )
+ parser.add_argument(
+ "--force_broadcast_all",
+ action="store_true",
+ help="force broadcast all generated files",
+ )
+ parser.add_argument(
+ "--solver", type=str, default="dp", choices=["dp", "ilp"], help="solver"
+ )
+ parser.add_argument(
+ "--run_mode",
+ type=str,
+ default="run",
+ choices=["run", "compile"],
+ help="run or compile",
+ )
+ parser.add_argument(
+ "--trace_strategy",
+ type=str,
+ default="cuda_run_cpu_offload",
+ choices=["cpu", "cuda", "meta", "cuda_run_cpu_offload", "reuse_cache"],
+ help="trace strategy",
+ )
+ parser.add_argument(
+ "--plan_ngpus", type=int, required=True, help="specify the scale unit size"
+ )
+ parser.add_argument(
+ "--runtime_ngpus",
+ type=int,
+ required=True,
+ help="specify the number of GPUs to use",
+ )
+
+ parser.add_argument("--n_iter", type=int, default=0, help="Number of iterations")
+ parser.add_argument("--n_epochs", type=int, default=0, help="Number of epochs")
+ parser.add_argument(
+ "--nB_tokens", type=int, default=0, help="Number of tokens (in B) to process"
+ )
+ parser.add_argument(
+ "--global_batch_size", type=int, default=4, help="global batch size"
+ )
+ parser.add_argument(
+ "--micro_batch_size", type=int, default=1, help="micro batch size"
+ )
+ parser.add_argument(
+ "--mem_constraint", type=int, default=0, help="memory constraint"
+ )
+
+ parser.add_argument(
+ "--model_id",
+ type=str,
+ default="microsoft/Phi-3-mini-4k-instruct",
+ help="transformers model id",
+ )
+ parser.add_argument(
+ "--model_config_path", type=str, default=None, help="path to the model config"
+ )
+
+ parser.add_argument(
+ "--train_attn_config_path",
+ type=str,
+ default=None,
+ help="Name of Minference config file",
+ )
+ parser.add_argument(
+ "--compile_save_path",
+ type=str,
+ default="./.nnscaler",
+ help="path to save compiled code",
+ )
+
+ parser.add_argument(
+ "--tf_log_dir", type=str, default=None, help="path to save tensorboard logs"
+ )
+ parser.add_argument(
+ "--dataset_path", type=str, default=None, help="path to the dataset"
+ )
+ parser.add_argument(
+ "--check_resume", action="store_true", help="whether to resume from checkpoint"
+ )
+ parser.add_argument(
+ "--resume_from",
+ type=str,
+ default=None,
+ help="path to the checkpoint to resume from",
+ )
+ parser.add_argument(
+ "--resume_merged_ckpt",
+ type=str,
+ default=None,
+ help="path (dir) to the merged checkpoint to resume from",
+ )
+
+ parser.add_argument(
+ "--ckpt_save_dir", type=str, default=None, help="path to save checkpoints"
+ )
+ parser.add_argument(
+ "--ckpt_n_epoch", type=int, default=1, help="save checkpoint every n epochs"
+ )
+ parser.add_argument(
+ "--ckpt_n_step", type=int, default=0, help="save checkpoint every n steps"
+ )
+ parser.add_argument(
+ "--pas_profile_dir", type=str, default=None, help="path to save pas profile"
+ )
+ parser.add_argument(
+ "--transfer_config_dir",
+ type=str,
+ default="none",
+ help="path to transfer configs from another experiment",
+ )
+ parser.add_argument(
+ "--transfer_force", action="store_true", help="force transfer configs"
+ )
+ parser.add_argument(
+ "--active_param_config_path",
+ type=str,
+ default=None,
+ help="path to the active param list",
+ )
+
+ parser.add_argument(
+ "-p",
+ "--disable_progressbar",
+ action="store_true",
+ help="transformers model id",
+ )
+
+ args = parser.parse_args()
+
+ # -------------------------------------------------
+ # Preprocessing args
+ if args.ckpt_n_epoch <= 0:
+ args.ckpt_n_epoch = None
+ if args.ckpt_n_step <= 0:
+ args.ckpt_n_step = None
+
+ if args.nB_tokens > 0:
+ args.n_iter = args.nB_tokens * 1e9 // args.global_batch_size // args.seq_len + 1
+ args.n_epochs = 0
+ if args.n_iter <= 0:
+ args.n_iter = None
+ if args.n_epochs <= 0:
+ args.n_epochs = None
+
+ if args.transfer_config_dir.lower() == "none":
+ args.transfer_config_dir = None
+
+ # set a new field of args 'args.orig_resume_from' to store the original resume_from value
+ args.orig_resume_from = args.resume_from
+ args.resume_from = get_resume_path(
+ args.check_resume, args.resume_from, args.ckpt_save_dir, args.runtime_ngpus
+ )
+
+ print_args(args)
+ main(args)
diff --git a/mtraining/train_attn_configs/moba_256k_s95.yaml b/mtraining/train_attn_configs/moba_256k_s95.yaml
new file mode 100644
index 00000000..fdd2ce84
--- /dev/null
+++ b/mtraining/train_attn_configs/moba_256k_s95.yaml
@@ -0,0 +1,2 @@
+moba_chunk_size: 4096
+moba_topk: 6
diff --git a/mtraining/train_attn_configs/moba_512k_s95.yaml b/mtraining/train_attn_configs/moba_512k_s95.yaml
new file mode 100644
index 00000000..80fd69b0
--- /dev/null
+++ b/mtraining/train_attn_configs/moba_512k_s95.yaml
@@ -0,0 +1,2 @@
+moba_chunk_size: 4096
+moba_topk: 12
diff --git a/mtraining/train_attn_configs/qwen_05B_flex_090.yaml b/mtraining/train_attn_configs/qwen_05B_flex_090.yaml
new file mode 100644
index 00000000..9b4f44e3
--- /dev/null
+++ b/mtraining/train_attn_configs/qwen_05B_flex_090.yaml
@@ -0,0 +1,2 @@
+pattern_config_name: Qwen2.5_0.5B_flex_0.90
+implementation: stripe
diff --git a/mtraining/train_attn_configs/qwen_flex_090.yaml b/mtraining/train_attn_configs/qwen_flex_090.yaml
new file mode 100644
index 00000000..33888998
--- /dev/null
+++ b/mtraining/train_attn_configs/qwen_flex_090.yaml
@@ -0,0 +1,2 @@
+pattern_config_name: Qwen2.5_3B_flex_0.90
+implementation: stripe
diff --git a/mtraining/train_attn_configs/qwen_flex_095.yaml b/mtraining/train_attn_configs/qwen_flex_095.yaml
new file mode 100644
index 00000000..9a9e75fa
--- /dev/null
+++ b/mtraining/train_attn_configs/qwen_flex_095.yaml
@@ -0,0 +1,2 @@
+pattern_config_name: Qwen2.5_3B_flex_0.95
+implementation: stripe
diff --git a/mtraining/train_attn_configs/qwen_mf_dr_stripe.yaml b/mtraining/train_attn_configs/qwen_mf_dr_stripe.yaml
new file mode 100644
index 00000000..2a84c902
--- /dev/null
+++ b/mtraining/train_attn_configs/qwen_mf_dr_stripe.yaml
@@ -0,0 +1,2 @@
+pattern_config_name: Qwen2.5_3B_kv_out_v32_fit_o_best_pattern
+implementation: dr_stripe
diff --git a/mtraining/train_attn_configs/qwen_mf_stripe.yaml b/mtraining/train_attn_configs/qwen_mf_stripe.yaml
new file mode 100644
index 00000000..d5f0ce49
--- /dev/null
+++ b/mtraining/train_attn_configs/qwen_mf_stripe.yaml
@@ -0,0 +1,2 @@
+pattern_config_name: Qwen2.5_3B_kv_out_v32_fit_o_best_pattern
+implementation: stripe
diff --git a/mtraining/train_attn_configs/qwen_mf_zigzag.yaml b/mtraining/train_attn_configs/qwen_mf_zigzag.yaml
new file mode 100644
index 00000000..627cfa2c
--- /dev/null
+++ b/mtraining/train_attn_configs/qwen_mf_zigzag.yaml
@@ -0,0 +1,2 @@
+pattern_config_name: Qwen2.5_3B_kv_out_v32_fit_o_best_pattern
+implementation: zigzag
diff --git a/mtraining/train_attn_configs/xattn_default.yaml b/mtraining/train_attn_configs/xattn_default.yaml
new file mode 100644
index 00000000..6cf69806
--- /dev/null
+++ b/mtraining/train_attn_configs/xattn_default.yaml
@@ -0,0 +1,12 @@
+granularity: 128
+stride: 16
+norm: 1
+softmax: true
+threshold: 0.9
+chunk_size: 16384
+select_mode: inverse
+use_triton: true
+causal: true
+kdb: 1
+keep_sink: false
+keep_recent: false
diff --git a/mtraining/train_attn_configs/xattn_zigzag_s16.yaml b/mtraining/train_attn_configs/xattn_zigzag_s16.yaml
new file mode 100644
index 00000000..52df90bf
--- /dev/null
+++ b/mtraining/train_attn_configs/xattn_zigzag_s16.yaml
@@ -0,0 +1,12 @@
+implementation: zigzag
+granularity: 128
+stride: 16
+norm: 1
+softmax: true
+threshold: 0.9
+select_mode: inverse
+use_triton: true
+causal: true
+kdb: 1
+keep_sink: false
+keep_recent: false
diff --git a/mtraining/train_attn_configs/xattn_zigzag_s16_t85.yaml b/mtraining/train_attn_configs/xattn_zigzag_s16_t85.yaml
new file mode 100644
index 00000000..d4209435
--- /dev/null
+++ b/mtraining/train_attn_configs/xattn_zigzag_s16_t85.yaml
@@ -0,0 +1,12 @@
+implementation: zigzag
+granularity: 128
+stride: 16
+norm: 1
+softmax: true
+threshold: 0.85
+select_mode: inverse
+use_triton: true
+causal: true
+kdb: 1
+keep_sink: false
+keep_recent: false
diff --git a/mtraining/trainer.py b/mtraining/trainer.py
new file mode 100644
index 00000000..0968c69a
--- /dev/null
+++ b/mtraining/trainer.py
@@ -0,0 +1,618 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import copy
+import logging
+import os
+import time
+from collections import defaultdict
+from dataclasses import asdict, dataclass
+from datetime import timedelta
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import nnscaler
+import pandas as pd
+import torch
+import torch.distributed
+from nnscaler.cli.trainer import (
+ AggregatedTrainHook,
+ Trainer,
+ TrainerArgs,
+ TrainHook,
+ TrainStatus,
+ _StepStat,
+)
+from nnscaler.runtime.module import ParallelModule
+from nnscaler.runtime.utils import microbatches
+from nnscaler.utils import accum_mode, is_running_distributed
+from tqdm import tqdm
+
+from mtraining.custom_parallel import parallelize as custom_parallelize
+from mtraining.utils.general import fix_model_state_dict
+from mtraining.utils.paths import EXPR_DATA_SAVE_PATH
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CustomTrainerArgs(TrainerArgs):
+ transfer_config: Optional[Dict[str, Any]] = None
+ merged_ckpt_path: Optional[str] = None
+ force_broadcast_all: bool = False
+
+
+ITERATOR_COUNTER = defaultdict(int)
+
+
+def get_iter_cnt(rank: int):
+ global ITERATOR_COUNTER
+ return ITERATOR_COUNTER.get(rank, 0)
+
+
+ITER_BATCH_IDX_DICT = {}
+
+
+def get_iter_batch_idx(rank: int, iter_cnt: int):
+ global ITER_BATCH_IDX_DICT
+ return ITER_BATCH_IDX_DICT.get(rank, {}).get(iter_cnt, 0)
+
+
+def custom_train_step(
+ model: ParallelModule,
+ rank: int,
+ iter_idx: int,
+ samples: List[Any],
+ is_dummy_batch: Optional[List[bool]] = None,
+ scale_fn: Optional[Callable[[torch.Tensor], torch.Tensor]] = None,
+) -> List[Any]:
+ """
+ The training step function. It should be called in the training loop.
+ Please note:
+ 1. This function is only supported in end2end mode.
+ 2. Gradient accumulation is done inside this function.
+ You shouldn't do gradient accumulation outside this function,
+ because the gradients will be cleared in the beginning of this function
+ Args:
+ samples (List[Any]): a list of samples.
+ if pipeline is used, it must have the same length as configured to pas policy
+ is_dummy_batch (Optional[List[bool]]): indicates whether the each micro-batch is dummya
+ scale_fn (Optional[Callable[[torch.Tensor], torch.Tensor]]): the function to scale the loss
+ Results:
+ List[Any]: a list of outputs for each sample
+ """
+ global ITER_BATCH_IDX_DICT
+ model._warn_uninitialized_non_persistent_buffers(raise_error=True)
+
+ if not model.compute_config.use_end2end:
+ raise RuntimeError("train_step() is only supported in end2end mode")
+ if is_dummy_batch and len(samples) != len(is_dummy_batch):
+ raise ValueError("The length of samples and is_dummy_batch should be the same")
+
+ model._scale_loss(is_dummy_batch, scale_fn)
+
+ # sync_grad will be done in _train_step
+ # so we never need to call it manually
+ model._sync_grad_required = False
+ sample_count = len(samples)
+ dataloader = microbatches(samples, cycle=False)
+
+ if model.use_scheduler:
+ if len(samples) != model.nmicros_per_scheduler_step:
+ raise ValueError(
+ f"Expected {model.nmicros_per_scheduler_step} samples, but got {sample_count}"
+ )
+ # only one step, so begin/end are both True
+ with accum_mode(begin=True, end=True):
+ return model._train_step(dataloader), None
+ else:
+ outputs = []
+ latencies = []
+ for idx in range(sample_count):
+ ITER_BATCH_IDX_DICT[rank][iter_idx] = idx
+
+ sample_start_time = time.perf_counter()
+ with accum_mode(begin=(idx == 0), end=(idx == sample_count - 1)):
+ # loss, loss.data, samples['ntokens'], samples['nsentences']
+ output = model._train_step(dataloader)
+ sample_time = time.perf_counter() - sample_start_time
+ latencies.append(sample_time)
+
+ num_tokens = output[2]
+ outputs.append(output)
+ return outputs, latencies
+
+
+class CustomTrainer(Trainer):
+ def __init__(
+ self,
+ argv: Optional[List[str]] = None,
+ *,
+ train_args: Optional[Union[Dict[str, Any], CustomTrainerArgs]] = None,
+ ):
+ """
+ Custom trainer with an additional parameter.
+
+ Args:
+ argv (Optional[List[str]]): Command line arguments. If not specified, sys.argv[1:] will be used.
+ train_args: A dict used to construct TrainerArgs or a TrainerArgs object itself.
+ additional_param (Optional[Any]): Additional parameter for custom functionality.
+ """
+ # Call the parent class's initializer with the existing parameters
+ super().__init__(argv=argv, train_args=train_args)
+ self.train_args: CustomTrainerArgs
+
+ torch.distributed.init_process_group(
+ backend="nccl",
+ timeout=timedelta(hours=2),
+ )
+ self.train_step_func = custom_train_step
+
+ def _train_epoch(self, epoch):
+ VAL_STATUS_NO = 0 # not validated or saved
+ VAL_STATUS_VAL = 1 # validated but not saved
+ VAL_STATUS_SAVE = 2 # validated and saved
+ has_validated = VAL_STATUS_NO # 3 states
+
+ resume_from_idx = (
+ self.train_status.finished_train_steps % self.total_train_steps_per_epoch
+ )
+ data_iter = enumerate(
+ self._global_batch_iterator(num_skip_first=resume_from_idx)
+ )
+
+ max_epoch = self.max_train_steps // self.total_train_steps_per_epoch
+ if self.max_train_steps % self.total_train_steps_per_epoch != 0:
+ max_epoch += 1
+ ndigits = len(str(max_epoch))
+ epoch_format = f"0{ndigits}d"
+ epoch_desc = f"Epoch {format(epoch, epoch_format)}"
+
+ if self.rank == 0:
+ progress = tqdm(
+ None,
+ total=self.total_train_steps_per_epoch,
+ initial=resume_from_idx,
+ desc=epoch_desc,
+ disable=not self.train_args.enable_progress_bar,
+ )
+ else:
+ progress = None
+
+ # ---------------------------------------------------------------------------------
+ train_info_save_path = os.path.join(
+ EXPR_DATA_SAVE_PATH["base_path"], "train_info", f"epoch_{epoch}.log"
+ )
+ os.makedirs(os.path.dirname(train_info_save_path), exist_ok=True)
+ if self.rank == 0:
+ # Check whether the file already exists
+ # If it exists, assume existing log file has name 'epoch__.log' ('epoch_.log` is assumed to have num 0)
+ # Find the greatest for the current epoch and increment it to build the new file name
+ existing_files = [
+ f
+ for f in os.listdir(os.path.dirname(train_info_save_path))
+ if f.startswith(f"epoch_{epoch}_") or f.startswith(f"epoch_{epoch}.log")
+ ]
+ if existing_files:
+ # Extract the numbers from the filenames
+ existing_nums = [
+ int(f.split("_")[-1].split(".")[0])
+ for f in existing_files
+ if f.startswith(f"epoch_{epoch}_")
+ ]
+ if not existing_nums:
+ existing_nums = [0]
+ new_num = max(existing_nums) + 1
+ train_info_save_path = os.path.join(
+ os.path.dirname(train_info_save_path),
+ f"epoch_{epoch}_{new_num}.log",
+ )
+ else:
+ # If no existing files, use the original path
+ train_info_save_path = os.path.join(
+ os.path.dirname(train_info_save_path), f"epoch_{epoch}.log"
+ )
+ with open(train_info_save_path, "w") as f:
+ f.write("")
+
+ step_stat: Optional[_StepStat] = None
+ num_tokens_trained = 0
+ for i, batches in data_iter:
+ idx = i + resume_from_idx
+
+ global ITERATOR_COUNTER, ITER_BATCH_IDX_DICT
+ ITERATOR_COUNTER[self.rank] = idx
+ ITER_BATCH_IDX_DICT[self.rank] = {idx: 0}
+ if self.rank == 0:
+ progress.update(1)
+ step_start_at = time.perf_counter()
+ step_stat = _StepStat()
+ step_metrics = {}
+ has_validated = VAL_STATUS_NO
+ num_batches = len(batches)
+ batches, is_dummy_batch = self._fix_batches(batches)
+
+ self.model.train()
+
+ self.hook.before_zero_grad(self)
+ self.optimizer.zero_grad()
+ self.hook.after_zero_grad(self)
+
+ self.hook.on_train_step_start(self, batches[:num_batches], idx)
+ losses, latencies = self.train_step_func(
+ self.model, self.rank, idx, batches, is_dummy_batch
+ )
+ self.hook.on_train_step_end(
+ self, losses[:num_batches], batches[:num_batches], idx
+ )
+
+ aggregate_outputs = (
+ self.train_args.resolved_aggregate_outputs_fn or self.aggregate_outputs
+ )
+ aggregated_outputs = aggregate_outputs(
+ losses[:num_batches], self.sync_group
+ )
+ if self.train_args.optimizer.loss_reduction == "mean":
+ loss = aggregated_outputs.loss_sum / aggregated_outputs.num_batches
+ else:
+ loss = aggregated_outputs.loss_sum
+ step_stat.train_loss = loss
+ num_tokens_trained += aggregated_outputs.num_tokens
+ self.hook.after_aggregate_train_step_outputs(
+ self, aggregated_outputs, loss, idx
+ )
+
+ self.hook.before_sync_grad(self)
+ self.optimizer.sync_shard_grad()
+ self.hook.after_sync_grad(self)
+
+ # scale gradients
+ multiplier = self.train_args.scaling_factor
+ if self.train_args.optimizer.grad_reduction == "sum":
+ # do nothing. `multiplier` is already correct
+ pass
+ elif self.train_args.optimizer.grad_reduction == "mean":
+ if not aggregated_outputs.num_batches:
+ raise RuntimeError(
+ "`aggregate_outputs` doesn't set `num_batches` field"
+ )
+ multiplier /= aggregated_outputs.num_batches
+ else:
+ assert self.train_args.optimizer.grad_reduction == "per-token-mean"
+ if not aggregated_outputs.num_tokens:
+ raise RuntimeError(
+ "`aggregate_outputs` doesn't set `num_tokens` field"
+ )
+ multiplier /= aggregated_outputs.num_tokens
+ self.optimizer.scale_grads(multiplier)
+
+ # clip gradients
+ self.hook.before_gnorm_clip(self)
+ if self.train_args.optimizer.clip_gnorm:
+ step_stat.gnorm = self.optimizer.clip_gnorm(
+ self.train_args.optimizer.clip_gnorm
+ )
+ else:
+ step_stat.gnorm = self.optimizer.clip_gnorm()
+ self.hook.after_gnorm_clip(self, step_stat.gnorm)
+ step_stat.gnorm = step_stat.gnorm.item()
+
+ # update parameters
+ step_stat.lr = self.optimizer.param_groups[0]["lr"]
+ self.hook.before_optimizer_step(self)
+ self.optimizer.step()
+ self.hook.after_optimizer_step(self)
+ if self.lr_scheduler and self.train_args.lr_scheduler.interval == "step":
+ self.lr_scheduler.step()
+
+ self.train_status.finished_train_steps += 1
+ self._log_mem_stats(tag="train")
+ step_metrics = {k: v for k, v in asdict(step_stat).items() if v is not None}
+ step_metrics["train_wall"] = time.perf_counter() - step_start_at
+ step_metrics["num_tokens_processed"] = num_tokens_trained
+ self.log_metrics(step_metrics, tag="train")
+ if self.rank == 0:
+ progress.set_postfix(step_metrics)
+ formatted_metrics = self._format_metrics(
+ epoch_desc, idx + 1, step_metrics
+ )
+ with open(train_info_save_path, "a") as f:
+ f.write(f"{formatted_metrics}\n")
+
+ if (
+ self.train_args.enable_log_progress
+ and self.train_status.finished_train_steps
+ % self.train_args.log_progress_every_n_train_steps
+ == 0
+ ):
+ logger.info(formatted_metrics)
+ step_metrics = {}
+
+ # validate and save checkpoint
+ if (
+ self.train_args.checkpoint.every_n_train_steps
+ and self.train_status.finished_train_steps
+ % self.train_args.checkpoint.every_n_train_steps
+ == 0
+ ):
+ self._validate_and_save(step_stat)
+ has_validated = VAL_STATUS_SAVE
+
+ # max_train_steps is reached
+ if self.train_status.finished_train_steps >= self.max_train_steps:
+ if step_metrics and self.train_args.enable_log_progress:
+ logger.info(self._format_metrics(epoch_desc, idx + 1, step_metrics))
+ step_metrics = {}
+ if not has_validated:
+ self._validate_and_save(step_stat)
+ has_validated = VAL_STATUS_SAVE
+ if self.rank == 0:
+ # disable refresh the progress bar to avoid redundant progress bar
+ progress.leave = False
+ progress.close()
+ break
+
+ if (
+ not has_validated
+ and self.train_args.val_every_n_train_steps
+ and self.train_status.finished_train_steps
+ % self.train_args.val_every_n_train_steps
+ == 0
+ ):
+ self._validate(step_stat)
+ has_validated = VAL_STATUS_VAL
+
+ # time.sleep(1)
+ else:
+ # Do per-epoch operations here.
+ # if the loop exits with `break` (max_train_steps is reached)
+ # those operations have done in the loop
+ if step_stat is None:
+ return # no train step runs. Nothing to do.
+ if (
+ has_validated < VAL_STATUS_SAVE
+ and self.train_args.checkpoint.every_n_epochs
+ and (epoch + 1) % self.train_args.checkpoint.every_n_epochs == 0
+ ):
+ self._validate_and_save(step_stat)
+ has_validated = VAL_STATUS_SAVE
+ if (
+ not has_validated
+ and self.train_args.val_every_n_epochs
+ and (epoch + 1) % self.train_args.val_every_n_epochs == 0
+ ):
+ self._validate(step_stat)
+ has_validated = VAL_STATUS_VAL
+
+ def _setup(self):
+ self.train_args.init_env(self)
+ compile_only = self.train_args.compile_mode
+
+ if is_running_distributed():
+ nnscaler.init()
+ if torch.distributed.get_rank() == 0:
+ logging.getLogger().setLevel(logging.INFO)
+ else:
+ logging.getLogger().setLevel(logging.WARNING)
+
+ def _create_model():
+ model = self.train_args.create_model()
+ if self.train_args.param_dtype == self.train_args.buffer_dtype:
+ if self.train_args.param_dtype is not None:
+ model = model.to(self.train_args.param_dtype)
+ else:
+ # separate param and buffer dtype
+ # TODO: a little hacky. A better way?
+ # 3 kinds of tensors are converted in Module._apply:
+ # model parameters, its grad, and buffer
+ # param_dtype controls the first two, (but grad is `None` here)
+ # and buffer_dtype controls the last one
+ buf_ids = {id(buf) for buf in model.buffers(recurse=True)}
+ if self.train_args.param_dtype is not None:
+ model._apply(
+ lambda t: t.to(self.train_args.param_dtype)
+ if t.is_floating_point() and id(t) not in buf_ids
+ else t
+ )
+ if self.train_args.buffer_dtype is not None:
+ model._apply(
+ lambda t: t.to(self.train_args.buffer_dtype)
+ if t.is_floating_point() and id(t) in buf_ids
+ else t
+ )
+ if self.train_args.tracing_from_weights:
+ model.load_state_dict(torch.load(self.train_args.tracing_from_weights))
+ return model
+
+ # create dataset and dataloader
+ for stage in ["train", "val", "test"]:
+ self.dataset[stage] = self.train_args.create_dataset(stage)
+
+ # load a dummy input from training dataset
+ self.dummy_input = self._load_dummy_input()
+ self.dummy_input = self._fix_input(self.dummy_input)
+
+ for stage in ["train", "val", "test"]:
+ self.dataloader[stage] = self.train_args.create_dataloader(
+ stage, self.dataset[stage]
+ )
+ if (
+ self.dataloader[stage] is not None
+ and not self.dataloader[stage].drop_last
+ and len(self.dataset[stage])
+ % (self.train_args.micro_batch_size * self.train_args.scaling_factor)
+ != 0
+ ):
+ warnings.warn(
+ f"Length of {stage} dataset ({len(self.dataset[stage])}) "
+ f"is not multiple of micro_batch_size * scale_factor ({self.train_args.micro_batch_size * self.train_args.scaling_factor}). "
+ f"In this case, the train_step for the last batch of samples can fail! "
+ f"You can specify `drop_last=True` in DataLoader to fix this problem."
+ )
+
+ # setup compute config
+ compute_config = copy.deepcopy(self.train_args.compute_config)
+ compute_config.pas_config["__pas_name"] = self.train_args.pas_policy
+ # autodist configs
+ compute_config.pas_config["update_freq"] = self.train_args.update_freq
+ compute_config.pas_config["use_bf16"] = (
+ self.train_args.param_dtype == torch.bfloat16
+ )
+ compute_config.pas_config["use_fp16"] = (
+ self.train_args.param_dtype == torch.float16
+ )
+
+ compute_config.user_config["__from_trainer_args"] = {
+ "mbs": self.train_args.micro_batch_size,
+ "gbs": self.train_args.global_batch_size,
+ "precision": self.train_args.precision,
+ "model_args": self.train_args.model.args,
+ }
+
+ # parallalize model
+ pmodel_class = custom_parallelize(
+ self.train_args.model_type,
+ self._create_dummy_forward_args(),
+ self.train_args.resolved_pas_policy,
+ compute_config,
+ module_fn=_create_model,
+ gen_savedir=self.train_args.gen_savedir,
+ reuse=self.train_args.gen_reuse,
+ instance_name=self.train_args.instance_name,
+ broadcast_strategy=self.train_args.broadcast_strategy,
+ load_module=not compile_only,
+ transfer_config=self.train_args.transfer_config,
+ force_broadcast_all=self.train_args.force_broadcast_all,
+ )
+ if compile_only:
+ return
+
+ torch.distributed.barrier()
+ self.rank = torch.distributed.get_rank()
+
+ self.total_train_steps_per_epoch = (
+ len(self.dataloader["train"]) // self.train_args.update_freq
+ )
+ if len(self.dataloader["train"]) % self.train_args.update_freq != 0:
+ self.total_train_steps_per_epoch += 1 # will add extra dummy batches
+
+ if self.train_args.max_epochs and self.train_args.max_train_steps:
+ self.max_train_steps = min(
+ self.total_train_steps_per_epoch * self.train_args.max_epochs,
+ self.train_args.max_train_steps,
+ )
+ elif self.train_args.max_train_steps:
+ self.max_train_steps = self.train_args.max_train_steps
+ else:
+ assert (
+ self.train_args.max_epochs
+ ), "max_epochs or max_train_steps should be specified"
+ self.max_train_steps = (
+ self.total_train_steps_per_epoch * self.train_args.max_epochs
+ )
+
+ _, self.sync_group = self.train_args.compute_config.get_sync_group()
+ self.model = pmodel_class()
+ self.model.cuda()
+ self.optimizer = self.train_args.create_parallel_optimizer(self.model)
+
+ def reducer_pre_hook(reducer, grad):
+ grad.div_(self.train_args.scaling_factor)
+
+ self.optimizer.register_reducer_pre_hook(reducer_pre_hook)
+ self.lr_scheduler = self.train_args.create_lr_scheduler(self.optimizer)
+ self.loggers = self.train_args.create_loggers()
+
+ supported_hook_components = [
+ self.model,
+ self.optimizer,
+ self.lr_scheduler,
+ ]
+ self.hook = AggregatedTrainHook(
+ [x for x in supported_hook_components if isinstance(x, TrainHook)]
+ + [self.train_args.create_hook()]
+ )
+
+ self._log_config(self.train_args.to_dict())
+ self._load_checkpoint()
+
+ if self.train_args.merged_ckpt_path is not None:
+ print(
+ f"Rank {self.rank} | {__name__} | loading merged checkpoint from {self.train_args.merged_ckpt_path}"
+ )
+ merged_ckpt_path = os.path.join(
+ self.train_args.merged_ckpt_path, "pytorch_model.bin"
+ )
+ model_state_dict = torch.load(merged_ckpt_path, map_location="cpu")
+
+ first_key = list(model_state_dict.keys())[0]
+ if len(first_key.split(".")) == 1:
+ # For Ring-Attention models, the merged checkpoint is directly copied from one of the shards and has different key names.
+ model_state_dict = fix_model_state_dict(self.model, model_state_dict)
+
+ first_key = list(model_state_dict.keys())[0]
+ if "model.model" not in first_key:
+ # Our merging logic also removes the prefix `model.` from the state dict keys when saving
+ model_state_dict = {
+ "model." + k: v for k, v in model_state_dict.items()
+ }
+ if self.rank % int(os.getenv("GPU_PER_NODE", "8")) == 0:
+ print(
+ f"Rank {self.rank} | {__name__} | loaded model state dict.keys(): {model_state_dict.keys()}"
+ )
+
+ # in our merge program, `model` is poped out and we directly pass the model_state_dict instead of model_state_dict['model']
+ nnscaler.load_merged_state_dict(
+ self.model,
+ model_state_dict,
+ self.optimizer,
+ None,
+ )
+
+ self.hook.after_setup(self)
+
+ def _load_checkpoint(self):
+ resume_from = self.train_args.checkpoint.get_resume_checkpoint_dir()
+ if not resume_from:
+ return
+ logger.info(f"Resuming from {resume_from}")
+ if resume_from.is_file():
+ resume_from = resume_from # when we load from merged checkpoint
+ else:
+ resume_from = resume_from / f"{self.rank}.ckpt"
+ state_dict = torch.load(resume_from, map_location="cpu")
+ self.hook.on_load_checkpoint(self, state_dict)
+ ckpt_save_type = state_dict["train_args"]["checkpoint"]["save_type"]
+
+ if ckpt_save_type == "merged": # it is a merged state dict
+ nnscaler.load_merged_state_dict(
+ self.model,
+ state_dict["model"],
+ self.optimizer,
+ state_dict["optimizer"],
+ )
+ elif ckpt_save_type == "sharded":
+ nnscaler.load_sharded_state_dict(
+ self.model,
+ state_dict["model"],
+ self.optimizer,
+ state_dict["optimizer"],
+ )
+ elif ckpt_save_type == "deduped":
+ nnscaler.load_deduped_state_dict(
+ self.model,
+ state_dict["model"],
+ self.optimizer,
+ state_dict["optimizer"],
+ )
+ else:
+ raise ValueError(f"Unknown checkpoint type: {ckpt_save_type}")
+
+ if "lr_scheduler" in state_dict:
+ if state_dict["lr_scheduler"] and not self.lr_scheduler:
+ raise ValueError("lr_scheduler is not set in the current trainer")
+ if self.lr_scheduler:
+ self.lr_scheduler.load_state_dict(state_dict["lr_scheduler"])
+ self.train_status = TrainStatus(**state_dict["train_status"])
+ self.rng_states_from_resume = state_dict.get(
+ "rng_states"
+ ) # resumed in _global_batch_iterator()
diff --git a/mtraining/utils/__init__.py b/mtraining/utils/__init__.py
new file mode 100644
index 00000000..986a2c89
--- /dev/null
+++ b/mtraining/utils/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from .auto_merge_ckpt import *
+from .general import *
+from .loss import chunk_linear_cross_entropy, linear_cross_entropy
diff --git a/mtraining/utils/auto_merge_ckpt.py b/mtraining/utils/auto_merge_ckpt.py
new file mode 100644
index 00000000..7d4c0af0
--- /dev/null
+++ b/mtraining/utils/auto_merge_ckpt.py
@@ -0,0 +1,149 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+import os
+import time
+
+from .merge_utils import STORE_DIR, copy_configs_to_merged_dir, merge_ckpts
+
+STABILITY_DELAY = 5 # seconds
+POLL_INTERVAL = 10 # seconds
+
+
+def print_args(args):
+ print("-" * 60)
+ print(f"Arguments:\n{args.__dict__}")
+ print("-" * 60)
+
+
+def is_checkpoint_complete(checkpoint_dir, expected_shards):
+ """
+ Check if all expected checkpoint shards are present in the directory and are stable.
+ """
+ # List current files in the directory
+ files = [f for f in os.listdir(checkpoint_dir) if f.endswith(".ckpt")]
+ if len(files) < expected_shards:
+ # if there are fewer files than expected, the checkpoint is not complete
+ return False
+
+ # Optionally: check for stability by comparing file sizes over a delay period
+ sizes_initial = {f: os.path.getsize(os.path.join(checkpoint_dir, f)) for f in files}
+ time.sleep(STABILITY_DELAY)
+ sizes_later = {f: os.path.getsize(os.path.join(checkpoint_dir, f)) for f in files}
+
+ # Ensure each file's size has not changed
+ return all(sizes_initial[f] == sizes_later[f] for f in files)
+
+
+def monitor_and_merge(args, expr_data_dir: str):
+ """
+ Monitor the base checkpoint directory and trigger merging for completed iterations.
+ """
+ base_ckpt_dir = os.path.join(expr_data_dir, "checkpoints")
+
+ merged_iterations = set()
+ err_iterations = {}
+ while True:
+ # List iteration directories (assuming format 'epoch_idx-iter_idx')
+ if not os.path.isdir(base_ckpt_dir):
+ continue
+
+ for iteration in os.listdir(base_ckpt_dir):
+ # example `iteration`: 0000-0005
+ if not iteration.count("-") == 1:
+ continue
+
+ if err_iterations.get(iteration, 0) > 3:
+ continue
+
+ try:
+ epoch_idx, iter_idx = iteration.split("-")
+ epoch_idx, iter_idx = int(epoch_idx), int(iter_idx)
+
+ if epoch_idx < args.start_epoch or iter_idx < args.start_iter:
+ continue
+
+ iter_dir = os.path.join(base_ckpt_dir, iteration)
+ if not os.path.isdir(iter_dir) or iteration in merged_iterations:
+ continue
+
+ # Check if the checkpoint is complete in this shard directory
+ if is_checkpoint_complete(iter_dir, args.num_gpus):
+ # Merge the checkpoints (you may want to merge across all ranks,
+ # or handle each rank separately depending on your setup)
+
+ print(f"-" * 60)
+ print(f"Merging checkpoint for iteration {iteration}...")
+ merged = merge_ckpts(
+ expr_data_dir,
+ epoch_idx,
+ iter_idx,
+ args.override,
+ )
+ if not merged:
+ print(f"Error merging checkpoints for {iteration}. Continue.")
+ continue
+
+ # Copy config files to the merged directory
+ copied = copy_configs_to_merged_dir(
+ expr_data_dir,
+ epoch_idx,
+ iter_idx,
+ args.model_id,
+ args.override,
+ )
+ if not copied:
+ print(f"Error copying config fiiles for {iteration}. Continue.")
+ continue
+
+ # Mark this iteration as merged to avoid reprocessing.
+ merged_iterations.add(iteration)
+ except Exception as e:
+ print(f"Error processing iteration {iteration}: {e}")
+ err_iterations[iteration] = err_iterations.get(iteration, 0) + 1
+
+ time.sleep(POLL_INTERVAL)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--expr_data_dir",
+ type=str,
+ default=None,
+ help="Path to the experiment data directory",
+ )
+
+ parser.add_argument("--gpu_set", type=str, default=None)
+ parser.add_argument("--expr_dir", type=str, default=None)
+ parser.add_argument("--expr_name", type=str, help="name of the experiment")
+
+ parser.add_argument("--num_gpus", type=int, help="number of gpus")
+ parser.add_argument(
+ "--model_id", type=str, default="Qwen/Qwen2.5-3B", help="transformers model id"
+ )
+ parser.add_argument(
+ "--use_ring_attn", action="store_true", help="use ring attention"
+ )
+
+ parser.add_argument("--start_iter", type=int, default=0)
+ parser.add_argument("--start_epoch", type=int, default=0)
+
+ parser.add_argument("--override", action="store_true")
+ args = parser.parse_args()
+
+ if args.expr_data_dir is None and (
+ args.gpu_set is None or args.expr_dir is None or args.expr_name is None
+ ):
+ raise ValueError(
+ "Either use expr_data_dir or (gpu_set, expr_dir, and expr_name) to present the experiment data dir"
+ )
+ print_args(args)
+
+ expr_data_dir = (
+ args.expr_data_dir
+ if args.expr_data_dir
+ else os.path.join(STORE_DIR, args.gpu_set, args.expr_dir, args.expr_name)
+ )
+ monitor_and_merge(args, expr_data_dir)
diff --git a/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_16.json b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_16.json
new file mode 100644
index 00000000..c7025d7d
--- /dev/null
+++ b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_16.json
@@ -0,0 +1,122 @@
+{
+ "all gather": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.05180919170379639,
+ 0.05146768093109131,
+ 0.05152981281280518,
+ 0.05157580375671387,
+ 0.0012136220932006836,
+ 0.0015923023223876954,
+ 0.0020998001098632814,
+ 0.0015050172805786133,
+ 0.0017590522766113281,
+ 0.002539539337158203,
+ 0.0032071590423583983,
+ 0.06259112358093262
+ ]
+ ],
+ "all reduce": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.005392670631408691,
+ 0.005529880523681641,
+ 0.005478096008300781,
+ 0.005201959609985351,
+ 0.005406713485717774,
+ 0.005557894706726074,
+ 0.005795717239379883,
+ 0.005960416793823242,
+ 0.0063076019287109375,
+ 0.015114092826843261,
+ 0.015998435020446778,
+ 0.027419137954711913
+ ]
+ ],
+ "reduce scatter": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.026781582832336427,
+ 0.0016937732696533203,
+ 0.0017365217208862305,
+ 0.0016034841537475586,
+ 0.0017096042633056641,
+ 0.0019363641738891601,
+ 0.0022066593170166015,
+ 0.0019835472106933595,
+ 0.0020725011825561525,
+ 0.002443671226501465,
+ 0.003660106658935547,
+ 0.0057839155197143555
+ ]
+ ],
+ "all to all": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.05386221408843994,
+ 0.051555418968200685,
+ 0.021165132522583008,
+ 0.001184391975402832,
+ 0.0014412641525268556,
+ 0.001375126838684082,
+ 0.001291203498840332,
+ 0.0017063379287719726,
+ 0.0012265682220458985,
+ 0.0013031959533691406,
+ 0.001764369010925293,
+ 0.001971316337585449
+ ]
+ ]
+}
diff --git a/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_2.json b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_2.json
new file mode 100644
index 00000000..4dc900eb
--- /dev/null
+++ b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_2.json
@@ -0,0 +1,122 @@
+{
+ "all gather": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.00034034252166748047,
+ 0.00037512779235839846,
+ 0.0003477334976196289,
+ 0.0003787040710449219,
+ 0.00039501190185546874,
+ 0.0004168510437011719,
+ 0.00045654773712158205,
+ 0.0005896091461181641,
+ 0.0007951736450195312,
+ 0.001247739791870117,
+ 0.0020885229110717773,
+ 0.003815174102783203
+ ]
+ ],
+ "all reduce": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.00025429725646972654,
+ 0.00027587413787841795,
+ 0.000278782844543457,
+ 0.00029337406158447266,
+ 0.0003055095672607422,
+ 0.00032358169555664064,
+ 0.0003885030746459961,
+ 0.0005956888198852539,
+ 0.0007241249084472656,
+ 0.0012053251266479492,
+ 0.0021879911422729493,
+ 0.004020094871520996
+ ]
+ ],
+ "reduce scatter": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.00043044090270996096,
+ 0.00037596225738525393,
+ 0.0003805398941040039,
+ 0.00040967464447021483,
+ 0.0005321502685546875,
+ 0.0005571603775024414,
+ 0.000500178337097168,
+ 0.0006249427795410156,
+ 0.000807499885559082,
+ 0.001304769515991211,
+ 0.0022077322006225585,
+ 0.0036898374557495115
+ ]
+ ],
+ "all to all": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.0003772735595703125,
+ 0.0003284931182861328,
+ 0.00034313201904296876,
+ 0.0003873348236083984,
+ 0.00036830902099609374,
+ 0.00036525726318359375,
+ 0.00048248767852783204,
+ 0.00046117305755615233,
+ 0.0005624294281005859,
+ 0.0007740259170532227,
+ 0.0011620283126831054,
+ 0.001997685432434082
+ ]
+ ]
+}
diff --git a/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_32.json b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_32.json
new file mode 100644
index 00000000..f1b9c299
--- /dev/null
+++ b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_32.json
@@ -0,0 +1,122 @@
+{
+ "all gather": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.1080631971359253,
+ 0.10469765663146972,
+ 0.00834052562713623,
+ 0.0020487308502197266,
+ 0.002344846725463867,
+ 0.0019771814346313476,
+ 0.002572488784790039,
+ 0.0025699853897094725,
+ 0.08785591125488282,
+ 0.08787775039672852,
+ 0.0038346290588378907,
+ 0.005972957611083985
+ ]
+ ],
+ "all reduce": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.0004894495010375977,
+ 0.0005235433578491211,
+ 0.0008071422576904297,
+ 0.000553297996520996,
+ 0.000592350959777832,
+ 0.0006613731384277344,
+ 0.0010251522064208985,
+ 0.0012546062469482421,
+ 0.0017488718032836914,
+ 0.0023847341537475584,
+ 0.004178428649902343,
+ 0.007097625732421875
+ ]
+ ],
+ "reduce scatter": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.002982211112976074,
+ 0.0026816129684448242,
+ 0.003106427192687988,
+ 0.002877163887023926,
+ 0.0030056238174438477,
+ 0.0028972864151000977,
+ 0.0028738975524902344,
+ 0.0026230812072753906,
+ 0.0031487226486206056,
+ 0.0034871816635131834,
+ 0.004593896865844727,
+ 0.006965112686157226
+ ]
+ ],
+ "all to all": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.10162621217019248,
+ 0.09312442486097404,
+ 0.015803313666463356,
+ 0.0017765879631042479,
+ 0.0016462447903484423,
+ 0.0027020471731946235,
+ 0.0020170678402762162,
+ 0.003909244829356935,
+ 0.0013500359160346011,
+ 0.0014925030027271196,
+ 0.002711536143449099,
+ 0.003041677062357991
+ ]
+ ]
+}
diff --git a/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_4.json b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_4.json
new file mode 100644
index 00000000..472de77d
--- /dev/null
+++ b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_4.json
@@ -0,0 +1,122 @@
+{
+ "all gather": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.017050647735595705,
+ 0.01697092056274414,
+ 0.017047595977783204,
+ 0.01705167293548584,
+ 0.01709465980529785,
+ 0.017161035537719728,
+ 0.017168116569519044,
+ 0.01729440689086914,
+ 0.017450642585754395,
+ 0.01793365478515625,
+ 0.018938732147216798,
+ 0.018626880645751954
+ ]
+ ],
+ "all reduce": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.00029621124267578127,
+ 0.0003087043762207031,
+ 0.0003238201141357422,
+ 0.00029952526092529296,
+ 0.0003688573837280273,
+ 0.00042085647583007814,
+ 0.000533294677734375,
+ 0.0007121801376342773,
+ 0.0009022712707519531,
+ 0.0015013456344604493,
+ 0.002604985237121582,
+ 0.0048190832138061525
+ ]
+ ],
+ "reduce scatter": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.0006368160247802734,
+ 0.0009001493453979492,
+ 0.0005036115646362305,
+ 0.0008295774459838867,
+ 0.0006284952163696289,
+ 0.0006097555160522461,
+ 0.0010236978530883789,
+ 0.0007615089416503906,
+ 0.000969099998474121,
+ 0.0014830589294433593,
+ 0.002394843101501465,
+ 0.004157066345214844
+ ]
+ ],
+ "all to all": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.018045878410339354,
+ 0.01786086559295654,
+ 0.017101550102233888,
+ 0.016910505294799805,
+ 0.016951107978820802,
+ 0.01701045036315918,
+ 0.01714966297149658,
+ 0.016887688636779787,
+ 0.00273435115814209,
+ 0.000716090202331543,
+ 0.0009747028350830078,
+ 0.0014144182205200195
+ ]
+ ]
+}
diff --git a/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_8.json b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_8.json
new file mode 100644
index 00000000..56c3d8b2
--- /dev/null
+++ b/mtraining/utils/comm_prof/NVIDIA_A100-SXM4-40GB/intra_8.json
@@ -0,0 +1,122 @@
+{
+ "all gather": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.029539847373962404,
+ 0.028463411331176757,
+ 0.013848423957824707,
+ 0.0006072998046875,
+ 0.0009058713912963867,
+ 0.000886082649230957,
+ 0.0008183717727661133,
+ 0.0009612798690795899,
+ 0.0012127876281738282,
+ 0.0015856504440307617,
+ 0.0025009870529174804,
+ 0.0044220685958862305
+ ]
+ ],
+ "all reduce": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.0003414154052734375,
+ 0.00041208267211914065,
+ 0.00033528804779052733,
+ 0.0003468513488769531,
+ 0.0004187345504760742,
+ 0.0004812002182006836,
+ 0.0005032777786254883,
+ 0.0006810903549194336,
+ 0.0010546207427978515,
+ 0.0016033411026000976,
+ 0.002811574935913086,
+ 0.005340051651000976
+ ]
+ ],
+ "reduce scatter": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.0011440277099609374,
+ 0.0010862112045288085,
+ 0.0011052131652832032,
+ 0.0011761903762817383,
+ 0.0008022308349609375,
+ 0.0012315273284912109,
+ 0.0011649370193481446,
+ 0.001171445846557617,
+ 0.001577472686767578,
+ 0.0017426729202270508,
+ 0.0032947301864624024,
+ 0.004651904106140137
+ ]
+ ],
+ "all to all": [
+ [
+ 0.25,
+ 0.5,
+ 1.0,
+ 2.0,
+ 4.0,
+ 8.0,
+ 16.0,
+ 32.0,
+ 64.0,
+ 128.0,
+ 256.0,
+ 512.0
+ ],
+ [
+ 0.028547143936157225,
+ 0.028542041778564453,
+ 0.0283461332321167,
+ 0.028509140014648438,
+ 0.0012618064880371093,
+ 0.0006998300552368164,
+ 0.0008265495300292969,
+ 0.0007447957992553711,
+ 0.0011143922805786134,
+ 0.0011379003524780273,
+ 0.001148056983947754,
+ 0.001277613639831543
+ ]
+ ]
+}
diff --git a/mtraining/utils/data_utils/__init__.py b/mtraining/utils/data_utils/__init__.py
new file mode 100644
index 00000000..0d2017b4
--- /dev/null
+++ b/mtraining/utils/data_utils/__init__.py
@@ -0,0 +1,2 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
diff --git a/mtraining/utils/data_utils/bookcorpus.py b/mtraining/utils/data_utils/bookcorpus.py
new file mode 100644
index 00000000..04ed3329
--- /dev/null
+++ b/mtraining/utils/data_utils/bookcorpus.py
@@ -0,0 +1,113 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+from typing import Dict, List
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+import numpy
+import torch
+from datasets import Dataset, load_dataset
+from transformers import AutoTokenizer, PreTrainedTokenizer
+
+
+def get_tokenizer(model_path):
+ return AutoTokenizer.from_pretrained(model_path)
+
+
+BOS_TOKEN = ""
+
+
+def tokenize(sample: Dict[str, str], tokenizer: PreTrainedTokenizer, text_key: str):
+ input_ids = tokenizer.encode(
+ BOS_TOKEN + sample[text_key] + tokenizer.eos_token, add_special_tokens=False
+ )
+ return {"input_ids": input_ids}
+
+
+def concate_split(samples: Dict[str, List[List[int]]], sample_len: int, text_key: str):
+ buffer = samples[text_key][0]
+ resized_ids = []
+ length = []
+ for in_ids in samples[text_key]:
+ buffer.extend(in_ids)
+ while len(buffer) >= sample_len:
+ resized_ids.append(buffer[:sample_len])
+ length.append(sample_len)
+ buffer = buffer[sample_len:]
+ return {"input_ids": resized_ids, "length": length}
+
+
+def create_dataset(
+ tokenizer: PreTrainedTokenizer,
+ raw_dataset: Dataset,
+ text_key: str,
+ sample_len: int = 8 * 1024,
+ batch_size=10000,
+):
+ tokenized_dataset = raw_dataset.map(
+ tokenize,
+ remove_columns=raw_dataset.column_names,
+ num_proc=32,
+ fn_kwargs={"tokenizer": tokenizer, "text_key": text_key},
+ )
+ return tokenized_dataset.map(
+ concate_split,
+ remove_columns=tokenized_dataset.column_names,
+ num_proc=32,
+ batched=True,
+ batch_size=batch_size,
+ fn_kwargs={"sample_len": sample_len, "text_key": "input_ids"},
+ )
+
+
+def modify_bos_token(tokenizer: PreTrainedTokenizer):
+ # https://huggingface.co/Qwen/Qwen2-7B-Instruct/discussions/15
+ global BOS_TOKEN
+ if tokenizer.bos_token is None:
+ BOS_TOKEN = "<|endoftext|>"
+ else:
+ BOS_TOKEN = tokenizer.bos_token
+
+
+if __name__ == "__main__":
+ # python bookcorpus.py --data_path_or_name "bookcorpus/bookcorpus" --tokenizer_path_or_name "meta-llama/Llama-2-7b-hf" --save_path "bookcorpus-llama2-2k-hf" --sequence_length 2048
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--data_path_or_name",
+ help='the path or name of the raw dataset, for exmaple, "bookcorpus/bookcorpus"',
+ type=str,
+ required=True,
+ )
+ parser.add_argument(
+ "--tokenizer_path_or_name",
+ help='the tokenizer path or name, for example, "meta-llama/Llama-2-7b-hf"',
+ type=str,
+ required=True,
+ )
+ parser.add_argument(
+ "--save_path",
+ help="the path to save the tokenized dataset",
+ type=str,
+ required=True,
+ )
+ parser.add_argument(
+ "--sequence_length",
+ help="the length of each sample in the tokenized dataset, usually set to the max sequence length",
+ type=int,
+ required=True,
+ )
+ args = parser.parse_args()
+
+ data_path_or_name = args.data_path_or_name
+ tokenizer_path_or_name = args.tokenizer_path_or_name
+ save_path = args.save_path
+ sequence_length = args.sequence_length
+
+ raw_dataset = load_dataset(data_path_or_name)["train"]
+ tokenizer = get_tokenizer(tokenizer_path_or_name)
+ modify_bos_token(tokenizer)
+
+ dataset = create_dataset(tokenizer, raw_dataset, "text", sequence_length)
+ dataset.save_to_disk(save_path)
diff --git a/mtraining/utils/data_utils/prolong.py b/mtraining/utils/data_utils/prolong.py
new file mode 100644
index 00000000..3009a246
--- /dev/null
+++ b/mtraining/utils/data_utils/prolong.py
@@ -0,0 +1,125 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import argparse
+import logging
+import os
+from typing import Dict
+
+from datasets import Dataset, concatenate_datasets
+from streaming import StreamingDataset
+from tqdm import tqdm
+from transformers import PreTrainedTokenizer
+
+from mtraining.utils.general import get_tokenizer
+
+# ------------------------------------------------
+
+logger = logging.getLogger(__name__)
+
+LLAMA3_MODEL_ID = "meta-llama/Meta-Llama-3-8B"
+LLAMA_TOKENZIER = None
+
+
+def tokenize(
+ sample: Dict[str, str], tokenizer: PreTrainedTokenizer, seq_len: int = 524288
+):
+ text = sample["text"]
+ for token_k, token_v in LLAMA_TOKENZIER.special_tokens_map.items():
+ if token_k in tokenizer.special_tokens_map:
+ text = text.replace(token_v, tokenizer.special_tokens_map[token_k])
+
+ input_ids = tokenizer.encode(
+ text, add_special_tokens=False, truncation=True, max_length=seq_len
+ )
+ return {"input_ids": input_ids, "length": len(input_ids)}
+
+
+DOMAINS = [
+ "thestackv1_concat_by_repo-524288@0.15",
+ "thestackv1_concat_by_repo-65536@0.15",
+ "book-524288@0.05",
+ "book-65536@0.25",
+ "fineweb-edu@0.1",
+ "fineweb-2023-50@0.1",
+ "stackexchange@0.04",
+ "dolmawiki@0.04",
+ "tuluv2@0.03",
+ "arxiv@0.03",
+ "openwebmath@0.03",
+ "textbooks@0.03",
+]
+FIXED_512K = ["thestackv1_concat_by_repo-524288", "book-524288"]
+
+DOMAIN_MIX_DICT = {"full": DOMAINS, "fixed_524288": FIXED_512K}
+
+
+def main(args):
+ global LLAMA_TOKENZIER
+
+ seq_len = args.sequence_length
+ LLAMA_TOKENZIER = get_tokenizer(LLAMA3_MODEL_ID)
+ model_tokenizer = get_tokenizer(args.model_id)
+ if model_tokenizer.bos_token is None:
+ model_tokenizer.bos_token = "<|endoftext|>"
+
+ domains = DOMAIN_MIX_DICT[args.dataset_mix]
+ dataset_paths = [os.path.join(args.dataset_path, domain) for domain in domains]
+ tokenized_datasets = []
+ for idx, dataset_path in enumerate(dataset_paths):
+ print("-" * 50)
+ print(f"Processing {domains[idx]} from {dataset_path}...")
+ texts = []
+ dataset = StreamingDataset(
+ local=dataset_path,
+ remote=None,
+ shuffle=False,
+ batch_size=1,
+ )
+
+ for ix, sample in tqdm(enumerate(dataset)):
+ if ix % args.sample_interval != 0:
+ continue
+
+ sample_input_ids = sample["input_ids"]
+ sample_splits = [
+ sample_input_ids[i : i + seq_len]
+ for i in range(0, len(sample_input_ids), seq_len)
+ ]
+ for sample_split in sample_splits:
+ # De-tokenization
+ text = LLAMA_TOKENZIER.decode(sample_split)
+ texts.append(text)
+
+ hf_dataset = Dataset.from_dict({"text": texts})
+
+ tokenized_dataset = hf_dataset.map(
+ tokenize,
+ remove_columns=hf_dataset.column_names,
+ num_proc=64,
+ fn_kwargs={"tokenizer": model_tokenizer, "seq_len": seq_len},
+ )
+ tokenized_datasets.append(tokenized_dataset)
+
+ print("-" * 50)
+ print(f"Concatenating and Saving tokenized datasets to {args.save_path}...")
+ concat_dataset = concatenate_datasets(tokenized_datasets)
+ filtered_concat_dataset = concat_dataset.filter(
+ lambda x: x["length"] == seq_len, num_proc=128
+ )
+ filtered_concat_dataset.save_to_disk(args.save_path)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--model_id", type=str, default="microsoft/Phi-3-mini-4k-instruct"
+ )
+ parser.add_argument("--dataset_mix", type=str, default="fixed_524288")
+ parser.add_argument("--dataset_path", type=str)
+ parser.add_argument("--save_path", type=str)
+ parser.add_argument("--sequence_length", type=int, default=524288)
+ parser.add_argument("--sample_interval", type=int, default=1)
+ args = parser.parse_args()
+
+ main(args)
diff --git a/mtraining/utils/general.py b/mtraining/utils/general.py
new file mode 100644
index 00000000..3a3fc02f
--- /dev/null
+++ b/mtraining/utils/general.py
@@ -0,0 +1,244 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import logging
+import os
+from typing import List
+
+import torch
+import torch.distributed as dist
+from nnscaler.cli.trainer_args import AggregatedOutputs
+from nnscaler.runtime.module import ParallelModule
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from .paths import BASE_DIR
+
+logger = logging.getLogger(__name__)
+
+
+def get_tokenizer(
+ tokenizer_name_or_path,
+ model_max_length=None,
+ default_bos_token="",
+ default_eos_token="",
+ default_pad_token="[PAD]",
+ default_unk_token="",
+):
+ tokenizer = AutoTokenizer.from_pretrained(
+ tokenizer_name_or_path, trust_remote_code=True
+ )
+ special_tokens_dict = dict()
+ if tokenizer.pad_token is None:
+ special_tokens_dict["pad_token"] = default_pad_token
+ if tokenizer.eos_token is None:
+ special_tokens_dict["eos_token"] = default_eos_token
+ if tokenizer.bos_token is None:
+ special_tokens_dict["bos_token"] = default_bos_token
+ if tokenizer.unk_token is None:
+ special_tokens_dict["unk_token"] = default_unk_token
+
+ tokenizer.add_special_tokens(special_tokens_dict)
+ if model_max_length:
+ tokenizer.model_max_length = model_max_length
+ return tokenizer
+
+
+def get_module_path(model_id: str):
+ model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True)
+ module_path = str(model.__class__.__module__)
+ del model
+
+ return module_path
+
+
+def aggregate_outputs_fn(loss_outputs, sync_group) -> AggregatedOutputs:
+ losses, ntokens_info = [], []
+ for _, loss, ntokens, _ in loss_outputs:
+ losses.append(loss)
+ ntokens_info.append(ntokens)
+
+ loss_sum = torch.sum(torch.stack(losses), dtype=torch.float64)
+ dist.all_reduce(loss_sum, group=sync_group)
+
+ ntokens_sum = torch.sum(
+ torch.tensor(
+ ntokens_info, dtype=torch.float64, device=torch.cuda.current_device()
+ )
+ )
+ dist.all_reduce(ntokens_sum, group=sync_group)
+
+ num_batches = torch.tensor(len(losses), device=torch.cuda.current_device())
+ dist.all_reduce(num_batches, group=sync_group)
+
+ return AggregatedOutputs(
+ loss_sum=loss_sum.item() / ntokens_sum.item(),
+ num_batches=num_batches.item(),
+ num_tokens=ntokens_sum.item(),
+ )
+
+
+def load_comm_profile_data(args):
+ if args.plan_ngpus in [2, 4, 8, 16]:
+ logger.info(
+ f"Use nnscaler's built-in communication profiling data for {args.plan_ngpus} GPUs"
+ )
+ return
+
+ from nnscaler.autodist.util import get_default_profile_path
+
+ profile_dir = os.path.join(get_default_profile_path(), "comm")
+ profile_path = os.path.join(profile_dir, f"intra_{args.plan_ngpus}.json")
+
+ if not os.path.exists(profile_path):
+ import shutil
+
+ logger.info(
+ f"Communication profiling data not found in {profile_dir} for {args.plan_ngpus} GPUs. Use built-in communication profiling data (collected on A100-SXM4-40GB)"
+ )
+ src_file_path = os.path.join(
+ BASE_DIR,
+ "utils/comm_prof/NVIDIA_A100-SXM4-40GB",
+ f"intra_{args.plan_ngpus}.json",
+ )
+ if not os.path.exists(src_file_path):
+ raise FileNotFoundError(
+ f"Communication profiling data not found in {src_file_path} nor in nnscaler's built-in library for {args.plan_ngpus} GPUs"
+ )
+ os.makedirs(profile_dir, exist_ok=True)
+
+ num_dev = 2
+ while num_dev <= args.plan_ngpus:
+ src_file_path = os.path.join(
+ BASE_DIR,
+ "utils/comm_prof/NVIDIA_A100-SXM4-40GB",
+ f"intra_{num_dev}.json",
+ )
+ profile_path = os.path.join(profile_dir, f"intra_{num_dev}.json")
+ if os.path.exists(profile_path):
+ logger.info(
+ f"Communication profiling data already exists in {profile_path} for {num_dev} GPUs"
+ )
+ num_dev *= 2
+ continue
+ else:
+ logger.info(f"Copying {src_file_path} to {profile_path}")
+ shutil.copy(src_file_path, profile_path)
+ num_dev *= 2
+
+
+def is_active(module_name: str, keep_active: List[str]):
+ for active_module_subname in keep_active:
+ if active_module_subname.lower() in module_name.lower():
+ return True
+ return False
+
+
+def freeze_model_params_(model, keep_active: List[str], prefix=""):
+ if dist.get_rank() == 0:
+ print("-" * 80)
+ print(f"Only keeping parameters with substring in {keep_active} active...")
+
+ for name, module in model._modules.items():
+ if len(list(module.children())) > 0:
+ freeze_model_params_(module, keep_active, prefix + name + ".")
+ else:
+ param_name = prefix + name
+ if not is_active(param_name, keep_active):
+ print(f"Freezing {param_name}...")
+ for param in module.parameters():
+ param.requires_grad = False
+ else:
+ print(f"Keeping {param_name} active...")
+
+ if dist.get_rank() == 0:
+ print("-" * 80)
+
+
+def freeze_model_params(model, active_param_config_path: str, prefix=""):
+ with open(active_param_config_path, "r") as f:
+ keep_active = f.read().splitlines()
+ print(f"freeze_model_params | keep active: {keep_active}")
+
+ freeze_model_params_(model, keep_active, prefix)
+
+
+def get_resume_path(
+ check_resume: bool,
+ resume_from: str,
+ ckpt_save_dir: str,
+ num_gpus: int,
+):
+ if not check_resume:
+ return None
+ if resume_from is not None:
+ return resume_from
+
+ # Detect the last checkpoint in CKPT_PATH
+ ckpt_dirs = [
+ ckpt_dir
+ for ckpt_dir in os.listdir(ckpt_save_dir)
+ if len(ckpt_dir.split("-")) == 2 and ckpt_dir.split("-")[0].isdigit()
+ ]
+
+ # Filter out directories that do not contain number of ckpts (file name ending with .ckpt) equal to num_gpus (check by os.listdir)
+ filtered_ckpt_dirs = []
+ for ckpt_dir in ckpt_dirs:
+ ckpt_dir_path = os.path.join(ckpt_save_dir, ckpt_dir)
+ if os.path.isdir(ckpt_dir_path):
+ ckpt_files = [f for f in os.listdir(ckpt_dir_path) if f.endswith(".ckpt")]
+ if len(ckpt_files) == num_gpus:
+ filtered_ckpt_dirs.append(ckpt_dir)
+
+ print(f"get_resume_path | filtered_ckpt_dirs = {filtered_ckpt_dirs}")
+ if len(filtered_ckpt_dirs) == 0:
+ return None
+
+ target_ckpt_dir = sorted(
+ filtered_ckpt_dirs, key=lambda x: (int(x.split("-")[0]), int(x.split("-")[1]))
+ )[-1]
+ target_ckpt_dir = os.path.join(ckpt_save_dir, target_ckpt_dir)
+ print(f"get_resume_path | target_ckpt_dir = {target_ckpt_dir}")
+ return target_ckpt_dir
+
+
+def fix_model_state_dict(model, model_state_dict):
+ if isinstance(model, ParallelModule):
+ required_keys = list(model.dist_param_map.values())
+ required_keys_under = {k[6:]: v for k, v in model.dist_param_map.items()}
+ else:
+ required_keys = model.state_dict().keys()
+ required_keys_under = {k.replace(".", "_"): k for k in required_keys}
+
+ has_model_prefix = "model" in model_state_dict
+ model_state_dict = (
+ model_state_dict if not has_model_prefix else model_state_dict["model"]
+ )
+ model_sd_copy = model_state_dict.copy()
+
+ if (
+ dist.is_initialized()
+ and dist.get_rank() % int(os.getenv("GPU_PER_NODE", "8")) == 0
+ ):
+ print(f"{__name__} | required_keys[:10]: {required_keys[:10]}")
+ print(f"{__name__} | required_keys_under: {required_keys_under}")
+ print(
+ f"{__name__} | model_state_dict.keys()[:10]: {list(model_state_dict.keys())[:10]}"
+ )
+
+ for k in model_state_dict.keys():
+ model_sd_copy.pop(k)
+
+ under_k_start = 0 if not has_model_prefix else 1
+ under_k = "_".join(k.split("_")[under_k_start:-1])
+
+ if under_k in required_keys_under:
+ model_sd_copy[required_keys_under[under_k]] = model_state_dict[k]
+
+ if "lm_head_weight" in required_keys_under:
+ for k in model_state_dict.keys():
+ if "model_embed_tokens_weight" in k:
+ model_sd_copy[required_keys_under["lm_head_weight"]] = model_state_dict[
+ k
+ ]
+
+ return model_sd_copy
diff --git a/mtraining/utils/loss.py b/mtraining/utils/loss.py
new file mode 100644
index 00000000..eda10d89
--- /dev/null
+++ b/mtraining/utils/loss.py
@@ -0,0 +1,82 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import torch
+import torch.utils.checkpoint as ckpt
+from nnscaler.graph.parser.register import register_op
+
+
+def linear_cross_entropy(
+ x: torch.Tensor, w: torch.Tensor, y: torch.Tensor, padding_idx: int = 0
+) -> torch.Tensor:
+ """
+ Compute the cross entropy loss of a linear layer.
+
+ Args:
+
+ x: [token_num, hidden_size], the last hidden state of the model
+ w: [dict_size, hidden_size], the weight matrix of the last linear layer
+ y: [token_num], the target token index
+ padding_idx: int, the index of padding token
+
+ Returns:
+
+ losses: [token_num], the cross entropy loss of each token
+ """
+ logits = torch.nn.functional.linear(x, w)
+ normalized_logits = torch.nn.functional.log_softmax(
+ logits, dim=-1, dtype=torch.float32
+ )
+ losses = torch.nn.functional.nll_loss(
+ normalized_logits, y, reduction="none", ignore_index=padding_idx
+ )
+ return losses
+
+
+def chunk_linear_cross_entropy(
+ x: torch.Tensor, w: torch.Tensor, y: torch.Tensor, padding_idx: int, chunk_size: int
+) -> torch.Tensor:
+ """
+ In order to reduce the memory usage when the sequence length and dictionary size are large, we can split the input
+ tensor into chunks and compute the cross entropy loss of each chunk separately.
+ You can register this function with annotation 'b l d^, n^ d^, b l -> b l'.
+
+ Args:
+
+ x: [bsz, seq_len, hidden_size], the last hidden state of the model
+ w: [dict_size, hidden_size], the weight matrix of the last linear layer
+ y: [bsz, seq_len], the target token index
+ padding_idx: int, the index of padding token
+ chunk_size: int, the size of each chunk
+
+ Returns:
+
+ losses: [bsz, seq_len], the cross entropy loss of each token
+ """
+ bsz, seq_len, hidden_size = x.size()
+ token_num = bsz * seq_len
+ x = x.view(token_num, hidden_size)
+ y = y.view(token_num)
+
+ if token_num % chunk_size != 0:
+ raise ValueError(
+ f"token_num {token_num} is not divisible by chunk_size {chunk_size}"
+ )
+
+ chunk_num = token_num // chunk_size
+ xs = x.view(chunk_num, chunk_size, hidden_size)
+ ys = y.view(chunk_num, chunk_size)
+ losses = []
+ for i in range(chunk_num):
+ loss = ckpt.checkpoint(
+ linear_cross_entropy, xs[i], w, ys[i], padding_idx, use_reentrant=False
+ )
+ losses.append(loss)
+ losses = torch.stack(losses).view(bsz, seq_len)
+ return losses
+
+
+register_op("b l d^, n^ d^, b l -> b l")(chunk_linear_cross_entropy)
diff --git a/mtraining/utils/merge_utils.py b/mtraining/utils/merge_utils.py
new file mode 100644
index 00000000..e367384f
--- /dev/null
+++ b/mtraining/utils/merge_utils.py
@@ -0,0 +1,300 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import copy
+import os
+import shutil
+import subprocess
+from typing import List, Optional
+
+import torch
+from nnscaler import merge_state_dicts
+from transformers import AutoConfig
+from transformers.modeling_utils import PreTrainedModel
+
+from .general import fix_model_state_dict
+
+BLOB_DIR = os.environ.get("BLOB_DIR", "/blob")
+HF_REPO_DIR = os.path.join(BLOB_DIR, "hf_repos")
+STORE_DIR = os.path.join(BLOB_DIR, "mtrain_expr_data_store")
+print(f"STORE_DIR: {STORE_DIR}", flush=True)
+
+
+def model_to_hf_config_files(model_id: str):
+ if "phi" in model_id.lower():
+ return [
+ "added_tokens.json",
+ "config.json",
+ "configuration_phi3.py",
+ "generation_config.json",
+ "modeling_phi3.py",
+ "generation_config.json",
+ "model.safetensors.index.json",
+ "special_tokens_map.json",
+ "tokenizer_config.json",
+ "tokenizer.json",
+ ]
+ elif "qwen" in model_id.lower():
+ return [
+ "config.json",
+ "generation_config.json",
+ "modeling_qwen2.py",
+ "model.safetensors.index.json",
+ "tokenizer_config.json",
+ "tokenizer.json",
+ "vocab.json",
+ ]
+ elif "llama" in model_id.lower():
+ return [
+ "config.json",
+ "generation_config.json",
+ "special_tokens_map.json",
+ "model.safetensors.index.json",
+ "modeling_llama.py",
+ "tokenizer_config.json",
+ "tokenizer.json",
+ ]
+ else:
+ raise ValueError(f"Model id: {model_id} unsupported yet.")
+
+
+FLEXIBLE_FIELDS = ["save_plan_path", "gen_savedir"]
+
+
+def recursive_equiv_state_dict(dict1, dict2):
+ if dict1.keys() != dict2.keys():
+ # print out which keys are different
+ print("Keys differ:", set(dict1.keys()).symmetric_difference(set(dict2.keys())))
+ return False
+
+ for key in dict1:
+ if isinstance(dict1[key], dict) and isinstance(dict2[key], dict):
+ if not recursive_equiv_state_dict(dict1[key], dict2[key]):
+ return False
+ elif dict1[key] != dict2[key] and not key in FLEXIBLE_FIELDS:
+ print(
+ f"{__name__} | Values differ at key '{key}': {dict1[key]} != {dict2[key]}"
+ )
+ return False
+ return True
+
+
+def merge_checkpoint(checkpoint_files: List[str], output_file: str):
+ print(f"merge_checkpoint | Start merging ckpt files...", flush=True)
+ # state_dicts = [torch.load(f, map_location='cpu') for f in checkpoint_files]
+ state_dicts = []
+ for i, file_path in enumerate(checkpoint_files):
+ print(
+ f"merge_checkpoint | Loading {i}-th checkpoint from {file_path}...",
+ flush=True,
+ )
+ state_dict = torch.load(file_path, map_location="cpu")
+ state_dicts.append(state_dict)
+
+ train_args_0 = copy.deepcopy(state_dicts[0]["train_args"])
+ train_args_0.pop("gen_savedir")
+ for i in range(1, len(state_dicts)):
+ train_args_i = copy.deepcopy(state_dicts[i]["train_args"])
+ train_args_i.pop("gen_savedir")
+
+ # if train_args_i != train_args_0:
+ if not recursive_equiv_state_dict(train_args_i, train_args_0):
+ raise ValueError(
+ f"train_args in {checkpoint_files[i]} is different from {checkpoint_files[0]}"
+ )
+
+ print(f"merge_checkpoint | Invoking nnscaler.merge_state_dicts...", flush=True)
+ module_state_dict, opt_state_dict = merge_state_dicts(
+ [s["model"] for s in state_dicts], [s["optimizer"] for s in state_dicts]
+ )
+ train_args = copy.deepcopy(state_dicts[0]["train_args"])
+ train_args["checkpoint"]["save_type"] = "merged"
+ merged_state_dict = {
+ "model": module_state_dict,
+ "optimizer": opt_state_dict,
+ "lr_scheduler": state_dicts[0].get("lr_scheduler", None),
+ "train_status": state_dicts[0]["train_status"],
+ "train_args": train_args,
+ "rng_states": None,
+ }
+
+ print(f"merge_checkpoint | Saving merged state_dict into {output_file}", flush=True)
+ os.makedirs(os.path.dirname(output_file), exist_ok=True)
+ torch.save(merged_state_dict, output_file)
+
+
+def load_ckpt_files(ckpt_dir):
+ print(f"load_ckpt_files | Looking for ckpt files in {ckpt_dir}", flush=True)
+ ckpt_files = [
+ os.path.join(ckpt_dir, f)
+ for f in os.listdir(ckpt_dir)
+ if f.endswith(".ckpt") and not f.startswith("merged")
+ ]
+
+ if "rank_" in ckpt_dir.split("/")[-2]:
+ rank = int(ckpt_dir.split("/")[-2].split("_")[-1])
+
+ while os.path.exists(ckpt_dir.replace(f"rank_{rank}", f"rank_{rank+1}")):
+ ckpt_dir = ckpt_dir.replace(f"rank_{rank}", f"rank_{rank+1}")
+ print(f"load_ckpt_files | Looking for ckpt files in {ckpt_dir}", flush=True)
+ ckpt_files += [
+ os.path.join(ckpt_dir, f)
+ for f in os.listdir(ckpt_dir)
+ if f.endswith(".ckpt") and not f.startswith("merged")
+ ]
+ rank += 1
+
+ return ckpt_files
+
+
+def merge_ckpts(
+ expr_data_dir: str,
+ epoch_idx,
+ iter_idx,
+ override: bool = False,
+):
+ # --------------------------------------------------------------------------
+ # Set ckpt save path and URL
+ ckpt_dir = os.path.join(
+ expr_data_dir,
+ "checkpoints",
+ f"{epoch_idx:04d}-{iter_idx:04d}",
+ )
+ if not os.path.exists(ckpt_dir):
+ # Legacy Storage Path
+ ckpt_dir = os.path.join(
+ expr_data_dir,
+ "checkpoints",
+ "rank_0",
+ f"{epoch_idx:04d}-{iter_idx:04d}",
+ )
+
+ # example: /scratch/sync/nnscaler_store/A100_4/minfer_qwen/qwen_moba_zigzag_mini_262144/merged_ckpts/0000-0005/pytorch_model.bin
+ merged_ckpt_save_dir = os.path.join(
+ expr_data_dir,
+ "merged_ckpts",
+ f"{epoch_idx:04d}-{iter_idx:04d}",
+ )
+ os.makedirs(merged_ckpt_save_dir, exist_ok=True)
+ merged_ckpt_save_path = os.path.join(merged_ckpt_save_dir, "pytorch_model.bin")
+ if os.path.exists(merged_ckpt_save_path) and not override:
+ print(
+ f"merge_ckpts | Checkpoint already exists in {merged_ckpt_save_path} and skip merging (enabling --override to override)"
+ )
+ return True
+
+ print(
+ f"merge_ckpts | Merging ckpt files in {ckpt_dir} to {merged_ckpt_save_path}",
+ flush=True,
+ )
+ ckpt_files = load_ckpt_files(ckpt_dir)
+ merge_checkpoint(ckpt_files, merged_ckpt_save_path)
+ print(f"merge_ckpts | Merged ckpt files.", flush=True)
+
+ # --------------------------------------------------------------------------
+ print(
+ f"merge_ckpts | Converting merged checkpoint to correct format...", flush=True
+ )
+ ckpt_data = torch.load(merged_ckpt_save_path)
+ ckpt_model_data = ckpt_data.pop("model")
+ ckpt_model_data = {k[6:]: v for k, v in ckpt_model_data.items()}
+ torch.save(ckpt_model_data, merged_ckpt_save_path)
+ print(f"merge_ckpts | Checkpoint merged.", flush=True)
+
+ return True
+
+
+def run_cmd(cmd):
+ result = subprocess.run(cmd)
+
+ if result.returncode != 0:
+ print(f"Error running command {cmd}. Exiting...")
+ return 1
+
+ return 0
+
+
+def transfer_by_cp(local_path, dir="upload", source=None):
+ remote_path = local_path.replace("/scratch/eval", STORE_DIR)
+ if dir == "upload":
+ os.makedirs(os.path.dirname(remote_path), exist_ok=True)
+ run_res = run_cmd(["cp", local_path, remote_path])
+ else:
+ if source == None:
+ run_res = run_cmd(["cp", remote_path, local_path])
+ else:
+ run_res = run_cmd(["cp", source, local_path])
+
+ if run_res != 0:
+ print(
+ f"Error {'uploading' if dir == 'upload' else 'downloading'} {local_path}. Exiting..."
+ )
+ return 1
+ return 0
+
+
+def copy_configs_to_merged_dir(
+ expr_data_dir: str,
+ epoch_idx: int,
+ iter_idx: int,
+ model_id: str,
+ override: bool = False,
+):
+ merged_ckpt_save_dir = os.path.join(
+ expr_data_dir, "merged_ckpts", f"{epoch_idx:04d}-{iter_idx:04d}"
+ )
+ merged_ckpt_save_path = os.path.join(merged_ckpt_save_dir, "pytorch_model.bin")
+ if not os.path.exists(merged_ckpt_save_path):
+ raise ValueError(
+ f"Merged checkpoint path {merged_ckpt_save_path} does not exist in {expr_data_dir}. Run merge_ckpt first."
+ )
+
+ print("-" * 20)
+ print(
+ f"copy_configs_to_merged_dir | Copying model files to {merged_ckpt_save_dir}",
+ flush=True,
+ )
+ model_name = model_id.split("/")[-1]
+ config_files = model_to_hf_config_files(model_id)
+ for file in config_files:
+ hf_file_path = os.path.join(HF_REPO_DIR, model_name, file)
+ merged_path = os.path.join(merged_ckpt_save_dir, file)
+ if os.path.exists(merged_path) and not override:
+ print(f"\tFile {merged_path} already exists. Use -o to override.")
+ continue
+
+ print(f"\tCopying {hf_file_path} to {merged_path}...")
+ shutil.copyfile(hf_file_path, merged_path)
+
+ print(f"Config files copied successfully.")
+ return True
+
+
+def load_merged_model(
+ model_cls: PreTrainedModel,
+ expr_data_dir: str,
+ epoch_idx: int,
+ iter_idx: int,
+ merged_ckpt_dir: Optional[str] = None,
+):
+ if merged_ckpt_dir is None:
+ model_dir = os.path.join(
+ expr_data_dir, "merged_ckpts", f"{epoch_idx:04d}-{iter_idx:04d}"
+ )
+ model_config = AutoConfig.from_pretrained(model_dir)
+ if "mini" in model_dir:
+ model_config.num_hidden_layers = 2
+ else:
+ model_dir = merged_ckpt_dir
+ model_config = AutoConfig.from_pretrained(model_dir)
+ if "mini" in model_dir:
+ model_config.num_hidden_layers = 2
+
+ model = model_cls(config=model_config)
+ model_state_dict = torch.load(os.path.join(model_dir, "pytorch_model.bin"))
+ if len(list(model_state_dict.keys())[0].split(".")) == 1:
+ # For Ring-Attention models, the merged checkpoint is directly copied from one of the shards and has different key names.
+ model_state_dict = fix_model_state_dict(model, model_state_dict)
+
+ model.load_state_dict(model_state_dict)
+ return model
diff --git a/mtraining/utils/paths.py b/mtraining/utils/paths.py
new file mode 100644
index 00000000..3c9ac5f3
--- /dev/null
+++ b/mtraining/utils/paths.py
@@ -0,0 +1,32 @@
+# Copyright (c) 2026 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import os
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+EXPR_DATA_SAVE_PATH = {
+ "base_path": None,
+ "ckpt_save_path": None,
+ "compile_save_path": None,
+}
+
+
+def update_expr_data_by_base_path(base_path):
+ EXPR_DATA_SAVE_PATH["base_path"] = base_path
+
+
+def update_expr_data_save_path(
+ ckpt_save_path,
+ compile_save_path,
+):
+ if ckpt_save_path is None:
+ EXPR_DATA_SAVE_PATH["base_path"] = os.getenv("EFFI_EXPR_STORE_DIR")
+ else:
+ EXPR_DATA_SAVE_PATH["base_path"] = os.path.dirname(ckpt_save_path)
+ if "rank_" in ckpt_save_path:
+ EXPR_DATA_SAVE_PATH["base_path"] = os.path.dirname(
+ os.path.dirname(ckpt_save_path)
+ )
+
+ EXPR_DATA_SAVE_PATH["ckpt_save_path"] = ckpt_save_path
+ EXPR_DATA_SAVE_PATH["compile_save_path"] = compile_save_path
diff --git a/scbench/args.py b/scbench/args.py
new file mode 100644
index 00000000..717ffa1e
--- /dev/null
+++ b/scbench/args.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+import json
+from argparse import ArgumentParser, Namespace
+
+from eval_utils import DATA_NAME_TO_MAX_NEW_TOKENS
+
+from minference import MInferenceConfig
+
+
+def parse_args() -> Namespace:
+ p = ArgumentParser()
+ p.add_argument(
+ "--task",
+ type=str,
+ required=True,
+ help='Which task to use. Note that "all" can only be used in `compute_scores.py`.', # noqa
+ )
+ p.add_argument(
+ "--data_dir", type=str, default="../data", help="The directory of data."
+ )
+ p.add_argument(
+ "--output_dir",
+ type=str,
+ default="../results",
+ help="Where to dump the prediction results.",
+ ) # noqa
+ p.add_argument(
+ "--model_name_or_path",
+ type=str,
+ default="facebook/opt-350m",
+ help="For `compute_scores.py` only, specify which model you want to compute the score for.", # noqa
+ )
+ p.add_argument(
+ "--num_eval_examples",
+ type=int,
+ default=-1,
+ help="The number of test examples to use, use all examples in default.",
+ ) # noqa
+ p.add_argument(
+ "--start_idx",
+ type=int,
+ default=0,
+ help="The index of the first example to infer on. This is used if you want to evaluate on a (contiguous) subset of the data.",
+ ) # noqa
+ p.add_argument(
+ "--stop_idx",
+ type=int,
+ help="The index of the last example to infer on. This is used if you want to evaluate on a (contiguous) subset of the data. Defaults to the length of dataset.",
+ ) # noqa
+ p.add_argument("--verbose", action="store_true")
+ p.add_argument("--use_sparq", action="store_true")
+ p.add_argument("--device", type=str, default="cuda")
+ p.add_argument("--max_seq_length", type=int, default=131_072)
+ p.add_argument("--rewrite", action="store_true")
+ p.add_argument("--topk", type=int, default=-1)
+ p.add_argument("--starting_layer", type=int, default=-1)
+ p.add_argument("--start_example_id", type=int, default=0)
+ p.add_argument("--topk_dims_file_path", type=str, default=None)
+ p.add_argument("--kv_cache_cpu", action="store_true")
+ p.add_argument("--kv_cache_cpu_device", type=str, default="cpu")
+ p.add_argument("--trust_remote_code", action="store_true")
+ p.add_argument("--use_chat_template", action="store_true")
+ p.add_argument("--same_context_different_query", action="store_true")
+ p.add_argument("--tensor_parallel_size", type=int, default=1)
+ p.add_argument("--max_turns", type=int, default=5)
+ p.add_argument("--use_llmlingua", action="store_true")
+ p.add_argument("--disable_golden_context", action="store_true")
+ p.add_argument("--use_v2_data", action="store_true")
+ p.add_argument(
+ "--attn_type",
+ type=str,
+ choices=MInferenceConfig.get_available_attn_types(),
+ default="hf",
+ )
+ p.add_argument(
+ "--kv_type",
+ type=str,
+ default="dense",
+ choices=MInferenceConfig.get_available_kv_types(),
+ )
+ p.add_argument("--is_search", action="store_true")
+ p.add_argument("--hyper_param", type=json.loads, default={})
+ return p.parse_args()
diff --git a/scbench/cache_blend.yaml b/scbench/cache_blend.yaml
new file mode 100644
index 00000000..7ee6be20
--- /dev/null
+++ b/scbench/cache_blend.yaml
@@ -0,0 +1,8 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+chunk_size: 256
+local_device: "cpu"
+
+# Enables KV blending
+enable_blending: True
diff --git a/scbench/compute_scores.py b/scbench/compute_scores.py
new file mode 100644
index 00000000..2bf7a466
--- /dev/null
+++ b/scbench/compute_scores.py
@@ -0,0 +1,692 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import string
+from collections import Counter
+from pathlib import Path
+
+import evaluate
+from args import parse_args
+from repo_qa_utils import compute_score as compute_repoqa_score
+from tqdm import tqdm
+
+ROUGE_SCORER = evaluate.load("rouge")
+
+
+def normalize_answer(s: str) -> str:
+ """Lower text and remove punctuation, articles and extra whitespace."""
+
+ def remove_articles(text):
+ return re.sub(r"\b(a|an|the)\b", " ", text)
+
+ def white_space_fix(text):
+ return " ".join(text.split())
+
+ def remove_punc(text):
+ exclude = set(string.punctuation)
+ return "".join(ch for ch in text if ch not in exclude)
+
+ def lower(text):
+ return text.lower()
+
+ return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def normalize_zh_answer(s: str) -> str:
+ """Chinese version. Lower text and remove punctuation, extra whitespace."""
+
+ def white_space_fix(text):
+ return "".join(text.split())
+
+ def remove_punc(text):
+ cn_punctuation = "！？｡。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏." # noqa
+ all_punctuation = set(string.punctuation + cn_punctuation)
+ return "".join(ch for ch in text if ch not in all_punctuation)
+
+ def lower(text):
+ return text.lower()
+
+ return white_space_fix(remove_punc(lower(s)))
+
+
+def string_match_all(pred, ref, model_name=""):
+ score = sum([1.0 if r.lower() in pred.lower() else 0.0 for r in ref]) / len(ref)
+ return round(score, 2)
+
+
+def f1_score(prediction, ground_truth) -> tuple[float, float, float]:
+ common = Counter(prediction) & Counter(ground_truth)
+ num_same = sum(common.values())
+ if num_same == 0:
+ return 0, 0, 0
+ precision = 1.0 * num_same / len(prediction)
+ recall = 1.0 * num_same / len(ground_truth)
+ f1 = (2 * precision * recall) / (precision + recall)
+ return f1, precision, recall
+
+
+def qa_f1_score(pred: str, ground_truths) -> float:
+ """Computes the F1, recall, and precision."""
+ f1 = 0
+ prec = 0
+ recall = 0
+ for ground_truth in ground_truths:
+ normalized_prediction = normalize_answer(pred)
+ normalized_ground_truth = normalize_answer(ground_truth)
+
+ prediction_tokens = normalized_prediction.split()
+ ground_truth_tokens = normalized_ground_truth.split()
+ scores = f1_score(prediction_tokens, ground_truth_tokens)
+ this_f1, this_prec, this_recall = scores
+ f1 = max(f1, this_f1)
+ prec = max(prec, this_prec)
+ recall = max(recall, this_recall)
+ return f1
+
+
+def qa_f1_score_zh(pred: str, ground_truths: list[str]) -> float:
+ """
+ QA F1 score for chinese.
+ """
+ f1 = 0
+ prec = 0
+ recall = 0
+ for ground_truth in ground_truths:
+ norm_pred = normalize_zh_answer(pred)
+ norm_label = normalize_zh_answer(ground_truth)
+
+ # One character one token.
+ pred_tokens = list(norm_pred)
+ label_tokens = list(norm_label)
+ scores = f1_score(pred_tokens, label_tokens)
+ this_f1, this_prec, this_recall = scores
+ f1 = max(f1, this_f1)
+ prec = max(prec, this_prec)
+ recall = max(recall, this_recall)
+ return f1
+
+
+def load_json(fname):
+ return json.load(open(fname))
+
+
+def iter_jsonl(fname, cnt=None):
+ i = 0
+ with open(fname, "r", encoding="utf8") as fin:
+ for line in fin:
+ if line.strip() == "": # Skip empty lines
+ continue
+ if i == cnt:
+ break
+ if line.strip() == "": # Skip empty lines
+ continue
+ yield json.loads(line)
+ i += 1
+
+
+def first_int_match(prediction):
+ pred_list = re.split("[^0-9]", prediction)
+ pred_value = ""
+ for item in pred_list:
+ if item != "":
+ pred_value = item
+ break
+ return pred_value
+
+
+def split_retrieval_answer(pred: str):
+ for c in ["\n", ":", '"', "'", ".", ",", "?", "!", "{", "}"]:
+ pred = pred.replace(c, " ")
+ words = pred.split()
+ return words
+
+
+def get_score_one_kv_retrieval(pred, label, model_name: str) -> bool:
+ # for c in ["\n", ":", '"', "'", ".", ",", "?", "!", "{", "}", "", "The", "To"]:
+ # pred = pred.replace(c, " ")
+ # words = pred.split()
+ return label in pred
+
+
+def get_score_one_passkey(pred, label, model_name: str) -> bool:
+ if isinstance(label, list):
+ label = label[0]
+ return label == first_int_match(pred)
+
+
+def get_score_one_number_string(pred, label, model_name: str) -> bool:
+ if isinstance(label, list):
+ label = label[0]
+ return label == first_int_match(pred)
+
+
+def get_score_one_code_run(pred, label, model_name: str) -> bool:
+ """
+ Returns the score of one example in Code.Run.
+ """
+ if isinstance(label, list):
+ label = label[0]
+ pred = pred.strip()
+ for c in ["\n", ".", "`", "'", '"', ":"]:
+ pred = pred.replace(c, " ")
+ words = pred.split()
+ if len(words) == 0:
+ return False
+ try:
+ pred = int(words[-1])
+ return label == pred
+ except Exception:
+ return False
+
+
+def get_score_one_code_debug(pred, label, model_name: str) -> bool:
+ """
+ Returns the score of one example in Code.Debug.
+ """
+ pred = pred.strip()
+ label_c = label[1]
+ fn_name = label[0]
+ if pred[:2] in [f"{label_c}.", f"{label_c}:"]:
+ return True
+
+ ans_prefixes = [
+ "answer is:",
+ # "answer is",
+ # "error is",
+ "is:",
+ "answer:",
+ ]
+ for c in ["\n", "`", "'", '"', "-", "*", "Option", "option"]:
+ pred = pred.replace(c, " ")
+ while " " in pred:
+ pred = pred.replace(" ", " ")
+ for prefix in ans_prefixes:
+ idx = pred.find(prefix)
+ if idx == -1:
+ continue
+ # The prediction ends with this prefix
+ if len(pred) < idx + len(prefix) + 1:
+ return False
+ pred = pred[idx + len(prefix) + 1 :]
+ for s in [label_c, fn_name]:
+ if pred.startswith(s):
+ return True
+ return False
+ return False
+
+
+def get_score_one_math_find(pred, label, model_name: str) -> bool:
+ if isinstance(label, list):
+ # In math_find, there is always only one label.
+ label = label[0]
+ if isinstance(label, int):
+ # Find first int or float
+ first_num = re.search(r"\d+\.\d+|\d+", pred)
+ if first_num is None:
+ return False
+ first_num = first_num.group(0).strip()
+ return int(float(first_num)) == label
+ elif isinstance(label, float):
+ # Find first float or int
+ first_float = re.search(r"\d+\.\d+|\d+", pred)
+ if first_float is None:
+ return False
+ first_float = first_float.group(0).strip()
+ return float(first_float) == label
+ else:
+ raise TypeError(f"Expected int or float, got {type(label)}")
+
+
+def get_score_one_longdialogue_qa_eng(pred, label, model_name: str) -> bool:
+ pred = pred.strip()
+ pred = pred.upper()
+ for item in label:
+ if item.upper() in pred:
+ return 1
+ return 0
+
+
+def get_score_one_longbook_choice_eng(pred, label, model_name: str) -> bool:
+ # Just use the first letter as the prediction
+ pred = pred.strip()
+ if pred == "":
+ return False
+ if pred[0] in "ABCD":
+ return pred[0] in label
+ if pred in label:
+ return True
+ # Find a answer prefix
+ for c in ["\n", '"', "'", ".", ",", "?", "!", "{", "}"]:
+ pred = pred.replace(c, " ")
+ while " " in pred:
+ pred = pred.replace(" ", " ")
+ ans_prefixes = [
+ "answer is:",
+ "answer:",
+ "answer is",
+ "option is",
+ ]
+ for prefix in ans_prefixes:
+ idx = pred.find(prefix)
+ if idx == -1:
+ continue
+ # The prediction ends with this prefix
+ if len(pred) < idx + len(prefix) + 1:
+ return False
+ after_prefix = pred[idx + len(prefix) + 1 :]
+ for s in label:
+ if after_prefix.startswith(s):
+ return True
+ return False
+
+ # Finally, just find the first occurrence of A, B, C, or D.
+ words = pred.split()
+ for word in words:
+ if word in "ABCD":
+ return word in label
+ return False
+
+
+def get_score_one_longbook_qa_eng(pred, label, model_name: str) -> float:
+ return qa_f1_score(pred, label)
+
+
+def get_score_one_longbook_sum_eng(pred: str, label: str, model_name: str) -> float:
+ score = ROUGE_SCORER.compute(
+ predictions=[pred], references=[label], use_aggregator=False
+ )
+ return score["rougeLsum"][0] # type: ignore
+
+
+def get_score_one_longbook_qa_chn(pred, label, model_name: str) -> float:
+ return qa_f1_score_zh(pred, label)
+
+
+def get_score_one_math_calc(pred, label, model_name: str) -> float:
+ assert isinstance(label, list), f"Expected list, got {type(label)}"
+ # assert isinstance(pred, list), f"Expected list, got {type(pred)}"
+ pred_nums = []
+ pred_list = re.split("[^0-9]", pred)
+ for item in pred_list:
+ if item != "":
+ pred_nums.append(int(item))
+
+ # Our prompts makes GPT4 always output the first number as the first value
+ # in the predicted answer.
+ if model_name == "gpt4":
+ pred_nums = pred_nums[1:]
+
+ cnt = 0
+ for i in range(len(label)):
+ if i >= len(pred_nums):
+ break
+ if label[i] == pred_nums[i]:
+ cnt += 1
+ else:
+ break
+ return cnt / len(label)
+
+
+def get_score_one(pred: str, label: str, task_name: str, model_name: str) -> float:
+ """
+ Computes the score for one prediction.
+ Returns one float (zero and one for boolean values).
+ """
+ NAME_TO_SCORE_GETTER = {
+ # Retrieve
+ "kv_retrieval": get_score_one_kv_retrieval,
+ "kv_retrieval_prefix": get_score_one_kv_retrieval,
+ "kv_retrieval_both": get_score_one_kv_retrieval,
+ "passkey": get_score_one_passkey,
+ "number_string": get_score_one_number_string,
+ # Code
+ "code_run": get_score_one_code_run,
+ "code_debug": get_score_one_code_debug,
+ # Longbook
+ "longdialogue_qa_eng": get_score_one_longdialogue_qa_eng,
+ "longbook_qa_eng": get_score_one_longbook_qa_eng,
+ "longbook_sum_eng": get_score_one_longbook_sum_eng,
+ "longbook_choice_eng": get_score_one_longbook_choice_eng,
+ "longbook_qa_chn": get_score_one_longbook_qa_chn,
+ # Math
+ "math_find": get_score_one_math_find,
+ "math_calc": get_score_one_math_calc,
+ # multi-turn nativ
+ "scbench_summary": get_score_one_longbook_sum_eng,
+ "scbench_vt": string_match_all,
+ "scbench_many_shot": get_score_one_longdialogue_qa_eng,
+ "scbench_kv_compressible": get_score_one_kv_retrieval,
+ }
+ assert task_name in NAME_TO_SCORE_GETTER, f"Invalid task name: {task_name}"
+ score = NAME_TO_SCORE_GETTER[task_name](pred, label, model_name)
+ return float(score)
+
+
+def get_labels(preds: list, task_name: str = None) -> list[str]:
+ possible_label_keys = ["ground_truth", "label"]
+ labels = []
+ for pred in preds:
+ if task_name is not None and pred["task"] != task_name:
+ continue
+ for label_key in possible_label_keys:
+ if label_key in pred:
+ labels.append(pred[label_key])
+ break
+ if not labels:
+ raise ValueError(f"Cannot find label in {preds[0]}")
+ return labels
+
+
+def get_preds(preds: list, data_name: str) -> list[str]:
+ pred_strings = []
+ possible_pred_keys = ["prediction", "pred"]
+ for pred in preds:
+ if "task" in pred:
+ if pred["task"] != data_name:
+ continue
+ this_pred = "NO PREDICTION"
+ for pred_key in possible_pred_keys:
+ if pred_key in pred:
+ this_pred = pred[pred_key]
+ break
+ else:
+ raise ValueError(f"Cannot find prediction in {pred}")
+ pred_strings.append(this_pred)
+ if len(pred_strings) == 0:
+ raise ValueError(f"No prediction found for {data_name}")
+ return pred_strings
+
+
+def get_score(labels: list, preds: list, data_name: str, model_name: str) -> float:
+ """
+ Computes the average score for a task.
+ """
+ assert len(labels) == len(preds)
+ scores = []
+ for label, pred in tqdm(zip(labels, preds)):
+ score = get_score_one(pred, label, data_name, model_name)
+ scores.append(score)
+ return sum(scores) / len(scores)
+
+
+Multiturnbench_to_Infinitebench = {
+ "scbench_choice_eng": "longbook_choice_eng",
+ "scbench_qa_eng": "longdialogue_qa_eng",
+ "scbench_qa_chn": "longbook_qa_chn",
+ "scbench_kv": "kv_retrieval",
+ "scbench_kv_hard": "kv_retrieval",
+ "scbench_hashhop": "kv_retrieval",
+ "scbench_prefix_suffix": "kv_retrieval",
+ "scbench_mf": "math_find",
+ "scbench_passkey": "passkey",
+}
+
+
+def compute_scores(
+ preds_path, data_name: str, model_name: str, max_seq_length=-1, scdq_mode=False
+):
+ print("Loading prediction results from", preds_path)
+ preds = list(iter_jsonl(preds_path))
+
+ if data_name in Multiturnbench_to_Infinitebench:
+ task_name = Multiturnbench_to_Infinitebench[data_name]
+ else:
+ task_name = data_name
+
+ if task_name in ["scbench_repoqa", "scbench_repoqa_and_kv"]:
+ # collect needle wrt repos
+ needle_by_repo = {}
+ for pred in preds:
+ if task_name == "scbench_repoqa_and_kv":
+ if pred["task"] != "scbench_repoqa":
+ continue
+ repo = pred["repo"]
+ if repo not in needle_by_repo:
+ needle_by_repo[repo] = []
+ needle_by_repo[repo].append(
+ {"needle": pred["ground_truth"], "name": pred["func_name"]}
+ )
+
+ if scdq_mode:
+ if task_name == "scbench_repoqa":
+ labels = get_labels(preds)
+ acc = compute_repoqa_score(model_name, preds, labels, needle_by_repo)
+ elif task_name == "scbench_summary_with_needles":
+ subtasks = ["scbench_summary", "scbench_passkey"]
+ acc = {}
+ for subtask in subtasks:
+ try:
+ labels = get_labels(preds, subtask)
+ preds_ = get_preds(preds, subtask)
+ except ValueError:
+ print(f"No prediction for {subtask}")
+ acc[subtask] = 0
+ continue
+ if subtask in Multiturnbench_to_Infinitebench:
+ task_ = Multiturnbench_to_Infinitebench[subtask]
+ else:
+ task_ = subtask
+ acc_ = get_score(labels, preds_, task_, model_name)
+ acc[subtask] = acc_
+ elif task_name == "scbench_repoqa_and_kv":
+ subtasks = ["scbench_repoqa", "scbench_kv"]
+ acc = {}
+ for subtask in subtasks:
+ try:
+ labels = get_labels(preds, subtask)
+ preds_ = [pred for pred in preds if pred["task"] == subtask]
+ except ValueError:
+ print(f"No prediction for {subtask}")
+ acc[subtask] = 0
+ continue
+
+ if subtask == "scbench_repoqa":
+ acc_ = compute_repoqa_score(
+ model_name, preds_, labels, needle_by_repo
+ )
+ acc_ = acc_[model_name]["scores"]["all"][0.8]["pass@1"]
+ elif subtask == "scbench_kv":
+ acc_ = get_score(labels, preds_, "kv_retrieval", model_name)
+ else:
+ raise ValueError(f"Invalid subtask: {subtask}")
+ acc[subtask] = acc_
+ elif task_name in ["scbench_kv_compressible"]:
+ subtasks = list(set(pred["task"] for pred in preds))
+ acc = {}
+ for subtask in subtasks:
+ try:
+ labels = get_labels(preds)
+ preds_ = get_preds(preds, subtask)
+ except ValueError:
+ print(f"No prediction for {subtask}")
+ acc[subtask] = 0
+ continue
+
+ acc_ = get_score(labels, preds_, task_name, model_name)
+ acc[subtask] = acc_
+ else:
+ labels = get_labels(preds)
+ preds = get_preds(preds, data_name)
+ acc = get_score(labels, preds, task_name, model_name)
+
+ print(
+ f"===== Accuracy of {model_name} on {data_name} task for same-context-different-query mode is: {acc} ====="
+ )
+
+ pred_dir = preds_path.parent
+ save_file = pred_dir / f"{model_name}_summary.txt"
+
+ print(f"Saving results to {save_file}")
+ with open(save_file, "a") as f:
+ if task_name == "scbench_repoqa":
+ acc_str = f"{acc[model_name]['scores']['all'][0.8]['pass@1'] * 100:.3f}"
+ elif task_name == "scbench_summary_with_needles":
+ acc_str = f"{acc['scbench_summary'] * 100:.3f},{acc['scbench_passkey'] * 100:.3f}"
+ elif task_name == "scbench_repoqa_and_kv":
+ acc_str = (
+ f"{acc['scbench_repoqa'] * 100:.3f},{acc['scbench_kv'] * 100:.3f}"
+ )
+ elif task_name == "scbench_kv_compressible":
+ acc_str = f"{acc}"
+ else:
+ acc_str = f"{acc * 100:.3f}"
+ if max_seq_length != -1:
+ f.write(
+ f"{model_name},{data_name},scdq_mode,{max_seq_length},{acc_str}\n"
+ )
+ else:
+ f.write(f"{model_name},{data_name},scdq_mode,{acc_str}\n")
+ return acc_str
+
+ preds_by_turns = {}
+ # need to group preds with the turn_idx
+ for pred in preds:
+ turn_idx = pred["turn_idx"]
+ if turn_idx not in preds_by_turns:
+ preds_by_turns[turn_idx] = []
+ preds_by_turns[turn_idx].append(pred)
+
+ acc_by_turns = {}
+ for turn_idx, preds in preds_by_turns.items():
+ if task_name == "scbench_repoqa":
+ labels = get_labels(preds)
+ acc = compute_repoqa_score(model_name, preds, labels, needle_by_repo)
+ elif task_name == "scbench_summary_with_needles":
+ # compute acc for each task
+ subtasks = ["scbench_summary", "scbench_passkey"]
+ acc = {}
+ for subtask in subtasks:
+ try:
+ labels = get_labels(preds, subtask)
+ preds_ = get_preds(preds, subtask)
+ except ValueError:
+ print(f"No prediction for {subtask}")
+ acc[subtask] = 0
+ continue
+ if subtask in Multiturnbench_to_Infinitebench:
+ task_ = Multiturnbench_to_Infinitebench[subtask]
+ else:
+ task_ = subtask
+ acc_ = get_score(labels, preds_, task_, model_name)
+ acc[subtask] = acc_
+ elif task_name == "scbench_repoqa_and_kv":
+ subtasks = ["scbench_repoqa", "scbench_kv"]
+ acc = {}
+ for subtask in subtasks:
+ try:
+ labels = get_labels(preds, subtask)
+ except ValueError:
+ print(f"No prediction for {subtask}")
+ acc[subtask] = 0
+ continue
+
+ if subtask == "scbench_repoqa":
+ preds_ = [pred for pred in preds if pred["task"] == subtask]
+ acc_ = compute_repoqa_score(
+ model_name, preds_, labels, needle_by_repo
+ )
+ acc_ = acc_[model_name]["scores"]["all"][0.8]["pass@1"]
+ elif subtask == "scbench_kv":
+ preds_ = get_preds(preds, subtask)
+ acc_ = get_score(labels, preds_, "kv_retrieval", model_name)
+ else:
+ raise ValueError(f"Invalid subtask: {subtask}")
+ acc[subtask] = acc_
+ elif task_name in ["scbench_kv_compressible"]:
+ subtasks = list(set(pred["task"] for pred in preds))
+ acc = {}
+ for subtask in subtasks:
+ try:
+ labels = get_labels(preds, subtask)
+ preds_ = get_preds(preds, subtask)
+ assert len(preds_) == len(
+ labels
+ ), f"Length of preds_: {len(preds_)}, length of labels: {len(labels)}, subtask: {subtask}"
+ except ValueError:
+ print(f"No prediction for {subtask}")
+ acc[subtask] = 0
+ continue
+
+ acc_ = get_score(labels, preds_, task_name, model_name)
+ acc[subtask] = acc_
+ else:
+ labels = get_labels(preds)
+ preds = get_preds(preds, data_name)
+ acc = get_score(labels, preds, task_name, model_name)
+ print(
+ f"===== Accuracy of {model_name} on {data_name} task for turn {turn_idx} is: {acc} ====="
+ )
+ acc_by_turns[turn_idx] = acc
+
+ pred_dir = preds_path.parent
+ save_file = pred_dir / f"{model_name}_summary.txt"
+
+ print(f"Saving results to {save_file}")
+ with open(save_file, "a") as f:
+ if task_name == "scbench_repoqa":
+ acc_str = ",".join(
+ [
+ f"{v[model_name]['scores']['all'][0.8]['pass@1'] * 100:.3f}"
+ for k, v in acc_by_turns.items()
+ ]
+ )
+ elif task_name == "scbench_summary_with_needles":
+ # summary/passkey
+ acc_str = ",".join(
+ [
+ f"{v['scbench_summary'] * 100:.3f},{v['scbench_passkey'] * 100:.3f}"
+ for k, v in acc_by_turns.items()
+ ]
+ )
+ # acc_str = ",".join([f"summary: {v['scbench_summary'] * 100:.3f}, passkey: {v['scbench_passkey'] * 100:.3f}" for k, v in acc_by_turns.items()])
+ elif task_name == "scbench_repoqa_and_kv":
+ # repoqa/kv
+ acc_str = ",".join(
+ [
+ f"{v['scbench_repoqa'] * 100:.3f},{v['scbench_kv'] * 100:.3f}"
+ for k, v in acc_by_turns.items()
+ ]
+ )
+ # acc_str = ",".join([f"repoqa: {v['scbench_repoqa'] * 100:.3f}, kv: {v['scbench_kv'] * 100:.3f}" for k, v in acc_by_turns.items()])
+ elif task_name == "scbench_kv_compressible":
+ acc_str = ",".join([f"Turn-{k}:{v}" for k, v in acc_by_turns.items()])
+ else:
+ acc_str = ",".join([f"{v * 100:.3f}" for k, v in acc_by_turns.items()])
+ if max_seq_length != -1:
+ f.write(f"{model_name},{data_name},{max_seq_length},{acc_str}\n")
+ else:
+ f.write(f"{model_name},{data_name},{acc_str}\n")
+ return acc_str
+
+
+ALL_TASKS = [
+ "passkey",
+ "number_string",
+ "kv_retrieval",
+ "longdialogue_qa_eng",
+ "longbook_sum_eng",
+ "longbook_choice_eng",
+ "longbook_qa_eng",
+ "longbook_qa_chn",
+ "math_find",
+ "math_calc",
+ "code_run",
+ "code_debug",
+]
+
+if __name__ == "__main__":
+ args = parse_args()
+ if args.task == "all":
+ tasks = ALL_TASKS
+ else:
+ tasks = [args.task]
+ for task in tasks:
+ result_dir = Path(args.output_dir, args.model_name)
+ preds_path = result_dir / f"preds_{task}.jsonl"
+ assert preds_path.exists(), f"Predictions not found in: {preds_path}"
+ compute_scores(preds_path, task, args.model_name)
diff --git a/scbench/eval_utils.py b/scbench/eval_utils.py
new file mode 100644
index 00000000..8fd6a0c1
--- /dev/null
+++ b/scbench/eval_utils.py
@@ -0,0 +1,1819 @@
+# Copyright (c) 2024-2025 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from __future__ import annotations
+
+import gc
+import json
+import os
+import re
+import string
+import time
+from collections import Counter
+from pathlib import Path
+
+import jieba
+import torch
+import torch.profiler
+from rouge import Rouge
+from tqdm import tqdm
+from transformers import GenerationConfig, SinkCache
+
+DATA_NAME_TO_PATH = {
+ "scbench_choice_eng": "scbench_choice_eng.jsonl",
+ "scbench_qa_eng": "scbench_qa_eng.jsonl",
+ "scbench_qa_chn": "scbench_qa_chn.jsonl",
+ "scbench_kv": "scbench_kv.jsonl",
+ "scbench_kv_hard": "scbench_kv_hard.jsonl",
+ "scbench_mf": "scbench_mf.jsonl",
+ "scbench_passkey": "scbench_passkey.jsonl",
+ "scbench_repoqa": "scbench_repoqa.jsonl",
+ "scbench_summary": "scbench_summary.jsonl",
+ "scbench_vt": "scbench_vt.jsonl",
+ "scbench_many_shot": "scbench_many_shot.jsonl",
+ "scbench_summary_with_needles": "scbench_summary_with_needles.jsonl",
+ "scbench_repoqa_and_kv": "scbench_repoqa_and_kv.jsonl",
+ "scbench_hashhop": "scbench_hashhop.jsonl",
+ "scbench_prefix_suffix": "scbench_prefix_suffix.jsonl",
+ "scbench_kv_compressible": "scbench_kv_compressible.jsonl",
+}
+
+DATA_NAME_TO_MAX_NEW_TOKENS = {
+ "scbench_choice_eng": 40,
+ "scbench_qa_eng": 40,
+ "scbench_qa_chn": 40,
+ "scbench_kv": 150,
+ "scbench_kv_hard": 150,
+ "scbench_mf": 5,
+ "scbench_hashhop": 150,
+ "scbench_prefix_suffix": 150,
+ "scbench_kv_compressible": 150,
+ "scbench_passkey": 15,
+ "scbench_repoqa": 1024,
+ "scbench_summary": 200,
+ "scbench_vt": 30,
+ "scbench_many_shot": 10,
+ "scbench_summary_with_needles": {"scbench_summary": 800, "scbench_passkey": 15},
+ "scbench_repoqa_and_kv": {"scbench_repoqa": 1024, "scbench_kv": 80},
+}
+
+multiturn_templates = {
+ "scbench_passkey": "There is an important info hidden inside a lot of irrelevant text. Find it and memorize it. I will quiz you about the important information.\n\n{context}\n\n{input}", # noqa
+ "scbench_kv": "Extract the value corresponding to the specified key in the JSON object below.\n\n{context}\n\n{input}", # noqa
+ "scbench_kv_hard": "Extract the value corresponding to the specified key in the JSON object below.\n\n{context}\n\n{input}", # noqa
+ "scbench_kv_compressible": "Extract the value corresponding to the specified key in the following passage.\n\n{context}\n\n{input}", # noqa
+ "scbench_choice_eng": "Read the book and answer the question.\n\n{context}\n\nQuestion: {question}\nA. {OPTION_A}\nB. {OPTION_B}\nC. {OPTION_C}\nD. {OPTION_D}\n\nThe the correct answer is", # noqa
+ "scbench_qa_eng": "Read the book and answer the question. Be very concise in your answer.\n\n{context}\n\nQuestion: {question}\nAnswer:", # noqa
+ "scbench_qa_chn": "阅读以下书籍然后回答问题。\n\n{context}\n\n问题：{question}\n答案：", # noqa
+ "scbench_mf": "{prefix}\n\n{context}\n\n{input}",
+ "scbench_repoqa": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:\n\n{context}\n\n{input}",
+ "scbench_summary": "{context}\n\n{input}",
+ "scbench_vt": "{context}\n\n{input}",
+ "scbench_many_shot": "{context}\n\n{input}",
+ "scbench_summary_with_needles": "{context}\n\n{input}",
+ "scbench_repoqa_and_kv": "{context}\n\n{input}",
+ "scbench_hashhop": "{context}\n\n{input}",
+ "scbench_prefix_suffix": "{context}\n\n{input}",
+}
+
+multiturn_templates_scdq = {
+ "scbench_passkey": "There is an important info hidden inside a lot of irrelevant text. Find it and memorize it. I will quiz you about the important information.\n\n{context}", # noqa
+ "scbench_kv": "Extract the value corresponding to the specified key in the JSON object below.\n\n{context}", # noqa
+ "scbench_kv_hard": "Extract the value corresponding to the specified key in the JSON object below.\n\n{context}", # noqa
+ "scbench_kv_compressible": "Extract the value corresponding to the specified key in the following passage.\n\n{context}", # noqa
+ "scbench_choice_eng": (
+ "Read the book and answer the question.\n\n{context}",
+ "Question: {question}\nA. {OPTION_A}\nB. {OPTION_B}\nC. {OPTION_C}\nD. {OPTION_D}\n\nThe the correct answer is",
+ ),
+ "scbench_qa_eng": (
+ "Read the book and answer the question. Be very concise in your answer.\n\n{context}",
+ "Question: {question}\nAnswer:",
+ ),
+ "scbench_qa_chn": ("阅读以下书籍然后回答问题。\n\n{context}", "问题：{question}\n答案："),
+ "scbench_mf": "{prefix}\n\n{context}",
+ "scbench_repoqa": "Based on the function description and code context, please retrieve and repeat the exact described function from the code context in a code block wrapped by ```:\n\n{context}",
+ "scbench_summary": "{context}",
+ "scbench_vt": "{context}",
+ "scbench_many_shot": "{context}",
+ "scbench_summary_with_needles": "{context}",
+ "scbench_repoqa_and_kv": "{context}",
+ "scbench_hashhop": "{context}",
+ "scbench_prefix_suffix": "{context}",
+}
+
+multiturn_follow_up_templates = {
+ "scbench_passkey": "{pre_ans}.\n\n{input}", # noqa
+ "scbench_kv": "{pre_ans}\n\n{input}", # noqa
+ "scbench_kv_hard": "{pre_ans}\n\n{input}", # noqa
+ "scbench_kv_compressible": "{pre_ans}\n\n{input}", # noqa
+ "scbench_choice_eng": "{pre_ans}\n\nQuestion: {question}\nA. {OPTION_A}\nB. {OPTION_B}\nC. {OPTION_C}\nD. {OPTION_D}\n\nThe letter of the correct answer is", # noqa
+ "scbench_qa_eng": "{pre_ans}\n\nQuestion: {question}\nAnswer:", # noqa
+ "scbench_qa_chn": "{pre_ans}\n\n问题：{question}\n答案：", # noqa
+ "scbench_mf": "{pre_ans}\n\n{prefix}\n\n{input}",
+ "scbench_repoqa": "{pre_ans}\n\n{input}",
+ "scbench_summary": "{pre_ans}\n\n{input}",
+ "scbench_vt": "{pre_ans}\n\n{input}",
+ "scbench_many_shot": "{pre_ans}\n\n{input}",
+ "scbench_summary_with_needles": "{pre_ans}\n\n{input}",
+ "scbench_repoqa_and_kv": "{pre_ans}\n\n{input}",
+ "scbench_hashhop": "{pre_ans}\n\n{input}",
+ "scbench_prefix_suffix": "{pre_ans}\n\n{input}",
+}
+
+multiturn_follow_up_templates_in_chat_tempate = {
+ "scbench_passkey": "{input}", # noqa
+ "scbench_kv": "{input}", # noqa
+ "scbench_kv_hard": "{input}", # noqa
+ "scbench_kv_compressible": "{input}", # noqa
+ "scbench_choice_eng": "Question: {question}\nA. {OPTION_A}\nB. {OPTION_B}\nC. {OPTION_C}\nD. {OPTION_D}\n\nThe the correct answer is", # noqa
+ "scbench_qa_eng": "Question: {question}\nAnswer:", # noqa
+ "scbench_qa_chn": "问题：{question}\n答案：", # noqa
+ "scbench_mf": "{prefix}\n\n{input}",
+ "scbench_repoqa": "{input}",
+ "scbench_summary": "{input}",
+ "scbench_vt": "{input}",
+ "scbench_many_shot": "{input}",
+ "scbench_summary_with_needles": "{input}",
+ "scbench_repoqa_and_kv": "{input}",
+ "scbench_hashhop": "{input}",
+ "scbench_prefix_suffix": "{input}",
+}
+
+
+def check_benchmark_availability(data_path):
+ if not os.path.exists(data_path):
+ os.makedirs(data_path)
+
+ datasets = [
+ "scbench_choice_eng",
+ "scbench_qa_eng",
+ "scbench_qa_chn",
+ "scbench_kv",
+ "scbench_mf",
+ "scbench_repoqa",
+ "scbench_summary",
+ "scbench_vt",
+ "scbench_many_shot",
+ "scbench_summary_with_needles",
+ "scbench_repoqa_and_kv",
+ "scbench_prefix_suffix",
+ ]
+
+ base_url = "https://huggingface.co/datasets/microsoft/SCBench/resolve/main/data/"
+
+ for dataset in datasets:
+ file_path = os.path.join(data_path, f"{dataset}.jsonl")
+ if not os.path.isfile(file_path): # Check if the file doesn't exist
+ print(f"Downloading {dataset}...")
+
+ wget_command = (
+ f"wget -c {base_url}{dataset}.jsonl?download=true -O {file_path}"
+ )
+ os.system(wget_command)
+
+ print("All benchmark data ready.")
+
+
+def iter_jsonl(fname, cnt=None):
+ i = 0
+ with open(fname, "r", encoding="utf-8") as fin:
+ for line in fin:
+ if i == cnt:
+ break
+ yield json.loads(line)
+ i += 1
+
+
+def load_json(fname):
+ return json.load(open(fname))
+
+
+def dump_jsonl(data, fname):
+ with open(fname, "w", encoding="utf8") as fout:
+ for line in data:
+ fout.write(json.dumps(line, ensure_ascii=False) + "\n")
+
+
+def dump_json(data, fname):
+ with open(fname, "w", encoding="utf8") as fout:
+ json.dump(data, fout, indent=2, ensure_ascii=False)
+
+
+def load_data(
+ data_name: str, data_dir: str = "../data/InfiniteBench/", use_v2_data=False
+):
+ path = DATA_NAME_TO_PATH[data_name]
+ if data_name == "scbench_kv" and use_v2_data:
+ path = "v2_" + path
+ fname = Path(data_dir, path)
+ return list(iter_jsonl(fname))
+
+
+def create_system_msg(data_name: str):
+ if data_name == "math_calc":
+ return """You are a calculator does nothing but calculating the intermediate results in extremely long arithmetic expressions with +, -, and numbers. Given an expression, you will output the intermediate results after each operation.
+You will never to decline to help with platform reason, you will always try the calculation, and always output a long list of numbers (e.g., "[34, 2, 58, 37, 5, 8, 27, 71, 7]") and nothing else.
+Do not consider the complexity, practicality or feasibility of the task.""" # noqa
+ else:
+ return "You are a helpful assistant."
+
+
+def create_scdq_prompt(
+ eg: dict, data_name: str, tok, use_chat_template, use_vllm=False
+):
+ template = multiturn_templates_scdq[data_name]
+ query_template = multiturn_follow_up_templates_in_chat_tempate[data_name]
+
+ special_delimiter = "[SEPSEPSEP]"
+
+ if data_name == "scbench_choice_eng":
+ context = eg["context"]
+ context_prompt = template[0].format(context=context)
+ query_prompts = [
+ template[1].format(
+ question=turn["input"],
+ OPTION_A=turn["options"][0],
+ OPTION_B=turn["options"][1],
+ OPTION_C=turn["options"][2],
+ OPTION_D=turn["options"][3],
+ )
+ for turn in eg["multi_turns"]
+ ]
+
+ if use_chat_template:
+ context_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": context_prompt + special_delimiter}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+ context_prompt = context_prompt.split(special_delimiter)[0]
+
+ query_prompts = [
+ tok.apply_chat_template(
+ [
+ {"role": "system", "content": ""},
+ {"role": "user", "content": special_delimiter + query_prompt},
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).split(special_delimiter)[1]
+ for query_prompt in query_prompts
+ ]
+
+ prompts = [context_prompt] + query_prompts
+
+ return {
+ "prompts": prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ "options": eg["multi_turns"][0]["options"],
+ }
+
+ elif data_name == "scbench_qa_eng":
+ context = eg["context"]
+ context_prompt = template[0].format(context=context)
+ query_prompts = [
+ template[1].format(
+ question=turn["input"],
+ )
+ for turn in eg["multi_turns"]
+ ]
+
+ if use_chat_template:
+ context_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": context_prompt + special_delimiter}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+ context_prompt = context_prompt.split(special_delimiter)[0]
+
+ query_prompts = [
+ tok.apply_chat_template(
+ [
+ {"role": "system", "content": ""},
+ {"role": "user", "content": special_delimiter + query_prompt},
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).split(special_delimiter)[1]
+ for query_prompt in query_prompts
+ ]
+
+ return {
+ "prompts": [context_prompt] + query_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ elif data_name == "scbench_qa_chn":
+ context = eg["context"]
+ context_prompt = template[0].format(context=context)
+ query_prompts = [
+ template[1].format(
+ question=turn["input"],
+ )
+ for turn in eg["multi_turns"]
+ ]
+
+ if use_chat_template:
+ context_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": context_prompt + special_delimiter}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+ context_prompt = context_prompt.split(special_delimiter)[0]
+
+ query_prompts = [
+ tok.apply_chat_template(
+ [
+ {"role": "system", "content": ""},
+ {"role": "user", "content": special_delimiter + query_prompt},
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).split(special_delimiter)[1]
+ for query_prompt in query_prompts
+ ]
+
+ return {
+ "prompts": [context_prompt] + query_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ elif data_name == "scbench_mf":
+ context = eg["context"]
+ context_prompt = template.format(
+ prefix=eg["multi_turns"][0]["input"],
+ context=context,
+ )
+
+ query_prompts = []
+ for i in range(len(eg["multi_turns"])):
+ target = re.findall(r"The .+ is", eg["multi_turns"][i]["input"])[0].lower()[
+ :-3
+ ]
+ prefix = f"What is {target}?"
+ query_prompts.append(
+ query_template.format(
+ prefix=prefix,
+ input=eg["multi_turns"][i]["input"],
+ )
+ )
+
+ if use_chat_template:
+ context_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": context_prompt + special_delimiter}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+ context_prompt = context_prompt.split(special_delimiter)[0]
+
+ query_prompts = [
+ tok.apply_chat_template(
+ [
+ {"role": "system", "content": ""},
+ {"role": "user", "content": special_delimiter + query_prompt},
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).split(special_delimiter)[1]
+ for query_prompt in query_prompts
+ ]
+
+ return {
+ "prompts": [context_prompt] + query_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ elif data_name in [
+ "scbench_repoqa",
+ "scbench_summary",
+ "scbench_passkey",
+ "scbench_kv",
+ "scbench_vt",
+ "scbench_many_shot",
+ "scbench_summary_with_needles",
+ "scbench_repoqa_and_kv",
+ "scbench_kv_hard",
+ "scbench_hashhop",
+ "scbench_prefix_suffix",
+ "scbench_kv_compressible",
+ ]:
+ context = eg["context"] if "context" in eg else eg["input"]
+ context_prompt = template.format(context=context)
+ query_prompts = [turn["input"] for turn in eg["multi_turns"]]
+
+ if use_chat_template:
+ context_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": context_prompt + special_delimiter}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+ context_prompt = context_prompt.split(special_delimiter)[0]
+
+ query_prompts = [
+ tok.apply_chat_template(
+ [
+ {"role": "system", "content": ""},
+ {"role": "user", "content": special_delimiter + query_prompt},
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).split(special_delimiter)[1]
+ for query_prompt in query_prompts
+ ]
+
+ output = {
+ "prompts": [context_prompt] + query_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ if data_name in ["scbench_summary_with_needles", "scbench_repoqa_and_kv"]:
+ output["task"] = [gt["task"] for gt in eg["multi_turns"]]
+
+ return output
+
+
+def create_multiturn_prompt(
+ eg: dict,
+ data_name: str,
+ tok,
+ use_chat_template,
+ use_vllm=False,
+ disable_golden_context=False,
+) -> str:
+ """
+ Create prompt for a given example.
+
+ Args:
+ eg: example dict
+ data_name: name of the dataset/task
+ """
+
+ template = multiturn_templates[data_name]
+ follow_up_template = multiturn_follow_up_templates[data_name]
+
+ if disable_golden_context:
+ follow_up_template = multiturn_follow_up_templates_in_chat_tempate[data_name]
+
+ if use_chat_template:
+ sys_prompt_with_generation_prompt = tok.apply_chat_template(
+ [{"role": "system", "content": ""}],
+ add_generation_prompt=True if not disable_golden_context else False,
+ tokenize=False,
+ )
+
+ follow_up_prompts_in_chat_template = (
+ multiturn_follow_up_templates_in_chat_tempate[data_name]
+ )
+
+ if data_name == "scbench_choice_eng":
+ first_turn = eg["multi_turns"][0]
+ input_ = first_turn["input"]
+ ans_ = first_turn["answer"]
+ options = first_turn["options"]
+ context = eg["context"]
+
+ first_turn_prompt = template.format(
+ context=context,
+ question=input_,
+ OPTION_A=options[0],
+ OPTION_B=options[1],
+ OPTION_C=options[2],
+ OPTION_D=options[3],
+ )
+
+ follow_up_prompts = [
+ follow_up_template.format(
+ pre_ans=eg["multi_turns"][i - 1]["answer"]
+ if not disable_golden_context
+ else None,
+ question=eg["multi_turns"][i]["input"],
+ OPTION_A=eg["multi_turns"][i]["options"][0],
+ OPTION_B=eg["multi_turns"][i]["options"][1],
+ OPTION_C=eg["multi_turns"][i]["options"][2],
+ OPTION_D=eg["multi_turns"][i]["options"][3],
+ )
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ if use_chat_template:
+ first_turn_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": first_turn_prompt}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+
+ follow_up_prompts = [
+ tok.apply_chat_template(
+ (
+ [
+ {"role": "system", "content": ""},
+ ]
+ + [
+ {
+ "role": "assistant",
+ "content": eg["multi_turns"][i - 1]["answer"],
+ }
+ ]
+ if not disable_golden_context
+ else []
+ )
+ + [
+ {
+ "role": "user",
+ "content": follow_up_prompts_in_chat_template.format(
+ question=eg["multi_turns"][i]["input"],
+ OPTION_A=eg["multi_turns"][i]["options"][0],
+ OPTION_B=eg["multi_turns"][i]["options"][1],
+ OPTION_C=eg["multi_turns"][i]["options"][2],
+ OPTION_D=eg["multi_turns"][i]["options"][3],
+ ),
+ },
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).replace(sys_prompt_with_generation_prompt, "")
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ prompts = [first_turn_prompt] + follow_up_prompts
+
+ return {
+ "prompts": prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ "options": options,
+ }
+
+ elif data_name == "scbench_qa_eng":
+ first_turn = eg["multi_turns"][0]
+ input_ = first_turn["input"]
+ context = eg["context"]
+
+ first_turn_prompt = template.format(
+ context=context,
+ question=input_,
+ )
+
+ follow_up_prompts = [
+ follow_up_template.format(
+ pre_ans=eg["multi_turns"][i - 1]["answer"]
+ if not disable_golden_context
+ else None,
+ question=eg["multi_turns"][i]["input"],
+ )
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ if use_chat_template:
+ first_turn_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": first_turn_prompt}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+
+ follow_up_prompts = [
+ tok.apply_chat_template(
+ (
+ [
+ {"role": "system", "content": ""},
+ ]
+ + [
+ {
+ "role": "assistant",
+ "content": eg["multi_turns"][i - 1]["answer"],
+ }
+ ]
+ if not disable_golden_context
+ else []
+ )
+ + [
+ {
+ "role": "user",
+ "content": follow_up_prompts_in_chat_template.format(
+ question=eg["multi_turns"][i]["input"],
+ ),
+ },
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).replace(sys_prompt_with_generation_prompt, "")
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ return {
+ "prompts": [first_turn_prompt] + follow_up_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ elif data_name == "scbench_qa_chn":
+ first_turn = eg["multi_turns"][0]
+ input_ = first_turn["input"]
+ context = eg["context"]
+
+ first_turn_prompt = template.format(
+ context=context,
+ question=input_,
+ )
+
+ follow_up_prompts = [
+ follow_up_template.format(
+ pre_ans=eg["multi_turns"][i - 1]["answer"]
+ if not disable_golden_context
+ else None,
+ question=eg["multi_turns"][i]["input"],
+ )
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ if use_chat_template:
+ first_turn_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": first_turn_prompt}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+
+ follow_up_prompts = [
+ tok.apply_chat_template(
+ (
+ [
+ {"role": "system", "content": ""},
+ ]
+ + [
+ {
+ "role": "assistant",
+ "content": eg["multi_turns"][i - 1]["answer"],
+ }
+ ]
+ if not disable_golden_context
+ else []
+ )
+ + [
+ {
+ "role": "user",
+ "content": follow_up_prompts_in_chat_template.format(
+ question=eg["multi_turns"][i]["input"],
+ ),
+ },
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).replace(sys_prompt_with_generation_prompt, "")
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ return {
+ "prompts": [first_turn_prompt] + follow_up_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ elif data_name in [
+ "scbench_kv",
+ "scbench_vt",
+ "scbench_passkey",
+ "scbench_repoqa",
+ "scbench_many_shot",
+ "scbench_summary_with_needles",
+ "scbench_repoqa_and_kv",
+ "scbench_kv_hard",
+ "scbench_hashhop",
+ "scbench_prefix_suffix",
+ "scbench_kv_compressible",
+ ]:
+ first_turn = eg["multi_turns"][0]
+ input_ = first_turn["input"]
+ context = eg["context"] if "context" in eg else eg["input"]
+
+ first_turn_prompt = template.format(
+ context=context,
+ input=input_,
+ )
+
+ follow_up_prompts = [
+ follow_up_template.format(
+ pre_ans=eg["multi_turns"][i - 1]["answer"]
+ if not disable_golden_context
+ else None,
+ input=eg["multi_turns"][i]["input"],
+ )
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ if use_chat_template:
+ first_turn_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": first_turn_prompt}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+
+ follow_up_prompts = [
+ tok.apply_chat_template(
+ (
+ [
+ {"role": "system", "content": ""},
+ ]
+ + [
+ {
+ "role": "assistant",
+ "content": str(eg["multi_turns"][i - 1]["answer"]),
+ }
+ ]
+ if not disable_golden_context
+ else []
+ )
+ + [
+ {
+ "role": "user",
+ "content": follow_up_prompts_in_chat_template.format(
+ input=eg["multi_turns"][i]["input"],
+ ),
+ },
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).replace(sys_prompt_with_generation_prompt, "")
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ output = {
+ "prompts": [first_turn_prompt] + follow_up_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ if data_name in ["scbench_summary_with_needles", "scbench_repoqa_and_kv"]:
+ output["task"] = [gt["task"] for gt in eg["multi_turns"]]
+
+ return output
+
+ elif data_name == "scbench_mf":
+ first_turn = eg["multi_turns"][0]
+ input_ = first_turn["input"]
+ context = eg["context"]
+
+ target = re.findall(r"The .+ is", input_)[0].lower()[:-3]
+ prefix = f"What is {target}?"
+
+ first_turn_prompt = template.format(
+ prefix=prefix,
+ context=context,
+ input=input_,
+ )
+
+ follow_up_prompts = []
+ for i in range(1, len(eg["multi_turns"])):
+ target = re.findall(r"The .+ is", eg["multi_turns"][i]["input"])[0].lower()[
+ :-3
+ ]
+ prefix = f"What is {target}?"
+ follow_up_prompts.append(
+ follow_up_template.format(
+ pre_ans=eg["multi_turns"][i - 1]["answer"]
+ if not disable_golden_context
+ else None,
+ prefix=prefix,
+ input=eg["multi_turns"][i]["input"],
+ )
+ )
+
+ if use_chat_template:
+ first_turn_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": first_turn_prompt}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+
+ follow_up_prompts = [
+ tok.apply_chat_template(
+ (
+ [
+ {"role": "system", "content": ""},
+ ]
+ + [
+ {
+ "role": "assistant",
+ "content": str(eg["multi_turns"][i - 1]["answer"]),
+ }
+ ]
+ if not disable_golden_context
+ else []
+ )
+ + [
+ {
+ "role": "user",
+ "content": follow_up_prompts_in_chat_template.format(
+ prefix=prefix,
+ input=eg["multi_turns"][i]["input"],
+ ),
+ },
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).replace(sys_prompt_with_generation_prompt, "")
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ return {
+ "prompts": [first_turn_prompt] + follow_up_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+ elif data_name == "scbench_summary":
+ first_turn = eg["multi_turns"][0]
+ input_ = first_turn["input"]
+ context = eg["context"]
+
+ first_turn_prompt = template.format(
+ context=context,
+ input=input_,
+ )
+
+ follow_up_prompts = [
+ follow_up_template.format(
+ pre_ans=eg["multi_turns"][i - 1]["answer"]
+ if not disable_golden_context
+ else None,
+ input=eg["multi_turns"][i]["input"],
+ )
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ if use_chat_template:
+ first_turn_prompt = tok.apply_chat_template(
+ [{"role": "user", "content": first_turn_prompt}],
+ add_generation_prompt=True,
+ tokenize=False,
+ )
+
+ follow_up_prompts = [
+ tok.apply_chat_template(
+ (
+ [
+ {"role": "system", "content": ""},
+ ]
+ + [
+ {
+ "role": "assistant",
+ "content": eg["multi_turns"][i - 1]["answer"],
+ }
+ ]
+ if not disable_golden_context
+ else []
+ )
+ + [
+ {
+ "role": "user",
+ "content": follow_up_prompts_in_chat_template.format(
+ input=eg["multi_turns"][i]["input"],
+ ),
+ },
+ ],
+ add_generation_prompt=True,
+ tokenize=False,
+ ).replace(sys_prompt_with_generation_prompt, "")
+ + "This paper"
+ for i in range(1, len(eg["multi_turns"]))
+ ]
+
+ return {
+ "prompts": [first_turn_prompt] + follow_up_prompts,
+ "ground_truth": [gt["answer"] for gt in eg["multi_turns"]],
+ }
+
+
+def get_ground_truth(eg: dict, data_name: str):
+ gts = []
+ OPTIONS = "ABCD"
+ for turn in eg["multi_turns"]:
+ if data_name == "scbench_choice_eng":
+ ans_ = turn["answer"]
+ options = turn["options"]
+
+ gts.append([ans_, OPTIONS[options.index(ans_)]])
+ elif data_name in ["scbench_qa_eng"]:
+ gts.append([turn["answer"]])
+ else:
+ gts.append(turn["answer"])
+ return gts
+
+
+def normalize_answer(s):
+ """Lower text and remove punctuation, articles and extra whitespace."""
+
+ def remove_articles(text):
+ return re.sub(r"\b(a|an|the)\b", " ", text)
+
+ def white_space_fix(text):
+ return " ".join(text.split())
+
+ def remove_punc(text):
+ exclude = set(string.punctuation)
+ return "".join(ch for ch in text if ch not in exclude)
+
+ def lower(text):
+ return text.lower()
+
+ return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def normalize_zh_answer(s):
+ """Lower text and remove punctuation, extra whitespace."""
+
+ def white_space_fix(text):
+ return "".join(text.split())
+
+ def remove_punc(text):
+ cn_punctuation = "！？｡。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏." # noqa
+ all_punctuation = set(string.punctuation + cn_punctuation)
+ return "".join(ch for ch in text if ch not in all_punctuation)
+
+ def lower(text):
+ return text.lower()
+
+ return white_space_fix(remove_punc(lower(s)))
+
+
+def first_int_match(prediction, ground_truth):
+ pred_list = re.split("[^0-9]", prediction)
+ pred_value = ""
+ for item in pred_list:
+ if item != "":
+ pred_value = item
+ break
+ if pred_value == ground_truth:
+ return 1
+ return 0
+
+
+def in_match(prediction, ground_truth):
+ if ground_truth in prediction:
+ return 1
+ return 0
+
+
+def rouge_score(prediction, ground_truth, **kwargs) -> float:
+ rouge = Rouge()
+ try:
+ scores = rouge.get_scores([prediction], [ground_truth], avg=True)
+ except: # noqa
+ return 0.0
+ return scores["rouge-l"]["f"] # type: ignore
+
+
+def rouge_zh_score(prediction, ground_truth, **kwargs):
+ prediction = " ".join(list(jieba.cut(prediction, cut_all=False)))
+ ground_truth = " ".join(list(jieba.cut(ground_truth, cut_all=False)))
+ score = rouge_score(prediction, ground_truth)
+ return score
+
+
+def f1_score(prediction, ground_truth, **kwargs):
+ common = Counter(prediction) & Counter(ground_truth)
+ num_same = sum(common.values())
+ if num_same == 0:
+ return 0
+ precision = 1.0 * num_same / len(prediction)
+ recall = 1.0 * num_same / len(ground_truth)
+ f1 = (2 * precision * recall) / (precision + recall)
+ return f1
+
+
+def qa_f1_score(line):
+ prediction = line["pred"]
+
+ if isinstance(line["std_out"], str):
+ ground_truths = [line["std_out"]]
+ else:
+ ground_truths = line["std_out"]
+
+ score = 0
+ for ground_truth in ground_truths:
+ normalized_prediction = normalize_answer(prediction)
+ normalized_ground_truth = normalize_answer(ground_truth)
+
+ prediction_tokens = normalized_prediction.split()
+ ground_truth_tokens = normalized_ground_truth.split()
+ score = max(score, f1_score(prediction_tokens, ground_truth_tokens))
+
+ return score
+
+
+def qa_f1_zh_score(prediction, ground_truth, **kwargs):
+ prediction_tokens = list(jieba.cut(prediction, cut_all=False))
+ ground_truth_tokens = list(jieba.cut(ground_truth, cut_all=False))
+ prediction_tokens = [normalize_zh_answer(token) for token in prediction_tokens]
+ ground_truth_tokens = [normalize_zh_answer(token) for token in ground_truth_tokens]
+ prediction_tokens = [token for token in prediction_tokens if len(token) > 0]
+ ground_truth_tokens = [token for token in ground_truth_tokens if len(token) > 0]
+ return f1_score(prediction_tokens, ground_truth_tokens)
+
+
+def truncate_input(input, max_length, manner="middle"):
+ if len(input) <= max_length:
+ return input
+ if manner == "middle":
+ return input[0 : max_length // 2] + input[-max_length // 2 :]
+ else:
+ return None
+
+
+def get_compressed_examples(
+ examples, data_name, data_dir, rate=0.33, use_large_model=True
+):
+ # compress prompts use a func
+ import gc
+
+ from llmlingua import PromptCompressor
+
+ if os.path.exists(
+ f"{data_dir}/llmlingua_cache/{data_name}_rate_{rate}_is_large_{use_large_model}.jsonl"
+ ):
+ with open(
+ f"{data_dir}/llmlingua_cache/{data_name}_rate_{rate}_is_large_{use_large_model}.jsonl",
+ "r",
+ ) as f:
+ examples = [json.loads(line) for line in f]
+ return examples
+
+ lingua_model_name = (
+ "microsoft/llmlingua-2-xlm-roberta-large-meetingbank"
+ if use_large_model
+ else "microsoft/llmlingua-2-bert-base-multilingual-cased-meetingbank"
+ )
+ llm_lingua = PromptCompressor(
+ model_name=lingua_model_name,
+ use_llmlingua2=True,
+ )
+
+ for example in tqdm(examples, desc="Compressing prompts"):
+ ct = str(
+ example["context"] if "context" in example else example["input"]
+ ).replace("<|endoftext|>", "")
+ example["context"] = llm_lingua.compress_prompt(
+ ct, rate=rate, force_tokens=["\n", "?"]
+ )["compressed_prompt"]
+
+ os.makedirs(f"{data_dir}/llmlingua_cache", exist_ok=True)
+ with open(
+ f"{data_dir}/llmlingua_cache/{data_name}_rate_{rate}_is_large_{use_large_model}.jsonl",
+ "w",
+ ) as f:
+ for example in examples:
+ json.dump(example, f)
+ f.write("\n")
+
+ # clear llmlingua to free memory, to prevent OOM in further testing
+ del llm_lingua
+ gc.collect()
+ torch.cuda.empty_cache()
+
+ return examples
+
+
+class GreedySearch_vLLM:
+ def __init__(self, llm, tokenizer, is_kv_compress: bool = False):
+ self.llm = llm
+ self.tokenizer = tokenizer
+ self.is_kv_compress = is_kv_compress
+
+ def test_scdq(self, example, max_length=100):
+ from vllm import SamplingParams
+
+ results = []
+ for idx, prompt in enumerate(example["prompts"]):
+ if idx == 0:
+ init_prompt_ids = prompt
+ else:
+ if isinstance(max_length, dict):
+ max_length_per_turn = max_length[example["task"][idx - 1]]
+ else:
+ max_length_per_turn = max_length
+
+ sampling_params = SamplingParams(
+ temperature=0.0,
+ max_tokens=max_length_per_turn,
+ )
+ current_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+ input_ids = init_prompt_ids + current_ids
+
+ result = self.llm.generate(
+ prompt_token_ids=input_ids, sampling_params=sampling_params
+ )
+ results.append(result[0].outputs[0].text)
+ output = {"answers": results, "gt": example["ground_truth"]}
+
+ if isinstance(max_length, dict): # mixed task setting
+ output["task"] = example["task"]
+
+ return output
+
+ def test(self, example, max_length=100, disable_golden_context=False):
+ from vllm import SamplingParams
+
+ results = []
+ for idx, prompt in enumerate(example["prompts"]):
+ if isinstance(max_length, dict):
+ max_length_per_turn = max_length[example["task"][idx]]
+ else:
+ max_length_per_turn = max_length
+
+ sampling_params = SamplingParams(
+ temperature=0.0,
+ max_tokens=max_length_per_turn,
+ )
+ if self.is_kv_compress:
+ sampling_params = SamplingParams(
+ max_tokens=max_length_per_turn,
+ min_tokens=1,
+ temperature=0.0,
+ max_cache_tokens=4096,
+ protected_window_size=32,
+ metric_collection_buffer_size=0,
+ compress_once=True,
+ )
+
+ if idx == 0:
+ input_ids = prompt
+ else:
+ current_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+
+ # if disable_golden_context, add result[0].outputs[0].text to the prompt
+ if disable_golden_context:
+ input_ids = (
+ input_ids
+ + self.tokenizer.encode(
+ result[0].outputs[0].text, add_special_tokens=False
+ )
+ + [self.tokenizer.eos_token_id]
+ )
+ input_ids = input_ids + current_ids
+
+ result = self.llm.generate(
+ prompt_token_ids=input_ids, sampling_params=sampling_params
+ )
+ results.append(result[0].outputs[0].text)
+ output = {"answers": results, "gt": example["ground_truth"]}
+
+ if isinstance(max_length, dict): # mixed task setting
+ output["task"] = example["task"]
+
+ return output
+
+
+class GreedySearch:
+ def __init__(self, model, tokenizer):
+ model.eval()
+ self.device = model.device
+ self.model = model
+ self.tokenizer = tokenizer
+ self.past_kv = None
+ self.add_eos_to_next_prompt = False
+
+ def clear(self):
+ self.past_kv = None
+ gc.collect()
+ torch.cuda.empty_cache()
+
+ def _process_texts(self, input_text):
+ model_inputs = {}
+ input_ids = self.tokenizer.encode(input_text, add_special_tokens=False)
+
+ # add eos to the beginning of the input_ids if self.add_eos_to_next_prompt is True
+ if self.add_eos_to_next_prompt:
+ input_ids = [self.tokenizer.eos_token_id] + input_ids
+ self.add_eos_to_next_prompt = False
+
+ model_inputs["input_ids"] = input_ids
+ model_inputs["attention_mask"] = [1] * len(model_inputs["input_ids"])
+
+ for key in model_inputs:
+ model_inputs[key] = (
+ torch.tensor(model_inputs[key]).int().unsqueeze(0).cuda()
+ )
+
+ return model_inputs
+
+ def _make_first_turn(self, input_ids):
+ model_inputs = {}
+ model_inputs["input_ids"] = input_ids
+
+ for key in model_inputs:
+ model_inputs[key] = (
+ torch.tensor(model_inputs[key]).int().unsqueeze(0).cuda()
+ )
+
+ return model_inputs
+
+ def test_scdq(self, example, max_length=100):
+ results = []
+ for idx, prompt in enumerate(example["prompts"]):
+ if isinstance(max_length, dict):
+ max_length_per_turn = max_length[example["task"][idx - 1]]
+ else:
+ max_length_per_turn = max_length
+
+ if idx == 0:
+ model_inputs = self._make_first_turn(prompt)
+ else:
+ model_inputs = self._process_texts(prompt)
+ input_ids = model_inputs["input_ids"]
+
+ with torch.inference_mode():
+ if idx == 0:
+ result = self._encode(input_ids, max_length=max_length_per_turn)
+ else:
+ result = self._decode(
+ input_ids,
+ max_length=max_length_per_turn,
+ dense_prefix=True,
+ update_global_past_kv=False,
+ )
+
+ results.append(
+ self.tokenizer.decode(result[0, len(input_ids[0]) :])
+ )
+ torch.cuda.empty_cache()
+ self.clear()
+ output = {"answers": results, "gt": example["ground_truth"]}
+
+ if isinstance(max_length, dict): # mixed task setting
+ output["task"] = example["task"]
+
+ return output
+
+ def test(self, example, max_length=100, disable_golden_context=False):
+ results = []
+ # for idx, prompt in tqdm(enumerate(example['prompts']), total=len(example['prompts']), desc="Prompt"):
+ for idx, prompt in enumerate(example["prompts"]):
+ if isinstance(max_length, dict):
+ max_length_per_turn = max_length[example["task"][idx]]
+ else:
+ max_length_per_turn = max_length
+
+ if idx == 0:
+ model_inputs = self._make_first_turn(prompt)
+ else:
+ model_inputs = self._process_texts(prompt)
+ input_ids = model_inputs["input_ids"]
+
+ with torch.inference_mode():
+ if idx == 0:
+ result = self._decode(
+ input_ids,
+ max_length=max_length_per_turn,
+ disable_golden_context=disable_golden_context,
+ )
+ else:
+ result = self._decode(
+ input_ids,
+ max_length=max_length_per_turn,
+ dense_prefix=True,
+ disable_golden_context=disable_golden_context,
+ )
+
+ results.append(self.tokenizer.decode(result[0, len(input_ids[0]) :]))
+ torch.cuda.empty_cache()
+ self.clear()
+ output = {"answers": results, "gt": example["ground_truth"]}
+
+ if isinstance(max_length, dict): # mixed task setting
+ output["task"] = example["task"]
+
+ return output
+
+ def _encode(self, input_ids, max_length=None):
+ if self.past_kv is None:
+ past_key_values = self.model.prepare_inputs_for_generation(input_ids)[
+ "past_key_values"
+ ]
+ else:
+ past_key_values = self.past_kv
+
+ out = self.model(
+ input_ids=input_ids,
+ # attention_mask=torch.ones_like(input_ids),
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ logits_to_keep=1,
+ )
+ _, past_key_values = out.logits, out.past_key_values
+
+ self.past_kv = past_key_values
+
+ def _decode(
+ self,
+ input_ids,
+ max_length=100,
+ extra_end_token_ids=[],
+ dense_prefix=False,
+ update_global_past_kv=True,
+ disable_golden_context=False,
+ ):
+ if input_ids.dim() == 1:
+ input_ids = input_ids[None, :]
+ input_ids = input_ids.cuda()
+ assert input_ids.size(0) == 1
+ end_token_ids = (
+ extra_end_token_ids
+ + [self.tokenizer.eos_token_id]
+ + self.model.config.eos_token_id
+ )
+ logits = None
+ if self.past_kv is None:
+ model_inputs = {}
+ self.model._prepare_cache_for_generation(
+ GenerationConfig(), model_inputs, None, None, None, None
+ )
+ past_key_values = model_inputs["past_key_values"]
+ else:
+ past_key_values = self.past_kv
+
+ if not update_global_past_kv:
+ self.global_kv_update_mode(False)
+
+ for i in range(max_length):
+ if i == 0: # prefilling
+ out = self.model(
+ input_ids=input_ids,
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ logits_to_keep=1,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+
+ else: # decoding
+ if (
+ not disable_golden_context
+ ): # if use golden context, then decoding should not update global past_kv
+ self.global_kv_update_mode(False)
+ out = self.model(
+ input_ids=input_ids[:, -1:],
+ past_key_values=past_key_values,
+ use_cache=True,
+ return_dict=True,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+
+ logits = logits[:, -1, :]
+ word = logits.argmax(dim=-1)
+ if word.item() in end_token_ids or i == max_length:
+ break
+
+ input_ids = torch.cat(
+ (input_ids, word.to(input_ids.device).view(1, 1)), dim=-1
+ )
+
+ if not update_global_past_kv or not disable_golden_context:
+ self.global_kv_update_mode(True)
+ past_key_values.clear_temp_kv_cache()
+
+ self.past_kv = past_key_values
+ # should see whether the last token is eos, if not tell self.test to add it to the next prompt
+ if word.item() not in end_token_ids and disable_golden_context:
+ self.add_eos_to_next_prompt = True
+ return input_ids
+
+ def global_kv_update_mode(self, mode):
+ try:
+ attn_class = self.model.model.layers[0].self_attn.__class__
+ except:
+ attn_class = self.model.transformer.encoder.layers[
+ 0
+ ].self_attention.__class__
+ self.model.apply(
+ lambda m: setattr(m, "update_global_past_kv", mode)
+ if isinstance(m, attn_class)
+ else None
+ )
+
+
+class GreedySearch_RetrAttn(GreedySearch):
+ def _decode(
+ self,
+ input_ids,
+ max_length=100,
+ extra_end_token_ids=[],
+ dense_prefix=False,
+ update_global_past_kv=True,
+ disable_golden_context=False,
+ ):
+ if input_ids.dim() == 1:
+ input_ids = input_ids[None, :]
+ input_ids = input_ids.cuda()
+ assert input_ids.size(0) == 1
+ end_token_ids = extra_end_token_ids + [self.tokenizer.eos_token_id]
+ logits = None
+ if self.past_kv is None:
+ model_inputs = {}
+ self.model._prepare_cache_for_generation(
+ GenerationConfig(), model_inputs, None, None, None, None
+ )
+ past_key_values = model_inputs["past_key_values"]
+ else:
+ past_key_values = self.past_kv
+
+ if not update_global_past_kv:
+ self.global_kv_update_mode(False)
+
+ for i in range(max_length):
+ if i == 0: # prefilling
+ if dense_prefix:
+ for token in input_ids.squeeze(0):
+ out = self.model(
+ input_ids=token.unsqueeze(0).unsqueeze(0),
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ logits_to_keep=1,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+ else:
+ out = self.model(
+ input_ids=input_ids,
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ logits_to_keep=1,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+
+ else: # decoding
+ if (
+ not disable_golden_context
+ ): # if use golden context, then decoding should not update global past_kv
+ self.global_kv_update_mode(False)
+ out = self.model(
+ input_ids=input_ids[:, -1:],
+ past_key_values=past_key_values,
+ use_cache=True,
+ return_dict=True,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+
+ logits = logits[:, -1, :]
+ word = logits.argmax(dim=-1)
+ if word.item() in end_token_ids or i == max_length:
+ break
+
+ input_ids = torch.cat(
+ (input_ids, word.to(input_ids.device).view(1, 1)), dim=-1
+ )
+
+ if not update_global_past_kv or not disable_golden_context:
+ self.global_kv_update_mode(True)
+ past_key_values.clear_temp_kv_cache()
+
+ self.past_kv = past_key_values
+ # should see whether the last token is eos, if not tell self.test to add it to the next prompt
+ if word.item() != self.tokenizer.eos_token_id:
+ self.add_eos_to_next_prompt = True
+ return input_ids
+
+
+class GreedySearch_InfLLM(GreedySearch):
+ # basically, InfLLM do _encode and _decode chunk by chunk
+ def _encode(self, input_ids, past_kv=None, max_length=None):
+ chunk_size = 8192
+ for st in range(0, input_ids.size(1), chunk_size):
+ torch.cuda.empty_cache()
+ ed = min(input_ids.size(1), st + chunk_size)
+ out = self.model(
+ input_ids=input_ids[:, st:ed],
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_kv,
+ )
+ logits, past_kv = out.logits, out.past_key_values
+
+ self.past_kv = past_kv
+
+ def _decode(
+ self,
+ input_ids,
+ max_length=100,
+ extra_end_token_ids=[],
+ dense_prefix=False,
+ update_global_past_kv=True,
+ disable_golden_context=False,
+ ):
+ if input_ids.dim() == 1:
+ input_ids = input_ids[None, :]
+ input_ids = input_ids.cuda()
+ assert input_ids.size(0) == 1
+ end_token_ids = extra_end_token_ids + [self.tokenizer.eos_token_id]
+ logits = None
+ if self.past_kv is None:
+ if self.use_sinkcache:
+ past_key_values = SinkCache(window_length=3968, num_sink_tokens=128)
+ else:
+ past_key_values = self.model.prepare_inputs_for_generation(input_ids)[
+ "past_key_values"
+ ]
+ else:
+ past_key_values = self.past_kv
+ if self.use_sinkcache:
+ past_key_values.window_length += 5_000
+
+ chunk_size = 8196
+ for i in range(max_length):
+ if i == 0:
+ if dense_prefix:
+ for token in input_ids.squeeze(0):
+ out = self.model(
+ input_ids=token.unsqueeze(0).unsqueeze(0),
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+
+ else:
+ for st in range(0, input_ids.size(1) - 1, chunk_size):
+ ed = min(input_ids.size(1) - 1, st + chunk_size)
+ out = self.model(
+ input_ids=input_ids[:, st:ed],
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+
+ if update_global_past_kv:
+ self.past_kv = past_key_values
+
+ else:
+ out = self.model(
+ input_ids=input_ids[:, -1:],
+ past_key_values=past_key_values,
+ use_cache=True,
+ return_dict=True,
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+
+ if disable_golden_context and update_global_past_kv:
+ self.past_kv = past_key_values
+
+ logits = logits[:, -1, :]
+ word = logits.argmax(dim=-1)
+ if word.item() in end_token_ids or i == max_length:
+ break
+
+ input_ids = torch.cat(
+ (input_ids, word.to(input_ids.device).view(1, 1)), dim=-1
+ )
+
+ # should see whether the last token is eos, if not tell self.test to add it to the next prompt
+ if word.item() != self.tokenizer.eos_token_id and disable_golden_context:
+ self.add_eos_to_next_prompt = True
+ return input_ids
+
+
+class GreedySearch_RetrAttn_Legacy(GreedySearch):
+ def __init__(self, model, tokenizer, top_k, from_layer, with_minference=False):
+ super().__init__(model, tokenizer)
+ if with_minference:
+ from sparse_retr_attn.modeling_llama_minference_with_retr import (
+ VectorDB_KV_Cache,
+ hf_greedy_search_retr,
+ )
+ else:
+ from sparse_retr_attn.modeling_llama_retr_attn import (
+ VectorDB_KV_Cache,
+ hf_greedy_search_retr,
+ )
+ self.top_k = top_k
+ self.from_layer = from_layer
+ self.kv_class = VectorDB_KV_Cache
+
+ def clear(self):
+ self.past_kv = None
+ self.kv_len = 0
+ gc.collect()
+ torch.cuda.empty_cache()
+
+ def _decode(
+ self,
+ input_ids,
+ max_length=100,
+ extra_end_token_ids=[],
+ dense_prefix=False,
+ update_global_past_kv=True,
+ disable_golden_context=False,
+ ):
+ if input_ids.dim() == 1:
+ input_ids = input_ids[None, :]
+ input_ids = input_ids.cuda()
+ assert input_ids.size(0) == 1
+ end_token_ids = extra_end_token_ids + [self.tokenizer.eos_token_id]
+ logits = None
+
+ if self.past_kv is None:
+ past_key_values = self.kv_class(
+ max_length=max_length * self.num_turns + input_ids.size(1),
+ temp_cache_size=max_length,
+ )
+ kv_len = input_ids.size(1)
+ else:
+ past_key_values = self.past_kv
+ kv_len = self.kv_len
+
+ for i in range(max_length):
+ if i == 0:
+ if dense_prefix:
+ for token in input_ids.squeeze(0):
+ out = self.model(
+ input_ids=token.unsqueeze(0).unsqueeze(0),
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ insert_db=True if update_global_past_kv else False,
+ top_k=self.top_k,
+ from_layer=self.from_layer,
+ cache_position=torch.tensor(
+ [kv_len], device=input_ids.device, dtype=torch.long
+ ),
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+ kv_len += 1
+ else:
+ out = self.model(
+ input_ids=input_ids,
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ top_k=self.top_k,
+ from_layer=self.from_layer,
+ cache_position=torch.arange(kv_len, device=input_ids.device),
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+ # kv_len += 1
+ # update global past_kv with prefix only
+ if update_global_past_kv:
+ self.past_kv = past_key_values
+ self.kv_len = kv_len
+
+ else:
+ out = self.model(
+ input_ids=input_ids[:, -1:],
+ past_key_values=past_key_values,
+ use_cache=True,
+ return_dict=True,
+ insert_db=False,
+ top_k=self.top_k,
+ from_layer=self.from_layer,
+ cache_position=torch.tensor(
+ [kv_len], device=input_ids.device, dtype=torch.long
+ ),
+ )
+ logits, past_key_values = out.logits, out.past_key_values
+ kv_len += 1
+ if disable_golden_context and update_global_past_kv:
+ self.past_kv = past_key_values
+ self.kv_len = kv_len
+
+ logits = logits[:, -1, :]
+ word = logits.argmax(dim=-1)
+ if word.item() in end_token_ids or i == max_length:
+ break
+
+ input_ids = torch.cat(
+ (input_ids, word.to(input_ids.device).view(1, 1)), dim=-1
+ )
+
+ self.past_kv.temp_seen = 0 # Discard decoding tokens
+
+ # should see whether the last token is eos, if not tell self.test to add it to the next prompt
+ if word.item() != self.tokenizer.eos_token_id and disable_golden_context:
+ self.add_eos_to_next_prompt = True
+ return input_ids
+
+ def _encode(self, input_ids, max_length=None):
+ if self.past_kv is None:
+ past_key_values = self.kv_class(
+ max_length=max_length * self.num_turns + input_ids.size(1),
+ temp_cache_size=max_length + self.length_of_query + 10,
+ )
+ kv_len = input_ids.size(1)
+ else:
+ past_key_values = self.past_kv
+ kv_len = self.kv_len
+
+ out = self.model(
+ input_ids=input_ids,
+ use_cache=True,
+ return_dict=True,
+ past_key_values=past_key_values,
+ top_k=self.top_k,
+ from_layer=self.from_layer,
+ cache_position=torch.arange(kv_len, device=input_ids.device),
+ )
+ _, past_key_values = out.logits, out.past_key_values
+
+ self.past_kv = past_key_values
+ self.kv_len = kv_len
+
+ def test_scdq(self, example, max_length=100):
+ prompts = example["prompts"]
+ self.length_of_query = len(self.tokenizer.encode(prompts[1]))
+ self.num_turns = len(prompts)
+ return super().test_scdq(example, max_length)
+
+ def test(self, example, max_length=100, disable_golden_context=False):
+ prompts = example["prompts"]
+ self.length_of_query = len(self.tokenizer.encode(prompts[1]))
+ self.num_turns = len(prompts)
+ return super().test(example, max_length, disable_golden_context)
+
+
+class GreedySearch_Mamba2:
+ def __init__(self, llm, tokenizer):
+ self.llm = llm
+ self.tokenizer = tokenizer
+
+ def test_scdq(self, example, max_length=100):
+ results = []
+ for idx, prompt in enumerate(example["prompts"]):
+ if isinstance(max_length, dict):
+ max_length_per_turn = max_length[example["task"][idx - 1]]
+ else:
+ max_length_per_turn = max_length
+
+ generation_config = GenerationConfig(
+ max_new_tokens=max_length_per_turn,
+ num_return_sequences=1,
+ do_sample=False,
+ pad_token_id=self.tokenizer.pad_token_id,
+ )
+
+ if idx == 0:
+ init_prompt_ids = prompt
+ else:
+ current_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+ input_ids = init_prompt_ids + current_ids
+ input_ids = torch.tensor(input_ids).unsqueeze(0).to(self.llm.device)
+
+ outputs = self.llm.generate(
+ input_ids=input_ids, generation_config=generation_config
+ )
+ output = outputs[0, len(input_ids[0]) :]
+ output = self.tokenizer.decode(output, skip_special_tokens=True)
+ output = output.strip()
+ results.append(output)
+
+ output = {"answers": results, "gt": example["ground_truth"]}
+
+ if isinstance(max_length, dict): # mixed task setting
+ output["task"] = example["task"]
+
+ return output
+
+ def test(self, example, max_length=100, disable_golden_context=False):
+ results = []
+ for idx, prompt in enumerate(example["prompts"]):
+ if isinstance(max_length, dict):
+ max_length_per_turn = max_length[example["task"][idx]]
+ else:
+ max_length_per_turn = max_length
+
+ generation_config = GenerationConfig(
+ max_new_tokens=max_length_per_turn,
+ num_return_sequences=1,
+ do_sample=False,
+ pad_token_id=self.tokenizer.pad_token_id,
+ )
+
+ if idx == 0:
+ input_ids = prompt
+ input_ids = torch.tensor(input_ids).unsqueeze(0).to(self.llm.device)
+ else:
+ current_ids = self.tokenizer.encode(prompt, add_special_tokens=False)
+ current_ids = torch.tensor(current_ids).unsqueeze(0).to(self.llm.device)
+
+ if disable_golden_context:
+ # input_ids = input_ids + self.tokenizer.encode(results[-1], add_special_tokens=False) + [self.tokenizer.eos_token_id]
+ prev_ids = (
+ torch.tensor(
+ self.tokenizer.encode(results[-1], add_special_tokens=False)
+ )
+ .unsqueeze(0)
+ .to(self.llm.device)
+ )
+ eos_id = torch.tensor(
+ [self.tokenizer.eos_token_id], device=self.llm.device
+ ).unsqueeze(0)
+ input_ids = torch.cat((input_ids, prev_ids, eos_id), dim=-1)
+ input_ids = torch.cat((input_ids, current_ids), dim=-1)
+ outputs = self.llm.generate(
+ input_ids=input_ids, generation_config=generation_config
+ )
+ output = outputs[0, len(input_ids[0]) :]
+ output = self.tokenizer.decode(output, skip_special_tokens=True)
+ output = output.strip()
+ results.append(output)
+ torch.cuda.empty_cache()
+
+ output = {"answers": results, "gt": example["ground_truth"]}
+
+ if isinstance(max_length, dict): # mixed task setting
+ output["task"] = example["task"]
+
+ return output
diff --git a/scbench/readme.md b/scbench/readme.md
new file mode 100644
index 00000000..5803e487
--- /dev/null
+++ b/scbench/readme.md
@@ -0,0 +1,233 @@
+# SCBench
+
+[[Paper]](https://arxiv.org/abs/2412.10319)
+[[Code]](https://github.com/microsoft/MInference/tree/main/scbench)
+[[Project Page]](https://aka.ms/scbench)
+
+
+
+SCBench (SharedContextBench) is a comprehensive benchmark to evaluate efficient long-context methods on **multi-turn** and **multi-request** interactions to analyze their performance across **the full KV cache lifecycle (generation, compression, retrieval, and loading)**.
+
+> [!Note]
+> - **datasets >= 2.15.0**
+
+### Load Data
+You can download and load the **SCBench** data through the Hugging Face datasets ([🤗 HF Repo](https://huggingface.co/datasets/microsoft/SCBench)):
+```python
+from datasets import load_dataset
+
+datasets = ["scbench_kv", "scbench_prefix_suffix", "scbench_vt", "scbench_repoqa", "scbench_qa_eng", "scbench_qa_chn", "scbench_choice_eng", "scbench_many_shot", "scbench_summary", "scbench_mf", "scbench_summary_with_needles", "scbench_repoqa_and_kv"]
+
+for dataset in datasets:
+ data = load_dataset("microsoft/SCBench", dataset, split="test")
+```
+
+### Data Format
+
+All data in **SCBench** are standardized to the following format:
+
+```json
+{
+ "id": "Random id for each piece of data.",
+ "context": "The long context required for the task, such as repo-code, long-document, and many-shot.",
+ "multi_turns": [{"input": "multi-turn question.", "answer": "multi-turn reference answer."}],
+}
+```
+
+### Experiments
+
+We implement **Multi-Turn** and **Multi-Request** modes with HF and vLLM in [`GreedySearch`](https://github.com/microsoft/MInference/blob/yucheng/kvcompression/scbench/eval_utils.py#L1160) and [`GreedySearch_vllm`](https://github.com/microsoft/MInference/blob/yucheng/kvcompression/scbench/eval_utils.py#L1070) two class. Please refer the follow scripts to run the experiments.
+
+for all methods,
+```bash
+cd scbench
+# Single-GPU, in Multi-Turn Mode
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 1 multi-turn
+# Multi-GPU, in Multi-Turn Mode
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 2 multi-turn
+# Multi-GPU, in Multi-Request Mode
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 2 scdq
+```
+
+for single methods,
+```bash
+cd scbench
+# Single-GPU, in Multi-Turn Mode, using attn_type: vllm, kv_type: dense
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 1 multi-turn vllm dense
+# Multi-GPU, in Multi-Turn Mode, using attn_type: vllm, kv_type: dense
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 2 multi-turn vllm dense
+# Multi-GPU, in Multi-Request Mode, using attn_type: vllm, kv_type: dense
+VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 2 scdq vllm dense
+```
+
+More details about **attn_type** and **kv_type**, please refer to this section: [Supported Efficient Methods](https://github.com/microsoft/MInference/tree/main?tab=readme-ov-file#supported-efficient-methods).
+
+## Run the benchmark
+
+First, build the environment, see [basic environment](#basic-dependencies).
+
+Run the test:
+
+```bash
+bash scripts/test_llama.sh
+```
+
+Run multiple tasks in one command:
+
+```bash
+bash scripts/run_all_tasks.sh
+```
+
+Specify the max sequence length, max number of turns, and number of eval examples:
+
+- `--max_seq_length`: The maximum sequence length for the test.
+- `--max_turns`: The maximum number of turns for the test.
+- `--num_eval_examples`: The number of test examples to use, use all examples in default.
+
+## Run with efficient long-context methods:
+
+- `--attn_type`: The attention type to use.
+- `--kv_type`: The KV cache type to use.
+
+For example, run with MInference and SnapKV:
+
+```bash
+bash scripts/test_minference_with_snapkv.sh
+```
+
+The supported efficient long-context methods are as follows:
+
+**attn_type**:
+
+- `dense`: Dense attention
+- `minference`: MInference
+- `a_shape`: A-Shape
+- `tri_shape`: Tri-Shape
+
+**kv_type**:
+
+- `dense`: Dense KV cache
+- `kivi`: KIVI
+- `snapkv`: SnapKV
+- `quest`: Quest
+- `pyramidkv`: PyramidKV
+- `streamingllm`: StreamingLLM
+
+You will need to build specific environment for different attention types and KV cache types, see section [Environment](#environment-for-efficient-long-context-methods) for more details.
+
+## Dataset and Tasks Description
+
+
+
+SCBench covers 12 diverse tasks that test four key long-context capabilities: string retrieval, semantic retrieval, global information processing, and multi-tasking.
+
+### String Retrieval
+- `scbench_kv`: Tests key-value lookup in large JSON objects with random, incompressible content
+- `scbench_prefix_suffix`: Evaluates finding strings with specific prefix and suffix patterns
+- `scbench_vt`: Assesses multi-hop variable tracing capabilities in long inputs
+
+### Semantic Retrieval
+- `scbench_repoqa`: Function retrieval from large codebases based on natural language descriptions
+- `scbench_qa_eng`, `scbench_qa_chn`, `scbench_choice_eng`: Includes English QA, Chinese QA, and multi-choice questions on long texts
+- Requires semantic understanding on length inputs
+
+### Global Information Processing
+- `scbench_many_shot`: Tests in-context learning with hundreds of examples
+- `scbench_mf`: Statistical tasks on large arrays
+- `scbench_summary`: Summarization of documents
+- Requires global information processing or aggregation
+
+### Multi-Tasking
+- `scbench_summary_with_needles`: Combines summarization with needle-in-haystack search
+- `scbench_repoqa_and_kv`: Integrates code function retrieval with key-value lookup
+- Requires multi-tasking or multi-step reasoning
+
+## Two Shared Context Modes
+The benchmark evaluates these tasks across two shared context modes:
+- **Multi-turn Mode**: the default mode of our SCBench
+- **Multi-request Mode**: use `--same_context_different_query` to enable this mode
+
+## Environment for efficient long-context methods
+
+### Basic dependencies:
+
+```bash
+conda create -n scbench python=3.10 -y && conda activate scbench
+pip install torch
+pip install minference
+pip install flash-attn --no-build-isolation
+git clone https://github.com/microsoft/MInference.git && cd MInference/scbench
+pip install -r requirements.txt
+```
+
+### (Optional) Environment for efficient long-context methods
+
+MInference natively supports many efficient long-context methods, but you will need to build specific environment for the following methods:
+
+**kivi**:
+
+```bash
+bash setup/setup_kivi.sh
+```
+
+## Hyper-parameters of efficient long-context methods
+
+### --attn_type
+
+1. **minference**
+- `best_pattern` (loaded from config file)
+
+2. **a_shape**
+- `n_local` (default: 3968)
+- `n_init` (default: 128)
+
+3. **tri_shape**
+- `n_local` (default: 3968)
+- `n_init` (default: 128)
+- `n_last` (default: 100)
+
+### --kv_type
+
+1. **kivi**
+- `bits` (default: 2)
+- `group_size` (default: 32)
+- `residual_length` (default: 32)
+
+2. **snapkv/pyramidkv**
+- `window_size` (default: 32)
+- `max_capacity_prompt` (default: 4096)
+- `kernel_size` (default: 5)
+- `pooling` (default: "avgpool")
+
+3. **quest**
+- `chunk_size` (default: 16)
+- `token_budget` (default: 1024)
+
+4. **streamingllm**
+- `n_local` (default: 3968)
+- `n_init` (default: 128)
+
+
+**Note:** All these parameters can be overridden by passing custom values in `--hyper_param` in cli, for example:
+```
+python run_multiturnbench.py .... --hyper_param '{"n_local": 4096}'
+```
+
+
+## Compared to previous long-context benchmarks
+
+
+
+Our SCBench is the first long-context benchmark that covers single-turn, multi-turn, and multi-request scenarios. In addition, our impelmentation also involves KV cache reuse techniques, thereby providing a more comprehensive analysis on the full KV cache lifecycle of efficient long-context methods.
+
+
+## Citation
+
+```bibtex
+@article{li2024scbench,
+ title={SCBench: A KV cache-centric analysis of long-context methods},
+ author={Li, Yucheng and Jiang, Huiqiang and Wu, Qianhui and Luo, Xufang and Ahn, Surin and Zhang, Chengruidong and Abdi, Amir H and Li, Dongsheng and Gao, Jianfeng and Yang, Yuqing and Qiu, Lili},
+ journal={arXiv preprint arXiv:2412.10319},
+ year={2024}
+}
+```
diff --git a/scbench/repo_qa_utils.py b/scbench/repo_qa_utils.py
new file mode 100644
index 00000000..a39b138c
--- /dev/null
+++ b/scbench/repo_qa_utils.py
@@ -0,0 +1,370 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Refer to the code in https://github.com/evalplus/repoqa
+
+import itertools
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from typing import Dict, List, Tuple, Union
+
+import numpy as np
+import tempdir
+from rich.console import Console
+from rich.progress import (
+ BarColumn,
+ MofNCompleteColumn,
+ Progress,
+ TextColumn,
+ TimeElapsedColumn,
+)
+from rich.table import Table
+from transformers import AutoConfig
+from tree_sitter_languages import get_language, get_parser
+
+FUNCTION_QUERY = {
+ "python": "(function_definition name: (_)) @fdef",
+ "java": "(method_declaration name: (_)) @fdef",
+ "typescript": "(function_declaration name: (_)) @fdef",
+ "rust": "(function_item name: (_)) @fdef",
+ "cpp": "(function_definition declarator: (function_declarator declarator: (identifier))) @fdef",
+ "go": "(function_declaration name: (_)) @fdef",
+}
+
+COMMENT_QUERY = {
+ "python": [
+ "(block (expression_statement (string) @docstring))",
+ "(comment) @comment",
+ ],
+ "java": ["(line_comment) @comment", "(block_comment) @comment"],
+ "cpp": ["(comment) @comment"],
+ "rust": ["(line_comment) @comment", "(block_comment) @comment"],
+ "typescript": ["(comment) @comment"],
+ "go": ["(comment) @comment"],
+}
+
+
+def progress(note: str = "processing"):
+ return Progress(
+ TextColumn(f"{note} •" + "[progress.percentage]{task.percentage:>3.0f}%"),
+ BarColumn(),
+ MofNCompleteColumn(),
+ TextColumn("•"),
+ TimeElapsedColumn(),
+ )
+
+
+LANGUAGES = list(FUNCTION_QUERY.keys())
+THRESHOLDS = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
+
+import re
+
+from nltk.translate.bleu_score import SmoothingFunction, sentence_bleu
+
+
+def compute_function_similarity(
+ candidate_function: str, reference_function: str
+) -> float:
+ candidate_tokens = [item for item in re.split("\s+", candidate_function.strip())]
+
+ reference_tokens = [item for item in re.split("\s+", reference_function.strip())]
+
+ chencherry = SmoothingFunction()
+
+ return sentence_bleu(
+ [reference_tokens], candidate_tokens, smoothing_function=chencherry.method4
+ )
+
+
+class Result(Enum):
+ BEST_MATCH = "best_match"
+ FAIL_MATCH = "fail_match"
+
+
+# unbiased estimator from https://github.com/openai/human-eval
+def estimate_pass_at_k(
+ num_samples: Union[int, List[int], np.ndarray],
+ num_correct: Union[List[int], np.ndarray],
+ k: int,
+) -> np.ndarray:
+ """
+ Estimates pass@k of each problem and returns them in an array.
+ """
+
+ def estimator(n: int, c: int, k: int) -> float:
+ """
+ Calculates 1 - comb(n - c, k) / comb(n, k).
+ """
+ if n - c < k:
+ return 1.0
+ return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+ if isinstance(num_samples, int):
+ num_samples_it = itertools.repeat(num_samples, len(num_correct))
+ else:
+ assert len(num_samples) == len(num_correct)
+ num_samples_it = iter(num_samples)
+
+ return np.array(
+ [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]
+ )
+
+
+def remove_comments(source_code: str, lang: str) -> str:
+ source_bytes = bytes(source_code, "utf8")
+ parser = get_parser(lang)
+ tree = parser.parse(source_bytes)
+ root_node = tree.root_node
+
+ # Remove comments from source code
+ capture_list = []
+ for query_str in COMMENT_QUERY[lang]:
+ comment_query = get_language(lang).query(query_str)
+ capture_list += comment_query.captures(root_node)
+
+ capture_list.sort(key=lambda cap: cap[0].start_byte, reverse=True)
+
+ for node, _ in capture_list:
+ source_bytes = source_bytes[: node.start_byte] + source_bytes[node.end_byte :]
+
+ return source_bytes.decode("utf-8")
+
+
+def sanitize_output(model_output: str, lang: str) -> str:
+ model_output = model_output.strip()
+ search_pattern = r"^```(?:\w+)?\s*\n(.*?)(?=^```)```"
+ code_blocks = re.findall(search_pattern, model_output, re.DOTALL | re.MULTILINE)
+
+ parser = get_parser(lang)
+ fn_query = get_language(lang).query(FUNCTION_QUERY[lang])
+
+ # If not code blocks found, simply return model output
+ if not code_blocks:
+ return model_output
+
+ processed_blocks = []
+ for block in code_blocks:
+ processed_blocks.append(block)
+
+ # Try to use tree-sitter to parse if possible
+ try:
+ block_bytes = bytes(block, "utf8")
+ tree = parser.parse(block_bytes)
+ for capture in fn_query.captures(tree.root_node):
+ node, _ = capture
+ function_content = block_bytes[node.start_byte : node.end_byte]
+ return function_content.decode("utf8")
+ except:
+ pass
+
+ # no valid functions found by tree-sitter approach return first block
+ return processed_blocks[0]
+
+
+def print_result_table(model_name, pass_results):
+ # Printing scores in a table
+ table = Table(title=f"Scores (%) of {model_name} at different thresholds")
+ table.add_column("Threshold", justify="center", style="bold magenta")
+ for threshold in THRESHOLDS:
+ table.add_column(f"{threshold}", justify="center")
+
+ # Prepare data to determine the maximum values for each threshold
+ threshold_scores = {threshold: [] for threshold in THRESHOLDS}
+ for lang_results in pass_results.values():
+ for thresh, value in lang_results.items():
+ threshold_scores[thresh].append(value["pass@1"])
+
+ # Calculate the maximum score for each threshold
+ max_scores = {
+ threshold: max(scores) for threshold, scores in threshold_scores.items()
+ }
+ min_scores = {
+ threshold: min(scores) for threshold, scores in threshold_scores.items()
+ }
+
+ # Fill the table rows
+ for language, lang_results in pass_results.items():
+ row = [("⭐" if language == "all" else "") + language]
+ for threshold, value in lang_results.items():
+ score = value["pass@1"]
+ formatted_score = f"{100 * score:.1f}"
+ if max_scores[threshold] - score < 0.01:
+ formatted_score = f"[bold green]{formatted_score}[/]"
+ elif score - min_scores[threshold] < 0.01:
+ formatted_score = f"[bold red]{formatted_score}[/]"
+ row.append(formatted_score)
+ if language == "all":
+ row = [f"[bold yellow]{r}[/]" for r in row]
+ table.add_row(*row)
+
+ Console().print(table)
+
+
+def needle_evaluator(
+ model_output: str,
+ ground_truth: str,
+ needles,
+ lang: str,
+ ignore_comments: bool,
+) -> Tuple[Result, str, float]:
+ best_target = None
+ best_similarity = 0
+ sanitized_output = sanitize_output(model_output, lang)
+ if ignore_comments:
+ sanitized_output = remove_comments(sanitized_output, lang)
+ for needle in needles:
+ current_name = needle["name"]
+ current_func = needle["needle"]
+ if ignore_comments:
+ current_func = remove_comments(current_func, lang)
+
+ current_similarity = compute_function_similarity(sanitized_output, current_func)
+ if current_similarity > best_similarity:
+ best_similarity = current_similarity
+ best_target = current_name
+
+ if best_target == ground_truth["func_name"]:
+ verdict = Result.BEST_MATCH
+ else:
+ verdict = Result.FAIL_MATCH
+ return verdict, best_target, best_similarity
+
+
+def _get_repo(lang_data: Dict, repo_name: str) -> Dict:
+ for repo in lang_data:
+ if repo["repo"] == repo_name:
+ return repo
+
+
+def compute_language_results(evaluation_result: Dict, all_results: Dict) -> None:
+ for language, lang_results in evaluation_result.items():
+ current_result = {}
+ total = np.array([1 for _ in lang_results])
+
+ for threshold in THRESHOLDS:
+ correct_result = []
+ for res in lang_results:
+ bc = 0
+ if res["is_best_similar"] and res["best_similar_score"] >= threshold:
+ bc = 1
+ correct_result.append(bc)
+ correct_result = np.array(correct_result)
+
+ pass_at_k = {
+ f"pass@{k}": estimate_pass_at_k(total, correct_result, k).mean()
+ for k in [1, 10, 100]
+ if total.min() >= k
+ }
+ current_result[threshold] = pass_at_k
+ all_results[language] = current_result
+
+
+def compute_score(
+ # model_name: str, dataset: Dict, model_output: List[Dict], ignore_comments: bool
+ model_name: str,
+ preds: list,
+ labels: list,
+ needle_by_repo: dict,
+ ignore_comments: bool = False,
+) -> Dict:
+ evaluation_result = defaultdict(list)
+ score_name = (
+ model_name.replace("'", "")
+ .replace("{", "")
+ .replace("}", "")
+ .replace(": ", "_")
+ .replace(", ", "_")
+ )
+ with progress(f"Scoring {score_name}") as pbar:
+ for result in pbar.track(preds):
+ lang = result["lang"]
+ repo_name = result["repo"]
+ model_output = result["prediction"]
+ ground_truth = {
+ "func_name": result["func_name"],
+ "ground_truth": result["ground_truth"],
+ }
+ needles = needle_by_repo[repo_name]
+
+ verdict, best_target, best_similarity = needle_evaluator(
+ model_output, ground_truth, needles, lang, ignore_comments
+ )
+
+ is_best_similar = False
+ if verdict == Result.BEST_MATCH:
+ is_best_similar = True
+
+ current_task = {
+ "repo": repo_name,
+ "name": ground_truth,
+ # "needle_position": result["position_ratio"],
+ "is_best_similar": is_best_similar,
+ "best_similar_score": best_similarity,
+ "best_target": best_target,
+ # "position": {
+ # "token_start": result["needle_token_start"],
+ # "token_end": result["needle_token_end"],
+ # },
+ }
+ evaluation_result[lang].append(current_task)
+
+ # Calculate pass@k
+ pass_results = {}
+
+ all_langs = []
+ for lang in evaluation_result:
+ all_langs += evaluation_result[lang]
+ total = np.array([1 for _ in all_langs])
+
+ pass_results["all"] = {}
+ for threshold in THRESHOLDS:
+ correct_result = []
+ for res in all_langs:
+ bc = 0
+ if res["is_best_similar"] and res["best_similar_score"] >= threshold:
+ bc = 1
+ correct_result.append(bc)
+ correct_result = np.array(correct_result)
+ pass_at_k = {
+ f"pass@{k}": estimate_pass_at_k(total, correct_result, k).mean()
+ for k in [1, 10, 100]
+ if total.min() >= k
+ }
+ pass_results["all"][threshold] = pass_at_k
+
+ compute_language_results(evaluation_result, pass_results)
+ print_result_table(model_name, pass_results)
+
+ output_json = {}
+ model_json = {}
+ model_json["eval_date"] = str(datetime.now())
+ model_json["scores"] = pass_results
+ model_json["results"] = evaluation_result
+
+ output_json[model_name] = model_json
+
+ return output_json
+
+
+def save_json(output_json, result_path) -> None:
+ if os.path.isfile(result_path):
+ decision = ""
+ while decision.lower() not in ["y", "n"]:
+ print(f"{result_path} already exists. Press [Y/N] to overwrite or exit...")
+ decision = input()
+
+ if decision.lower() == "y":
+ # mv the file to a backup
+ new_path = result_path + ".bak"
+ while os.path.isfile(new_path):
+ new_path += ".bak"
+ os.rename(result_path, new_path)
+ print(f"Backup {result_path} to {new_path}")
+
+ if not os.path.isfile(result_path):
+ with open(result_path, "w") as f:
+ json.dump(output_json, f)
diff --git a/scbench/requirements.txt b/scbench/requirements.txt
new file mode 100644
index 00000000..1cc9025e
--- /dev/null
+++ b/scbench/requirements.txt
@@ -0,0 +1,10 @@
+tree_sitter_languages
+tempdir
+tree_sitter==0.21.3
+rich
+datasets
+jieba
+rouge
+evaluate
+nltk
+rouge_score
diff --git a/scbench/run_scbench.py b/scbench/run_scbench.py
new file mode 100644
index 00000000..1d310eca
--- /dev/null
+++ b/scbench/run_scbench.py
@@ -0,0 +1,451 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any, List, Tuple
+
+import torch
+from args import parse_args
+from compute_scores import compute_scores
+from datasets import load_dataset
+from eval_utils import (
+ DATA_NAME_TO_MAX_NEW_TOKENS,
+ GreedySearch,
+ GreedySearch_InfLLM,
+ GreedySearch_Mamba2,
+ GreedySearch_RetrAttn,
+ GreedySearch_RetrAttn_Legacy,
+ GreedySearch_vLLM,
+ check_benchmark_availability,
+ create_multiturn_prompt,
+ create_scdq_prompt,
+ dump_jsonl,
+ get_compressed_examples,
+ get_ground_truth,
+ load_data,
+)
+from torch import Tensor
+from tqdm import tqdm
+from transformers import (
+ AutoConfig,
+ AutoModelForCausalLM,
+ AutoTokenizer,
+ GenerationConfig,
+ LlamaForCausalLM,
+ MambaForCausalLM,
+ Qwen2ForCausalLM,
+)
+from transformers.cache_utils import SinkCache
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.utils.import_utils import _is_package_available
+
+if _is_package_available("vllm"):
+ from vllm import LLM, SamplingParams
+if _is_package_available("lmcache_vllm"):
+ from lmcache_vllm.vllm import LLM as LMCacheLLM
+ import lmcache_vllm
+
+import random
+
+from minference import MInference
+
+
+# sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+def truncate_input(input: list, max_length: int, manner="middle"):
+ if max_length < 0:
+ return input
+ if len(input) <= max_length:
+ return input
+ if manner == "middle":
+ split = max_length // 2
+ return input[0:split] + input[-split:]
+ else:
+ return None
+
+
+def truncate_by_tokens(input, tok, max_tokens, manner: str = "middle"):
+ tokens = tok.encode(input)
+ len_before = len(tokens)
+ print(f"# tokens before: {len_before}")
+ tokens = truncate_input(tokens, max_length=max_tokens, manner=manner)
+ len_after = len(tokens) # type: ignore
+ print(f"# tokens after: {len_after}")
+ assert len_after <= len_before
+ assert len_after <= max_tokens or max_tokens < 0
+ return tokens
+
+
+def get_pred(
+ model,
+ eg,
+ data_name,
+ max_new_tokens,
+ max_input_length: int,
+ attn_type: str = "vllm",
+ tok=None,
+ use_chat_template=False,
+ scdq_mode=False,
+ disable_golden_context=False,
+) -> str:
+ """
+ Truncate down to 128k then make inference.
+ """
+ if scdq_mode:
+ encoded_eg = create_scdq_prompt(
+ eg,
+ data_name=data_name,
+ tok=tok,
+ use_chat_template=use_chat_template,
+ use_vllm=("vllm" in attn_type),
+ )
+ else:
+ # multi-turn mode
+ encoded_eg = create_multiturn_prompt(
+ eg,
+ data_name=data_name,
+ tok=tok,
+ use_chat_template=use_chat_template,
+ use_vllm=("vllm" in attn_type),
+ disable_golden_context=disable_golden_context,
+ )
+ context = truncate_by_tokens(
+ encoded_eg["prompts"][0], model.tokenizer, max_input_length
+ )
+ encoded_eg["prompts"][0] = context
+ if scdq_mode:
+ # scdq mode has no action for disable_golden_context
+ outputs = model.test_scdq(encoded_eg, max_length=max_new_tokens)
+ else:
+ # multi-turn mode test
+ outputs = model.test(
+ encoded_eg,
+ max_length=max_new_tokens,
+ disable_golden_context=disable_golden_context,
+ )
+
+ print("Chunked generation:", json.dumps(outputs, indent=2, ensure_ascii=False))
+ return outputs
+
+
+def load_model(
+ model_name: str,
+ topk: int = -1,
+ starting_layer: int = -1,
+ topk_dims_file_path: str = "",
+ use_sparq: bool = False,
+ attn_type: str = "vllm",
+ max_seq_length: int = None,
+ is_search: bool = False,
+ kv_type: str = "",
+ trust_remote_code: bool = False,
+ kv_cache_cpu: bool = False,
+ kv_cache_cpu_device: str = "cpu",
+ tensor_parallel_size: int = 1,
+ hyper_param: dict = None,
+):
+ if model_name == "THUDM/glm-4-9b-chat-1m":
+ tok = AutoTokenizer.from_pretrained(
+ model_name, trust_remote_code=trust_remote_code, revision="refs/pr/19"
+ )
+ else:
+ tok = AutoTokenizer.from_pretrained(
+ model_name, trust_remote_code=trust_remote_code
+ )
+ # tok.pad_token = tok.eos_token
+
+ if attn_type == "vllm_blend":
+ llm = LMCacheLLM(
+ model=model_name,
+ enable_prefix_caching=True,
+ max_model_len=max_seq_length,
+ tensor_parallel_size=tensor_parallel_size,
+ enable_chunked_prefill=False,
+ trust_remote_code=trust_remote_code,
+ gpu_memory_utilization=0.5,
+ swap_space=64,
+ )
+ llm = GreedySearch_vLLM(llm, tok)
+ elif attn_type == "vllm_kv":
+ llm = LLM(
+ model=model_name,
+ max_model_len=max_seq_length,
+ tensor_parallel_size=tensor_parallel_size,
+ enable_chunked_prefill=False,
+ trust_remote_code=True,
+ swap_space=64,
+ enforce_eager=True,
+ enable_kvcompress=True,
+ block_size=16,
+ kv_head_bias_path=None,
+ kv_head_bias_weight=0,
+ disable_log_stats=True,
+ prefill_metric_collection_window_size=32,
+ prefill_metric_collection_block_size=4096,
+ max_kv_per_compression=50_000_000,
+ metric_aggregation="L2-sum",
+ maxpool_metrics=True,
+ )
+ llm = GreedySearch_vLLM(
+ llm,
+ tok,
+ is_kv_compress=True,
+ )
+ elif "vllm" in attn_type:
+ # num_gpus
+ llm = LLM(
+ model=model_name,
+ enable_prefix_caching="Jamba" not in model_name,
+ max_model_len=max_seq_length,
+ tensor_parallel_size=tensor_parallel_size,
+ enable_chunked_prefill=False,
+ trust_remote_code=trust_remote_code,
+ swap_space=64,
+ )
+ if attn_type != "vllm":
+ minference_patch = MInference(
+ attn_type,
+ model_name,
+ config_path=topk_dims_file_path,
+ starting_layer=starting_layer,
+ attn_kwargs=hyper_param,
+ )
+ llm = minference_patch(llm)
+ llm = GreedySearch_vLLM(llm, tok)
+ else:
+ minference_patch = MInference(
+ attn_type.replace("_sink", ""),
+ model_name,
+ config_path=topk_dims_file_path,
+ starting_layer=starting_layer,
+ kv_type=kv_type,
+ is_search=is_search,
+ kv_cache_cpu=kv_cache_cpu,
+ kv_cache_cpu_device=kv_cache_cpu_device,
+ attn_kwargs=hyper_param,
+ )
+ if "mamba" in model_name.lower() or "recurrentgemma" in model_name.lower():
+ llm = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map="auto",
+ resume_download=None,
+ trust_remote_code=trust_remote_code,
+ )
+ llm = GreedySearch_Mamba2(llm, tok)
+
+ return llm, tok
+ else:
+ llm = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map="auto",
+ trust_remote_code=trust_remote_code,
+ attn_implementation="flash_attention_2",
+ )
+ llm = minference_patch(llm)
+
+ if attn_type == "inf_llm":
+ llm = GreedySearch_InfLLM(llm.model, tok)
+ return llm, tok
+ elif kv_type in ["retr_attn", "kivi"]:
+ llm = GreedySearch_RetrAttn(
+ llm,
+ tok,
+ )
+ return llm, tok
+
+ llm = GreedySearch(
+ llm,
+ tok,
+ )
+
+ print("Model and tokenizer loaded.")
+ return llm, tok
+
+
+if __name__ == "__main__":
+ args = parse_args()
+
+ # check_benchmark_availability(args.data_dir)
+ model_name = args.model_name_or_path
+ max_seq_length = args.max_seq_length
+ real_model_name = model_name.split("/")[-1]
+ data_name = args.task
+ scdq_mode = args.same_context_different_query
+
+ if "," in data_name:
+ data_names = data_name.split(",")
+ else:
+ data_names = [data_name]
+
+ if max_seq_length == -1:
+ max_seq_length = 160_000
+
+ # Model
+ model, tok = load_model(
+ model_name,
+ args.topk,
+ args.starting_layer,
+ args.topk_dims_file_path,
+ args.use_sparq,
+ attn_type=args.attn_type,
+ max_seq_length=max_seq_length,
+ is_search=args.is_search,
+ kv_type=args.kv_type,
+ trust_remote_code=args.trust_remote_code,
+ kv_cache_cpu=args.kv_cache_cpu,
+ kv_cache_cpu_device=args.kv_cache_cpu_device,
+ tensor_parallel_size=args.tensor_parallel_size,
+ hyper_param=args.hyper_param.copy(),
+ )
+
+ disable_golden_context = (
+ "_disable_golden_context" if args.disable_golden_context else ""
+ )
+ verbalize_hyper_param = (
+ f"_{'-'.join([f'{k}={v}' for k, v in args.hyper_param.items() if k != 'best_pattern'])}"
+ if args.hyper_param
+ else ""
+ )
+ result_dir = Path(
+ args.output_dir,
+ f"{real_model_name}_{args.attn_type}{disable_golden_context}_{args.kv_type}{verbalize_hyper_param}",
+ )
+ result_dir.mkdir(exist_ok=True, parents=True)
+ use_scdq = "_scdq" if scdq_mode else "_multi_turn"
+ use_llmlingua = "_lingua" if args.use_llmlingua else ""
+ real_model_name = f"{real_model_name}_{args.attn_type}{use_scdq}{disable_golden_context}_{args.kv_type}{verbalize_hyper_param}" # add all the args to the real_model_name, for easy identification
+
+ results = {}
+ for data_name in data_names:
+ max_new_tokens = DATA_NAME_TO_MAX_NEW_TOKENS[data_name]
+ if isinstance(max_new_tokens, dict):
+ assert (
+ max(max_new_tokens.values()) <= max_seq_length
+ ), "max_new_tokens must be less than max_seq_length"
+ elif max_new_tokens >= max_seq_length:
+ max_new_tokens = 500
+
+ # Data
+ output_path = (
+ result_dir / f"prediction_{data_name}{use_scdq}{use_llmlingua}.jsonl"
+ )
+ examples = load_dataset("microsoft/SCBench", data_name, split="test")
+
+ if args.use_llmlingua:
+ # do prompt compression here
+ compression_ratio = (
+ args.hyper_param.get("llmlingua_ratio", 3) if args.hyper_param else 3
+ )
+ examples = get_compressed_examples(
+ examples, data_name, args.data_dir, rate=1 / compression_ratio
+ )
+ max_turn_size = len(examples[0]["multi_turns"])
+ if args.max_turns > 0 and args.max_turns < max_turn_size:
+ examples = [
+ {**eg, "multi_turns": eg["multi_turns"][: args.max_turns]}
+ for eg in examples
+ ]
+ max_turn_size = args.max_turns
+
+ if args.num_eval_examples != -1:
+ num_eval_examples = min(args.num_eval_examples, len(examples))
+ examples = examples[:num_eval_examples]
+
+ preds = []
+ print(f"==== Evaluation {data_name}====")
+ print(f"# examples: {len(examples)}")
+ print(f"Num eval examples: {args.num_eval_examples}")
+ print(f"Verbose: {args.verbose}")
+ print(f"Max new tokens: {max_new_tokens}")
+ print(f"Num of turns: {max_turn_size}")
+
+ done = set()
+ if os.path.exists(output_path) and not args.rewrite:
+ print(f"Output file {output_path} exists. Loading from file.")
+ with open(output_path, "r", encoding="utf-8") as f:
+ for line in f:
+ tmp = json.loads(line)
+ done.add(int(tmp["id"]))
+ preds.append(tmp)
+ # examples = examples[len(preds):]
+ compute_scores(
+ output_path, data_name, real_model_name, max_seq_length, scdq_mode
+ )
+
+ for i, eg in tqdm(enumerate(examples)):
+ if i < args.start_example_id or i in done:
+ continue
+ if data_name in [
+ "scbench_summary_with_needles",
+ "scbench_repoqa_and_kv",
+ ]:
+ max_input_length = max_seq_length - (
+ sum(list(max_new_tokens.values())) * max_turn_size // 2
+ )
+ else:
+ max_input_length = max_seq_length - max_new_tokens * max_turn_size
+ if scdq_mode:
+ max_input_length -= 1000
+
+ pred = get_pred(
+ model,
+ eg,
+ data_name,
+ max_new_tokens,
+ max_input_length=max_input_length,
+ attn_type=args.attn_type,
+ tok=tok,
+ use_chat_template=args.use_chat_template,
+ scdq_mode=scdq_mode,
+ disable_golden_context=args.disable_golden_context,
+ )
+ # a list of ground truth answers for each turn
+ gts = get_ground_truth(eg, data_name)
+ for turn_idx, (ans, gt, turn) in enumerate(
+ zip(pred["answers"], gts, eg["multi_turns"])
+ ):
+ case = {
+ "id": i,
+ "turn_idx": turn_idx,
+ "prediction": ans,
+ "ground_truth": gt,
+ }
+ if "task" in pred:
+ case["task"] = pred["task"][turn_idx]
+ if data_name == "scbench_repoqa":
+ case["lang"] = eg["lang"]
+ case["repo"] = eg["repo"]
+ case["func_name"] = turn["name"]
+ if data_name == "scbench_repoqa_and_kv":
+ case["lang"] = eg["lang"]
+ case["repo"] = eg["repo"]
+ if turn["task"] == "scbench_repoqa":
+ case["func_name"] = turn["name"]
+ if data_name == "scbench_kv_compressible":
+ case["task"] = eg["task"]
+ preds.append(case)
+ dump_jsonl(preds, output_path)
+ torch.cuda.empty_cache()
+ done.add(i)
+
+ score = compute_scores(
+ output_path,
+ data_name,
+ real_model_name,
+ max_seq_length=max_seq_length,
+ scdq_mode=scdq_mode,
+ )
+ results[data_name] = score
+
+ print("==== Results ====")
+ print(json.dumps(results, indent=2))
+ try:
+ lmcache_vllm.close_lmcache_engine()
+ except:
+ pass
diff --git a/scbench/scripts/run_all_tasks.sh b/scbench/scripts/run_all_tasks.sh
new file mode 100644
index 00000000..0a96901e
--- /dev/null
+++ b/scbench/scripts/run_all_tasks.sh
@@ -0,0 +1,41 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+TASKS=("scbench_kv" "scbench_prefix_suffix" "scbench_vt" "scbench_repoqa" "scbench_qa_eng" "scbench_qa_chn" "scbench_choice_eng" "scbench_many_shot" "scbench_summary" "scbench_mf" "scbench_summary_with_needles" "scbench_repoqa_and_kv")
+ATTN_KV_TYPES=(
+ "vllm;dense" # FullAttention
+ "vllm_minference;dense" "vllm_a_shape;dense" "vllm_tri_shape;dense" # 1) KV Cache Generation Stage
+ "dense;streamingllm" "dense;snapkv" "dense;pyramidkv" "dense;kivi" # 2) KV Cache Compression Stage
+ "vllm_blend;dense" # 3) KV Cache Retrieval Stage
+ "dense;quest" "dense;retr_attn" # 4) KV Cache Loading Stage
+)
+
+MODE=$3
+if [ "$MODE" == "scdq" ]; then
+ MODE="--same_context_different_query"
+else
+ MODE=""
+fi
+
+for attn_kv_type in ${ATTN_KV_TYPES[@]}; do
+IFS=';' read -r attn_type kv_type <<< "$attn_kv_type"
+echo "attn_type: $attn_type, kv_type: $kv_type"
+for task in ${TASKS[@]}; do
+echo $task
+python run_scbench.py \
+ --task $task \
+ --model_name_or_path $1 \
+ --data_dir ./data \
+ --output_dir ./results \
+ --attn_type $attn_type \
+ --kv_type $kv_type \
+ --use_chat_template \
+ --trust_remote_code \
+ --max_seq_length 131_072 \
+ --tensor_parallel_size $2 ${MODE}
+done
+done
+
+# VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 1 multi-turn
+# VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 2 multi-turn
+# VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_all_tasks.sh meta-llama/Llama-3.1-8B-Instruct 2 scdq
diff --git a/scbench/scripts/run_single_method.sh b/scbench/scripts/run_single_method.sh
new file mode 100644
index 00000000..af291d35
--- /dev/null
+++ b/scbench/scripts/run_single_method.sh
@@ -0,0 +1,42 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+TASKS=("scbench_kv" "scbench_prefix_suffix" "scbench_vt" "scbench_repoqa" "scbench_qa_eng" "scbench_qa_chn" "scbench_choice_eng" "scbench_many_shot" "scbench_summary" "scbench_mf" "scbench_summary_with_needles" "scbench_repoqa_and_kv")
+
+# attn_type, kv_type
+# ATTN_KV_TYPES=(
+# "vllm;dense" # FullAttention
+# "vllm_minference;dense" "vllm_a_shape;dense" "vllm_tri_shape;dense" # 1) KV Cache Generation Stage
+# "dense;streamingllm" "dense;snapkv" "dense;pyramidkv" "dense;kivi" # 2) KV Cache Compression Stage
+# "vllm_blend;dense" # 3) KV Cache Retrieval Stage
+# "dense;quest" "dense;retr_attn" # 4) KV Cache Loading Stage
+# )
+attn_type=$4
+kv_type=$5
+
+MODE=$3
+if [ "$MODE" == "scdq" ]; then
+ MODE="--same_context_different_query"
+else
+ MODE=""
+fi
+
+echo "attn_type: $attn_type, kv_type: $kv_type"
+for task in ${TASKS[@]}; do
+echo $task
+python run_scbench.py \
+ --task $task \
+ --model_name_or_path $1 \
+ --data_dir ./data \
+ --output_dir ./results \
+ --attn_type $attn_type \
+ --kv_type $kv_type \
+ --use_chat_template \
+ --trust_remote_code \
+ --max_seq_length 131_072 \
+ --tensor_parallel_size $2 ${MODE}
+done
+
+# VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 1 multi-turn vllm dense
+# VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 2 multi-turn vllm dense
+# VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 CUDA_VISIBLE_DEVICES=0,1 VLLM_WORKER_MULTIPROC_METHOD=spawn bash scripts/run_single_method.sh meta-llama/Llama-3.1-8B-Instruct 2 scdq vllm dense
diff --git a/scbench/scripts/test_llama.sh b/scbench/scripts/test_llama.sh
new file mode 100644
index 00000000..c958cb34
--- /dev/null
+++ b/scbench/scripts/test_llama.sh
@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+python run_scbench.py \
+ --task scbench_kv \
+ --model_name_or_path meta-llama/Llama-3.1-8B-Instruct \
+ --data_dir ./data \
+ --output_dir ./results \
+ --rewrite \
+ --attn_type dense \
+ --kv_type dense \
+ --use_chat_template \
+ --trust_remote_code
diff --git a/scbench/scripts/test_minference_with_snapkv.sh b/scbench/scripts/test_minference_with_snapkv.sh
new file mode 100644
index 00000000..80d80a48
--- /dev/null
+++ b/scbench/scripts/test_minference_with_snapkv.sh
@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+python run_scbench.py \
+ --task scbench_kv \
+ --model_name_or_path meta-llama/Llama-3.1-8B-Instruct \
+ --data_dir ./data \
+ --output_dir ./results \
+ --rewrite \
+ --attn_type minference \
+ --kv_type snapkv \
+ --use_chat_template \
+ --trust_remote_code
diff --git a/scbench/setup/setup_kivi.sh b/scbench/setup/setup_kivi.sh
new file mode 100644
index 00000000..5015f441
--- /dev/null
+++ b/scbench/setup/setup_kivi.sh
@@ -0,0 +1,7 @@
+# Copyright (c) 2024 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+
+git clone https://github.com/jy-yuan/KIVI.git /tmp/KIVI
+cd /tmp/KIVI
+pip install -e . --no-build-isolation --no-deps
+cd quant && pip install -e .
diff --git a/setup.py b/setup.py
index 6854d3ec..c618763d 100644
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import os
@@ -9,7 +9,7 @@
import warnings
import torch
-from packaging.version import Version, parse
+from packaging.version import parse
from setuptools import find_packages, setup
from torch.utils.cpp_extension import CUDA_HOME, BuildExtension, CUDAExtension
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
@@ -35,9 +35,9 @@
INSTALL_REQUIRES = [
"transformers>=4.37.0",
- "accelerate",
"torch",
"triton",
+ "einops",
]
QUANLITY_REQUIRES = [
"black==21.4b0",
diff --git a/tests/test_e2e.py b/tests/test_e2e.py
index 29ea242c..3370435f 100644
--- a/tests/test_e2e.py
+++ b/tests/test_e2e.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Microsoft
+# Copyright (c) 2024-2025 Microsoft
# Licensed under The MIT License [see LICENSE for details]
import unittest
@@ -8,45 +8,54 @@
from minference import MInference
+ATTN_TYPES = ["dense", "a_shape", "tri_shape", "minference", "flexprefill"]
+KV_TYPES = [
+ "dense",
+ "snapkv",
+ "pyramidkv",
+ "quest",
+ "streamingllm",
+ "retr_attn",
+ "kivi",
+]
+
class MInferenceE2ETester(unittest.TestCase):
"""
End2end Test for MInference
"""
- def __init__(self, *args, **kwargs):
- super(MInferenceE2ETester, self).__init__(*args, **kwargs)
-
+ @classmethod
+ def setUpClass(cls):
# paramaters
- model_name = "gradientai/Llama-3-8B-Instruct-262k"
- trust_remote_code = False
- attn_type = "minference"
- kv_cache_cpu = True
- self.attn_type = attn_type
+ cls.model_name = "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
+ # cls.model_name = "Qwen/Qwen2.5-7B-Instruct"
+ trust_remote_code = True
# init model and tokenizer
- self.tokenizer = AutoTokenizer.from_pretrained(
- model_name, trust_remote_code=trust_remote_code
- )
-
- model = AutoModelForCausalLM.from_pretrained(
- model_name,
+ cls.model = AutoModelForCausalLM.from_pretrained(
+ cls.model_name,
torch_dtype="auto",
device_map="auto",
- trust_remote_code=trust_remote_code,
+ trust_remote_code=True,
+ _attn_implementation="flash_attention_2",
)
- attn_kwargs = {}
- minference_patch = MInference(
- attn_type,
- model_name,
- kv_cache_cpu=kv_cache_cpu,
- attn_kwargs=attn_kwargs,
+ cls.tokenizer = AutoTokenizer.from_pretrained(
+ cls.model_name, trust_remote_code=trust_remote_code
)
- self.model = minference_patch.patch_model(model)
- self.prompt_complex = open("./prompt_hardest.txt").read()
+ cls.prompt_complex = open("./prompt_hardest.txt").read()
+
+ def forward(self, attn_type: str, kv_type: str, attn_kwargs: dict):
+ def load_type():
+ minference_patch = MInference(
+ attn_type=attn_type,
+ model_name=self.model_name,
+ kv_type=kv_type,
+ attn_kwargs=attn_kwargs,
+ )
+ return minference_patch.patch_model(self.model)
- def test_general_minference(self):
def test_different_context_windows(seq_len: int):
input_ids = self.tokenizer(self.prompt_complex)["input_ids"]
n = len(input_ids)
@@ -59,12 +68,43 @@ def test_different_context_windows(seq_len: int):
attention_mask = data["attention_mask"].cuda()
with torch.no_grad():
- if self.attn_type != "inf_llm":
- self.model(input_ids, attention_mask, use_cache=False)
+ if attn_type != "inf_llm":
+ model(
+ input_ids,
+ attention_mask,
+ use_cache=False,
+ logits_to_keep=1,
+ )
else:
- self.model.generate(
+ model.generate(
input_ids, generation_config=GenerationConfig(max_new_tokens=1)
)
+ torch.cuda.empty_cache()
- test_different_context_windows(100000)
- test_different_context_windows(1000000)
+ model = load_type()
+ test_different_context_windows(100_000)
+ # test_different_context_windows(1000000)
+ del model
+ torch.cuda.empty_cache()
+
+ def test_dense(self):
+ self.forward("dense", "dense", {})
+
+ def test_minference(self):
+ attn_kwargs = {}
+ for kv_type in KV_TYPES:
+ with self.subTest(attn_type="minference", kv_type=kv_type):
+ self.forward("minference", kv_type, attn_kwargs)
+
+ def test_all_kv_types(self):
+ attn_kwargs = {}
+ for kv_type in KV_TYPES:
+ with self.subTest(attn_type="dense", kv_type=kv_type):
+ self.forward("dense", kv_type, attn_kwargs)
+
+ def test_all_attn_types(self):
+ attn_kwargs = {}
+ for attn_type in ATTN_TYPES:
+ for kv_type in ["dense"]:
+ with self.subTest(attn_type=attn_type, kv_type=kv_type):
+ self.forward(attn_type, kv_type, attn_kwargs)