> For the complete documentation index, see [llms.txt](https://remi-calizzano.gitbook.io/benchmark-for-transformers/llms.txt). Markdown versions of documentation pages are available by appending `.md` to page URLs; this page is available as [Markdown](https://remi-calizzano.gitbook.io/benchmark-for-transformers/examples-of-benchmark-json-files.md).

# Examples of benchmark json files

## Summarization

```
{
    "scenarios": [
        {
            "name": "google pegasus",
            "model_class": "summarization",
            "model_name": "google/pegasus-xsum",
            "tokenizer_name": "google/pegasus-xsum",
            "init_kwargs": {
                "generation_parameters": {
                    "num_beams": 4,
                    "length_penalty": 0.5,
                    "min_length": 11,
                    "max_length": 62
                }
            },
            "batch_size": 1,
            "device": "cuda"
        },
        {
            "name": "student pegasus 16-8",
            "model_class": "summarization",
            "model_name": "sshleifer/student_pegasus_xsum_16_8",
            "tokenizer_name": "google/pegasus-xsum",
            "init_kwargs": {
                "generation_parameters": {
                    "num_beams": 4,
                    "length_penalty": 0.5,
                    "min_length": 11,
                    "max_length": 62
                }
            },
            "batch_size": 1,
            "device": "cuda"
        },
        {
            "name": "student pegasus 16-4",
            "model_class": "summarization",
            "model_name": "sshleifer/student_pegasus_xsum_16_4",
            "tokenizer_name": "google/pegasus-xsum",
            "init_kwargs": {
                "generation_parameters": {
                    "num_beams": 4,
                    "length_penalty": 0.5,
                    "min_length": 11,
                    "max_length": 62
                }
            },
            "batch_size": 1,
            "device": "cuda"
        }
    ],
    "dataset": {
        "dataset_name": "xsum",
        "split": "test[:10]",
        "x_column_name": ["document"],
        "y_column_name": "summary"
    },
    "metrics": [
        {
            "metric_name": "rouge",
            "values": ["rouge1", "rouge2", "rougeL"],
            "run_kwargs": {"rouge_types": ["rouge1", "rouge2", "rougeL"]}
        }
    ]
}
```

## NER

```
{
    "scenarios": [
        {
            "name": "albert",
            "model_class": "ner",
            "model_name": "KB/albert-base-v2-ner",
            "batch_size": 1
        },
        {
            "name": "bert",
            "model_class": "ner",
            "model_name": "dslim/bert-base-NER",
            "batch_size": 1
        }
    ],
    "dataset": {
        "dataset_name": "wnut_17",
        "split": "validation[:20]",
        "x_column_name": ["tokens"],
        "y_column_name": "labels"
    },
    "metrics": [
        {
            "metric_name": "seqeval",
            "values": ["precision", "recall", "f1", "accuracy"]
        }
    ]
}
```

## Classification

```
{
    "scenarios": [
        {
            "name": "distilbert",
            "model_class": "classification",
            "model_name": "distilbert-base-uncased-finetuned-sst-2-english",
            "tokenizer_name": "distilbert-base-uncased",
            "batch_size": 1
        },
        {
            "name": "bert base",
            "model_class": "classification",
            "model_name": "textattack/bert-base-uncased-SST-2",
            "batch_size": 1
        },
        {
            "name": "quantized distilbert",
            "model_class": "classification",
            "model_name": "distilbert-base-uncased-finetuned-sst-2-english",
            "tokenizer_name": "distilbert-base-uncased",
            "batch_size": 1,
            "quantization": true
        },
        {
            "name": "quantized bert base",
            "model_class": "classification",
            "model_name": "textattack/bert-base-uncased-SST-2",
            "batch_size": 1,
            "quantization": true
        },
        {
            "name": "onnx distilbert",
            "model_class": "classification",
            "model_name": "distilbert-base-uncased-finetuned-sst-2-english",
            "tokenizer_name": "distilbert-base-uncased",
            "onnx": true,
            "batch_size": 1
        },
        {
            "name": "onnx bert base",
            "model_class": "classification",
            "model_name": "textattack/bert-base-uncased-SST-2",
            "onnx": true,
            "batch_size": 1
        },
        {
            "name": "onnx quantized distilbert",
            "model_class": "classification",
            "model_name": "distilbert-base-uncased-finetuned-sst-2-english",
            "tokenizer_name": "distilbert-base-uncased",
            "batch_size": 1,
            "onnx": true,
            "quantization": true
        },
        {
            "name": "onnx quantized bert base",
            "model_class": "classification",
            "model_name": "textattack/bert-base-uncased-SST-2",
            "batch_size": 1,
            "onnx": true,
            "quantization": true
        }
    ],
    "dataset": {
        "dataset_name": "glue",
        "split": "validation",
        "x_column_name": ["sentence"],
        "y_column_name": "label",
        "init_kwargs": {"name": "sst2"}
    },
    "metrics": [
        {
            "metric_name": "glue",
            "values": ["accuracy"],
            "init_kwargs": {"name": "sst2"}
        }
    ]
}
```