{
    "bert_config": {
        "hidden_size": 768,
        "vocab_size": 30400,
        "num_attention_heads": 12,
        "num_hidden_layers": 12,
        "intermediate_size": 3072,
        "hidden_dropout_prob": 0.1,
        "attention_probs_dropout_prob": 0.1,
        "max_position_embeddings": 512,
        "type_vocab_size": 2,
        "initializer_range": 0.02,
        "layer_norm_eps": 1e-12,
        "position_embedding_type": "absolute"
    },
    "global_batch": {
        "replicas": 4,
        "micro_batch_size": 4,
        "grad_acc_steps_per_replica": 20
    },
    "num_epochs": 3,
    "ipu_config": {
        "pipeline_stages": [
            ["emb"],
            ["hid","hid", "hid", "hid"],
            ["hid","hid", "hid", "hid"],
            ["hid","hid", "hid", "hid"],
            ["qa_head"]
        ],
       "pipeline_device_mapping": [0, 1, 2, 3, 0],
       "matmul_available_memory_proportion_per_pipeline_stage": [0.25, 0.23, 0.28, 0.25],
       "replicated_tensor_sharding": false
    },
    "optimizer_opts": {
        "name": "LAMB",
        "weight_decay_rate": 0.01,
        "learning_rate": {
            "schedule_name": "up_down",
            "max_learning_rate": 1e-3,
            "warmup_frac": 0.3
        }
    },
    "seed": 1984,
    "global_batches_per_log": 1,
    "enable_wandb": false,
    "wandb_tags": ["squad_fine_tuning"],
    "dataset_dir": "./cache/",
    "output_dir": "./output_dir_hf/"
}