---
common_options: &common_options
  data:
    throughput:
      reduction_type: 'mean'
      regexp: 'throughput: *(.*?) samples\/sec'
      skip: 2
    accuracy:
      reduction_type: 'final'
      regexp: 'accuracy: *(.*?) \%'
      skip: 1
    loss:
      reduction_type: 'final'
      regexp: 'loss: *(\d*\.\d*)'
      skip: 1
  output:
    - [samples/sec, 'throughput']
    - [accuracy, 'accuracy']
    - [loss, 'loss']

common_options_inference: &common_options_inference
  data:
    throughput:
      reduction_type: 'mean'
      regexp: 'throughput: *(.*?) samples\/sec'
      skip: 0
  output:
    - [samples/sec, 'throughput']

config_options: &config_options
  requirements_path: requirements.txt

tf2_resnet50_train_real_pod4:
  <<: [*common_options, *config_options]
  description:
    Resnet50 8k batch size, 16.16 precision, 1 replica, 4 pipeline stages
  cmd: >-
    python3 train.py
        --config resnet50_mlperf_pod4_bs20
        --num-epochs 5
        --logs-per-epoch 1
        --dataset-path $DATASETS_DIR/imagenet-data
        --validation false

tf2_resnet50_train_real_pod16:
  <<: [*common_options, *config_options]
  description:
    Resnet50 8k batch size, 16.16 precision, 4 replicas, 4 pipeline stages
  cmd: >-
    python3 train.py
        --config resnet50_mlperf_pod16_bs20
        --num-epochs 5
        --logs-per-epoch 1
        --dataset-path $DATASETS_DIR/imagenet-data
        --validation false

tf2_resnet50_train_real_pod16_conv:
  <<: [*common_options, *config_options]
  description:
    Resnet50 8k batch size, 16.16 precision, 4 replicas, 4 pipeline stages
  cmd: >-
    python3 train.py
        --config resnet50_mlperf_pod16_bs20
        --logs-per-epoch 1
        --dataset-path $DATASETS_DIR/imagenet-data
        --checkpoint-output-dir ./checkpoints
        --checkpoint-input-dir ./checkpoints
        --clean-dir False
        --wandb
        --wandb-run-name tf2_resnet50_train_real_pod16_conv

tf2_resnet50_serving_throughput:
  <<: [*common_options_inference, *config_options]
  description:
    Resnet50 serving throughput, batch size 1 and 32
  parameters:
    batchsize: 1,32
  cmd: >-
    python3 send_request.py
        --config resnet50_infer_bs{batchsize}
        --dataset-path $DATASETS_DIR/imagenet-data
        --batch-size {batchsize}
        --port 8502
        --use-async
        --num-threads 16

tf2_resnet50_serving_latency:
  <<: *config_options
  data:
    latency:
        regexp: 'Latency for batch: *(.*?) ms'
        skip: 20
        reduction_type: 'mean'
  output:
    - [latency(ms), 'latency']
  description:
    Resnet50 serving latency, batch size 1
  cmd: >-
    python3 send_request.py
        --config resnet50_infer_bs1
        --dataset-path $DATASETS_DIR/imagenet-data
        --batch-size 1
        --port 8502
        --num-threads 1
        --verbose
