diff --git a/tftrt/benchmarking-python/CHANGELOG.md b/tftrt/benchmarking-python/CHANGELOG.md index ea417f3b5..5845993e9 100644 --- a/tftrt/benchmarking-python/CHANGELOG.md +++ b/tftrt/benchmarking-python/CHANGELOG.md @@ -46,6 +46,17 @@ Description of the change +## [1.2.0] - 2022.07.31 - @DEKHTIARJonathan + +Setting up the benchmarking suite to allow remote upload and storage of the +metrics and experiments. + +Adding arguments: +* `--model_name`: Name of the model being executed. +* `--model_source`: Name of the model's source that originally published the model. +* `--experiment_name`: Name of the experiment being conducted. +* `--upload_metrics_endpoint`: Distant endpoint being used to push metrics to. + ## [1.1.0] - 2022.07.25 - @DEKHTIARJonathan Replacing all `print()` calls by `logging.()` calls diff --git a/tftrt/benchmarking-python/benchmark_args.py b/tftrt/benchmarking-python/benchmark_args.py index 7567daebf..042602843 100644 --- a/tftrt/benchmarking-python/benchmark_args.py +++ b/tftrt/benchmarking-python/benchmark_args.py @@ -175,6 +175,15 @@ def __init__(self): "generated and used at every iteration." ) + self._parser.add_argument( + "--precision", + type=str, + choices=self.ALLOWED_PRECISION_MODES, + default="FP32", + help="Precision mode to use. FP16 and INT8 modes only works if " + "--use_tftrt is used." + ) + # =========================== TF-TRT Flags ========================== # self._add_bool_argument( @@ -232,15 +241,6 @@ def __init__(self): help="If set to True, TensorRT engines are built before runtime." ) - self._parser.add_argument( - "--precision", - type=str, - choices=self.ALLOWED_PRECISION_MODES, - default="FP32", - help="Precision mode to use. FP16 and INT8 modes only works if " - "--use_tftrt is used." - ) - self._add_bool_argument( name="use_dynamic_shape", default=False, @@ -248,7 +248,31 @@ def __init__(self): help="Whether to use implicit batch mode or dynamic shape mode." ) - # =========================== DEBUG Flags ========================== # + # =========================== Metric Flags ========================== # + + self._parser.add_argument( + "--experiment_name", + type=str, + default=None, + help="Name of the experiment being run, only used for archiving " + "objectives: exports in JSON or CSV." + ) + + self._parser.add_argument( + "--model_name", + type=str, + required=True, + default=None, + help="Name of the model being benchmarked." + ) + + self._parser.add_argument( + "--model_source", + type=str, + required=True, + default=None, + help="Source of the model where it was originally published." + ) self._parser.add_argument( "--export_metrics_json_path", @@ -266,6 +290,16 @@ def __init__(self): "to the set location in CSV format for further processing." ) + self._parser.add_argument( + "--upload_metrics_endpoint", + type=str, + default=None, + help="If set, the benchmark will upload the metrics in JSON format " + "to the set endpoint using a PUT requests." + ) + + # =========================== TF Profiling =========================== # + self._parser.add_argument( "--tf_profile_export_path", type=str, @@ -281,6 +315,8 @@ def __init__(self): help="If set to True, will add extra information to the TF Profile." ) + # ============================ Debug Flags =========================== # + self._add_bool_argument( name="debug", default=False, @@ -388,6 +424,9 @@ def _validate_args(self, args): "doesn't exist or is not a directory" ) + if args.upload_metrics_endpoint is not None: + raise NotImplementedError("This feature is not yet implemented.") + def _post_process_args(self, args): if args.use_synthetic_data: # This variable is not used in synthetic data mode. diff --git a/tftrt/benchmarking-python/benchmark_info.py b/tftrt/benchmarking-python/benchmark_info.py index c42a01571..8ba860fae 100644 --- a/tftrt/benchmarking-python/benchmark_info.py +++ b/tftrt/benchmarking-python/benchmark_info.py @@ -10,7 +10,7 @@ # The `__version__` number shall be updated everytime core benchmarking files # are updated. # Please update CHANGELOG.md with a description of what this version changed. -__version__ = "1.1.0" +__version__ = "1.2.0" def get_commit_id(): diff --git a/tftrt/benchmarking-python/huggingface/bert/base_run_inference.sh b/tftrt/benchmarking-python/huggingface/bert/base_run_inference.sh index a95cd2cad..e2a6b9f95 100755 --- a/tftrt/benchmarking-python/huggingface/bert/base_run_inference.sh +++ b/tftrt/benchmarking-python/huggingface/bert/base_run_inference.sh @@ -143,6 +143,8 @@ python ${BASE_DIR}/infer.py \ --calib_data_dir ${DATA_DIR} \ --input_saved_model_dir ${INPUT_SAVED_MODEL_DIR} \ --output_saved_model_dir /tmp/$RANDOM \ + --model_name "${MODEL_NAME}" \ + --model_source "huggingface" \ --batch_size ${BATCH_SIZE} \ --vocab_size ${VOCAB_SIZE} \ --sequence_length=${SEQ_LEN} \ diff --git a/tftrt/benchmarking-python/huggingface/gpt2/base_run_inference.sh b/tftrt/benchmarking-python/huggingface/gpt2/base_run_inference.sh index 1e5f70a8c..71e890f6a 100755 --- a/tftrt/benchmarking-python/huggingface/gpt2/base_run_inference.sh +++ b/tftrt/benchmarking-python/huggingface/gpt2/base_run_inference.sh @@ -105,6 +105,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ + --model_name "${MODEL_NAME}" \ + --model_source "huggingface" \ --tokenizer_model_dir=${TOKENIZER_DIR} \ --batch_size=${BATCH_SIZE} \ --sequence_length=${SEQ_LEN} \ diff --git a/tftrt/benchmarking-python/huggingface/t5/base_run_inference.sh b/tftrt/benchmarking-python/huggingface/t5/base_run_inference.sh index e39737869..98024e876 100755 --- a/tftrt/benchmarking-python/huggingface/t5/base_run_inference.sh +++ b/tftrt/benchmarking-python/huggingface/t5/base_run_inference.sh @@ -116,6 +116,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ + --model_name "${MODEL_NAME}" \ + --model_source "huggingface" \ --tokenizer_model_dir=${TOKENIZER_DIR}\ --vocab_model_dir=${TOKENIZER_DIR}\ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ diff --git a/tftrt/benchmarking-python/image_classification/base_run_inference.sh b/tftrt/benchmarking-python/image_classification/base_run_inference.sh index 1c4a44836..15bf02601 100755 --- a/tftrt/benchmarking-python/image_classification/base_run_inference.sh +++ b/tftrt/benchmarking-python/image_classification/base_run_inference.sh @@ -79,12 +79,6 @@ case ${MODEL_NAME} in NUM_CLASSES=1000 ;; - "resnet50-v1.5_tf1_ngc" ) - NUM_CLASSES=1000 - OUTPUT_TENSORS_NAME="classes" - PREPROCESS_METHOD="resnet50_v1_5_tf1_ngc" - ;; - "resnet50v2_backbone" | "resnet50v2_sparse_backbone" ) INPUT_SIZE=256 OUTPUT_TENSORS_NAME="outputs" @@ -151,6 +145,8 @@ python ${BASE_DIR}/infer.py \ --calib_data_dir ${DATA_DIR} \ --input_saved_model_dir ${INPUT_SAVED_MODEL_DIR} \ --output_saved_model_dir /tmp/$RANDOM \ + --model_name "${MODEL_NAME}" \ + --model_source "tf_models_image" \ --input_size ${INPUT_SIZE} \ --preprocess_method ${PREPROCESS_METHOD} \ --num_classes ${NUM_CLASSES} \ diff --git a/tftrt/benchmarking-python/image_classification/infer.py b/tftrt/benchmarking-python/image_classification/infer.py index ecba62ace..645d1e068 100644 --- a/tftrt/benchmarking-python/image_classification/infer.py +++ b/tftrt/benchmarking-python/image_classification/infer.py @@ -64,7 +64,7 @@ def __init__(self): self._parser.add_argument( '--preprocess_method', type=str, - choices=['vgg', 'inception', 'resnet50_v1_5_tf1_ngc'], + choices=['vgg', 'inception'], default='vgg', help='The image preprocessing method used in dataloading.' ) diff --git a/tftrt/benchmarking-python/image_classification/models/resnet50-v1.5_tf1_ngc/run_inference.sh b/tftrt/benchmarking-python/image_classification/models/resnet50-v1.5_tf1_ngc/run_inference.sh deleted file mode 100755 index f01361b3f..000000000 --- a/tftrt/benchmarking-python/image_classification/models/resnet50-v1.5_tf1_ngc/run_inference.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." - -bash ${BASE_DIR}/base_run_inference.sh --model_name="resnet50-v1.5_tf1_ngc" ${@} diff --git a/tftrt/benchmarking-python/nvidia_examples/bert_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/bert_tf2/run_inference.sh index 0e2e2122f..a193b62c5 100755 --- a/tftrt/benchmarking-python/nvidia_examples/bert_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/bert_tf2/run_inference.sh @@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/squad/bert_tf2 \ --calib_data_dir=/data/squad/bert_tf2 \ --input_saved_model_dir=/models/nvidia_examples/bert_tf2 \ + --model_name "bert_tf2" \ + --model_source "nvidia_examples" \ --batch_size=64 \ --output_tensors_name="end_positions,start_positions" \ --total_max_samples=11000 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/dlrm_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/dlrm_tf2/run_inference.sh index 13f01b797..4b97869b9 100755 --- a/tftrt/benchmarking-python/nvidia_examples/dlrm_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/dlrm_tf2/run_inference.sh @@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/criteo \ --calib_data_dir=/data/criteo \ --input_saved_model_dir=/models/nvidia_examples/dlrm_tf2 \ + --model_name "dlrm_tf2" \ + --model_source "nvidia_examples" \ --batch_size=65536 \ --output_tensors_name="output_1" \ --total_max_samples=92000000 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B0_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B0_tf2/run_inference.sh index 4fd28349c..f3200505f 100755 --- a/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B0_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B0_tf2/run_inference.sh @@ -12,6 +12,8 @@ python ${BASE_DIR}/infer.py \ --calib_data_dir=/data/imagenet \ --input_saved_model_dir=/models/nvidia_examples/efficientnet_v1_B0_tf2 \ --batch_size=128 \ + --model_name "efficientnet_v1_B0_tf2" \ + --model_source "nvidia_examples" \ --output_tensors_name="output_1" \ --total_max_samples=55000 \ --input_size=224 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B4_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B4_tf2/run_inference.sh index 74569aafc..d19f692ef 100755 --- a/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B4_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B4_tf2/run_inference.sh @@ -11,6 +11,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/imagenet \ --calib_data_dir=/data/imagenet \ --input_saved_model_dir=/models/nvidia_examples/efficientnet_v1_B4_tf2 \ + --model_name "efficientnet_v1_B4_tf2" \ + --model_source "nvidia_examples" \ --batch_size=128 \ --output_tensors_name="output_1" \ --total_max_samples=55000 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/efficientnet_v2_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/efficientnet_v2_tf2/run_inference.sh index ba5115207..725c1446a 100755 --- a/tftrt/benchmarking-python/nvidia_examples/efficientnet_v2_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/efficientnet_v2_tf2/run_inference.sh @@ -11,6 +11,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/imagenet \ --calib_data_dir=/data/imagenet \ --input_saved_model_dir=/models/nvidia_examples/efficientnet_v2_tf2 \ + --model_name "efficientnet_v2_tf2" \ + --model_source "nvidia_examples" \ --batch_size=128 \ --output_tensors_name="output_1" \ --total_max_samples=55000 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/electra_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/electra_tf2/run_inference.sh index 72f7dcd71..5d41307c3 100755 --- a/tftrt/benchmarking-python/nvidia_examples/electra_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/electra_tf2/run_inference.sh @@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/squad/electra \ --calib_data_dir=/data/squad/electra \ --input_saved_model_dir=/models/nvidia_examples/electra_tf2 \ + --model_name "electra_tf2" \ + --model_source "nvidia_examples" \ --batch_size=64 \ --do_lower_case \ --output_tensors_name="tf_electra_for_question_answering,tf_electra_for_question_answering_1,tf_electra_for_question_answering_2,tf_electra_for_question_answering_3,tf_electra_for_question_answering_4" \ diff --git a/tftrt/benchmarking-python/nvidia_examples/mrcnn_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/mrcnn_tf2/run_inference.sh index 83561204f..02671fbf2 100755 --- a/tftrt/benchmarking-python/nvidia_examples/mrcnn_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/mrcnn_tf2/run_inference.sh @@ -17,6 +17,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/coco2017/tfrecord \ --calib_data_dir=/data/coco2017/tfrecord \ --input_saved_model_dir=/models/nvidia_examples/mrcnn_tf2 \ + --model_name "mrcnn_tf2" \ + --model_source "nvidia_examples" \ --batch_size=8 \ --output_tensors_name="detection_boxes,detection_classes,detection_scores,image_info,num_detections,source_ids" \ --total_max_samples=5200 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/nnunet2d_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/nnunet2d_tf2/run_inference.sh index 24354dfd1..0699f1314 100755 --- a/tftrt/benchmarking-python/nvidia_examples/nnunet2d_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/nnunet2d_tf2/run_inference.sh @@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/msd_task01/numpy_2d_bigger_tf2 \ --calib_data_dir=/data/msd_task01/numpy_2d_bigger_tf2 \ --input_saved_model_dir=/models/nvidia_examples/nnunet2d_tf2 \ + --model_name "nnunet2d_tf2" \ + --model_source "nvidia_examples" \ --batch_size=1 \ --num_image_slices=32 \ --output_tensors_name="output_1" \ diff --git a/tftrt/benchmarking-python/nvidia_examples/nnunet3d_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/nnunet3d_tf2/run_inference.sh index 273c0812f..a41f79e6b 100755 --- a/tftrt/benchmarking-python/nvidia_examples/nnunet3d_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/nnunet3d_tf2/run_inference.sh @@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/msd_task01/numpy_3d_bigger_tf2 \ --calib_data_dir=/data/msd_task01/numpy_3d_bigger_tf2 \ --input_saved_model_dir=/models/nvidia_examples/nnunet3d_tf2 \ + --model_name "nnunet3d_tf2" \ + --model_source "nvidia_examples" \ --batch_size=1 \ --output_tensors_name="output_1" \ --total_max_samples=500 \ diff --git a/tftrt/benchmarking-python/image_classification/models/resnet50-v1.5_tf1_ngc/analysis.txt b/tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/analysis.txt similarity index 100% rename from tftrt/benchmarking-python/image_classification/models/resnet50-v1.5_tf1_ngc/analysis.txt rename to tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/analysis.txt diff --git a/tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/infer.py b/tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/infer.py new file mode 100644 index 000000000..5f0fc9134 --- /dev/null +++ b/tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/infer.py @@ -0,0 +1,165 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= + +import os +import sys + +import numpy as np + +import tensorflow as tf + +# Allow import of top level python files +import inspect + +currentdir = os.path.dirname( + os.path.abspath(inspect.getfile(inspect.currentframe())) +) + +benchmark_base_dir = os.path.dirname(os.path.dirname(currentdir)) +sys.path.insert(0, benchmark_base_dir) +sys.path.insert(0, os.path.join(benchmark_base_dir, "image_classification")) + +from benchmark_args import BaseCommandLineAPI +from benchmark_runner import BaseBenchmarkRunner + +from image_classification.dataloading import get_dataloader +from image_classification import preprocessing + + +class CommandLineAPI(BaseCommandLineAPI): + + def __init__(self): + super(CommandLineAPI, self).__init__() + + self._parser.add_argument( + '--input_size', + type=int, + default=224, + help='Size of input images expected by the ' + 'model' + ) + + self._parser.add_argument( + '--num_classes', + type=int, + default=1000, + help='Number of classes used when training ' + 'the model' + ) + + self._parser.add_argument( + '--preprocess_method', + type=str, + choices=['resnet50_v1_5_tf1_ngc'], + default='resnet50_v1_5_tf1_ngc', + help='The image preprocessing method used in dataloading.' + ) + + def _post_process_args(self, args): + args = super(CommandLineAPI, self)._post_process_args(args) + args.labels_shift = 1 if args.num_classes == 1001 else 0 + + return args + + def _validate_args(self, args): + super(CommandLineAPI, self)._validate_args(args) + + if args.input_size != 224: + raise ValueError( + "The argument --input_size must be equal to 224 for this model." + ) + + if args.num_classes != 1000: + raise ValueError( + "The argument --num_classes must be equal to 1000 for this model." + ) + + +class BenchmarkRunner(BaseBenchmarkRunner): + + def get_dataset_batches(self): + """Returns a list of batches of input samples. + + Each batch should be in the form [x, y], where + x is a numpy array of the input samples for the batch, and + y is a numpy array of the expected model outputs for the batch + + Returns: + - dataset: a TF Dataset object + - bypass_data_to_eval: any object type that will be passed unmodified to + `evaluate_result()`. If not necessary: `None` + + Note: script arguments can be accessed using `self._args.attr` + """ + + dataset = get_dataloader(self._args) + + return dataset, None + + def preprocess_model_inputs(self, data_batch): + """This function prepare the `data_batch` generated from the dataset. + Returns: + x: input of the model + y: data to be used for model evaluation + + Note: script arguments can be accessed using `self._args.attr` + """ + + x, y = data_batch + return x, y + + def postprocess_model_outputs(self, predictions, expected): + """Post process if needed the predictions and expected tensors. At the + minimum, this function transforms all TF Tensors into a numpy arrays. + Most models will not need to modify this function. + + Note: script arguments can be accessed using `self._args.attr` + """ + + predictions = predictions.numpy() + + if len(predictions.shape) != 1: + predictions = tf.math.argmax(predictions, axis=1) + predictions = predictions.numpy().reshape(-1) + + predictions - self._args.labels_shift + + return predictions - self._args.labels_shift, expected.numpy() + + def evaluate_model(self, predictions, expected, bypass_data_to_eval): + """Evaluate result predictions for entire dataset. + + This computes overall accuracy, mAP, etc. Returns the + metric value and a metric_units string naming the metric. + + Note: script arguments can be accessed using `self._args.attr` + """ + + return ( + np.mean(predictions["data"] == expected["data"]) * 100.0, + "Top-1 Accuracy %" + ) + + +if __name__ == '__main__': + + cmdline_api = CommandLineAPI() + args = cmdline_api.parse_args() + + runner = BenchmarkRunner(args) + + runner.execute_benchmark() diff --git a/tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/run_inference.sh new file mode 100755 index 000000000..1ad008ed6 --- /dev/null +++ b/tftrt/benchmarking-python/nvidia_examples/resnet50-v1.5_tf1/run_inference.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +nvidia-smi + +BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Default Argument Values +BATCH_SIZE=128 +OUTPUT_TENSOR_NAMES="classes" +TOTAL_MAX_SAMPLES=50000 +DATA_DIR="/data/imagenet" +MODEL_DIR="/models/nvidia_examples/resnet50-v1.5_tf1" + +BYPASS_ARGUMENTS="" + +# Loop through arguments and process them +for arg in "$@" +do + case $arg in + --model_name=*) + MODEL_NAME="${arg#*=}" + shift # Remove --model_name from processing + ;; + --batch_size=*) + BATCH_SIZE="${arg#*=}" + shift # Remove --batch_size= from processing + ;; + --output_tensors_name=*) + OUTPUT_TENSOR_NAMES="${arg#*=}" + shift # Remove --output_tensors_name= from processing + ;; + --data_dir=*) + DATA_DIR="${arg#*=}" + shift # Remove --data_dir= from processing + ;; + --input_saved_model_dir=*) + MODEL_DIR="${arg#*=}" + shift # Remove --input_saved_model_dir= from processing + ;; + --total_max_samples=*) + TOTAL_MAX_SAMPLES="${arg#*=}" + shift # Remove --total_max_samples= from processing + ;; + *) + BYPASS_ARGUMENTS="${BYPASS_ARGUMENTS} ${arg}" + ;; + esac +done + +# Trimming front and back whitespaces +BYPASS_ARGUMENTS=$(echo ${BYPASS_ARGUMENTS} | tr -s " ") + +echo -e "\n********************************************************************" +echo "" +echo "[*] DATA_DIR: ${DATA_DIR}" +echo "[*] MODEL_DIR: ${MODEL_DIR}" +echo "" +echo "[*] BATCH_SIZE: ${BATCH_SIZE}" +echo "[*] OUTPUT_TENSOR_NAMES: ${OUTPUT_TENSOR_NAMES}" +echo "" +echo "[*] BYPASS_ARGUMENTS: ${BYPASS_ARGUMENTS}" + +echo -e "********************************************************************\n" + +if [[ ! -d ${DATA_DIR} ]]; then + echo "ERROR: \`--data_dir=/path/to/directory\` does not exist. [Received: \`${DATA_DIR}\`]" + exit 1 +fi + +if [[ ! -d ${MODEL_DIR} ]]; then + echo "ERROR: \`--input_saved_model_dir=/path/to/directory\` does not exist. [Received: \`${MODEL_DIR}\`]" + exit 1 +fi + +set -x + +python ${BASE_DIR}/infer.py \ + --data_dir=${DATA_DIR} \ + --calib_data_dir=${DATA_DIR} \ + --input_saved_model_dir=${MODEL_DIR} \ + --model_name "resnet50-v1.5_tf1" \ + --model_source "nvidia_examples" \ + --batch_size=${BATCH_SIZE} \ + --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ + --input_size 224 \ + --num_classes 1000 \ + --preprocess_method resnet50_v1_5_tf1_ngc \ + --total_max_samples=${TOTAL_MAX_SAMPLES} \ + ${BYPASS_ARGUMENTS} diff --git a/tftrt/benchmarking-python/nvidia_examples/sim_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/sim_tf2/run_inference.sh index 1b8b644a4..0515faeda 100755 --- a/tftrt/benchmarking-python/nvidia_examples/sim_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/sim_tf2/run_inference.sh @@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/seq_20 \ --calib_data_dir=/data/seq_20 \ --input_saved_model_dir=/models/nvidia_examples/sim_tf2/21.10.0_fp32 \ + --model_name "sim_tf2" \ + --model_source "nvidia_examples" \ --batch_size=256 \ --output_tensors_name="sim_model,sim_model_1" \ --total_max_samples=220000 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/unet_medical_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/unet_medical_tf2/run_inference.sh index 15c4cf670..9006a9423 100755 --- a/tftrt/benchmarking-python/nvidia_examples/unet_medical_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/unet_medical_tf2/run_inference.sh @@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/em_segmentation \ --calib_data_dir=/data/em_segmentation \ --input_saved_model_dir=/models/nvidia_examples/unet_medical_tf2 \ + --model_name "unet_medical_tf2" \ + --model_source "nvidia_examples" \ --batch_size=8 \ --output_tensors_name="output_1" \ --total_max_samples=6500 \ diff --git a/tftrt/benchmarking-python/nvidia_examples/wide_deep_tf2/run_inference.sh b/tftrt/benchmarking-python/nvidia_examples/wide_deep_tf2/run_inference.sh index 9d3c81aba..cc2eeadd6 100755 --- a/tftrt/benchmarking-python/nvidia_examples/wide_deep_tf2/run_inference.sh +++ b/tftrt/benchmarking-python/nvidia_examples/wide_deep_tf2/run_inference.sh @@ -16,6 +16,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=/data/outbrain \ --calib_data_dir=/data/outbrain \ --input_saved_model_dir=/models/nvidia_examples/wide_deep_tf2 \ + --model_name "wide_deep_tf2" \ + --model_source "nvidia_examples" \ --batch_size=32768 \ --output_tensors_name "output_1" \ --total_max_samples=28000000 \ diff --git a/tftrt/benchmarking-python/object_detection/base_run_inference.sh b/tftrt/benchmarking-python/object_detection/base_run_inference.sh index fb80dd6b8..03d0cff69 100755 --- a/tftrt/benchmarking-python/object_detection/base_run_inference.sh +++ b/tftrt/benchmarking-python/object_detection/base_run_inference.sh @@ -147,6 +147,8 @@ python ${BASE_DIR}/infer.py \ --annotation_path ${ANNOTATIONS_DATA_FILE} \ --input_saved_model_dir ${INPUT_SAVED_MODEL_DIR} \ --output_saved_model_dir /tmp/$RANDOM \ + --model_name "${MODEL_NAME}" \ + --model_source "tf_models_object" \ --batch_size ${BATCH_SIZE} \ --input_size ${INPUT_SIZE} \ --max_workspace_size ${MAX_WORKSPACE_SIZE} \ diff --git a/tftrt/benchmarking-python/tf_hub/albert/base_run_inference.sh b/tftrt/benchmarking-python/tf_hub/albert/base_run_inference.sh index 9cdf6ae4a..3afec0190 100755 --- a/tftrt/benchmarking-python/tf_hub/albert/base_run_inference.sh +++ b/tftrt/benchmarking-python/tf_hub/albert/base_run_inference.sh @@ -111,6 +111,8 @@ python ${BASE_DIR}/infer.py \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ --tokenizer_dir=${TOKENIZER_DIR}\ + --model_name "${MODEL_NAME}" \ + --model_source "tf_hub" \ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ `# The following is set because we will be running synthetic benchmarks` \ --total_max_samples=1 \ diff --git a/tftrt/benchmarking-python/tf_hub/electra/base_run_inference.sh b/tftrt/benchmarking-python/tf_hub/electra/base_run_inference.sh index dd8fce722..fede9752f 100755 --- a/tftrt/benchmarking-python/tf_hub/electra/base_run_inference.sh +++ b/tftrt/benchmarking-python/tf_hub/electra/base_run_inference.sh @@ -108,10 +108,12 @@ set -x python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ + --tokenizer_dir=${TOKENIZER_DIR}\ + --input_saved_model_dir=${MODEL_DIR} \ + --model_name "${MODEL_NAME}" \ + --model_source "tf_hub" \ --sequence_length=${SEQ_LEN} \ --vocab_size=${VOCAB_SIZE} \ - --input_saved_model_dir=${MODEL_DIR} \ - --tokenizer_dir=${TOKENIZER_DIR}\ --batch_size=${BATCH_SIZE} \ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ --total_max_samples=${TOTAL_MAX_SAMPLES} \ diff --git a/tftrt/benchmarking-python/tf_hub/movinet/base_run_inference.sh b/tftrt/benchmarking-python/tf_hub/movinet/base_run_inference.sh index e5d711798..b51ef3e4b 100755 --- a/tftrt/benchmarking-python/tf_hub/movinet/base_run_inference.sh +++ b/tftrt/benchmarking-python/tf_hub/movinet/base_run_inference.sh @@ -94,6 +94,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ + --model_name "movinet_${MODEL_NAME}" \ + --model_source "tf_hub" \ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ `# The following is set because we will be running synthetic benchmarks` \ --total_max_samples=1 \ diff --git a/tftrt/benchmarking-python/tf_hub/spice/run_inference.sh b/tftrt/benchmarking-python/tf_hub/spice/run_inference.sh index cc849f746..60f614ca1 100755 --- a/tftrt/benchmarking-python/tf_hub/spice/run_inference.sh +++ b/tftrt/benchmarking-python/tf_hub/spice/run_inference.sh @@ -77,6 +77,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ + --model_name "spice" \ + --model_source "tf_hub" \ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ --batch_size=${BATCH_SIZE} \ --samples_per_input=${SAMPLES_PER_INPUT} \ diff --git a/tftrt/benchmarking-python/tf_hub/swin_transformers/base_run_inference.sh b/tftrt/benchmarking-python/tf_hub/swin_transformers/base_run_inference.sh index 7c1d0e859..8731974dc 100755 --- a/tftrt/benchmarking-python/tf_hub/swin_transformers/base_run_inference.sh +++ b/tftrt/benchmarking-python/tf_hub/swin_transformers/base_run_inference.sh @@ -90,6 +90,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ + --model_name "${MODEL_NAME}" \ + --model_source "tf_hub" \ --input_size=${INPUT_SIZE} \ --batch_size=${BATCH_SIZE} \ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ diff --git a/tftrt/benchmarking-python/tf_hub/vision_transformer/base_run_inference.sh b/tftrt/benchmarking-python/tf_hub/vision_transformer/base_run_inference.sh index 4f029305b..21b09a1bf 100755 --- a/tftrt/benchmarking-python/tf_hub/vision_transformer/base_run_inference.sh +++ b/tftrt/benchmarking-python/tf_hub/vision_transformer/base_run_inference.sh @@ -82,6 +82,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ + --model_name "${MODEL_NAME}" \ + --model_source "tf_hub" \ --batch_size=${BATCH_SIZE} \ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ --total_max_samples=${TOTAL_MAX_SAMPLES} \ diff --git a/tftrt/benchmarking-python/tf_hub/vision_transformer/models/vit_r26_s32_lightaug_classification/run_inference.sh b/tftrt/benchmarking-python/tf_hub/vision_transformer/models/vit_r26_s32_lightaug_classification/run_inference.sh deleted file mode 100755 index 2077a83db..000000000 --- a/tftrt/benchmarking-python/tf_hub/vision_transformer/models/vit_r26_s32_lightaug_classification/run_inference.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -x - -BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." - -bash ${BASE_DIR}/base_run_inference.sh --model_name="vit_r26_s32_lightaug_classification" ${@} \ No newline at end of file diff --git a/tftrt/benchmarking-python/tf_hub/vision_transformer/models/vit_r26_s32_medaug_classification/run_inference.sh b/tftrt/benchmarking-python/tf_hub/vision_transformer/models/vit_r26_s32_medaug_classification/run_inference.sh deleted file mode 100755 index 3aa4f2c33..000000000 --- a/tftrt/benchmarking-python/tf_hub/vision_transformer/models/vit_r26_s32_medaug_classification/run_inference.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -x - -BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/../.." - -bash ${BASE_DIR}/base_run_inference.sh --model_name="vit_r26_s32_medaug_classification" ${@} \ No newline at end of file diff --git a/tftrt/benchmarking-python/tf_hub/yamnet/run_inference.sh b/tftrt/benchmarking-python/tf_hub/yamnet/run_inference.sh index e124e125f..98092fe1e 100755 --- a/tftrt/benchmarking-python/tf_hub/yamnet/run_inference.sh +++ b/tftrt/benchmarking-python/tf_hub/yamnet/run_inference.sh @@ -77,6 +77,8 @@ python ${BASE_DIR}/infer.py \ --data_dir=${DATA_DIR} \ --calib_data_dir=${DATA_DIR} \ --input_saved_model_dir=${MODEL_DIR} \ + --model_name "yamnet" \ + --model_source "tf_hub" \ --output_tensors_name=${OUTPUT_TENSOR_NAMES} \ --batch_size=${BATCH_SIZE} \ --frame_length=${FRAME_LENGTH} \