[Benchmarking-Py] 1.2.0 - Adding Metric Push to Remote

tftrt/benchmarking-python/CHANGELOG.md

@@ -46,6 +46,17 @@ Description of the change
 
 <!-- YOU CAN EDIT FROM HERE -->
 
+## [1.2.0] - 2022.07.31 - @DEKHTIARJonathan
+
+Setting up the benchmarking suite to allow remote upload and storage of the
+metrics and experiments.
+
+Adding arguments:
+* `--model_name`: Name of the model being executed.
+* `--model_source`: Name of the model's source that originally published the model.
+* `--experiment_name`: Name of the experiment being conducted.
+* `--upload_metrics_endpoint`: Distant endpoint being used to push metrics to.
+
 ## [1.1.0] - 2022.07.25 - @DEKHTIARJonathan
 
 Replacing all `print()` calls by `logging.<level>()` calls

tftrt/benchmarking-python/benchmark_args.py

@@ -175,6 +175,15 @@ def __init__(self):
             "generated and used at every iteration."
         )
 
+        self._parser.add_argument(
+            "--precision",
+            type=str,
+            choices=self.ALLOWED_PRECISION_MODES,
+            default="FP32",
+            help="Precision mode to use. FP16 and INT8 modes only works if "
+            "--use_tftrt is used."
+        )
+
         # =========================== TF-TRT Flags ========================== #
 
         self._add_bool_argument(
@@ -232,23 +241,38 @@ def __init__(self):
             help="If set to True, TensorRT engines are built before runtime."
         )
 
-        self._parser.add_argument(
-            "--precision",
-            type=str,
-            choices=self.ALLOWED_PRECISION_MODES,
-            default="FP32",
-            help="Precision mode to use. FP16 and INT8 modes only works if "
-            "--use_tftrt is used."
-        )
-
         self._add_bool_argument(
             name="use_dynamic_shape",
             default=False,
             required=False,
             help="Whether to use implicit batch mode or dynamic shape mode."
         )
 
-        # =========================== DEBUG Flags ========================== #
+        # =========================== Metric Flags ========================== #
+
+        self._parser.add_argument(
+            "--experiment_name",
+            type=str,
+            default=None,
+            help="Name of the experiment being run, only used for archiving "
+            "objectives: exports in JSON or CSV."
+        )
+
+        self._parser.add_argument(
+            "--model_name",
+            type=str,
+            required=True,
+            default=None,
+            help="Name of the model being benchmarked."
+        )
+
+        self._parser.add_argument(
+            "--model_source",
+            type=str,
+            required=True,
+            default=None,
+            help="Source of the model where it was originally published."
+        )
 
         self._parser.add_argument(
             "--export_metrics_json_path",
@@ -266,6 +290,16 @@ def __init__(self):
             "to the set location in CSV format for further processing."
         )
 
+        self._parser.add_argument(
+            "--upload_metrics_endpoint",
+            type=str,
+            default=None,
+            help="If set, the benchmark will upload the metrics in JSON format "
+            "to the set endpoint using a PUT requests."
+        )
+
+        # =========================== TF Profiling =========================== #
+
         self._parser.add_argument(
             "--tf_profile_export_path",
             type=str,
@@ -281,6 +315,8 @@ def __init__(self):
             help="If set to True, will add extra information to the TF Profile."
         )
 
+        # ============================ Debug Flags =========================== #
+
         self._add_bool_argument(
             name="debug",
             default=False,
@@ -388,6 +424,9 @@ def _validate_args(self, args):
                         "doesn't exist or is not a directory"
                     )
 
+        if args.upload_metrics_endpoint is not None:
+            raise NotImplementedError("This feature is not yet implemented.")
+
     def _post_process_args(self, args):
         if args.use_synthetic_data:
             # This variable is not used in synthetic data mode.

tftrt/benchmarking-python/benchmark_info.py

@@ -10,7 +10,7 @@
 # The `__version__` number shall be updated everytime core benchmarking files
 # are updated.
 # Please update CHANGELOG.md with a description of what this version changed.
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 
 
 def get_commit_id():

tftrt/benchmarking-python/huggingface/bert/base_run_inference.sh

@@ -143,6 +143,8 @@ python ${BASE_DIR}/infer.py \
     --calib_data_dir ${DATA_DIR} \
     --input_saved_model_dir ${INPUT_SAVED_MODEL_DIR} \
     --output_saved_model_dir /tmp/$RANDOM \
+    --model_name "${MODEL_NAME}" \
+    --model_source "huggingface" \
     --batch_size ${BATCH_SIZE} \
     --vocab_size ${VOCAB_SIZE} \
     --sequence_length=${SEQ_LEN} \

tftrt/benchmarking-python/huggingface/gpt2/base_run_inference.sh

@@ -105,6 +105,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=${DATA_DIR} \
     --calib_data_dir=${DATA_DIR} \
     --input_saved_model_dir=${MODEL_DIR} \
+    --model_name "${MODEL_NAME}" \
+    --model_source "huggingface" \
     --tokenizer_model_dir=${TOKENIZER_DIR} \
     --batch_size=${BATCH_SIZE} \
     --sequence_length=${SEQ_LEN} \

tftrt/benchmarking-python/huggingface/t5/base_run_inference.sh

@@ -116,6 +116,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=${DATA_DIR} \
     --calib_data_dir=${DATA_DIR} \
     --input_saved_model_dir=${MODEL_DIR} \
+    --model_name "${MODEL_NAME}" \
+    --model_source "huggingface" \
     --tokenizer_model_dir=${TOKENIZER_DIR}\
     --vocab_model_dir=${TOKENIZER_DIR}\
     --output_tensors_name=${OUTPUT_TENSOR_NAMES} \

tftrt/benchmarking-python/image_classification/base_run_inference.sh

@@ -79,12 +79,6 @@ case ${MODEL_NAME} in
     NUM_CLASSES=1000
     ;;
 
-  "resnet50-v1.5_tf1_ngc" )
-    NUM_CLASSES=1000
-    OUTPUT_TENSORS_NAME="classes"
-    PREPROCESS_METHOD="resnet50_v1_5_tf1_ngc"
-    ;;
-
   "resnet50v2_backbone" | "resnet50v2_sparse_backbone" )
     INPUT_SIZE=256
     OUTPUT_TENSORS_NAME="outputs"
@@ -151,6 +145,8 @@ python ${BASE_DIR}/infer.py \
     --calib_data_dir ${DATA_DIR} \
     --input_saved_model_dir ${INPUT_SAVED_MODEL_DIR} \
     --output_saved_model_dir /tmp/$RANDOM \
+    --model_name "${MODEL_NAME}" \
+    --model_source "tf_models_image" \
     --input_size ${INPUT_SIZE} \
     --preprocess_method ${PREPROCESS_METHOD} \
     --num_classes ${NUM_CLASSES} \

tftrt/benchmarking-python/image_classification/infer.py

@@ -64,7 +64,7 @@ def __init__(self):
         self._parser.add_argument(
             '--preprocess_method',
             type=str,
-            choices=['vgg', 'inception', 'resnet50_v1_5_tf1_ngc'],
+            choices=['vgg', 'inception'],
             default='vgg',
             help='The image preprocessing method used in dataloading.'
         )

tftrt/benchmarking-python/nvidia_examples/bert_tf2/run_inference.sh

@@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/squad/bert_tf2 \
     --calib_data_dir=/data/squad/bert_tf2 \
     --input_saved_model_dir=/models/nvidia_examples/bert_tf2 \
+    --model_name "bert_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=64 \
     --output_tensors_name="end_positions,start_positions" \
     --total_max_samples=11000 \

tftrt/benchmarking-python/nvidia_examples/dlrm_tf2/run_inference.sh

@@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/criteo \
     --calib_data_dir=/data/criteo \
     --input_saved_model_dir=/models/nvidia_examples/dlrm_tf2 \
+    --model_name "dlrm_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=65536 \
     --output_tensors_name="output_1" \
     --total_max_samples=92000000 \

tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B0_tf2/run_inference.sh

@@ -12,6 +12,8 @@ python ${BASE_DIR}/infer.py \
     --calib_data_dir=/data/imagenet \
     --input_saved_model_dir=/models/nvidia_examples/efficientnet_v1_B0_tf2 \
     --batch_size=128 \
+    --model_name "efficientnet_v1_B0_tf2" \
+    --model_source "nvidia_examples" \
     --output_tensors_name="output_1" \
     --total_max_samples=55000 \
     --input_size=224 \

tftrt/benchmarking-python/nvidia_examples/efficientnet_v1_B4_tf2/run_inference.sh

@@ -11,6 +11,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/imagenet \
     --calib_data_dir=/data/imagenet \
     --input_saved_model_dir=/models/nvidia_examples/efficientnet_v1_B4_tf2 \
+    --model_name "efficientnet_v1_B4_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=128 \
     --output_tensors_name="output_1" \
     --total_max_samples=55000 \

tftrt/benchmarking-python/nvidia_examples/efficientnet_v2_tf2/run_inference.sh

@@ -11,6 +11,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/imagenet \
     --calib_data_dir=/data/imagenet \
     --input_saved_model_dir=/models/nvidia_examples/efficientnet_v2_tf2 \
+    --model_name "efficientnet_v2_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=128 \
     --output_tensors_name="output_1" \
     --total_max_samples=55000 \

tftrt/benchmarking-python/nvidia_examples/electra_tf2/run_inference.sh

@@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/squad/electra \
     --calib_data_dir=/data/squad/electra \
     --input_saved_model_dir=/models/nvidia_examples/electra_tf2 \
+    --model_name "electra_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=64 \
     --do_lower_case \
     --output_tensors_name="tf_electra_for_question_answering,tf_electra_for_question_answering_1,tf_electra_for_question_answering_2,tf_electra_for_question_answering_3,tf_electra_for_question_answering_4" \

tftrt/benchmarking-python/nvidia_examples/mrcnn_tf2/run_inference.sh

@@ -17,6 +17,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/coco2017/tfrecord \
     --calib_data_dir=/data/coco2017/tfrecord \
     --input_saved_model_dir=/models/nvidia_examples/mrcnn_tf2 \
+    --model_name "mrcnn_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=8 \
     --output_tensors_name="detection_boxes,detection_classes,detection_scores,image_info,num_detections,source_ids" \
     --total_max_samples=5200 \

tftrt/benchmarking-python/nvidia_examples/nnunet2d_tf2/run_inference.sh

@@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/msd_task01/numpy_2d_bigger_tf2 \
     --calib_data_dir=/data/msd_task01/numpy_2d_bigger_tf2 \
     --input_saved_model_dir=/models/nvidia_examples/nnunet2d_tf2 \
+    --model_name "nnunet2d_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=1 \
     --num_image_slices=32 \
     --output_tensors_name="output_1" \

tftrt/benchmarking-python/nvidia_examples/nnunet3d_tf2/run_inference.sh

@@ -10,6 +10,8 @@ python ${BASE_DIR}/infer.py \
     --data_dir=/data/msd_task01/numpy_3d_bigger_tf2 \
     --calib_data_dir=/data/msd_task01/numpy_3d_bigger_tf2 \
     --input_saved_model_dir=/models/nvidia_examples/nnunet3d_tf2 \
+    --model_name "nnunet3d_tf2" \
+    --model_source "nvidia_examples" \
     --batch_size=1 \
     --output_tensors_name="output_1" \
     --total_max_samples=500 \