update

Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com>

src/transformers/benchmark/benchmark_utils_generic.py

@@ -0,0 +1,151 @@
+# Copyright 2024 The HuggingFace Team and the AllenNLP authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Benchmark utilities - module containing base classes for benchmarking
+"""
+import json
+import os.path
+import timeit
+
+
+class BenchMark:
+    """Base class specifying the methods to be implemented in benchmark subclasses.
+
+    All the methods except `run` are designed to be private: only the `run` method should be used by an end user.
+    """
+
+    def __init__(self, *arg, **kwargs):
+        self._buffer = {"init_kwargs": {}, "runs": []}
+        self._run_buffer = {
+            "config": {
+                "inputs_kwargs": {},
+                "target_kwargs": {},
+                "measure_kwargs": {},
+                "report_kwargs": {},
+            },
+            "result": None,
+        }
+
+    def _reset_run_buffer(self):
+        self._run_buffer = {
+            "config": {
+                "inputs_kwargs": {},
+                "target_kwargs": {},
+                "measure_kwargs": {},
+                "report_kwargs": {},
+            },
+            "result": None,
+        }
+
+    def _measure(self, func, **measure_kwargs):
+        """Return a callable that, when called, will return some measurement results for the argument `func`.
+
+        See `SpeedBenchMark` for an example implementation.
+        """
+        raise NotImplementedError
+
+    def _target(self, **target_kwargs):
+        """Return a callable against which we would like to perform benchmark.
+
+        See `FromPretrainedBenchMark` and `CacheBenchMark` for example implementations.
+        """
+        raise NotImplementedError
+
+    def _inputs(self, **inputs_kwargs):
+        return {}
+
+    def _convert_to_json(self, report):
+        if isinstance(report, list):
+            return [self._convert_to_json(x) for x in report]
+        if isinstance(report, dict):
+            return {k: self._convert_to_json(v) for k, v in report.items()}
+        if isinstance(report, type):
+            return report.__name__
+        return report
+
+    def _report(self, result, output_path=None, only_result=False):
+        self._run_buffer["config"]["report_kwargs"]["output_path"] = output_path
+        self._run_buffer["config"]["report_kwargs"]["only_result"] = only_result
+
+        self._run_buffer["result"] = result
+        self._buffer["runs"].append(self._run_buffer)
+
+        report = {"result": result}
+        if not only_result:
+            report = self._buffer["runs"][-1]
+
+        complete_report = self._convert_to_json(self._buffer)
+        if output_path is not None:
+            if not os.path.isdir(output_path):
+                os.makedirs(output_path)
+            output_path = os.path.join(output_path, "benchmark_report.json")
+
+            with open(output_path, "w", encoding="UTF-8") as fp:
+                json.dump(complete_report, fp, ensure_ascii=False, indent=4)
+
+        report = self._convert_to_json(report)
+
+        return report
+
+    def run(self, measure_kwargs=None, target_kwargs=None, inputs_kwargs=None, report_kwargs=None):
+        self._reset_run_buffer()
+
+        if measure_kwargs is None:
+            measure_kwargs = {}
+        if target_kwargs is None:
+            target_kwargs = {}
+        if inputs_kwargs is None:
+            inputs_kwargs = {}
+        if report_kwargs is None:
+            report_kwargs = {}
+        if measure_kwargs is None:
+            measure_kwargs = {}
+
+        target = self._target(**target_kwargs)
+
+        all_inputs_kwargs = [inputs_kwargs] if isinstance(inputs_kwargs, dict) else inputs_kwargs
+        results = []
+        for _inputs_kwargs in all_inputs_kwargs:
+            inputs = self._inputs(**_inputs_kwargs)
+            result = self._measure(target, **measure_kwargs)(**inputs)
+            results.append(result)
+
+        if isinstance(inputs_kwargs, dict):
+            results = results[0]
+
+        return self._report(results, **report_kwargs)
+
+
+class SpeedBenchMark(BenchMark):
+    """A simple class used to benchmark the running time of a callable."""
+
+    def _measure(self, func, number=3, repeat=1):
+        self._run_buffer["config"]["measure_kwargs"]["number"] = number
+        self._run_buffer["config"]["measure_kwargs"]["repeat"] = repeat
+
+        def wrapper(*args, **kwargs):
+            # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
+
+            def _func():
+                func(*args, **kwargs)
+
+            runtimes = timeit.repeat(
+                _func,
+                repeat=repeat,
+                number=number,
+            )
+
+            return {"time": min(runtimes) / number}
+
+        return wrapper

src/transformers/benchmark/from_pretrained_benchmark.py

@@ -0,0 +1,75 @@
+# Copyright 2024 The HuggingFace Team and the AllenNLP authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Benchmark for models' `from_pretrained` method
+"""
+import argparse
+import json
+
+from benchmark_utils_generic import BenchMark, SpeedBenchMark
+
+import transformers
+
+
+class FromPretrainedBenchMark(BenchMark):
+    def _target(self, model_class, repo_id):
+        self._run_buffer["config"]["target_kwargs"]["model_class"] = model_class
+        self._run_buffer["config"]["target_kwargs"]["repo_id"] = repo_id
+
+        def target():
+            _ = getattr(transformers, model_class).from_pretrained(repo_id)
+
+        return target
+
+
+class FromPretrainedSpeedBenchMark(SpeedBenchMark, FromPretrainedBenchMark):
+    pass
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--config_path",
+        default=None,
+        type=str,
+        required=False,
+        help="Path to a prepared run file or a previously run output file.",
+    )
+    parser.add_argument(
+        "--output_path", type=str, required=True, help="Path to the output file where the run's info. will be saved."
+    )
+    args = parser.parse_args()
+
+    if args.config_path is None:
+        init_kwargs = {}
+
+        repo_id = "bert-base-uncased"
+        run_kwargs = {
+            "measure_kwargs": {"number": 2, "repeat": 3},
+            "target_kwargs": {"model_class": "AutoModel", "repo_id": repo_id},
+            "inputs_kwargs": [{}],
+            "report_kwargs": {},
+        }
+        run_configs = [run_kwargs]
+    else:
+        with open(args.config_path) as fp:
+            config = json.load(fp)
+            init_kwargs = config["init_kwargs"]
+            run_configs = [run["config"] for run in config["runs"]]
+
+    benchmark = FromPretrainedSpeedBenchMark(**init_kwargs)
+
+    for run_config in run_configs:
+        run_config["report_kwargs"]["output_path"] = args.output_path
+        result = benchmark.run(**run_config)

utils/benchmark.py

@@ -0,0 +1,98 @@
+# Copyright 2024 The HuggingFace Team and the AllenNLP authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+A handy script to run benchmark script(s) against a list of git commits.
+
+Example: python utils/benchmark.py --benchmark_path src/transformers/benchmark/from_pretrained_benchmark.py --base_output_path "./bench_reports" --commits "62408e13,6c97e80b0"
+"""
+
+
+import argparse
+import os
+from contextlib import contextmanager
+from pathlib import Path
+
+from git import Repo
+
+
+PATH_TO_REPO = Path(__file__).parent.parent.resolve()
+
+
+@contextmanager
+def checkout_commit(repo: Repo, commit_id: str):
+    """
+    Context manager that checks out a given commit when entered, but gets back to the reference it was at on exit.
+
+    Args:
+        repo (`git.Repo`): A git repository (for instance the Transformers repo).
+        commit_id (`str`): The commit reference to checkout inside the context manager.
+    """
+    current_head = repo.head.commit if repo.head.is_detached else repo.head.ref
+
+    try:
+        repo.git.checkout(commit_id)
+        yield
+
+    finally:
+        repo.git.checkout(current_head)
+
+
+if __name__ == "__main__":
+
+    def list_str(values):
+        return values.split(",")
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--benchmark_path",
+        type=str,
+        required=True,
+        help="Path to the benchmark script to run.",
+    )
+    parser.add_argument(
+        "--base_output_path",
+        type=str,
+        required=True,
+        help="Base path to the output file where the run's info. will be saved.",
+    )
+    parser.add_argument(
+        "--config_path",
+        default=None,
+        type=str,
+        required=False,
+        help="Path to a prepared run file or a previously run output file.",
+    )
+    parser.add_argument(
+        "--commits",
+        type=list_str,
+        required=True,
+        help="Comma-separated list of commit SHA values against which the benchmark will be run",
+    )
+
+    args = parser.parse_args()
+
+    repo = Repo(PATH_TO_REPO)
+
+    for commit in args.commits:
+        with checkout_commit(repo, commit):
+            print(f"benchmark against commit: {repo.head.commit}")
+
+            output_path = os.path.join(args.base_output_path, f"{commit}")
+            commandline_args = f"--output_path {output_path}"
+            if args.config_path is not None:
+                commandline_args += " --config_path {args.config_path}"
+
+            # TODO: use `subprocess`
+            os.system(f"python {args.benchmark_path} {commandline_args}")

utils/not_doctested.txt

@@ -332,6 +332,9 @@ src/transformers/benchmark/benchmark_args_tf.py
 src/transformers/benchmark/benchmark_args_utils.py
 src/transformers/benchmark/benchmark_tf.py
 src/transformers/benchmark/benchmark_utils.py
+src/transformers/benchmark/benchmark_utils_generic.py
+src/transformers/benchmark/cache_benchmark.py
+src/transformers/benchmark/from_pretrained_benchmark.py
 src/transformers/commands/add_new_model.py
 src/transformers/commands/add_new_model_like.py
 src/transformers/commands/convert.py