Fix examples (#323)

IlyasMoutawwakil · web-flow · commit f8cc4f1fcfab · 2025-08-17T14:36:24.000+02:00
diff --git a/.github/workflows/test_api_cpu.yaml b/.github/workflows/test_api_cpu.yaml
@@ -55,16 +55,3 @@ jobs:
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           PUSH_REPO_ID: optimum-benchmark/cpu
-
-      # no examples for now
-      # - if: ${{
-      #     (github.event_name == 'push') ||
-      #     (github.event_name == 'workflow_dispatch') ||
-      #     contains( github.event.pull_request.labels.*.name, 'examples')
-      #     }}
-      #   name: Run examples
-      #   run: |
-      #     pytest tests/test_examples.py -s -k "api and cpu"
-      #   env:
-      #     HF_TOKEN: ${{ secrets.HF_TOKEN }}
-      #     PUSH_REPO_ID: optimum-benchmark/cpu
diff --git a/.github/workflows/test_api_cuda.yaml b/.github/workflows/test_api_cuda.yaml
@@ -51,15 +51,10 @@ jobs:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           PUSH_REPO_ID: optimum-benchmark/cuda
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: |
           pip install -e .[testing,torchao,gptqmodel]
-          pytest tests/test_examples.py -x -s -k "api and cuda and pytorch"
+          pytest tests/test_examples.py -x -s -k "api and cuda"
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           PUSH_REPO_ID: optimum-benchmark/cuda
diff --git a/.github/workflows/test_cli_cpu_ipex.yaml b/.github/workflows/test_cli_cpu_ipex.yaml
@@ -53,10 +53,5 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and ipex"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: pytest tests/test_examples.py -s -k "cli and cpu and ipex"
diff --git a/.github/workflows/test_cli_cpu_llama_cpp.yaml b/.github/workflows/test_cli_cpu_llama_cpp.yaml
@@ -53,10 +53,5 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and llama_cpp"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: pytest tests/test_examples.py -s -k "cli and cpu and llama_cpp"
diff --git a/.github/workflows/test_cli_cpu_onnxruntime.yaml b/.github/workflows/test_cli_cpu_onnxruntime.yaml
@@ -53,10 +53,5 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and onnxruntime"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: pytest tests/test_examples.py -s -k "cli and cpu and onnxruntime"
diff --git a/.github/workflows/test_cli_cpu_openvino.yaml b/.github/workflows/test_cli_cpu_openvino.yaml
@@ -53,10 +53,5 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and openvino"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: pytest tests/test_examples.py -s -k "cli and cpu and openvino"
diff --git a/.github/workflows/test_cli_cpu_py_txi.yaml b/.github/workflows/test_cli_cpu_py_txi.yaml
@@ -46,18 +46,9 @@ jobs:
           pip install uv
           uv pip install --upgrade pip
           uv pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          uv pip install -e .[testing,py-txi] git+https://github.com/IlyasMoutawwakil/py-txi.git
+          uv pip install -e .[testing,py-txi]
         env:
           UV_SYSTEM_PYTHON: 1
 
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and py_txi"
-
-      # no examples for now
-      # - if: ${{
-      #     (github.event_name == 'push') ||
-      #     (github.event_name == 'workflow_dispatch') ||
-      #     contains( github.event.pull_request.labels.*.name, 'examples')
-      #     }}
-      #   name: Run examples
-      #   run: pytest tests/test_examples.py -s -k "cli and cpu and (tgi or tei)"
diff --git a/.github/workflows/test_cli_cpu_pytorch.yaml b/.github/workflows/test_cli_cpu_pytorch.yaml
@@ -52,12 +52,3 @@ jobs:
 
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and pytorch"
-
-      # no examples for now
-      # - if: ${{
-      #     (github.event_name == 'push') ||
-      #     (github.event_name == 'workflow_dispatch') ||
-      #     contains( github.event.pull_request.labels.*.name, 'examples')
-      #     }}
-      #   name: Run examples
-      #   run: pytest tests/test_examples.py -s -k "cli and cpu and pytorch"
diff --git a/.github/workflows/test_cli_cuda_onnxruntime.yaml b/.github/workflows/test_cli_cuda_onnxruntime.yaml
@@ -48,12 +48,3 @@ jobs:
       - name: Run tests
         run: |
           pytest tests/test_cli.py -x -s -k "cli and cuda and onnxruntime"
-
-      # no examples for now
-      # - if: ${{
-      #     (github.event_name == 'push') ||
-      #     (github.event_name == 'workflow_dispatch') ||
-      #     contains( github.event.pull_request.labels.*.name, 'examples')
-      #     }}
-      #   name: Run examples
-      #   run: pytest tests/test_examples.py -x -s -k "cli and cuda and onnxruntime"
diff --git a/.github/workflows/test_cli_cuda_py_txi.yaml b/.github/workflows/test_cli_cuda_py_txi.yaml
@@ -47,19 +47,14 @@ jobs:
           pip install uv
           uv pip install --upgrade pip
           uv pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-          uv pip install -e .[testing,py-txi] git+https://github.com/IlyasMoutawwakil/py-txi.git
+          uv pip install -e .[testing,py-txi]
         env:
           UV_SYSTEM_PYTHON: 1
 
       - name: Run tests
         run: |
           FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and py_txi"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: |
           FORCE_SEQUENTIAL=1 pytest tests/test_examples.py -x -s -k "cli and cuda and (tgi or tei)"
diff --git a/.github/workflows/test_cli_cuda_pytorch.yaml b/.github/workflows/test_cli_cuda_pytorch.yaml
@@ -50,14 +50,8 @@ jobs:
         run: |
           pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and not (dp or tp or ddp or device_map or deepspeed) and not (bnb or awq or gptq)"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: |
-          pip install accelerate@git+https://github.com/huggingface/accelerate.git
           pytest tests/test_examples.py -x -s -k "cli and cuda and pytorch"
 
   run_cli_cuda_pytorch_multi_gpu_tests:
diff --git a/.github/workflows/test_cli_cuda_tensorrt_llm.yaml b/.github/workflows/test_cli_cuda_tensorrt_llm.yaml
@@ -50,12 +50,7 @@ jobs:
         run: |
           pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and not (tp or pp)"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: |
           pytest tests/test_examples.py -x -s -k "cli and cuda and trt"
 
diff --git a/.github/workflows/test_cli_cuda_vllm.yaml b/.github/workflows/test_cli_cuda_vllm.yaml
@@ -50,12 +50,7 @@ jobs:
         run: |
           FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and not (tp or pp)"
 
-      - if: ${{
-          (github.event_name == 'push') ||
-          (github.event_name == 'workflow_dispatch') ||
-          contains( github.event.pull_request.labels.*.name, 'examples')
-          }}
-        name: Run examples
+      - name: Run examples
         run: |
           pytest tests/test_examples.py -x -s -k "cli and cuda and vllm"
 
@@ -87,4 +82,4 @@ jobs:
 
       - name: Run tests (sequential)
         run: |
-          FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and (tp)" # add "or pp"
+          FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and (tp or pp)"
diff --git a/examples/cpu_ipex_bert.yaml b/examples/cpu_ipex_bert.yaml
@@ -16,8 +16,7 @@ launcher:
 
 backend:
   device: cpu
-  export: true
-  no_weights: false # on multi-node machines, intializing weights in the benchmark could harm performance
+  no_weights: false # on multi-node machines, inline weights initialization harms performance
   torch_dtype: float32 # use bfloat16 on compatible Intel CPUs
   model: google-bert/bert-base-uncased
 
diff --git a/examples/cpu_ipex_llama.yaml b/examples/cpu_ipex_llama.yaml
@@ -16,7 +16,6 @@ launcher:
 
 backend:
   device: cpu
-  export: true
   no_weights: false # on multi-node machines, intializing weights in the benchmark could harm performance
   torch_dtype: float32 # use bfloat16 on compatible Intel CPUs
   model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
diff --git a/examples/cpu_openvino_8bit_bert.yaml b/examples/cpu_openvino_8bit_bert.yaml
@@ -6,7 +6,7 @@ defaults:
   - _base_
   - _self_
 
-name: openvino_static_quant
+name: cpu_openvino_8bit_bert
 
 backend:
   device: cpu
diff --git a/optimum_benchmark/backends/base.py b/optimum_benchmark/backends/base.py
@@ -118,8 +118,11 @@ def create_no_weights_model_fast(self) -> None:
             HF_API.hf_hub_download(self.config.model, filename="config.json", cache_dir=self.tmpdir.name)
         ).parent
         save_model(model=torch.nn.Linear(1, 1), filename=model_path / "model.safetensors", metadata={"format": "pt"})
-        self.pretrained_processor.save_pretrained(save_directory=model_path)
-        self.pretrained_config.save_pretrained(save_directory=model_path)
+
+        if self.pretrained_processor is not None:
+            self.pretrained_processor.save_pretrained(save_directory=model_path)
+        if self.pretrained_config is not None:
+            self.pretrained_config.save_pretrained(save_directory=model_path)
 
         if self.config.task in TEXT_GENERATION_TASKS:
             self.generation_config.eos_token_id = None
diff --git a/optimum_benchmark/backends/ipex/backend.py b/optimum_benchmark/backends/ipex/backend.py
@@ -55,18 +55,13 @@ def load_ipexmodel_from_pretrained(self) -> None:
     def load_ipexmodel_with_no_weights(self) -> None:
         with fast_weights_init():
             original_model, self.config.model = self.config.model, self.no_weights_model_path.as_posix()
-            original_export, self.config.export = self.config.export, True
             self.load_ipexmodel_from_pretrained()
-            self.config.export = original_export
             self.config.model = original_model
 
     @property
     def ipexmodel_kwargs(self) -> Dict[str, Any]:
         kwargs = {}
 
-        if self.config.export:
-            kwargs["export"] = self.config.export
-
         if self.config.torch_dtype is not None:
             kwargs["torch_dtype"] = getattr(torch, self.config.torch_dtype)
 
diff --git a/optimum_benchmark/backends/ipex/config.py b/optimum_benchmark/backends/ipex/config.py
@@ -16,7 +16,6 @@ class IPEXConfig(BackendConfig):
     no_weights: bool = False
 
     # ipexmodel kwargs
-    export: Optional[bool] = None
     torch_dtype: Optional[str] = None
 
     def __post_init__(self):
diff --git a/optimum_benchmark/backends/onnxruntime/backend.py b/optimum_benchmark/backends/onnxruntime/backend.py
@@ -197,6 +197,7 @@ def optimize_onnx_files(self) -> None:
             one_external_file=True,
             file_suffix="",
         )
+
         if self.pretrained_processor is not None:
             self.pretrained_processor.save_pretrained(self.optimized_model)
         if self.pretrained_config is not None:
@@ -278,7 +279,6 @@ def quantize_onnx_files(self) -> None:
 
         if self.pretrained_processor is not None:
             self.pretrained_processor.save_pretrained(self.quantized_model)
-
         if self.pretrained_config is not None:
             self.pretrained_config.save_pretrained(self.quantized_model)
 
@@ -295,9 +295,9 @@ def prepare_inputs(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
             if isinstance(value, torch.Tensor):
                 inputs[key] = value.to(self.config.device)
 
-        for key in list(inputs.keys()):
-            if hasattr(self.pretrained_model, "input_names") and key not in self.pretrained_model.input_names:
-                inputs.pop(key)
+        for input_name in list(inputs.keys()):
+            if hasattr(self.pretrained_model, "input_names") and input_name not in self.pretrained_model.input_names:
+                inputs.pop(input_name)
 
         return inputs
 
diff --git a/optimum_benchmark/backends/vllm/backend.py b/optimum_benchmark/backends/vllm/backend.py
@@ -71,17 +71,6 @@ def prepare_inputs(self, inputs: Dict[str, Any]) -> Dict[str, Any]:
 
         return inputs
 
-    def batch_offline_engine_generate(self, inputs: Dict[str, Any], kwargs: Dict[str, Any]) -> Any:
-        for i, prompt in enumerate(inputs["prompts"]):
-            self.pretrained_model.add_request(
-                prompt=prompt,
-                request_id=str(i),
-                params=self.get_sampling_params(kwargs),
-            )
-
-        while self.pretrained_model.has_unfinished_requests():
-            self.pretrained_model.step()
-
     def get_sampling_params(self, kwargs: Dict[str, Any]) -> SamplingParams:
         return SamplingParams(
             ignore_eos=True,
@@ -93,8 +82,19 @@ def get_sampling_params(self, kwargs: Dict[str, Any]) -> SamplingParams:
             logits_processors=kwargs.get("logits_processors", None),
         )
 
+    def batch_offline_engine_generate(self, inputs: Dict[str, Any], kwargs: Dict[str, Any]) -> Any:
+        for i, prompt in enumerate(inputs["prompts"]):
+            self.pretrained_model.add_request(
+                prompt=prompt,
+                request_id=str(i),
+                params=self.get_sampling_params(kwargs),
+            )
+
+        while self.pretrained_model.has_unfinished_requests():
+            self.pretrained_model.step()
+
     async def single_online_engine_generate(self, prompt: str, request_id: str, kwargs: Dict[str, Any]) -> Any:
-        stream = await self.pretrained_model.add_request(
+        stream = self.pretrained_model.add_request(
             prompt=prompt,
             request_id=request_id,
             params=self.get_sampling_params(kwargs),
diff --git a/setup.py b/setup.py
@@ -68,9 +68,9 @@
     "onnxruntime": [f"optimum[onnxruntime]>={MIN_OPTIMUM_VERSION}"],
     "onnxruntime-gpu": [f"optimum[onnxruntime-gpu]>={MIN_OPTIMUM_VERSION}"],
     # other backends
+    "py-txi": ["py-txi@git+https://github.com/IlyasMoutawwakil/py-txi.git"],
     "llama-cpp": ["llama-cpp-python"],
     "llm-swarm": ["llm-swarm"],
-    "py-txi": ["py-txi"],
     "vllm": ["vllm"],
     # optional dependencies
     "sentence-transformers": ["sentence-transformers"],
diff --git a/tests/test_energy_star.py b/tests/test_energy_star.py
@@ -68,12 +68,12 @@ def test_cli_configs(config_name):
         "scenario.energy=true",
         "scenario.memory=true",
         "scenario.latency=true",
-        "scenario.num_samples=2",
-        "scenario.warmup_runs=2",
-        "scenario.input_shapes.batch_size=2",
-        "++scenario.generate_kwargs.max_new_tokens=10",
-        "++scenario.generate_kwargs.min_new_tokens=10",
-        "++scenario.call_kwargs.num_inference_steps=2",
+        "scenario.num_samples=1",
+        "scenario.warmup_runs=1",
+        "scenario.input_shapes.batch_size=1",
+        "++scenario.generate_kwargs.max_new_tokens=16",
+        "++scenario.generate_kwargs.min_new_tokens=16",
+        "++scenario.call_kwargs.num_inference_steps=4",
         "launcher.device_isolation=false",
         f"backend.model={model}",
     ]
diff --git a/tests/test_examples.py b/tests/test_examples.py
