From 018859f94d46c4e2bc8f895a31acf998f35f7ecd Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 08:42:01 -0400 Subject: [PATCH 01/11] explicitly putting precision --- energy_star/text_generation.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index cef9bf39..f313170a 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -18,7 +18,7 @@ backend: no_weights: False task: text-generation model: openai/gpt-oss-20b - torch_dtype: auto + torch_dtype: bfloat16 device_map: auto scenario: @@ -27,8 +27,6 @@ scenario: num_samples: 1000 truncation: True reasoning: True - reasoning_params: - reasoning_effort: high input_shapes: batch_size: 1 From eb1157e66047cc08ccfbdbac03bcdd4a71ec9705 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 08:43:00 -0400 Subject: [PATCH 02/11] removing reasoning --- energy_star/text_generation.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index f313170a..18cc1080 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -26,8 +26,6 @@ scenario: text_column_name: text num_samples: 1000 truncation: True - reasoning: True - input_shapes: batch_size: 1 From 48e5c6d269b340da9e390a7381d81ef207bc3544 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 08:45:46 -0400 Subject: [PATCH 03/11] is it the length thing again --- optimum_benchmark/preprocessors/dataset_preprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum_benchmark/preprocessors/dataset_preprocessor.py b/optimum_benchmark/preprocessors/dataset_preprocessor.py index ab0a92a4..84c4f647 100644 --- a/optimum_benchmark/preprocessors/dataset_preprocessor.py +++ b/optimum_benchmark/preprocessors/dataset_preprocessor.py @@ -287,7 +287,7 @@ def tokenize_function(examples): return pretrained_processor( examples[scenario_config.text_column_name], truncation=scenario_config.truncation, - max_length=max_length - new_tokens, + max_length=min(max_length, 2048) - new_tokens, return_token_type_ids=False, padding=padding, ) From 9e10f681714c6a760e83c71e4069a1ec1372fb21 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 08:56:13 -0400 Subject: [PATCH 04/11] and if we remove --- energy_star/text_generation.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index 18cc1080..3de926c7 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -26,6 +26,8 @@ scenario: text_column_name: text num_samples: 1000 truncation: True + reasoning: False + input_shapes: batch_size: 1 From 1af746bf62798b7e5710c0f70bbe45c323945d2c Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 08:57:13 -0400 Subject: [PATCH 05/11] another model --- energy_star/text_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index 3de926c7..348a6b51 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -17,7 +17,7 @@ backend: device_ids: 0 no_weights: False task: text-generation - model: openai/gpt-oss-20b + model: HuggingFaceTB/SmolLM3-3B torch_dtype: bfloat16 device_map: auto From e24e8265ccb2b9b32b819c8b09b37247fbb23a06 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 09:35:29 -0400 Subject: [PATCH 06/11] how about high --- energy_star/text_generation.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index 348a6b51..83db96c9 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -17,7 +17,7 @@ backend: device_ids: 0 no_weights: False task: text-generation - model: HuggingFaceTB/SmolLM3-3B + model: openai/gpt-oss-20b torch_dtype: bfloat16 device_map: auto @@ -26,7 +26,9 @@ scenario: text_column_name: text num_samples: 1000 truncation: True - reasoning: False + reasoning: True + reasoning_params: + reasoning_effort: high input_shapes: batch_size: 1 From 06b9b52c42849adaf3cac904e7cf52cb7d588bd8 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 09:51:36 -0400 Subject: [PATCH 07/11] testing low reasoning --- energy_star/text_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index 83db96c9..76258d50 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -28,7 +28,7 @@ scenario: truncation: True reasoning: True reasoning_params: - reasoning_effort: high + reasoning_effort: low input_shapes: batch_size: 1 From 772f7576854da83b9584a19425fdc9e0890954d1 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 10:03:28 -0400 Subject: [PATCH 08/11] do I need to put it twice --- energy_star/text_generation.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index 76258d50..154771ae 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -17,7 +17,7 @@ backend: device_ids: 0 no_weights: False task: text-generation - model: openai/gpt-oss-20b + model: Qwen/Qwen3-30B-A3B torch_dtype: bfloat16 device_map: auto @@ -28,7 +28,7 @@ scenario: truncation: True reasoning: True reasoning_params: - reasoning_effort: low + enable_thinking: True input_shapes: batch_size: 1 From 0b97279c845472c5d2604a352944102fbd3214ac Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 10:05:15 -0400 Subject: [PATCH 09/11] nope --- energy_star/text_generation.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index 154771ae..7452db44 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -27,8 +27,6 @@ scenario: num_samples: 1000 truncation: True reasoning: True - reasoning_params: - enable_thinking: True input_shapes: batch_size: 1 From 96b92e1c073e44ef43ed165336169b071dd627a4 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 10:11:37 -0400 Subject: [PATCH 10/11] smaller model --- energy_star/text_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index 7452db44..a5a4ca14 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -17,7 +17,7 @@ backend: device_ids: 0 no_weights: False task: text-generation - model: Qwen/Qwen3-30B-A3B + model: HuggingFaceTB/SmolLM3-3B torch_dtype: bfloat16 device_map: auto From 2b0163fd657bf196a47d0f0e87ff56c14a90d8b6 Mon Sep 17 00:00:00 2001 From: Sasha Date: Wed, 20 Aug 2025 10:24:52 -0400 Subject: [PATCH 11/11] why is it so slow --- energy_star/text_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/energy_star/text_generation.yaml b/energy_star/text_generation.yaml index a5a4ca14..348a6b51 100644 --- a/energy_star/text_generation.yaml +++ b/energy_star/text_generation.yaml @@ -26,7 +26,7 @@ scenario: text_column_name: text num_samples: 1000 truncation: True - reasoning: True + reasoning: False input_shapes: batch_size: 1