nemo-evaluator-launcher-configs/local_nvidia_nemotron_3_nano_30b_a3b.yaml

# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

defaults:
  - execution: local
  - deployment: none
  - _self_

execution:
  output_dir: ./results_nvidia_nemotron_3_nano_30b_a3b
  mounts:
    evaluation:
      ./hf_cache: /root/.cache/huggingface
  env_vars:
    evaluation: {}

target:
  api_endpoint:
    model_id: nvidia/nemotron-nano-3-30b-a3b
    url: https://integrate.api.nvidia.com/v1/chat/completions
    api_key_name: NGC_API_KEY # API Key with access to build.nvidia.com

evaluation:
  env_vars:
    HF_TOKEN: HF_TOKEN
    JUDGE_API_KEY: JUDGE_API_KEY # API Key with access to gpt-4o for HLE
    HF_HOME: HF_HOME
  nemo_evaluator_config:
    config:
      params:
        max_new_tokens: 131072
        temperature: 0.99999
        top_p: 0.99999
        parallelism: 512
        request_timeout: 3600
        max_retries: 10
        extra:
          tokenizer: NVIDIA-Nemotron-Nano-3-30B-A3B-BF16
          tokenizer_backend: huggingface
    target:
      api_endpoint:
        adapter_config:
          use_caching: true
          tracking_requests_stats: true
          log_failed_requests: true
          use_request_logging: true
          max_logged_requests: 10
          use_response_logging: true
          max_logged_responses: 10
  tasks:
  - name: ns_bfcl_v3
    env_vars:
      HF_TOKEN: HF_TOKEN
    nemo_evaluator_config:
      config:
        params:
          temperature: 0.6
          top_p: 0.95
          parallelism: 32
          extra:
            num_repeats: 1
            args: ++use_client_parsing=False
      target:
        api_endpoint:
          adapter_config:
            use_caching: false
  - name: ns_bfcl_v4
    env_vars:
      HF_TOKEN: HF_TOKEN
    nemo_evaluator_config:
      config:
        params:
          max_new_tokens: 8192
          parallelism: 128
          temperature: 0.6
          top_p: 0.95
          extra:
            num_repeats: 1
            args: ++use_client_parsing=False
  - name: ns_livecodebench
    env_vars:
      HF_TOKEN: HF_TOKEN
    nemo_evaluator_config:
      config:
        params:
          extra:
            num_repeats: 8
            dataset_split: test_v5_2407_2412
  - name: ns_mmlu_pro
    env_vars:
      HF_TOKEN: HF_TOKEN
    nemo_evaluator_config:
      config:
        params:
          extra:
            num_repeats: 1
            args: "++prompt_config=eval/aai/mcq-10choices-boxed"
  - name: ns_gpqa
    env_vars:
      HF_TOKEN: HF_TOKEN
    nemo_evaluator_config:
      config:
        params:
          extra:
            num_repeats: 8
            args: "++prompt_config=eval/aai/mcq-4choices"
  - name: ns_aime2025
    env_vars:
      HF_TOKEN: HF_TOKEN
      JUDGE_API_KEY: JUDGE_API_KEY
    nemo_evaluator_config:
      config:
        params:
          extra:
            num_repeats: 64
            args: ++prompt_config=/prompt_templates/math-oai.yaml
  - name: ns_scicode
    env_vars:
      HF_TOKEN: HF_TOKEN
      JUDGE_API_KEY: JUDGE_API_KEY
    nemo_evaluator_config:
      config:
        params:
          extra:
            num_repeats: 8
  - name: ns_ifbench
    env_vars:
      HF_TOKEN: HF_TOKEN
    nemo_evaluator_config:
      config:
        params:
          extra:
            num_repeats: 8
  - name: ns_hle
    env_vars:
      HF_TOKEN: HF_TOKEN
      JUDGE_API_KEY: JUDGE_API_KEY
    nemo_evaluator_config:
      config:
        params:
          extra:
            num_repeats: 1
            judge_support: true
            judge:
              parallelism: 16
              model_id: openai/gpt-4o
              url: <OPENAI_API_URL_FOR_JUDGE>
              api_key: JUDGE_API_KEY