Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
394 changes: 394 additions & 0 deletions agentplatform/_genai/evals.py

Large diffs are not rendered by default.

22 changes: 22 additions & 0 deletions agentplatform/_genai/types/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
from .common import _DeleteAgentEngineSessionRequestParameters
from .common import _DeleteAgentEngineTaskRequestParameters
from .common import _DeleteDatasetRequestParameters
from .common import _DeleteEvaluationExperimentParameters
from .common import _DeleteEvaluationMetricParameters
from .common import _DeleteMultimodalDatasetRequestParameters
from .common import _DeletePromptVersionRequestParameters
Expand Down Expand Up @@ -159,6 +160,7 @@
from .common import _UpdateAgentEngineRequestParameters
from .common import _UpdateAgentEngineSessionRequestParameters
from .common import _UpdateDatasetParameters
from .common import _UpdateEvaluationExperimentParameters
from .common import _UpdateMultimodalDatasetParameters
from .common import _UpdateRagConfigRequestParameters
from .common import _UpdateRagCorpusRequestParameters
Expand Down Expand Up @@ -440,6 +442,12 @@
from .common import DeleteAgentEngineTaskConfig
from .common import DeleteAgentEngineTaskConfigDict
from .common import DeleteAgentEngineTaskConfigOrDict
from .common import DeleteEvaluationExperimentConfig
from .common import DeleteEvaluationExperimentConfigDict
from .common import DeleteEvaluationExperimentConfigOrDict
from .common import DeleteEvaluationExperimentOperation
from .common import DeleteEvaluationExperimentOperationDict
from .common import DeleteEvaluationExperimentOperationOrDict
from .common import DeleteEvaluationMetricConfig
from .common import DeleteEvaluationMetricConfigDict
from .common import DeleteEvaluationMetricConfigOrDict
Expand Down Expand Up @@ -1903,6 +1911,9 @@
from .common import UpdateAgentEngineSessionConfig
from .common import UpdateAgentEngineSessionConfigDict
from .common import UpdateAgentEngineSessionConfigOrDict
from .common import UpdateEvaluationExperimentConfig
from .common import UpdateEvaluationExperimentConfigDict
from .common import UpdateEvaluationExperimentConfigOrDict
from .common import UpdatePromptConfig
from .common import UpdatePromptConfigDict
from .common import UpdatePromptConfigOrDict
Expand Down Expand Up @@ -2173,6 +2184,12 @@
"EvaluationSet",
"EvaluationSetDict",
"EvaluationSetOrDict",
"DeleteEvaluationExperimentConfig",
"DeleteEvaluationExperimentConfigDict",
"DeleteEvaluationExperimentConfigOrDict",
"DeleteEvaluationExperimentOperation",
"DeleteEvaluationExperimentOperationDict",
"DeleteEvaluationExperimentOperationOrDict",
"DeleteEvaluationMetricConfig",
"DeleteEvaluationMetricConfigDict",
"DeleteEvaluationMetricConfigOrDict",
Expand Down Expand Up @@ -2371,6 +2388,9 @@
"ListEvaluationMetricsResponse",
"ListEvaluationMetricsResponseDict",
"ListEvaluationMetricsResponseOrDict",
"UpdateEvaluationExperimentConfig",
"UpdateEvaluationExperimentConfigDict",
"UpdateEvaluationExperimentConfigOrDict",
"OptimizeConfig",
"OptimizeConfigDict",
"OptimizeConfigOrDict",
Expand Down Expand Up @@ -3720,6 +3740,7 @@
"_CreateEvaluationMetricParameters",
"_CreateEvaluationRunParameters",
"_CreateEvaluationSetParameters",
"_DeleteEvaluationExperimentParameters",
"_DeleteEvaluationMetricParameters",
"_EvaluateInstancesRequestParameters",
"_GenerateUserScenariosParameters",
Expand All @@ -3732,6 +3753,7 @@
"_GetEvaluationItemParameters",
"_ListEvaluationExperimentsParameters",
"_ListEvaluationMetricsParameters",
"_UpdateEvaluationExperimentParameters",
"_OptimizeRequestParameters",
"_CustomJobParameters",
"_GetCustomJobParameters",
Expand Down
165 changes: 165 additions & 0 deletions agentplatform/_genai/types/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -4137,6 +4137,92 @@ class EvaluationSetDict(TypedDict, total=False):
EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict]


class DeleteEvaluationExperimentConfig(_common.BaseModel):
"""Config for deleting an evaluation experiment."""

http_options: Optional[genai_types.HttpOptions] = Field(
default=None, description="""Used to override HTTP request options."""
)


class DeleteEvaluationExperimentConfigDict(TypedDict, total=False):
"""Config for deleting an evaluation experiment."""

http_options: Optional[genai_types.HttpOptions]
"""Used to override HTTP request options."""


DeleteEvaluationExperimentConfigOrDict = Union[
DeleteEvaluationExperimentConfig, DeleteEvaluationExperimentConfigDict
]


class _DeleteEvaluationExperimentParameters(_common.BaseModel):
"""Parameters for deleting an evaluation experiment."""

name: Optional[str] = Field(default=None, description="""""")
config: Optional[DeleteEvaluationExperimentConfig] = Field(
default=None, description=""""""
)


class _DeleteEvaluationExperimentParametersDict(TypedDict, total=False):
"""Parameters for deleting an evaluation experiment."""

name: Optional[str]
""""""

config: Optional[DeleteEvaluationExperimentConfigDict]
""""""


_DeleteEvaluationExperimentParametersOrDict = Union[
_DeleteEvaluationExperimentParameters, _DeleteEvaluationExperimentParametersDict
]


class DeleteEvaluationExperimentOperation(_common.BaseModel):
"""Operation for deleting an evaluation experiment."""

name: Optional[str] = Field(
default=None,
description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""",
)
metadata: Optional[dict[str, Any]] = Field(
default=None,
description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""",
)
done: Optional[bool] = Field(
default=None,
description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""",
)
error: Optional[dict[str, Any]] = Field(
default=None,
description="""The error result of the operation in case of failure or cancellation.""",
)


class DeleteEvaluationExperimentOperationDict(TypedDict, total=False):
"""Operation for deleting an evaluation experiment."""

name: Optional[str]
"""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`."""

metadata: Optional[dict[str, Any]]
"""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any."""

done: Optional[bool]
"""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available."""

error: Optional[dict[str, Any]]
"""The error result of the operation in case of failure or cancellation."""


DeleteEvaluationExperimentOperationOrDict = Union[
DeleteEvaluationExperimentOperation, DeleteEvaluationExperimentOperationDict
]


class DeleteEvaluationMetricConfig(_common.BaseModel):
"""Config for deleting an evaluation metric."""

Expand Down Expand Up @@ -6350,6 +6436,85 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False):
]


class UpdateEvaluationExperimentConfig(_common.BaseModel):
"""Config for updating an evaluation experiment."""

http_options: Optional[genai_types.HttpOptions] = Field(
default=None, description="""Used to override HTTP request options."""
)
update_mask: Optional[str] = Field(
default=None,
description="""The update mask to apply. For the `FieldMask` definition, see
https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""",
)
display_name: Optional[str] = Field(
default=None, description="""The display name of the evaluation experiment."""
)
labels: Optional[dict[str, str]] = Field(
default=None, description="""Labels for the evaluation experiment."""
)
merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field(
default=None, description="""Merge strategy for the evaluation experiment."""
)
metadata: Optional[dict[str, Any]] = Field(
default=None, description="""Metadata about the evaluation experiment."""
)


class UpdateEvaluationExperimentConfigDict(TypedDict, total=False):
"""Config for updating an evaluation experiment."""

http_options: Optional[genai_types.HttpOptions]
"""Used to override HTTP request options."""

update_mask: Optional[str]
"""The update mask to apply. For the `FieldMask` definition, see
https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask."""

display_name: Optional[str]
"""The display name of the evaluation experiment."""

labels: Optional[dict[str, str]]
"""Labels for the evaluation experiment."""

merge_strategy: Optional[EvaluationExperimentMergeStrategy]
"""Merge strategy for the evaluation experiment."""

metadata: Optional[dict[str, Any]]
"""Metadata about the evaluation experiment."""


UpdateEvaluationExperimentConfigOrDict = Union[
UpdateEvaluationExperimentConfig, UpdateEvaluationExperimentConfigDict
]


class _UpdateEvaluationExperimentParameters(_common.BaseModel):
"""Parameters for updating an evaluation experiment."""

name: Optional[str] = Field(
default=None, description="""The resource name of the EvaluationExperiment."""
)
config: Optional[UpdateEvaluationExperimentConfig] = Field(
default=None, description=""""""
)


class _UpdateEvaluationExperimentParametersDict(TypedDict, total=False):
"""Parameters for updating an evaluation experiment."""

name: Optional[str]
"""The resource name of the EvaluationExperiment."""

config: Optional[UpdateEvaluationExperimentConfigDict]
""""""


_UpdateEvaluationExperimentParametersOrDict = Union[
_UpdateEvaluationExperimentParameters, _UpdateEvaluationExperimentParametersDict
]


class OptimizeConfig(_common.BaseModel):
"""Config for Prompt Optimizer."""

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=protected-access,bad-continuation,missing-function-docstring

from agentplatform._genai import types
from tests.unit.agentplatform.genai.replays import pytest_helper
from google.genai._api_client import HttpOptions
import pytest

pytestmark = pytest_helper.setup(
file=__file__,
globals_for_file=globals(),
test_method="evals.update_evaluation_experiment",
http_options=HttpOptions(
api_version="v1beta1",
),
)

pytest_plugins = ("pytest_asyncio",)


def test_update_and_delete(client):
experiment = client.evals.create_evaluation_experiment(
display_name="sdk-update-delete-test"
)
assert isinstance(experiment, types.EvaluationExperiment)

updated = client.evals.update_evaluation_experiment(
name=experiment.name,
config={
"display_name": "sdk-update-delete-test-renamed",
"update_mask": "display_name",
},
)
assert isinstance(updated, types.EvaluationExperiment)
assert updated.display_name == "sdk-update-delete-test-renamed"

delete_operation = client.evals.delete_evaluation_experiment(
name=experiment.name
)
assert isinstance(
delete_operation, types.DeleteEvaluationExperimentOperation
)


@pytest.mark.asyncio
async def test_update_and_delete_async(client):
experiment = await client.aio.evals.create_evaluation_experiment(
display_name="sdk-update-delete-test-async"
)
assert isinstance(experiment, types.EvaluationExperiment)

updated = await client.aio.evals.update_evaluation_experiment(
name=experiment.name,
config={
"display_name": "sdk-update-delete-test-async-renamed",
"update_mask": "display_name",
},
)
assert isinstance(updated, types.EvaluationExperiment)
assert updated.display_name == "sdk-update-delete-test-async-renamed"

delete_operation = await client.aio.evals.delete_evaluation_experiment(
name=experiment.name
)
assert isinstance(
delete_operation, types.DeleteEvaluationExperimentOperation
)
Loading
Loading