From 52b1c24f5db291a190b5e1023d2128a700a36c71 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Thu, 23 Jul 2026 17:10:48 -0700 Subject: [PATCH] feat: GenAI Client(evals) - add get/list EvaluationExperiment PiperOrigin-RevId: 953036852 --- agentplatform/_genai/evals.py | 408 +++++++++++++++++++ agentplatform/_genai/types/__init__.py | 30 +- agentplatform/_genai/types/common.py | 305 ++++++++++---- tests/unit/agentplatform/genai/test_evals.py | 146 +++++-- vertexai/_genai/evals.py | 404 ++++++++++++++++++ vertexai/_genai/types/__init__.py | 22 + vertexai/_genai/types/common.py | 149 +++++++ 7 files changed, 1354 insertions(+), 110 deletions(-) diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 1f4aa7e1de..1e1fd5c617 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -733,6 +733,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -793,6 +807,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1754,6 +1806,95 @@ def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Example Usage: + + .. code-block:: python + + eval_experiment = client.evals.get_evaluation_experiment( + name="projects/my-project/locations/us-central1/evaluationExperiments/my-experiment" + ) + + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _get_evaluation_metric( self, *, @@ -2039,6 +2180,93 @@ def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + def list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + + Args: + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. + + Returns: + The list evaluation experiments response. + + Example Usage: + + .. code-block:: python + + response = client.evals.list_evaluation_experiments( + config={"filter": 'display_name="my-experiment"'} + ) + + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: @@ -3929,6 +4157,97 @@ async def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + async def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Example Usage: + + .. code-block:: python + + eval_experiment = client.evals.get_evaluation_experiment( + name="projects/my-project/locations/us-central1/evaluationExperiments/my-experiment" + ) + + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _get_evaluation_metric( self, *, @@ -4222,6 +4541,95 @@ async def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + async def list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + + Args: + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. + + Returns: + The list evaluation experiments response. + + Example Usage: + + .. code-block:: python + + response = client.evals.list_evaluation_experiments( + config={"filter": 'display_name="my-experiment"'} + ) + + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index a731e74a0c..a8db3865fc 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -90,6 +90,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -126,6 +127,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListPublisherModelsRequestParameters @@ -718,6 +720,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -891,6 +896,12 @@ from .common import ListDeployableModelsConfig from .common import ListDeployableModelsConfigDict from .common import ListDeployableModelsConfigOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -2323,6 +2334,12 @@ "GenerateInstanceRubricsResponse", "GenerateInstanceRubricsResponseDict", "GenerateInstanceRubricsResponseOrDict", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", "GetEvaluationMetricConfig", "GetEvaluationMetricConfigDict", "GetEvaluationMetricConfigOrDict", @@ -2335,6 +2352,12 @@ "GetEvaluationItemConfig", "GetEvaluationItemConfigDict", "GetEvaluationItemConfigOrDict", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "ListEvaluationMetricsConfig", "ListEvaluationMetricsConfigDict", "ListEvaluationMetricsConfigOrDict", @@ -3538,9 +3561,6 @@ "ObservabilityEvalCase", "ObservabilityEvalCaseDict", "ObservabilityEvalCaseOrDict", - "EvaluationExperiment", - "EvaluationExperimentDict", - "EvaluationExperimentOrDict", "RubricGroup", "RubricGroupDict", "RubricGroupOrDict", @@ -3652,13 +3672,13 @@ "EvaluationItemType", "SamplingMethod", "EvaluationRunState", + "EvaluationExperimentMergeStrategy", "OptimizeTarget", "MemoryMetadataMergeStrategy", "GenerateMemoriesResponseGeneratedMemoryAction", "RagFileState", "SkillRevisionState", "PromptOptimizerMethod", - "EvaluationExperimentMergeStrategy", "OptimizationMethod", "PromptData", "PromptDataDict", @@ -3697,10 +3717,12 @@ "_GenerateUserScenariosParameters", "_GenerateLossClustersParameters", "_GenerateInstanceRubricsRequest", + "_GetEvaluationExperimentParameters", "_GetEvaluationMetricParameters", "_GetEvaluationRunParameters", "_GetEvaluationSetParameters", "_GetEvaluationItemParameters", + "_ListEvaluationExperimentsParameters", "_ListEvaluationMetricsParameters", "_OptimizeRequestParameters", "_CustomJobParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index dd57c62b1e..9188c69be1 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -539,6 +539,17 @@ class EvaluationRunState(_common.CaseInSensitiveEnum): """Evaluation run is performing rubric generation.""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class OptimizeTarget(_common.CaseInSensitiveEnum): """Specifies the method for calling the optimize_prompt.""" @@ -613,17 +624,6 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum): """The data driven prompt optimizer designer for prompts from Android core API.""" -class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): - """Merge strategy for the evaluation experiment.""" - - MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" - """Unspecified merge strategy.""" - SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" - """Default. Runs are treated as an independent, sequential history.""" - SHARED_RESULT_SET = "SHARED_RESULT_SET" - """Runs are parallel iterations contributing to a shared result set.""" - - class OptimizationMethod(_common.CaseInSensitiveEnum): """The method for data driven prompt optimization.""" @@ -5805,6 +5805,117 @@ class GenerateInstanceRubricsResponseDict(TypedDict, total=False): ] +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + class GetEvaluationMetricConfig(_common.BaseModel): """Config for getting an evaluation metric.""" @@ -5971,6 +6082,111 @@ class _GetEvaluationItemParametersDict(TypedDict, total=False): ] +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class ListEvaluationMetricsConfig(_common.BaseModel): """Config for listing evaluation metrics.""" @@ -24912,73 +25128,6 @@ class ObservabilityEvalCaseDict(TypedDict, total=False): ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict] -class EvaluationExperiment(_common.BaseModel): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] = Field( - default=None, - description="""The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", - ) - display_name: Optional[str] = Field( - default=None, description="""The display name of the evaluation experiment.""" - ) - evaluation_runs: Optional[list[str]] = Field( - default=None, - description="""The EvaluationRuns that are part of this experiment.""", - ) - labels: Optional[dict[str, str]] = Field( - default=None, description="""Labels for the evaluation experiment.""" - ) - merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( - default=None, description="""Merge strategy for the evaluation experiment.""" - ) - metadata: Optional[dict[str, Any]] = Field( - default=None, - description="""Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""", - ) - create_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was created.""" - ) - update_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was last updated.""" - ) - - -class EvaluationExperimentDict(TypedDict, total=False): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] - """The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" - - display_name: Optional[str] - """The display name of the evaluation experiment.""" - - evaluation_runs: Optional[list[str]] - """The EvaluationRuns that are part of this experiment.""" - - labels: Optional[dict[str, str]] - """Labels for the evaluation experiment.""" - - merge_strategy: Optional[EvaluationExperimentMergeStrategy] - """Merge strategy for the evaluation experiment.""" - - metadata: Optional[dict[str, Any]] - """Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""" - - create_time: Optional[datetime.datetime] - """Timestamp when this experiment was created.""" - - update_time: Optional[datetime.datetime] - """Timestamp when this experiment was last updated.""" - - -EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] - - class RubricGroup(_common.BaseModel): """A group of rubrics. diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index b93b6db39b..0e3e69f9d3 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -4154,17 +4154,17 @@ def test_run_inference_with_litellm_parsing( @mock.patch.object(_evals_common, "_get_interactions_client") @mock.patch.object(_evals_utils, "EvalDatasetLoader") def test_run_inference_with_gemini_agent( - self, mock_eval_dataset_loader, mock_get_interactions_client, - mock_fetch_agent_config + self, + mock_eval_dataset_loader, + mock_get_interactions_client, + mock_fetch_agent_config, ): mock_fetch_agent_config.return_value = ( agentplatform_genai_types.evals.AgentConfig( agent_id="test-agent", instruction="You are helpful.", tools=[ - genai_types.Tool( - code_execution=genai_types.ToolCodeExecution() - ), + genai_types.Tool(code_execution=genai_types.ToolCodeExecution()), ], ) ) @@ -4234,7 +4234,9 @@ def make_interaction(interaction_id, prompt_text, output_text): @mock.patch.object(_evals_common, "_get_interactions_client") @mock.patch.object(_evals_utils, "EvalDatasetLoader") def test_run_inference_gemini_agent_continues_on_failure( - self, mock_eval_dataset_loader, mock_get_interactions_client, + self, + mock_eval_dataset_loader, + mock_get_interactions_client, mock_fetch_agent_config, ): mock_fetch_agent_config.return_value = ( @@ -7764,9 +7766,7 @@ def test_merge_single_dataset_with_interactions_data_source(self): ] ) - with mock.patch.object( - _evals_data_converters, "logger" - ) as mock_logger: + with mock.patch.object(_evals_data_converters, "logger") as mock_logger: merged = _evals_data_converters.merge_evaluation_datasets([dataset]) assert len(merged.eval_cases) == 1 @@ -7797,9 +7797,7 @@ def test_merge_two_datasets_with_interactions_data_source(self): ] ) - with mock.patch.object( - _evals_data_converters, "logger" - ) as mock_logger: + with mock.patch.object(_evals_data_converters, "logger") as mock_logger: merged = _evals_data_converters.merge_evaluation_datasets( [dataset_1, dataset_2] ) @@ -7845,9 +7843,7 @@ def test_merge_interactions_data_source_with_response_dataset(self): ] ) - with mock.patch.object( - _evals_data_converters, "logger" - ) as mock_logger: + with mock.patch.object(_evals_data_converters, "logger") as mock_logger: merged = _evals_data_converters.merge_evaluation_datasets( [dataset_interactions, dataset_response] ) @@ -11616,7 +11612,8 @@ def test_code_execution_expands_to_run_command(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) assert len(result.tools) == 1 decls = result.tools[0].function_declarations @@ -11630,13 +11627,18 @@ def test_filesystem_expands_to_file_tools(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) assert len(result.tools) == 1 names = {fd.name for fd in result.tools[0].function_declarations} assert names == { - "view_file", "create_file", "edit_file", - "list_dir", "delete_file", "move_file", + "view_file", + "create_file", + "edit_file", + "list_dir", + "delete_file", + "move_file", } def test_environment_adds_sandbox_tools(self): @@ -11648,7 +11650,8 @@ def test_environment_adds_sandbox_tools(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) # code_execution + sandbox tool assert len(result.tools) == 2 @@ -11673,12 +11676,14 @@ def test_mcp_server_kept_as_named_declaration(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) assert len(result.tools) == 2 assert any(t.google_search is not None for t in result.tools) mcp_tool = [ - t for t in result.tools + t + for t in result.tools if t.function_declarations and t.function_declarations[0].name == "mcp_server" ] @@ -11698,9 +11703,7 @@ def test_catalog_in_sync_with_server(self): try: from cloud.ai.platform.evaluation.utils import interaction_converter except ImportError: - pytest.skip( - "interaction_converter not available outside google3" - ) + pytest.skip("interaction_converter not available outside google3") # pylint: enable=g-import-not-at-top # --- Built-in tool types: keys must match --- @@ -11718,7 +11721,9 @@ def test_catalog_in_sync_with_server(self): for tool_type in server_builtin_keys: server_names = { fd.name - for fd in interaction_converter._BUILTIN_TOOL_FUNCTION_DECLARATIONS[tool_type] + for fd in interaction_converter._BUILTIN_TOOL_FUNCTION_DECLARATIONS[ + tool_type + ] } sdk_names = { fd.name @@ -11732,8 +11737,7 @@ def test_catalog_in_sync_with_server(self): # --- Sandbox declarations: names must match --- server_sandbox_names = { - fd.name - for fd in interaction_converter.sandbox_function_declarations() + fd.name for fd in interaction_converter.sandbox_function_declarations() } sdk_sandbox_names = { fd.name for fd in _evals_builtin_tools.SANDBOX_DECLARATIONS @@ -11743,3 +11747,89 @@ def test_catalog_in_sync_with_server(self): f" Server: {sorted(server_sandbox_names)}\n" f" SDK: {sorted(sdk_sandbox_names)}" ) + + +class TestGetEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": self.experiment_name, + "displayName": "my_experiment", + "evaluationRuns": [ + "projects/123/locations/us-central1/evaluationRuns/789" + ], + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_get_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.get_evaluation_experiment(name=self.experiment_name) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.name == self.experiment_name + assert experiment.display_name == "my_experiment" + assert experiment.evaluation_runs == [ + "projects/123/locations/us-central1/evaluationRuns/789" + ] + + def test_get_evaluation_experiment_uses_full_name_in_url(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.get_evaluation_experiment(name=self.experiment_name) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path == self.experiment_name + + +class TestListEvaluationExperiments: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "evaluationExperiments": [ + { + "name": "projects/123/locations/us-central1/evaluationExperiments/1", + "displayName": "exp_1", + }, + { + "name": "projects/123/locations/us-central1/evaluationExperiments/2", + "displayName": "exp_2", + }, + ] + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_list_evaluation_experiments_returns_experiments(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + response = evals_module.list_evaluation_experiments() + + assert len(response.evaluation_experiments) == 2 + assert response.evaluation_experiments[0].display_name == "exp_1" + assert response.evaluation_experiments[1].display_name == "exp_2" + + def test_list_evaluation_experiments_passes_filter_and_order_by(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.list_evaluation_experiments( + config={"filter": 'display_name="exp_1"', "order_by": "create_time desc"} + ) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path.startswith("evaluationExperiments?") + assert "orderBy=create_time+desc" in path diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index eb55f9a0dd..5078d0265a 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -721,6 +721,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -781,6 +795,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -3034,6 +3086,180 @@ def create_evaluation_metric( # result.name is Optional[str], but we know it's always returned on creation return cast(str, result.name) + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Example Usage: + + .. code-block:: python + + eval_experiment = client.evals.get_evaluation_experiment( + name="projects/my-project/locations/us-central1/evaluationExperiments/my-experiment" + ) + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + + Args: + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. + + Returns: + The list evaluation experiments response. + + Example Usage: + + .. code-block:: python + + response = client.evals.list_evaluation_experiments( + config={"filter": 'display_name="my-experiment"'} + ) + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def get_evaluation_metric( self, *, @@ -4784,6 +5010,184 @@ async def create_evaluation_metric( ) return cast(str, result.name) + async def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Example Usage: + + .. code-block:: python + + eval_experiment = client.evals.get_evaluation_experiment( + name="projects/my-project/locations/us-central1/evaluationExperiments/my-experiment" + ) + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{name}".format_map(request_url_dict) + else: + path = "{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + async def list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + + Args: + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. + + Returns: + The list evaluation experiments response. + + Example Usage: + + .. code-block:: python + + response = client.evals.list_evaluation_experiments( + config={"filter": 'display_name="my-experiment"'} + ) + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def get_evaluation_metric( self, *, diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index d088967399..040a6323d3 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -83,6 +83,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -107,6 +108,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListSandboxEnvironmentSnapshotsRequestParameters @@ -612,6 +614,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -715,6 +720,12 @@ from .common import ListDatasetVersionsResponse from .common import ListDatasetVersionsResponseDict from .common import ListDatasetVersionsResponseOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -2642,6 +2653,17 @@ "EvaluationExperimentDict", "EvaluationExperimentOrDict", "EvaluationExperimentMergeStrategy", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", + "_GetEvaluationExperimentParameters", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "_ListEvaluationExperimentsParameters", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "EvaluateDatasetConfig", "EvaluateDatasetConfigDict", "EvaluateDatasetConfigOrDict", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 59490f34fa..0cc660614a 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -19195,6 +19195,155 @@ class EvaluationExperimentDict(TypedDict, total=False): EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponseDict] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class RubricGroup(_common.BaseModel): """A group of rubrics.