From e1f77a82ec1c269d66ff5f22015822d0e0034a3b Mon Sep 17 00:00:00 2001 From: Matthew Roberson Date: Fri, 10 Jul 2026 12:59:24 -0500 Subject: [PATCH] Remove LLM prompt/response + response-creation project support Mirrors intelligence #30108 (e4feccc093, "Remove LLM prompt/response data-creation editors"), which dropped LLM_PROMPT_CREATION / LLM_PROMPT_RESPONSE_CREATION from the MediaType enum and RESPONSE_CREATION from the EditorTaskType enum. Staging now rejects these values, so the SDK's project-creation paths for them are dead. Removed (creation path only): - MediaType.LLMPromptCreation, MediaType.LLMPromptResponseCreation - OntologyKind.ResponseCreation; EditorTaskType.ResponseCreation (+ mapper branches) - Client.create_prompt_response_generation_project, Client.create_response_creation_project - Project.is_prompt_response and labeling_service_dashboard branches for the removed enums - Project-creation tests + fixtures (test_prompt_response_generation_project.py, test_response_creation_project.py, LLM/ResponseCreation params in test_generic_data_types.py, and the prompt/response conftest fixtures) Kept (still needed to read/export existing data, which the backend preserves): - PromptResponseClassification (ontology), PromptClassificationAnnotation/PromptText (annotation types), and NDPromptClassification ndjson serialization - MediaType.LLM and OntologyKind.ModelEvaluation / EditorTaskType.ModelChatEvaluation Co-Authored-By: Claude Opus 4.8 (1M context) --- libs/labelbox/src/labelbox/client.py | 119 +----- .../schema/labeling_service_dashboard.py | 12 - .../src/labelbox/schema/media_type.py | 2 - .../src/labelbox/schema/ontology_kind.py | 11 - libs/labelbox/src/labelbox/schema/project.py | 11 - .../tests/data/annotation_import/conftest.py | 339 +----------------- .../test_generic_data_types.py | 56 --- libs/labelbox/tests/integration/conftest.py | 171 --------- .../tests/integration/test_project.py | 4 +- ...test_prompt_response_generation_project.py | 185 ---------- .../test_response_creation_project.py | 30 -- libs/labelbox/tests/unit/test_project.py | 1 - libs/labelbox/tests/unit/test_utils.py | 2 +- 13 files changed, 13 insertions(+), 930 deletions(-) delete mode 100644 libs/labelbox/tests/integration/test_prompt_response_generation_project.py delete mode 100644 libs/labelbox/tests/integration/test_response_creation_project.py diff --git a/libs/labelbox/src/labelbox/client.py b/libs/labelbox/src/labelbox/client.py index 60fb8016d..a00d2c7f0 100644 --- a/libs/labelbox/src/labelbox/client.py +++ b/libs/labelbox/src/labelbox/client.py @@ -755,120 +755,6 @@ def create_offline_model_evaluation_project( } return self._create_project(_CoreProjectInput(**input)) - def create_prompt_response_generation_project( - self, - name: str, - media_type: MediaType, - description: Optional[str] = None, - auto_audit_percentage: Optional[float] = None, - auto_audit_number_of_labels: Optional[int] = None, - quality_modes: Optional[Set[QualityMode]] = { - QualityMode.Benchmark, - QualityMode.Consensus, - }, - is_benchmark_enabled: Optional[bool] = None, - is_consensus_enabled: Optional[bool] = None, - dataset_id: Optional[str] = None, - dataset_name: Optional[str] = None, - data_row_count: int = 100, - ) -> Project: - """ - Use this method exclusively to create a prompt and response generation project. - - Args: - dataset_name: When creating a new dataset, pass the name - dataset_id: When using an existing dataset, pass the id - data_row_count: The number of data row assets to use for the project - media_type: The type of assets that this project will accept. Limited to LLMPromptCreation and LLMPromptResponseCreation - See create_project for additional parameters - Returns: - Project: The created project - - NOTE: Only a dataset_name or dataset_id should be included - - Examples: - >>> client.create_prompt_response_generation_project(name=project_name, dataset_name="new data set", media_type=MediaType.LLMPromptResponseCreation) - >>> This creates a new dataset with a default number of rows (100), creates new prompt and response creation project and assigns a batch of the newly created data rows to the project. - - >>> client.create_prompt_response_generation_project(name=project_name, dataset_name="new data set", data_row_count=10, media_type=MediaType.LLMPromptCreation) - >>> This creates a new dataset with 10 data rows, creates new prompt creation project and assigns a batch of the newly created datarows to the project. - - >>> client.create_prompt_response_generation_project(name=project_name, dataset_id="clr00u8j0j0j0", media_type=MediaType.LLMPromptCreation) - >>> This creates a new prompt creation project, and adds 100 datarows to the dataset with id "clr00u8j0j0j0" and assigns a batch of the newly created data rows to the project. - - >>> client.create_prompt_response_generation_project(name=project_name, dataset_id="clr00u8j0j0j0", data_row_count=10, media_type=MediaType.LLMPromptResponseCreation) - >>> This creates a new prompt and response creation project, and adds 100 datarows to the dataset with id "clr00u8j0j0j0" and assigns a batch of the newly created 10 data rows to the project. - - """ - if not dataset_id and not dataset_name: - raise ValueError( - "dataset_name or dataset_id must be present and not be an empty string." - ) - - if dataset_id and dataset_name: - raise ValueError( - "Only provide a dataset_name or dataset_id, not both." - ) - - if dataset_id: - append_to_existing_dataset = True - dataset_name_or_id = dataset_id - else: - append_to_existing_dataset = False - dataset_name_or_id = dataset_name - - if media_type not in [ - MediaType.LLMPromptCreation, - MediaType.LLMPromptResponseCreation, - ]: - raise ValueError( - "media_type must be either LLMPromptCreation or LLMPromptResponseCreation" - ) - - input = { - "name": name, - "description": description, - "media_type": media_type, - "auto_audit_percentage": auto_audit_percentage, - "auto_audit_number_of_labels": auto_audit_number_of_labels, - "quality_modes": quality_modes, - "is_benchmark_enabled": is_benchmark_enabled, - "is_consensus_enabled": is_consensus_enabled, - "dataset_name_or_id": dataset_name_or_id, - "append_to_existing_dataset": append_to_existing_dataset, - "data_row_count": data_row_count, - } - return self._create_project(_CoreProjectInput(**input)) - - def create_response_creation_project( - self, - name: str, - description: Optional[str] = None, - quality_modes: Optional[Set[QualityMode]] = { - QualityMode.Benchmark, - QualityMode.Consensus, - }, - is_benchmark_enabled: Optional[bool] = None, - is_consensus_enabled: Optional[bool] = None, - ) -> Project: - """ - Creates a project for response creation. - Args: - See create_project for parameters - Returns: - Project: The created project - """ - input = { - "name": name, - "description": description, - "media_type": MediaType.Text, # Only Text is supported - "quality_modes": quality_modes, - "is_benchmark_enabled": is_benchmark_enabled, - "is_consensus_enabled": is_consensus_enabled, - "editor_task_type": EditorTaskType.ResponseCreation.value, # Special editor task type for response creation projects - } - return self._create_project(_CoreProjectInput(**input)) - def _create_project(self, input: _CoreProjectInput) -> Project: params = input.model_dump(exclude_none=True) @@ -1375,13 +1261,12 @@ def create_ontology( name (str): Name of the ontology normalized (dict): A normalized ontology payload. See above for details. media_type (MediaType or None): Media type of a new ontology - ontology_kind (OntologyKind or None): set to OntologyKind.ModelEvaluation if the ontology is for chat evaluation or - OntologyKind.ResponseCreation if ontology is for response creation, leave as None otherwise. + ontology_kind (OntologyKind or None): set to OntologyKind.ModelEvaluation if the ontology is for chat evaluation, leave as None otherwise. Returns: The created Ontology - NOTE for chat evaluation, we currently force media_type to Conversational and for response creation, we force media_type to Text. + NOTE for chat evaluation, we currently force media_type to Conversational. """ media_type_value = None diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index 9899212e5..40797541c 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -111,18 +111,6 @@ def service_type(self): ): return "Live chat evaluation" - if ( - self.editor_task_type == EditorTaskType.ResponseCreation - and self.media_type == MediaType.Text - ): - return "Response creation" - - if ( - self.media_type == MediaType.LLMPromptCreation - or self.media_type == MediaType.LLMPromptResponseCreation - ): - return "Prompt response creation" - return sentence_case(self.media_type.value) @classmethod diff --git a/libs/labelbox/src/labelbox/schema/media_type.py b/libs/labelbox/src/labelbox/schema/media_type.py index f55a65daf..6fe95d6c2 100644 --- a/libs/labelbox/src/labelbox/schema/media_type.py +++ b/libs/labelbox/src/labelbox/schema/media_type.py @@ -10,8 +10,6 @@ class MediaType(Enum): Geospatial_Tile = "TMS_GEO" Html = "HTML" Image = "IMAGE" - LLMPromptCreation = "LLM_PROMPT_CREATION" - LLMPromptResponseCreation = "LLM_PROMPT_RESPONSE_CREATION" Pdf = "PDF" Simple_Tile = "TMS_SIMPLE" Text = "TEXT" diff --git a/libs/labelbox/src/labelbox/schema/ontology_kind.py b/libs/labelbox/src/labelbox/schema/ontology_kind.py index 79ef7d7a3..023ebb314 100644 --- a/libs/labelbox/src/labelbox/schema/ontology_kind.py +++ b/libs/labelbox/src/labelbox/schema/ontology_kind.py @@ -10,7 +10,6 @@ class OntologyKind(Enum): """ ModelEvaluation = "MODEL_EVALUATION" - ResponseCreation = "RESPONSE_CREATION" Missing = None @classmethod @@ -34,10 +33,6 @@ def evaluate_ontology_kind_with_media_type( MediaType.Conversational, "For chat evaluation, media_type must be Conversational.", ), - OntologyKind.ResponseCreation: ( - MediaType.Text, - "For response creation, media_type must be Text.", - ), } if ontology_kind in ontology_to_media: @@ -55,7 +50,6 @@ def evaluate_ontology_kind_with_media_type( class EditorTaskType(str, Enum): ModelChatEvaluation = "MODEL_CHAT_EVALUATION" - ResponseCreation = "RESPONSE_CREATION" OfflineModelChatEvaluation = "OFFLINE_MODEL_CHAT_EVALUATION" Missing = None @@ -107,11 +101,6 @@ def map_to_editor_task_type( and media_type == MediaType.Conversational ): return EditorTaskType.ModelChatEvaluation - elif ( - onotology_kind == OntologyKind.ResponseCreation - and media_type == MediaType.Text - ): - return EditorTaskType.ResponseCreation else: return EditorTaskType.Missing diff --git a/libs/labelbox/src/labelbox/schema/project.py b/libs/labelbox/src/labelbox/schema/project.py index 1fab9aaee..bd51e8b20 100644 --- a/libs/labelbox/src/labelbox/schema/project.py +++ b/libs/labelbox/src/labelbox/schema/project.py @@ -202,17 +202,6 @@ def is_chat_evaluation(self) -> bool: and self.editor_task_type == EditorTaskType.ModelChatEvaluation ) - def is_prompt_response(self) -> bool: - """ - Returns: - True if this project is a prompt response project, False otherwise - """ - return ( - self.media_type == MediaType.LLMPromptResponseCreation - or self.media_type == MediaType.LLMPromptCreation - or self.editor_task_type == EditorTaskType.ResponseCreation - ) - def is_auto_data_generation(self) -> bool: return self.upload_type == UploadType.Auto # type: ignore diff --git a/libs/labelbox/tests/data/annotation_import/conftest.py b/libs/labelbox/tests/data/annotation_import/conftest.py index 5be0094b6..d009d6d3a 100644 --- a/libs/labelbox/tests/data/annotation_import/conftest.py +++ b/libs/labelbox/tests/data/annotation_import/conftest.py @@ -524,27 +524,6 @@ def normalized_ontology_by_media_type(): free_form_text_index, ], }, - MediaType.LLMPromptResponseCreation: { - "tools": [], - "classifications": [ - prompt_text, - response_text, - response_radio, - response_checklist, - ], - }, - MediaType.LLMPromptCreation: { - "tools": [], - "classifications": [prompt_text], - }, - OntologyKind.ResponseCreation: { - "tools": [], - "classifications": [ - response_text, - response_radio, - response_checklist, - ], - }, OntologyKind.ModelEvaluation: { "tools": [ message_single_selection_task, @@ -634,111 +613,6 @@ def get_global_key(): ##### Integration test strategies ##### -def _create_response_creation_project( - client: Client, - rand_gen, - data_row_json_by_media_type, - ontology_kind, - normalized_ontology_by_media_type, -) -> Tuple[Project, Ontology, Dataset]: - "For response creation projects" - - dataset = create_dataset_robust(client, name=rand_gen(str)) - - project = client.create_response_creation_project( - name=f"{ontology_kind}-{rand_gen(str)}" - ) - - ontology = client.create_ontology( - name=f"{ontology_kind}-{rand_gen(str)}", - normalized=normalized_ontology_by_media_type[ontology_kind], - media_type=MediaType.Text, - ontology_kind=ontology_kind, - ) - - project.connect_ontology(ontology) - - data_row_data = [] - - for _ in range(DATA_ROW_COUNT): - data_row_data.append( - data_row_json_by_media_type[MediaType.Text](rand_gen(str)) - ) - - task = dataset.create_data_rows(data_row_data) - task.wait_till_done() - global_keys = [row["global_key"] for row in task.result] - data_row_ids = [row["id"] for row in task.result] - - project.create_batch( - rand_gen(str), - data_row_ids, # sample of data row objects - 5, # priority between 1(Highest) - 5(lowest) - ) - project.data_row_ids = data_row_ids - project.global_keys = global_keys - - return project, ontology, dataset - - -@pytest.fixture -def llm_prompt_response_creation_dataset_with_data_row( - client: Client, rand_gen -): - dataset = create_dataset_robust(client, name=rand_gen(str)) - global_key = str(uuid.uuid4()) - - convo_data = { - "row_data": "https://storage.googleapis.com/lb-test-data/cataflow/media/sample-conversational-v2-4.json", - "global_key": global_key, - } - - task = dataset.create_data_rows([convo_data]) - task.wait_till_done() - assert task.status == "COMPLETE" - yield dataset - - dataset.delete() - - -def _create_prompt_response_project( - client: Client, - rand_gen, - media_type, - normalized_ontology_by_media_type, - export_v2_test_helpers, - llm_prompt_response_creation_dataset_with_data_row, -) -> Tuple[Project, Ontology]: - """For prompt response data row auto gen projects""" - dataset = llm_prompt_response_creation_dataset_with_data_row - prompt_response_project = client.create_prompt_response_generation_project( - name=f"{media_type.value}-{rand_gen(str)}", - dataset_id=dataset.uid, - media_type=media_type, - ) - - ontology = client.create_ontology( - name=f"{media_type}-{rand_gen(str)}", - normalized=normalized_ontology_by_media_type[media_type], - media_type=media_type, - ) - - prompt_response_project.connect_ontology(ontology) - - # We have to export to get data row ids - result = export_v2_test_helpers.run_project_export_v2_task( - prompt_response_project - ) - - data_row_ids = [dr["data_row"]["id"] for dr in result] - global_keys = [dr["data_row"]["global_key"] for dr in result] - - prompt_response_project.data_row_ids = data_row_ids - prompt_response_project.global_keys = global_keys - - return prompt_response_project, ontology - - def _create_offline_mmc_project( client: Client, rand_gen, data_row_json, normalized_ontology ) -> Tuple[Project, Ontology, Dataset]: @@ -831,7 +705,6 @@ def configured_project( request: FixtureRequest, normalized_ontology_by_media_type, export_v2_test_helpers, - llm_prompt_response_creation_dataset_with_data_row, teardown_helpers, ): """Configure project for test. Request.param will contain the media type if not present will use Image MediaType. The project will have 10 data rows.""" @@ -842,27 +715,7 @@ def configured_project( dataset = None - if ( - media_type == MediaType.LLMPromptCreation - or media_type == MediaType.LLMPromptResponseCreation - ): - project, ontology = _create_prompt_response_project( - client, - rand_gen, - media_type, - normalized_ontology_by_media_type, - export_v2_test_helpers, - llm_prompt_response_creation_dataset_with_data_row, - ) - elif media_type == OntologyKind.ResponseCreation: - project, ontology, dataset = _create_response_creation_project( - client, - rand_gen, - data_row_json_by_media_type, - media_type, - normalized_ontology_by_media_type, - ) - elif media_type == OntologyKind.ModelEvaluation: + if media_type == OntologyKind.ModelEvaluation: project, ontology, dataset = _create_offline_mmc_project( client, rand_gen, @@ -901,26 +754,7 @@ def configured_project_by_global_key( media_type = getattr(request, "param", MediaType.Image) dataset = None - if ( - media_type == MediaType.LLMPromptCreation - or media_type == MediaType.LLMPromptResponseCreation - ): - project, ontology = _create_prompt_response_project( - client, - rand_gen, - media_type, - normalized_ontology_by_media_type, - export_v2_test_helpers, - ) - elif media_type == OntologyKind.ResponseCreation: - project, ontology, dataset = _create_response_creation_project( - client, - rand_gen, - data_row_json_by_media_type, - media_type, - normalized_ontology_by_media_type, - ) - elif media_type == OntologyKind.ModelEvaluation: + if media_type == OntologyKind.ModelEvaluation: project, ontology, dataset = _create_offline_mmc_project( client, rand_gen, @@ -959,29 +793,13 @@ def module_project( media_type = getattr(request, "param", MediaType.Image) dataset = None - if ( - media_type == MediaType.LLMPromptCreation - or media_type == MediaType.LLMPromptResponseCreation - ): - project, ontology = _create_prompt_response_project( - client, rand_gen, media_type, normalized_ontology_by_media_type - ) - elif media_type == OntologyKind.ResponseCreation: - project, ontology, dataset = _create_response_creation_project( - client, - rand_gen, - data_row_json_by_media_type, - media_type, - normalized_ontology_by_media_type, - ) - else: - project, ontology, dataset = _create_project( - client, - rand_gen, - data_row_json_by_media_type, - media_type, - normalized_ontology_by_media_type, - ) + project, ontology, dataset = _create_project( + client, + rand_gen, + data_row_json_by_media_type, + media_type, + normalized_ontology_by_media_type, + ) yield project @@ -1781,18 +1599,6 @@ def annotations_by_media_type( ], MediaType.Text: [checklist_inference, text_inference, entity_inference], MediaType.Video: [video_checklist_inference], - MediaType.LLMPromptResponseCreation: [ - prompt_text_inference, - text_response_inference, - checklist_response_inference, - radio_response_inference, - ], - MediaType.LLMPromptCreation: [prompt_text_inference], - OntologyKind.ResponseCreation: [ - text_response_inference, - checklist_response_inference, - radio_response_inference, - ], OntologyKind.ModelEvaluation: [ message_single_selection_inference, message_multi_selection_inference, @@ -2420,121 +2226,6 @@ def expected_export_v2_document(): return expected_annotations -@pytest.fixture() -def expected_export_v2_llm_prompt_response_creation(): - expected_annotations = { - "objects": [], - "classifications": [ - { - "name": "prompt-text", - "value": "prompt-text", - "text_answer": { - "content": "free form text...", - "classifications": [], - }, - }, - { - "name": "response-text", - "text_answer": { - "content": "free form text...", - "classifications": [], - }, - "value": "response-text", - }, - { - "checklist_answers": [ - { - "classifications": [], - "name": "first_checklist_answer", - "value": "first_checklist_answer", - }, - { - "classifications": [], - "name": "second_checklist_answer", - "value": "second_checklist_answer", - }, - ], - "name": "checklist-response", - "value": "checklist-response", - }, - { - "name": "radio-response", - "radio_answer": { - "classifications": [], - "name": "first_radio_answer", - "value": "first_radio_answer", - }, - "value": "radio-response", - }, - ], - "relationships": [], - } - return expected_annotations - - -@pytest.fixture() -def expected_export_v2_llm_prompt_creation(): - expected_annotations = { - "objects": [], - "classifications": [ - { - "name": "prompt-text", - "value": "prompt-text", - "text_answer": { - "content": "free form text...", - "classifications": [], - }, - }, - ], - "relationships": [], - } - return expected_annotations - - -@pytest.fixture() -def expected_export_v2_llm_response_creation(): - expected_annotations = { - "objects": [], - "relationships": [], - "classifications": [ - { - "name": "response-text", - "text_answer": { - "content": "free form text...", - "classifications": [], - }, - "value": "response-text", - }, - { - "checklist_answers": [ - { - "classifications": [], - "name": "first_checklist_answer", - "value": "first_checklist_answer", - }, - { - "classifications": [], - "name": "second_checklist_answer", - "value": "second_checklist_answer", - }, - ], - "name": "checklist-response", - "value": "checklist-response", - }, - { - "name": "radio-response", - "radio_answer": { - "classifications": [], - "name": "first_radio_answer", - "value": "first_radio_answer", - }, - "value": "radio-response", - }, - ], - } - return expected_annotations - - @pytest.fixture def expected_exports_v2_mmc(mmc_example_data_row_message_ids): some_parent_id, some_child_ids = next( @@ -2669,9 +2360,6 @@ def exports_v2_by_media_type( expected_export_v2_video, expected_export_v2_conversation, expected_export_v2_document, - expected_export_v2_llm_prompt_response_creation, - expected_export_v2_llm_prompt_creation, - expected_export_v2_llm_response_creation, expected_exports_v2_mmc, ): return { @@ -2682,9 +2370,6 @@ def exports_v2_by_media_type( MediaType.Video: expected_export_v2_video, MediaType.Conversational: expected_export_v2_conversation, MediaType.Document: expected_export_v2_document, - MediaType.LLMPromptResponseCreation: expected_export_v2_llm_prompt_response_creation, - MediaType.LLMPromptCreation: expected_export_v2_llm_prompt_creation, - OntologyKind.ResponseCreation: expected_export_v2_llm_response_creation, OntologyKind.ModelEvaluation: expected_exports_v2_mmc, } @@ -2728,12 +2413,6 @@ def to_pascal_case(name: str) -> str: media_type = to_pascal_case(data_type_string) if media_type == "Conversational": media_type = "Conversational" - elif media_type == "Llmpromptcreation": - media_type = "LLMPromptCreation" - elif media_type == "Llmpromptresponsecreation": - media_type = "LLMPromptResponseCreation" - elif media_type == "Llmresponsecreation": - media_type = "Text" elif media_type == "Genericdatarow": media_type = "Image" project.update(media_type=MediaType[media_type]) diff --git a/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py b/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py index 70759357c..0e779055e 100644 --- a/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py +++ b/libs/labelbox/tests/data/annotation_import/test_generic_data_types.py @@ -34,7 +34,6 @@ def validate_iso_format(date_string: str): (MediaType.Video, MediaType.Video), (MediaType.Conversational, MediaType.Conversational), (MediaType.Document, MediaType.Document), - (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation), (OntologyKind.ModelEvaluation, OntologyKind.ModelEvaluation), ], indirect=["configured_project"], @@ -103,53 +102,6 @@ def test_import_media_types( assert exported_annotations == expected_data -@pytest.mark.parametrize( - "configured_project, media_type", - [ - ( - MediaType.LLMPromptResponseCreation, - MediaType.LLMPromptResponseCreation, - ), - (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), - ], - indirect=["configured_project"], -) -def test_import_media_types_llm( - client: Client, - configured_project: Project, - annotations_by_media_type, - exports_v2_by_media_type, - export_v2_test_helpers, - helpers, - media_type, - wait_for_label_processing, -): - annotations_ndjson = list( - itertools.chain.from_iterable(annotations_by_media_type[media_type]) - ) - - label_import = lb.LabelImport.create_from_objects( - client, - configured_project.uid, - f"test-import-{media_type}", - annotations_ndjson, - ) - label_import.wait_until_done() - - assert label_import.state == AnnotationImportState.FINISHED - assert len(label_import.errors) == 0 - - all_annotations = sorted([a["uuid"] for a in annotations_ndjson]) - successful_annotations = sorted( - [ - status["uuid"] - for status in label_import.statuses - if status["status"] == "SUCCESS" - ] - ) - assert successful_annotations == all_annotations - - @pytest.mark.parametrize( "configured_project_by_global_key, media_type", [ @@ -160,7 +112,6 @@ def test_import_media_types_llm( (MediaType.Video, MediaType.Video), (MediaType.Conversational, MediaType.Conversational), (MediaType.Document, MediaType.Document), - (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation), (OntologyKind.ModelEvaluation, OntologyKind.ModelEvaluation), ], indirect=["configured_project_by_global_key"], @@ -239,12 +190,6 @@ def test_import_media_types_by_global_key( (MediaType.Video, MediaType.Video), (MediaType.Conversational, MediaType.Conversational), (MediaType.Document, MediaType.Document), - ( - MediaType.LLMPromptResponseCreation, - MediaType.LLMPromptResponseCreation, - ), - (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), - (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation), pytest.param( OntologyKind.ModelEvaluation, OntologyKind.ModelEvaluation, @@ -284,7 +229,6 @@ def test_import_mal_annotations( (MediaType.Video, MediaType.Video), (MediaType.Conversational, MediaType.Conversational), (MediaType.Document, MediaType.Document), - (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation), pytest.param( OntologyKind.ModelEvaluation, OntologyKind.ModelEvaluation, diff --git a/libs/labelbox/tests/integration/conftest.py b/libs/labelbox/tests/integration/conftest.py index c0d4ef8cc..6233f6245 100644 --- a/libs/labelbox/tests/integration/conftest.py +++ b/libs/labelbox/tests/integration/conftest.py @@ -17,8 +17,6 @@ OntologyBuilder, Option, PromptIssueTool, - PromptResponseClassification, - ResponseOption, StepReasoningTool, Tool, ) @@ -389,175 +387,6 @@ def _upload_invalid_data_rows_for_dataset(dataset: Dataset): return _upload_invalid_data_rows_for_dataset -@pytest.fixture -def prompt_response_generation_project_with_new_dataset( - client: Client, rand_gen, request -): - """fixture is parametrize and needs project_type in request""" - media_type = request.param - prompt_response_project = client.create_prompt_response_generation_project( - name=f"{media_type.value}-{rand_gen(str)}", - dataset_name=f"{media_type.value}-{rand_gen(str)}", - data_row_count=1, - media_type=media_type, - ) - - yield prompt_response_project - - prompt_response_project.delete() - - -@pytest.fixture -def prompt_response_generation_project_with_dataset_id( - client: Client, dataset, rand_gen, request -): - """fixture is parametrized and needs project_type in request""" - media_type = request.param - prompt_response_project = client.create_prompt_response_generation_project( - name=f"{media_type.value}-{rand_gen(str)}", - dataset_id=dataset.uid, - data_row_count=1, - media_type=media_type, - ) - - yield prompt_response_project - - prompt_response_project.delete() - - -@pytest.fixture -def response_creation_project(client: Client, rand_gen): - project_name = f"response-creation-project-{rand_gen(str)}" - project = client.create_response_creation_project(name=project_name) - - yield project - - project.delete() - - -@pytest.fixture -def prompt_response_features(rand_gen): - prompt_text = PromptResponseClassification( - class_type=PromptResponseClassification.Type.PROMPT, - name=f"{rand_gen(str)}-prompt text", - ) - - response_radio = PromptResponseClassification( - class_type=PromptResponseClassification.Type.RESPONSE_RADIO, - name=f"{rand_gen(str)}-response radio classification", - options=[ - ResponseOption(value=f"{rand_gen(str)}-first radio option answer"), - ResponseOption(value=f"{rand_gen(str)}-second radio option answer"), - ], - ) - - response_checklist = PromptResponseClassification( - class_type=PromptResponseClassification.Type.RESPONSE_CHECKLIST, - name=f"{rand_gen(str)}-response checklist classification", - options=[ - ResponseOption( - value=f"{rand_gen(str)}-first checklist option answer" - ), - ResponseOption( - value=f"{rand_gen(str)}-second checklist option answer" - ), - ], - ) - - response_text_with_char = PromptResponseClassification( - class_type=PromptResponseClassification.Type.RESPONSE_TEXT, - name=f"{rand_gen(str)}-response text with character min and max", - character_min=1, - character_max=10, - ) - - response_text = PromptResponseClassification( - class_type=PromptResponseClassification.Type.RESPONSE_TEXT, - name=f"{rand_gen(str)}-response text", - ) - - nested_response_radio = PromptResponseClassification( - class_type=PromptResponseClassification.Type.RESPONSE_RADIO, - name=f"{rand_gen(str)}-nested response radio classification", - options=[ - ResponseOption( - f"{rand_gen(str)}-first_radio_answer", - options=[ - PromptResponseClassification( - class_type=PromptResponseClassification.Type.RESPONSE_RADIO, - name=f"{rand_gen(str)}-sub_radio_question", - options=[ - ResponseOption( - f"{rand_gen(str)}-first_sub_radio_answer" - ) - ], - ) - ], - ) - ], - ) - yield { - "prompts": [prompt_text], - "responses": [ - response_text, - response_radio, - response_checklist, - response_text_with_char, - nested_response_radio, - ], - } - - -@pytest.fixture -def prompt_response_ontology( - client: Client, rand_gen, prompt_response_features, request -): - """fixture is parametrize and needs project_type in request""" - - project_type = request.param - if project_type == MediaType.LLMPromptCreation: - ontology_builder = OntologyBuilder( - tools=[], classifications=prompt_response_features["prompts"] - ) - elif project_type == MediaType.LLMPromptResponseCreation: - ontology_builder = OntologyBuilder( - tools=[], - classifications=prompt_response_features["prompts"] - + prompt_response_features["responses"], - ) - else: - ontology_builder = OntologyBuilder( - tools=[], classifications=prompt_response_features["responses"] - ) - - ontology_name = f"prompt-response-{rand_gen(str)}" - - if project_type in MediaType: - ontology = client.create_ontology( - ontology_name, ontology_builder.asdict(), media_type=project_type - ) - else: - ontology = client.create_ontology( - ontology_name, - ontology_builder.asdict(), - media_type=MediaType.Text, - ontology_kind=OntologyKind.ResponseCreation, - ) - yield ontology - - featureSchemaIds = [ - feature["featureSchemaId"] - for feature in ontology.normalized["classifications"] - ] - - try: - client.delete_unused_ontology(ontology.uid) - for featureSchemaId in featureSchemaIds: - client.delete_unused_feature_schema(featureSchemaId) - except Exception as e: - print(f"Failed to delete ontology {ontology.uid}: {str(e)}") - - @pytest.fixture def point(): return Tool( diff --git a/libs/labelbox/tests/integration/test_project.py b/libs/labelbox/tests/integration/test_project.py index cbb6198e7..a1bb23d52 100644 --- a/libs/labelbox/tests/integration/test_project.py +++ b/libs/labelbox/tests/integration/test_project.py @@ -247,10 +247,8 @@ def test_media_type(client, project: Project, rand_gen): project.delete() for media_type in MediaType.get_supported_members(): - # Exclude LLM media types for now, as they are not supported + # Exclude LLM media type for now, as it is not supported if MediaType[media_type] in [ - MediaType.LLMPromptCreation, - MediaType.LLMPromptResponseCreation, MediaType.LLM, ]: continue diff --git a/libs/labelbox/tests/integration/test_prompt_response_generation_project.py b/libs/labelbox/tests/integration/test_prompt_response_generation_project.py deleted file mode 100644 index f5003f061..000000000 --- a/libs/labelbox/tests/integration/test_prompt_response_generation_project.py +++ /dev/null @@ -1,185 +0,0 @@ -from unittest.mock import patch - -import pytest - -from labelbox import MediaType - - -@pytest.mark.parametrize( - "prompt_response_ontology, prompt_response_generation_project_with_new_dataset", - [ - (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), - ( - MediaType.LLMPromptResponseCreation, - MediaType.LLMPromptResponseCreation, - ), - ], - indirect=True, -) -def test_prompt_response_generation_ontology_project( - client, - prompt_response_ontology, - prompt_response_generation_project_with_new_dataset, - response_data_row, - rand_gen, -): - ontology = prompt_response_ontology - - assert ontology - assert ontology.name - - for classification in ontology.classifications(): - assert classification.schema_id - assert classification.feature_schema_id - - project = prompt_response_generation_project_with_new_dataset - - project.connect_ontology(ontology) - - assert project.labeling_frontend().name == "Editor" - assert project.ontology().name == ontology.name - - with pytest.raises( - ValueError, - match="Cannot create batches for auto data generation projects", - ): - project.create_batch( - rand_gen(str), - [response_data_row.uid], # sample of data row objects - ) - - with pytest.raises( - ValueError, - match="Cannot create batches for auto data generation projects", - ): - with patch( - "labelbox.schema.project.MAX_SYNC_BATCH_ROW_COUNT", new=0 - ): # force to async - project.create_batch( - rand_gen(str), - [response_data_row.uid], # sample of data row objects - ) - - -@pytest.mark.parametrize( - "prompt_response_ontology, prompt_response_generation_project_with_dataset_id", - [ - (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation), - ( - MediaType.LLMPromptResponseCreation, - MediaType.LLMPromptResponseCreation, - ), - ], - indirect=True, -) -def test_prompt_response_generation_ontology_project_with_existing_dataset( - prompt_response_ontology, prompt_response_generation_project_with_dataset_id -): - ontology = prompt_response_ontology - - project = prompt_response_generation_project_with_dataset_id - assert project - project.connect_ontology(ontology) - - assert project.labeling_frontend().name == "Editor" - assert project.ontology().name == ontology.name - - -@pytest.fixture -def classification_json(): - classifications = [ - { - "featureSchemaId": None, - "kind": "Prompt", - "minCharacters": 2, - "maxCharacters": 10, - "name": "prompt text", - "instructions": "prompt text", - "required": True, - "schemaNodeId": None, - "scope": "global", - "type": "prompt", - "options": [], - }, - { - "featureSchemaId": None, - "kind": "ResponseCheckboxQuestion", - "name": "response checklist", - "instructions": "response checklist", - "options": [ - { - "featureSchemaId": None, - "kind": "ResponseCheckboxOption", - "label": "response checklist option", - "schemaNodeId": None, - "position": 0, - "value": "option_1", - } - ], - "required": True, - "schemaNodeId": None, - "scope": "global", - "type": "response-checklist", - }, - { - "featureSchemaId": None, - "kind": "ResponseText", - "maxCharacters": 10, - "minCharacters": 1, - "name": "response text", - "instructions": "response text", - "required": True, - "schemaNodeId": None, - "scope": "global", - "type": "response-text", - "options": [], - }, - ] - - return classifications - - -@pytest.fixture -def features_from_json(client, classification_json): - classifications = classification_json - features = {client.create_feature_schema(t) for t in classifications if t} - - yield features - - for f in features: - client.delete_unused_feature_schema(f.uid) - - -@pytest.fixture -def ontology_from_feature_ids(client, features_from_json): - feature_ids = {f.uid for f in features_from_json} - ontology = client.create_ontology_from_feature_schemas( - name="test-prompt_response_creation{rand_gen(str)}", - feature_schema_ids=feature_ids, - media_type=MediaType.LLMPromptResponseCreation, - ) - - yield ontology - - client.delete_unused_ontology(ontology.uid) - - -def test_ontology_create_feature_schema( - ontology_from_feature_ids, features_from_json, classification_json -): - created_ontology = ontology_from_feature_ids - feature_schema_ids = {f.uid for f in features_from_json} - classifications_normalized = created_ontology.normalized["classifications"] - classifications = classification_json - - for classification in classifications: - generated_tool = next( - c - for c in classifications_normalized - if c["name"] == classification["name"] - ) - assert generated_tool["schemaNodeId"] is not None - assert generated_tool["featureSchemaId"] in feature_schema_ids - assert generated_tool["type"] == classification["type"] - assert generated_tool["name"] == classification["name"] - assert generated_tool["required"] == classification["required"] diff --git a/libs/labelbox/tests/integration/test_response_creation_project.py b/libs/labelbox/tests/integration/test_response_creation_project.py deleted file mode 100644 index d7f9a1e46..000000000 --- a/libs/labelbox/tests/integration/test_response_creation_project.py +++ /dev/null @@ -1,30 +0,0 @@ -from labelbox.schema.project import Project -import pytest - -from labelbox.schema.ontology_kind import OntologyKind - - -@pytest.mark.parametrize( - "prompt_response_ontology", [OntologyKind.ResponseCreation], indirect=True -) -def test_create_response_creation_project( - client, - rand_gen, - response_creation_project, - prompt_response_ontology, - response_data_row, -): - project: Project = response_creation_project - assert project - - ontology = prompt_response_ontology - project.connect_ontology(ontology) - - assert project.labeling_frontend().name == "Editor" - assert project.ontology().name == ontology.name - - batch = project.create_batch( - rand_gen(str), - [response_data_row.uid], # sample of data row objects - ) - assert batch diff --git a/libs/labelbox/tests/unit/test_project.py b/libs/labelbox/tests/unit/test_project.py index 201f8e9e1..0946edbcf 100644 --- a/libs/labelbox/tests/unit/test_project.py +++ b/libs/labelbox/tests/unit/test_project.py @@ -120,7 +120,6 @@ def test_get_overview_rejects_empty_batch_ids(project_entity): [ (None, EditorTaskType.Missing), ("MODEL_CHAT_EVALUATION", EditorTaskType.ModelChatEvaluation), - ("RESPONSE_CREATION", EditorTaskType.ResponseCreation), ( "OFFLINE_MODEL_CHAT_EVALUATION", EditorTaskType.OfflineModelChatEvaluation, diff --git a/libs/labelbox/tests/unit/test_utils.py b/libs/labelbox/tests/unit/test_utils.py index 969f3a46b..cfdf41684 100644 --- a/libs/labelbox/tests/unit/test_utils.py +++ b/libs/labelbox/tests/unit/test_utils.py @@ -27,7 +27,7 @@ def test_datetime_parsing(datetime_str, expected_datetime_str): "str, expected_str", [ ("AUDIO", "Audio"), - ("LLM_PROMPT_RESPONSE_CREATION", "Llm prompt response creation"), + ("MODEL_CHAT_EVALUATION", "Model chat evaluation"), ], ) def test_sentence_case(str, expected_str):