Mark VertexAI AutoMLText deprecation (#42251)

- Label deprecated items with deprecated decorator.
- Refactor vertex_ai AutoMLHook.create_auto_ml_text_training_job, to
  avoid calling get_auto_ml_text_training_job, to not trigger mypy
errors after applying @deprecate on hook method.
- Update docs.
- Delete irrelevant system tests.

Co-authored-by: Oleg Kachur 

Author: olegkachur-e

airflow/providers/google/cloud/hooks/vertex_ai/auto_ml.py

@@ -36,6 +36,7 @@
 from google.cloud.aiplatform_v1 import JobServiceClient, PipelineServiceClient
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.providers.google.common.deprecated import deprecated
 from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
 
 if TYPE_CHECKING:
@@ -185,6 +186,11 @@ def get_auto_ml_image_training_job(
             model_encryption_spec_key_name=model_encryption_spec_key_name,
         )
 
+    @deprecated(
+        planned_removal_date="June 15, 2025",
+        category=AirflowProviderDeprecationWarning,
+        reason="Deprecation of AutoMLText API",
+    )
     def get_auto_ml_text_training_job(
         self,
         display_name: str,
@@ -197,7 +203,12 @@ def get_auto_ml_text_training_job(
         training_encryption_spec_key_name: str | None = None,
         model_encryption_spec_key_name: str | None = None,
     ) -> AutoMLTextTrainingJob:
-        """Return AutoMLTextTrainingJob object."""
+        """
+        Return AutoMLTextTrainingJob object.
+
+        WARNING: Text creation API is deprecated since September 15, 2024
+        (https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/overview).
+        """
         return AutoMLTextTrainingJob(
             display_name=display_name,
             prediction_type=prediction_type,
@@ -980,6 +991,11 @@ def create_auto_ml_image_training_job(
         return model, training_id
 
     @GoogleBaseHook.fallback_to_default_project_id
+    @deprecated(
+        planned_removal_date="September 15, 2025",
+        category=AirflowProviderDeprecationWarning,
+        reason="Deprecation of AutoMLText API",
+    )
     def create_auto_ml_text_training_job(
         self,
         project_id: str,
@@ -1009,6 +1025,9 @@ def create_auto_ml_text_training_job(
         """
         Create an AutoML Text Training Job.
 
+        WARNING: Text creation API is deprecated since September 15, 2024
+        (https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/overview).
+
         :param project_id: Required. Project to run training in.
         :param region: Required. Location to run training in.
         :param display_name: Required. The user-defined name of this TrainingPipeline.
@@ -1101,13 +1120,14 @@ def create_auto_ml_text_training_job(
             concurrent Future and any downstream object will be immediately returned and synced when the
             Future has completed.
         """
-        self._job = self.get_auto_ml_text_training_job(
-            project=project_id,
-            location=region,
+        self._job = AutoMLTextTrainingJob(
             display_name=display_name,
             prediction_type=prediction_type,
             multi_label=multi_label,
             sentiment_max=sentiment_max,
+            project=project_id,
+            location=region,
+            credentials=self.get_credentials(),
             labels=labels,
             training_encryption_spec_key_name=training_encryption_spec_key_name,
             model_encryption_spec_key_name=model_encryption_spec_key_name,
@@ -1117,13 +1137,13 @@ def create_auto_ml_text_training_job(
             raise AirflowException("AutoMLTextTrainingJob was not created")
 
         model = self._job.run(
-            dataset=dataset,
-            training_fraction_split=training_fraction_split,
-            validation_fraction_split=validation_fraction_split,
+            dataset=dataset,  # type: ignore[arg-type]
+            training_fraction_split=training_fraction_split,  # type: ignore[call-arg]
+            validation_fraction_split=validation_fraction_split,  # type: ignore[call-arg]
             test_fraction_split=test_fraction_split,
             training_filter_split=training_filter_split,
             validation_filter_split=validation_filter_split,
-            test_filter_split=test_filter_split,
+            test_filter_split=test_filter_split,  # type: ignore[call-arg]
             model_display_name=model_display_name,
             model_labels=model_labels,
             sync=sync,

airflow/providers/google/cloud/operators/vertex_ai/auto_ml.py

@@ -455,8 +455,22 @@ def execute(self, context: Context):
         return result
 
 
+@deprecated(
+    planned_removal_date="September 15, 2024",
+    use_instead="SupervisedFineTuningTrainOperator",
+    instructions=(
+        "Please consider using Fine Tuning over the Gemini model. "
+        "More info: https://cloud.google.com/vertex-ai/docs/start/automl-gemini-comparison"
+    ),
+    category=AirflowProviderDeprecationWarning,
+)
 class CreateAutoMLTextTrainingJobOperator(AutoMLTrainingJobBaseOperator):
-    """Create Auto ML Text Training job."""
+    """
+    Create Auto ML Text Training job.
+
+    WARNING: Text creation API is deprecated since September 15, 2024
+        (https://cloud.google.com/vertex-ai/docs/tutorials/text-classification-automl/overview).
+    """
 
     template_fields = [
         "parent_model",

docs/apache-airflow-providers-google/operators/cloud/automl.rst

@@ -109,15 +109,12 @@ available on the Vertex AI platform. Please use
 :class:`~airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLImageTrainingJobOperator` or
 :class:`~airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLVideoTrainingJobOperator`.
 
-You can find example on how to use VertexAI operators for AutoML Natural Language classification here:
+The Vertex AutoMLText API for model training is deprecated on September 15, 2024 and the other part will be deprecated
+on June 15, 2025.
+Please consider using fine tuning with Gemini model -
+https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini-tuning.
 
-.. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py
-    :language: python
-    :dedent: 4
-    :start-after: [START howto_cloud_create_text_classification_training_job_operator]
-    :end-before: [END howto_cloud_create_text_classification_training_job_operator]
-
-Additionally, you can find example on how to use VertexAI operators for AutoML Vision classification here:
+You can find example on how to use VertexAI operators for AutoML Vision classification here:
 
 .. exampleinclude:: /../../tests/system/providers/google/cloud/automl/example_automl_vision_classification.py
     :language: python

docs/apache-airflow-providers-google/operators/cloud/vertex_ai.rst

@@ -260,14 +260,11 @@ put dataset id to ``dataset_id`` parameter in operator.
 How to run AutoML Text Training Job
 :class:`~airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLTextTrainingJobOperator`
 
-Before start running this Job you must prepare and create ``Text`` dataset. After that you should
-put dataset id to ``dataset_id`` parameter in operator.
+Operator is deprecated, please use
+:class:`~airflow.providers.google.cloud.operators.vertex_ai.generative_model.SupervisedFineTuningTrainOperator` over
+the Gemini model.
+More info: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini-tuning#tuning-gemini
 
-.. exampleinclude:: /../../tests/system/providers/google/cloud/vertex_ai/example_vertex_ai_auto_ml_text_training.py
-    :language: python
-    :dedent: 4
-    :start-after: [START how_to_cloud_vertex_ai_create_auto_ml_text_training_job_operator]
-    :end-before: [END how_to_cloud_vertex_ai_create_auto_ml_text_training_job_operator]
 
 How to run AutoML Video Training Job
 :class:`~airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLVideoTrainingJobOperator`

tests/always/test_project_structure.py

@@ -389,6 +389,7 @@ class TestGoogleProviderProjectStructure(ExampleCoverageTest, AssetsCoverageTest
         "airflow.providers.google.cloud.operators.bigquery.BigQueryPatchDatasetOperator",
         "airflow.providers.google.cloud.operators.dataflow.DataflowCreatePythonJobOperator",
         "airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator",
+        "airflow.providers.google.cloud.operators.vertex_ai.auto_ml.CreateAutoMLTextTrainingJobOperator",
         "airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceAsyncSensor",
         "airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistencePartitionAsyncSensor",
         "airflow.providers.google.cloud.sensors.cloud_composer.CloudComposerEnvironmentSensor",

tests/providers/google/cloud/operators/test_vertex_ai.py

@@ -1833,20 +1833,21 @@ class TestVertexAICreateAutoMLTextTrainingJobOperator:
     @mock.patch(VERTEX_AI_PATH.format("auto_ml.AutoMLHook"))
     def test_execute(self, mock_hook, mock_dataset):
         mock_hook.return_value.create_auto_ml_text_training_job.return_value = (None, "training_id")
-        op = CreateAutoMLTextTrainingJobOperator(
-            task_id=TASK_ID,
-            gcp_conn_id=GCP_CONN_ID,
-            impersonation_chain=IMPERSONATION_CHAIN,
-            display_name=DISPLAY_NAME,
-            dataset_id=TEST_DATASET_ID,
-            prediction_type=None,
-            multi_label=False,
-            sentiment_max=10,
-            sync=True,
-            region=GCP_LOCATION,
-            project_id=GCP_PROJECT,
-            parent_model=TEST_PARENT_MODEL,
-        )
+        with pytest.warns(AirflowProviderDeprecationWarning):
+            op = CreateAutoMLTextTrainingJobOperator(
+                task_id=TASK_ID,
+                gcp_conn_id=GCP_CONN_ID,
+                impersonation_chain=IMPERSONATION_CHAIN,
+                display_name=DISPLAY_NAME,
+                dataset_id=TEST_DATASET_ID,
+                prediction_type=None,
+                multi_label=False,
+                sentiment_max=10,
+                sync=True,
+                region=GCP_LOCATION,
+                project_id=GCP_PROJECT,
+                parent_model=TEST_PARENT_MODEL,
+            )
         op.execute(context={"ti": mock.MagicMock()})
         mock_hook.assert_called_once_with(gcp_conn_id=GCP_CONN_ID, impersonation_chain=IMPERSONATION_CHAIN)
         mock_dataset.assert_called_once_with(dataset_name=TEST_DATASET_ID)
@@ -1880,20 +1881,21 @@ def test_execute(self, mock_hook, mock_dataset):
     @mock.patch(VERTEX_AI_PATH.format("auto_ml.AutoMLHook"))
     def test_execute__parent_model_version_index_is_removed(self, mock_hook, mock_dataset):
         mock_hook.return_value.create_auto_ml_text_training_job.return_value = (None, "training_id")
-        op = CreateAutoMLTextTrainingJobOperator(
-            task_id=TASK_ID,
-            gcp_conn_id=GCP_CONN_ID,
-            impersonation_chain=IMPERSONATION_CHAIN,
-            display_name=DISPLAY_NAME,
-            dataset_id=TEST_DATASET_ID,
-            prediction_type=None,
-            multi_label=False,
-            sentiment_max=10,
-            sync=True,
-            region=GCP_LOCATION,
-            project_id=GCP_PROJECT,
-            parent_model=VERSIONED_TEST_PARENT_MODEL,
-        )
+        with pytest.warns(AirflowProviderDeprecationWarning):
+            op = CreateAutoMLTextTrainingJobOperator(
+                task_id=TASK_ID,
+                gcp_conn_id=GCP_CONN_ID,
+                impersonation_chain=IMPERSONATION_CHAIN,
+                display_name=DISPLAY_NAME,
+                dataset_id=TEST_DATASET_ID,
+                prediction_type=None,
+                multi_label=False,
+                sentiment_max=10,
+                sync=True,
+                region=GCP_LOCATION,
+                project_id=GCP_PROJECT,
+                parent_model=VERSIONED_TEST_PARENT_MODEL,
+            )
         op.execute(context={"ti": mock.MagicMock()})
         mock_hook.return_value.create_auto_ml_text_training_job.assert_called_once_with(
             project_id=GCP_PROJECT,