feat: Add null_markers property to LoadJobConfig and CSVOptions (#2239)

* feat: Add null_markers property to LoadJobConfig and CSVOptions

* feat: adds null_markers as a load and external_config option

---------

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>

Author: chalmerlowe

google/cloud/bigquery/external_config.py

@@ -474,6 +474,27 @@ def skip_leading_rows(self):
     def skip_leading_rows(self, value):
         self._properties["skipLeadingRows"] = str(value)
 
+    @property
+    def null_markers(self) -> Optional[Iterable[str]]:
+        """Optional[Iterable[str]]: A list of strings represented as SQL NULL values in a CSV file.
+
+        .. note::
+            null_marker and null_markers can't be set at the same time.
+            If null_marker is set, null_markers has to be not set.
+            If null_markers is set, null_marker has to be not set.
+            If both null_marker and null_markers are set at the same time, a user error would be thrown.
+            Any strings listed in null_markers, including empty string would be interpreted as SQL NULL.
+            This applies to all column types.
+
+        See
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#CsvOptions.FIELDS.null_markers
+        """
+        return self._properties.get("nullMarkers")
+
+    @null_markers.setter
+    def null_markers(self, value: Optional[Iterable[str]]):
+        self._properties["nullMarkers"] = value
+
     def to_api_repr(self) -> dict:
         """Build an API representation of this object.
 

google/cloud/bigquery/job/load.py

@@ -386,6 +386,27 @@ def null_marker(self):
     def null_marker(self, value):
         self._set_sub_prop("nullMarker", value)
 
+    @property
+    def null_markers(self) -> Optional[List[str]]:
+        """Optional[List[str]]: A list of strings represented as SQL NULL values in a CSV file.
+
+        .. note::
+            null_marker and null_markers can't be set at the same time.
+            If null_marker is set, null_markers has to be not set.
+            If null_markers is set, null_marker has to be not set.
+            If both null_marker and null_markers are set at the same time, a user error would be thrown.
+            Any strings listed in null_markers, including empty string would be interpreted as SQL NULL.
+            This applies to all column types.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.null_markers
+        """
+        return self._get_sub_prop("nullMarkers")
+
+    @null_markers.setter
+    def null_markers(self, value: Optional[List[str]]):
+        self._set_sub_prop("nullMarkers", value)
+
     @property
     def preserve_ascii_control_characters(self):
         """Optional[bool]: Preserves the embedded ASCII control characters when sourceFormat is set to CSV.
@@ -854,6 +875,13 @@ def null_marker(self):
         """
         return self.configuration.null_marker
 
+    @property
+    def null_markers(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.null_markers`.
+        """
+        return self.configuration.null_markers
+
     @property
     def quote_character(self):
         """See

tests/unit/job/test_load.py

@@ -42,6 +42,7 @@ def _setUpConstants(self):
         self.TIME_ZONE = "UTC"
         self.TIME_FORMAT = "%H:%M:%S"
         self.TIMESTAMP_FORMAT = "YYYY-MM-DD HH:MM:SS.SSSSSSZ"
+        self.NULL_MARKERS = ["", "NA"]
 
     def _make_resource(self, started=False, ended=False):
         resource = super(TestLoadJob, self)._make_resource(started, ended)
@@ -52,6 +53,7 @@ def _make_resource(self, started=False, ended=False):
         config["timeZone"] = self.TIME_ZONE
         config["timeFormat"] = self.TIME_FORMAT
         config["timestampFormat"] = self.TIMESTAMP_FORMAT
+        config["nullMarkers"] = self.NULL_MARKERS
 
         config["destinationTable"] = {
             "projectId": self.PROJECT,
@@ -140,6 +142,10 @@ def _verifyResourceProperties(self, job, resource):
             self.assertEqual(job.null_marker, config["nullMarker"])
         else:
             self.assertIsNone(job.null_marker)
+        if "nullMarkers" in config:
+            self.assertEqual(job.null_markers, config["nullMarkers"])
+        else:
+            self.assertIsNone(job.null_markers)
         if "quote" in config:
             self.assertEqual(job.quote_character, config["quote"])
         else:
@@ -211,6 +217,7 @@ def test_ctor(self):
         self.assertIsNone(job.ignore_unknown_values)
         self.assertIsNone(job.max_bad_records)
         self.assertIsNone(job.null_marker)
+        self.assertIsNone(job.null_markers)
         self.assertIsNone(job.quote_character)
         self.assertIsNone(job.skip_leading_rows)
         self.assertIsNone(job.source_format)

tests/unit/job/test_load_config.py

@@ -469,6 +469,22 @@ def test_null_marker_setter(self):
         config.null_marker = null_marker
         self.assertEqual(config._properties["load"]["nullMarker"], null_marker)
 
+    def test_null_markers_missing(self):
+        config = self._get_target_class()()
+        self.assertIsNone(config.null_markers)
+
+    def test_null_markers_hit(self):
+        null_markers = ["", "NA"]
+        config = self._get_target_class()()
+        config._properties["load"]["nullMarkers"] = null_markers
+        self.assertEqual(config.null_markers, null_markers)
+
+    def test_null_markers_setter(self):
+        null_markers = ["", "NA"]
+        config = self._get_target_class()()
+        config.null_markers = null_markers
+        self.assertEqual(config._properties["load"]["nullMarkers"], null_markers)
+
     def test_preserve_ascii_control_characters_missing(self):
         config = self._get_target_class()()
         self.assertIsNone(config.preserve_ascii_control_characters)

tests/unit/test_external_config.py

@@ -277,6 +277,7 @@ def test_from_api_repr_csv(self):
                     "allowJaggedRows": False,
                     "encoding": "encoding",
                     "preserveAsciiControlCharacters": False,
+                    "nullMarkers": ["", "NA"],
                 },
             },
         )
@@ -293,6 +294,7 @@ def test_from_api_repr_csv(self):
         self.assertEqual(ec.options.allow_jagged_rows, False)
         self.assertEqual(ec.options.encoding, "encoding")
         self.assertEqual(ec.options.preserve_ascii_control_characters, False)
+        self.assertEqual(ec.options.null_markers, ["", "NA"])
 
         got_resource = ec.to_api_repr()
 
@@ -314,6 +316,7 @@ def test_to_api_repr_csv(self):
         options.skip_leading_rows = 123
         options.allow_jagged_rows = False
         options.preserve_ascii_control_characters = False
+        options.null_markers = ["", "NA"]
         ec.csv_options = options
 
         exp_resource = {
@@ -326,6 +329,7 @@ def test_to_api_repr_csv(self):
                 "allowJaggedRows": False,
                 "encoding": "encoding",
                 "preserveAsciiControlCharacters": False,
+                "nullMarkers": ["", "NA"],
             },
         }