Skip to content

Commit dd2c41a

Browse files
Neenu1995Neenu1995
authored
fix(bigquery-jdbc): handle null parameters in Storage Write API bulk inserts (#14270)
Fixes #14066
1 parent e8cbd42 commit dd2c41a

3 files changed

Lines changed: 126 additions & 9 deletions

File tree

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.google.cloud.bigquery.storage.v1.TableName;
3535
import com.google.gson.Gson;
3636
import com.google.gson.JsonArray;
37+
import com.google.gson.JsonNull;
3738
import com.google.gson.JsonObject;
3839
import com.google.protobuf.Descriptors.DescriptorValidationException;
3940
import java.io.IOException;
@@ -67,6 +68,7 @@
6768
import java.util.Arrays;
6869
import java.util.Calendar;
6970
import java.util.LinkedList;
71+
import java.util.List;
7072
import java.util.Queue;
7173

7274
class BigQueryPreparedStatement extends BigQueryStatement implements PreparedStatement {
@@ -389,15 +391,7 @@ private long bulkInsertWithWriteAPI(BigQueryWriteClient bigQueryWriteClient)
389391
FieldList fieldLists = this.insertSchema.getFields();
390392
if (fieldLists.size() == parameterList.size()) {
391393

392-
JsonObject rowObject = new JsonObject();
393-
for (int j = 0; j < parameterList.size(); j++) {
394-
BigQueryJdbcParameter parameter = parameterList.get(j);
395-
if (parameter.getSqlType() == StandardSQLTypeName.STRING) {
396-
rowObject.addProperty(fieldLists.get(j).getName(), parameter.getValue().toString());
397-
} else {
398-
rowObject.addProperty(fieldLists.get(j).getName(), gson.toJson(parameter.getValue()));
399-
}
400-
}
394+
JsonObject rowObject = createJsonRow(fieldLists, parameterList, gson);
401395
jsonArray.add(rowObject);
402396

403397
if (jsonArray.size() == this.querySettings.getWriteAPIAppendRowCount()
@@ -431,6 +425,22 @@ private long bulkInsertWithWriteAPI(BigQueryWriteClient bigQueryWriteClient)
431425
return rowCount;
432426
}
433427

428+
static JsonObject createJsonRow(
429+
FieldList fieldLists, List<BigQueryJdbcParameter> parameterList, Gson gson) {
430+
JsonObject rowObject = new JsonObject();
431+
for (int j = 0; j < parameterList.size(); j++) {
432+
BigQueryJdbcParameter parameter = parameterList.get(j);
433+
if (parameter.getValue() == null) {
434+
rowObject.add(fieldLists.get(j).getName(), JsonNull.INSTANCE);
435+
} else if (parameter.getSqlType() == StandardSQLTypeName.STRING) {
436+
rowObject.addProperty(fieldLists.get(j).getName(), parameter.getValue().toString());
437+
} else {
438+
rowObject.addProperty(fieldLists.get(j).getName(), gson.toJson(parameter.getValue()));
439+
}
440+
}
441+
return rowObject;
442+
}
443+
434444
private void setInsertMetadata(QueryStatistics statistics) throws SQLException {
435445
LOG.finer("++enter++");
436446
if (!statistics.getStatementType().equals(StatementType.INSERT)

‎java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatementSettersTest.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,18 @@
2121
import static org.junit.jupiter.api.Assertions.assertNotNull;
2222
import static org.junit.jupiter.api.Assertions.assertNull;
2323
import static org.junit.jupiter.api.Assertions.assertThrows;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
2425
import static org.mockito.Mockito.mock;
26+
import static org.mockito.Mockito.when;
2527

2628
import com.google.cloud.bigquery.Field;
29+
import com.google.cloud.bigquery.FieldList;
30+
import com.google.cloud.bigquery.QueryJobConfiguration;
2731
import com.google.cloud.bigquery.Schema;
2832
import com.google.cloud.bigquery.StandardSQLTypeName;
33+
import com.google.gson.Gson;
34+
import com.google.gson.JsonNull;
35+
import com.google.gson.JsonObject;
2936
import java.sql.Array;
3037
import java.sql.Date;
3138
import java.sql.ParameterMetaData;
@@ -38,6 +45,7 @@
3845
import java.time.LocalDate;
3946
import java.time.LocalDateTime;
4047
import java.time.LocalTime;
48+
import java.util.ArrayList;
4149
import java.util.Calendar;
4250
import java.util.TimeZone;
4351
import org.junit.jupiter.api.BeforeEach;
@@ -51,6 +59,7 @@ public class BigQueryPreparedStatementSettersTest {
5159
@BeforeEach
5260
public void setUp() throws Exception {
5361
connection = mock(BigQueryConnection.class);
62+
when(connection.getQueryDialect()).thenReturn("SQL");
5463
preparedStatement = new BigQueryPreparedStatement(connection, "SELECT ?, ?, ?, ?, ?");
5564
}
5665

@@ -269,4 +278,51 @@ public void testSetObjectWithJavaTime() throws Exception {
269278
preparedStatement.setObject(4, instant);
270279
assertEquals(Timestamp.class, preparedStatement.parameterHandler.getType(4));
271280
}
281+
282+
// Verifies standard DML QueryJobConfiguration creates positional parameters with null values when
283+
// setObject is called with null.
284+
@Test
285+
public void testBatchJobConfigurationWithSetObjectNull() throws Exception {
286+
preparedStatement.setObject(1, null);
287+
preparedStatement.setInt(2, 42);
288+
preparedStatement.setString(3, "test");
289+
preparedStatement.setDouble(4, 3.14);
290+
preparedStatement.setBoolean(5, true);
291+
preparedStatement.addBatch();
292+
293+
ArrayList<BigQueryJdbcParameter> batchParams =
294+
preparedStatement.parameterHandler.parametersList;
295+
QueryJobConfiguration config = preparedStatement.getWriteBatchJobConfiguration(batchParams);
296+
assertNotNull(config);
297+
assertEquals(5, config.getPositionalParameters().size());
298+
assertNull(config.getPositionalParameters().get(0).getValue());
299+
assertEquals(StandardSQLTypeName.STRING, config.getPositionalParameters().get(0).getType());
300+
}
301+
302+
// Verifies Storage Write API JSON row serialization maps null parameter values to
303+
// JsonNull.INSTANCE.
304+
@Test
305+
public void testCreateJsonRowWithSetObjectNull() throws Exception {
306+
preparedStatement.setObject(1, null);
307+
preparedStatement.setInt(2, 42);
308+
preparedStatement.setString(3, "test");
309+
preparedStatement.setDouble(4, 3.14);
310+
preparedStatement.setBoolean(5, true);
311+
312+
FieldList fieldList =
313+
FieldList.of(
314+
Field.of("col1", StandardSQLTypeName.STRING),
315+
Field.of("col2", StandardSQLTypeName.INT64),
316+
Field.of("col3", StandardSQLTypeName.STRING),
317+
Field.of("col4", StandardSQLTypeName.FLOAT64),
318+
Field.of("col5", StandardSQLTypeName.BOOL));
319+
320+
ArrayList<BigQueryJdbcParameter> params = preparedStatement.parameterHandler.parametersList;
321+
JsonObject jsonRow = BigQueryPreparedStatement.createJsonRow(fieldList, params, new Gson());
322+
323+
assertNotNull(jsonRow);
324+
assertEquals(JsonNull.INSTANCE, jsonRow.get("col1"));
325+
assertTrue(jsonRow.get("col1").isJsonNull());
326+
assertEquals("42", jsonRow.get("col2").getAsString());
327+
}
272328
}

‎java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITNightlyBigQueryTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1198,6 +1198,57 @@ public void testBulkInsertOperation() throws SQLException {
11981198
}
11991199
}
12001200

1201+
// Verifies batch inserts via BigQuery Storage Write API succeed when parameters are bound to
1202+
// null.
1203+
@Test
1204+
public void testBulkInsertOperationWithSetObjectNull() throws SQLException {
1205+
String TABLE_NAME = "JDBC_BULK_INSERT_NULL_TABLE_" + randomNumber;
1206+
String createQuery =
1207+
String.format(
1208+
"CREATE OR REPLACE TABLE %s.%s (`StringField` STRING,\n"
1209+
+ " `IntegerField` INTEGER,"
1210+
+ " `FloatField` FLOAT64,"
1211+
+ " `NumericField` NUMERIC,"
1212+
+ " `BigNumericField` BIGNUMERIC,"
1213+
+ " `BooleanField` BOOLEAN"
1214+
+ " );",
1215+
DATASET, TABLE_NAME);
1216+
String insertQuery =
1217+
String.format("INSERT INTO %s.%s VALUES(?, ?, ?, ?, ?, ?);", DATASET, TABLE_NAME);
1218+
String dropQuery = String.format("DROP TABLE %s.%s", DATASET, TABLE_NAME);
1219+
String selectQuery = String.format("SELECT * FROM %s.%s", DATASET, TABLE_NAME);
1220+
1221+
String connection_uri =
1222+
ITNightlyBigQueryTest.connection_uri
1223+
+ "EnableWriteAPI=1;"
1224+
+ "SWA_ActivationRowCount=5;"
1225+
+ "SWA_AppendRowCount=500";
1226+
1227+
try (Connection connection = DriverManager.getConnection(connection_uri)) {
1228+
bigQueryStatement.execute(createQuery);
1229+
PreparedStatement statement = connection.prepareStatement(insertQuery);
1230+
for (int i = 0; i < 20; ++i) {
1231+
statement.setObject(1, null);
1232+
statement.setInt(2, i);
1233+
statement.setFloat(3, (float) (i + .6));
1234+
statement.setInt(4, random.nextInt());
1235+
statement.setInt(5, random.nextInt());
1236+
statement.setBoolean(6, true);
1237+
1238+
statement.addBatch();
1239+
}
1240+
int[] result = statement.executeBatch();
1241+
1242+
ResultSet resultSet = bigQueryStatement.executeQuery(selectQuery);
1243+
assertEquals(result.length, resultSetRowCount(resultSet));
1244+
1245+
bigQueryStatement.execute(dropQuery);
1246+
1247+
} catch (SQLException e) {
1248+
throw new BigQueryJdbcException(e);
1249+
}
1250+
}
1251+
12011252
@Test
12021253
public void testBulkInsertOperationStandard() throws SQLException {
12031254
String TABLE_NAME = "JDBC_BULK_INSERT_STANDARD_TABLE_" + randomNumber;

0 commit comments

Comments
 (0)