Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -1731,7 +1731,10 @@ public TableDataList call() throws IOException {
new PageImpl<>(
new TableDataPageFetcher(tableId, schema, serviceOptions, cursor, pageOptionMap),
cursor,
transformTableData(result.getRows(), schema, serviceOptions.getUseInt64Timestamps())),
transformTableData(
result.getRows(),
schema,
serviceOptions.getDataFormatOptions().useInt64Timestamp())),
result.getTotalRows());
} catch (BigQueryRetryHelperException e) {
throw BigQueryException.translateAndThrow(e);
Expand Down Expand Up @@ -2007,7 +2010,9 @@ public com.google.api.services.bigquery.model.QueryResponse call()
new QueryPageFetcher(jobId, schema, getOptions(), cursor, optionMap(options)),
cursor,
transformTableData(
results.getRows(), schema, getOptions().getUseInt64Timestamps())))
results.getRows(),
schema,
getOptions().getDataFormatOptions().useInt64Timestamp())))
.setJobId(jobId)
.setQueryId(results.getQueryId())
.build();
Expand All @@ -2021,7 +2026,9 @@ public com.google.api.services.bigquery.model.QueryResponse call()
new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)),
null,
transformTableData(
results.getRows(), schema, getOptions().getUseInt64Timestamps())))
results.getRows(),
schema,
getOptions().getDataFormatOptions().useInt64Timestamp())))
// Return the JobID of the successful job
.setJobId(
results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null)
Expand Down Expand Up @@ -2066,10 +2073,9 @@ && getOptions().getOpenTelemetryTracer() != null) {
}
try (Scope queryScope = querySpan != null ? querySpan.makeCurrent() : null) {
// If all parameters passed in configuration are supported by the query() method on the
// backend,
// put on fast path
// backend, put on fast path
QueryRequestInfo requestInfo =
new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps());
new QueryRequestInfo(configuration, getOptions().getDataFormatOptions());
if (requestInfo.isFastQuerySupported(jobId)) {
// Be careful when setting the projectID in JobId, if a projectID is specified in the JobId,
// the job created by the query method will use that project. This may cause the query to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.google.cloud.bigquery;

import com.google.api.core.BetaApi;
import com.google.api.core.ObsoleteApi;
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.cloud.ServiceDefaults;
import com.google.cloud.ServiceOptions;
Expand All @@ -26,6 +27,7 @@
import com.google.cloud.bigquery.spi.BigQueryRpcFactory;
import com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc;
import com.google.cloud.http.HttpTransportOptions;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import io.opentelemetry.api.trace.Tracer;
import java.util.Set;
Expand All @@ -41,6 +43,7 @@ public class BigQueryOptions extends ServiceOptions<BigQuery, BigQueryOptions> {
// set the option ThrowNotFound when you want to throw the exception when the value not found
private boolean setThrowNotFound;
private boolean useInt64Timestamps;
private DataFormatOptions dataFormatOptions;
private JobCreationMode defaultJobCreationMode = JobCreationMode.JOB_CREATION_MODE_UNSPECIFIED;
private boolean enableOpenTelemetryTracing;
private Tracer openTelemetryTracer;
Expand Down Expand Up @@ -70,6 +73,7 @@ public static class Builder extends ServiceOptions.Builder<BigQuery, BigQueryOpt

private String location;
private boolean useInt64Timestamps;
private DataFormatOptions dataFormatOptions;
private boolean enableOpenTelemetryTracing;
private Tracer openTelemetryTracer;
private ResultRetryAlgorithm<?> resultRetryAlgorithm;
Expand All @@ -94,11 +98,32 @@ public Builder setLocation(String location) {
return this;
}

/**
* This setter is marked as Obsolete. Prefer {@link #setDataFormatOptions(DataFormatOptions)} to
* set the int64timestamp configuration instead.
*
* <p>If useInt64Timestamps value is set in here and via DataFormatOptions, the
* DataFormatOptions configuration value is used.
*
* <p>{@code DataFormatOptions.newBuilder().setUseInt64Timestamp(...).build()}
*/
@ObsoleteApi("Use setDataFormatOptions(DataFormatOptions) instead")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason you would prefer @ObsoleteApi over @Deprecated?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Deprecated could result downstream customer's CI jobs failing based on their compiler settings. We prefer @ObsoleteApi as a first warning and then moving to @Deprecated in a future major version.

public Builder setUseInt64Timestamps(boolean useInt64Timestamps) {
this.useInt64Timestamps = useInt64Timestamps;
return this;
}

/**
* Set the format options for the BigQuery data types
*
* @param dataFormatOptions Configuration of the formatting options
*/
public Builder setDataFormatOptions(DataFormatOptions dataFormatOptions) {
Preconditions.checkNotNull(dataFormatOptions, "DataFormatOptions cannot be null");
this.dataFormatOptions = dataFormatOptions;
return this;
}

/**
* Enables OpenTelemetry tracing functionality for this BigQuery instance
*
Expand Down Expand Up @@ -143,6 +168,15 @@ private BigQueryOptions(Builder builder) {
} else {
this.resultRetryAlgorithm = BigQueryBaseService.DEFAULT_BIGQUERY_EXCEPTION_HANDLER;
}

// If dataFormatOptions is not set, then create a new instance and set it with the
// useInt64Timestamps configured in BigQueryOptions
if (builder.dataFormatOptions == null) {
this.dataFormatOptions =
DataFormatOptions.newBuilder().useInt64Timestamp(builder.useInt64Timestamps).build();
} else {
this.dataFormatOptions = builder.dataFormatOptions;
}
}

private static class BigQueryDefaults implements ServiceDefaults<BigQuery, BigQueryOptions> {
Expand Down Expand Up @@ -191,8 +225,23 @@ public void setThrowNotFound(boolean setThrowNotFound) {
this.setThrowNotFound = setThrowNotFound;
}

/**
* This setter is marked as Obsolete. Prefer {@link
* Builder#setDataFormatOptions(DataFormatOptions)} to set the int64timestamp configuration
* instead.
*
* <p>If useInt64Timestamps is set via DataFormatOptions, then the value in DataFormatOptions will
* be used. Otherwise, this value will be passed to DataFormatOptions.
*
* <p>Alternative: {@code DataFormatOptions.newBuilder().setUseInt64Timestamp(...).build()}
*/
@ObsoleteApi("Use Builder#setDataFormatOptions(DataFormatOptions) instead")
public void setUseInt64Timestamps(boolean useInt64Timestamps) {
this.useInt64Timestamps = useInt64Timestamps;
// Because this setter exists outside the Builder, DataFormatOptions needs be rebuilt to
// account for this setting.
this.dataFormatOptions =
dataFormatOptions.toBuilder().useInt64Timestamp(useInt64Timestamps).build();
}

@Deprecated
Expand All @@ -206,8 +255,22 @@ public boolean getThrowNotFound() {
return setThrowNotFound;
}

/**
* This getter is marked as Obsolete. Prefer {@link
* DataFormatOptions.Builder#useInt64Timestamp(boolean)} to set the int64timestamp configuration
* instead.
*
* <p>Warning: DataFormatOptions values have precedence. Use {@link
* DataFormatOptions#useInt64Timestamp()} to get `useInt64Timestamp` value used by the BigQuery
* client.
*/
@ObsoleteApi("Use getDataFormatOptions().isUseInt64Timestamp() instead")
public boolean getUseInt64Timestamps() {
return useInt64Timestamps;
return dataFormatOptions.useInt64Timestamp();
}

public DataFormatOptions getDataFormatOptions() {
return dataFormatOptions;
}

public JobCreationMode getDefaultJobCreationMode() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright 2025 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.cloud.bigquery;

import com.google.auto.value.AutoValue;
import java.io.Serializable;

/**
* Google BigQuery DataFormatOptions. Configures the output format for data types returned from
* BigQuery.
*/
@AutoValue
public abstract class DataFormatOptions implements Serializable {
public enum TimestampFormatOptions {
TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED("TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED"),
FLOAT64("FLOAT64"),
INT64("INT64"),
ISO8601_STRING("ISO8601_STRING");

private final String format;

TimestampFormatOptions(String format) {
this.format = format;
}

@Override
public String toString() {
return format;
}
}

public abstract boolean useInt64Timestamp();

public abstract TimestampFormatOptions timestampFormatOptions();

public static Builder newBuilder() {
return new AutoValue_DataFormatOptions.Builder()
.useInt64Timestamp(false)
.timestampFormatOptions(TimestampFormatOptions.TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED);
}

public abstract Builder toBuilder();

@AutoValue.Builder
public abstract static class Builder {
public abstract Builder useInt64Timestamp(boolean useInt64Timestamp);

public abstract Builder timestampFormatOptions(TimestampFormatOptions timestampFormatOptions);

public abstract DataFormatOptions build();
}

com.google.api.services.bigquery.model.DataFormatOptions toPb() {
com.google.api.services.bigquery.model.DataFormatOptions request =
new com.google.api.services.bigquery.model.DataFormatOptions();
request.setUseInt64Timestamp(useInt64Timestamp());
request.setTimestampOutputFormat(timestampFormatOptions().toString());
return request;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ final class QueryRequestInfo {
private final DataFormatOptions formatOptions;
private final String reservation;

QueryRequestInfo(QueryJobConfiguration config, Boolean useInt64Timestamps) {
QueryRequestInfo(
QueryJobConfiguration config, com.google.cloud.bigquery.DataFormatOptions dataFormatOptions) {
this.config = config;
this.connectionProperties = config.getConnectionProperties();
this.defaultDataset = config.getDefaultDataset();
Expand All @@ -61,7 +62,7 @@ final class QueryRequestInfo {
this.useLegacySql = config.useLegacySql();
this.useQueryCache = config.useQueryCache();
this.jobCreationMode = config.getJobCreationMode();
this.formatOptions = new DataFormatOptions().setUseInt64Timestamp(useInt64Timestamps);
this.formatOptions = dataFormatOptions.toPb();
this.reservation = config.getReservation();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@

package com.google.cloud.bigquery;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import com.google.cloud.TransportOptions;
import org.junit.Assert;
import org.junit.Test;
Expand All @@ -35,4 +40,55 @@ public void testInvalidTransport() {
Assert.assertNotNull(expected.getMessage());
}
}

@Test
public void dataFormatOptions_createdByDefault() {
BigQueryOptions options = BigQueryOptions.newBuilder().setProjectId("project-id").build();

assertNotNull(options.getDataFormatOptions());
assertFalse(options.getDataFormatOptions().useInt64Timestamp());
assertEquals(
DataFormatOptions.TimestampFormatOptions.TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED,
options.getDataFormatOptions().timestampFormatOptions());
}

@Test
public void nonBuilderSetUseInt64Timestamp_capturedInDataFormatOptions() {
BigQueryOptions options =
BigQueryOptions.newBuilder()
.setDataFormatOptions(DataFormatOptions.newBuilder().useInt64Timestamp(false).build())
.setProjectId("project-id")
.build();
options.setUseInt64Timestamps(true);

assertTrue(options.getDataFormatOptions().useInt64Timestamp());
}

@Test
public void nonBuilderSetUseInt64Timestamp_overridesEverything() {
BigQueryOptions options = BigQueryOptions.newBuilder().setProjectId("project-id").build();
options.setUseInt64Timestamps(true);

assertTrue(options.getDataFormatOptions().useInt64Timestamp());
}

@Test
public void noDataFormatOptions_capturesUseInt64TimestampSetInBuilder() {
BigQueryOptions options =
BigQueryOptions.newBuilder().setUseInt64Timestamps(true).setProjectId("project-id").build();

assertTrue(options.getDataFormatOptions().useInt64Timestamp());
}

@Test
public void dataFormatOptionsSetterHasPrecedence() {
BigQueryOptions options =
BigQueryOptions.newBuilder()
.setProjectId("project-id")
.setDataFormatOptions(DataFormatOptions.newBuilder().useInt64Timestamp(true).build())
.setUseInt64Timestamps(false)
.build();

assertTrue(options.getDataFormatOptions().useInt64Timestamp());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@ public class QueryRequestInfoTest {
.setJobCreationMode(jobCreationModeRequired)
.setReservation(RESERVATION)
.build();
QueryRequestInfo REQUEST_INFO = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false);
QueryRequestInfo REQUEST_INFO =
new QueryRequestInfo(QUERY_JOB_CONFIGURATION, DataFormatOptions.newBuilder().build());
private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION_SUPPORTED =
QueryJobConfiguration.newBuilder(QUERY)
.setUseQueryCache(USE_QUERY_CACHE)
Expand All @@ -156,7 +157,8 @@ public class QueryRequestInfoTest {
.setReservation(RESERVATION)
.build();
QueryRequestInfo REQUEST_INFO_SUPPORTED =
new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED, false);
new QueryRequestInfo(
QUERY_JOB_CONFIGURATION_SUPPORTED, DataFormatOptions.newBuilder().build());

@Test
public void testIsFastQuerySupported() {
Expand All @@ -177,17 +179,25 @@ public void testToPb() {
@Test
public void equalTo() {
compareQueryRequestInfo(
new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED, false), REQUEST_INFO_SUPPORTED);
compareQueryRequestInfo(new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false), REQUEST_INFO);
new QueryRequestInfo(
QUERY_JOB_CONFIGURATION_SUPPORTED, DataFormatOptions.newBuilder().build()),
REQUEST_INFO_SUPPORTED);
compareQueryRequestInfo(
new QueryRequestInfo(QUERY_JOB_CONFIGURATION, DataFormatOptions.newBuilder().build()),
REQUEST_INFO);
}

@Test
public void testInt64Timestamp() {
QueryRequestInfo requestInfo = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false);
QueryRequestInfo requestInfo =
new QueryRequestInfo(QUERY_JOB_CONFIGURATION, DataFormatOptions.newBuilder().build());
QueryRequest requestPb = requestInfo.toPb();
assertFalse(requestPb.getFormatOptions().getUseInt64Timestamp());

QueryRequestInfo requestInfoLosslessTs = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, true);
QueryRequestInfo requestInfoLosslessTs =
new QueryRequestInfo(
QUERY_JOB_CONFIGURATION,
DataFormatOptions.newBuilder().useInt64Timestamp(true).build());
QueryRequest requestLosslessTsPb = requestInfoLosslessTs.toPb();
assertTrue(requestLosslessTsPb.getFormatOptions().getUseInt64Timestamp());
}
Expand Down
Loading