Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/it/java/io/weaviate/integration/DataITest.java
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,11 @@ public void testDataTypes() throws IOException {

var types = client.collections.use(nsDataTypes);

var now = OffsetDateTime.now();
// Truncated to the minute on purpose: OffsetDateTime.toString() used to drop
// the seconds when second and nano are both zero, producing a non-RFC3339
// string the server rejects. A plain OffsetDateTime.now() practically never
// lands on a minute boundary, so it never caught it.
var now = OffsetDateTime.now().withSecond(0).withNano(0);
var uuid = UUID.randomUUID();

Map<String, Object> want = Map.ofEntries(
Expand Down
4 changes: 3 additions & 1 deletion src/it/java/io/weaviate/integration/SearchITest.java
Original file line number Diff line number Diff line change
Expand Up @@ -722,7 +722,9 @@ public void test_filterIsNull() throws IOException {
@Test
public void test_filterCreateUpdateTime() throws IOException {
// Arrange
var now = OffsetDateTime.now().minusHours(1);
// On a minute boundary: the filter operand used to serialize without the
// seconds, and the server failed to parse it as RFC3339.
var now = OffsetDateTime.now().minusHours(1).withSecond(0).withNano(0);
var nsCounter = ns("Counter");

var counter = client.collections.create(nsCounter,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import io.weaviate.client6.v1.api.collections.GeoCoordinates;
import io.weaviate.client6.v1.api.collections.PhoneNumber;
import io.weaviate.client6.v1.api.collections.WeaviateObject;
import io.weaviate.client6.v1.internal.DateUtil;
import io.weaviate.client6.v1.internal.MapUtil;
import io.weaviate.client6.v1.internal.grpc.ByteStringUtil;
import io.weaviate.client6.v1.internal.grpc.Rpc;
Expand Down Expand Up @@ -175,7 +176,7 @@ private static com.google.protobuf.Value marshalValue(Object value) {
} else if (value instanceof UUID v) {
protoValue.setStringValue(v.toString());
} else if (value instanceof OffsetDateTime v) {
protoValue.setStringValue(v.toString());
protoValue.setStringValue(DateUtil.toRFC3339(v));
} else if (value instanceof Boolean v) {
protoValue.setBoolValue(v.booleanValue());
} else if (value instanceof Number v) {
Expand Down Expand Up @@ -208,7 +209,7 @@ private static com.google.protobuf.Value marshalValue(Object value) {
} else if (listValue instanceof UUID lv) {
protoListValue.setStringValue(lv.toString());
} else if (listValue instanceof OffsetDateTime lv) {
protoListValue.setStringValue(lv.toString());
protoListValue.setStringValue(DateUtil.toRFC3339(lv));
} else if (listValue instanceof Boolean lv) {
protoListValue.setBoolValue(lv);
} else if (listValue instanceof Number lv) {
Expand Down Expand Up @@ -238,7 +239,7 @@ private static com.google.protobuf.Value marshalValue(Object value) {
.map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(lv.toString()).build()).toList();
} else if (value instanceof OffsetDateTime[] v) {
values = Arrays.stream(v)
.map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(lv.toString()).build()).toList();
.map(lv -> com.google.protobuf.Value.newBuilder().setStringValue(DateUtil.toRFC3339(lv)).build()).toList();
} else if (value instanceof Boolean[] v) {
values = Arrays.stream(v)
.map(lv -> com.google.protobuf.Value.newBuilder().setBoolValue(lv).build()).toList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.util.Arrays;
import java.util.List;

import io.weaviate.client6.v1.internal.DateUtil;
import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase;
import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase.Filters;

Expand Down Expand Up @@ -908,7 +909,7 @@ private DateOperand(OffsetDateTime value) {

@Override
public void appendTo(WeaviateProtoBase.Filters.Builder filter) {
filter.setValueText(value.toString());
filter.setValueText(DateUtil.toRFC3339(value));
}

@Override
Expand All @@ -930,7 +931,7 @@ private DateArrayOperand(OffsetDateTime... values) {
}

private List<String> formatted() {
return values.stream().map(OffsetDateTime::toString).toList();
return values.stream().map(DateUtil::toRFC3339).toList();
}

@Override
Expand Down
39 changes: 38 additions & 1 deletion src/main/java/io/weaviate/client6/v1/internal/DateUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

import java.io.IOException;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.temporal.ChronoField;

import com.google.gson.Gson;
import com.google.gson.TypeAdapter;
Expand All @@ -11,6 +14,29 @@
import com.google.gson.stream.JsonWriter;

public final class DateUtil {
/**
* RFC 3339 date-time with the seconds always present.
*
* <p>
* {@link OffsetDateTime#toString()} and {@link DateTimeFormatter}'s ISO
* constants omit {@code :ss} when the second and the nanosecond are both zero,
* producing {@code 2024-03-01T00:00Z}. RFC 3339's {@code partial-time} requires
* {@code hour ":" minute ":" second}, and Weaviate rejects the shorter form, so
* the seconds are written unconditionally here. The fraction stays optional and
* variable-width so sub-second precision is neither invented nor truncated.
*/
private static final DateTimeFormatter RFC3339 = new DateTimeFormatterBuilder()
.append(DateTimeFormatter.ISO_LOCAL_DATE)
.appendLiteral('T')
.appendValue(ChronoField.HOUR_OF_DAY, 2)
.appendLiteral(':')
.appendValue(ChronoField.MINUTE_OF_HOUR, 2)
.appendLiteral(':')
.appendValue(ChronoField.SECOND_OF_MINUTE, 2)
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
.appendOffsetId()
.toFormatter();

/** Prevent public initialization. */
private DateUtil() {
}
Expand All @@ -20,6 +46,17 @@ public static OffsetDateTime fromISO8601(String iso8601) {
return OffsetDateTime.parse(iso8601);
}

/**
* Format the timestamp for the wire as RFC 3339.
*
* <p>
* Use this rather than {@link OffsetDateTime#toString()} anywhere a timestamp
* is sent to Weaviate: over REST, in a gRPC batch, or as a filter operand.
*/
public static String toRFC3339(OffsetDateTime dateTime) {
return RFC3339.format(dateTime);
}

public static enum CustomTypeAdapterFactory implements TypeAdapterFactory {
INSTANCE;

Expand All @@ -34,7 +71,7 @@ public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {

@Override
public void write(JsonWriter out, OffsetDateTime value) throws IOException {
out.value(value.toString());
out.value(toRFC3339(value));
}

@Override
Expand Down
127 changes: 127 additions & 0 deletions src/test/java/io/weaviate/client6/v1/internal/Rfc3339DateTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
package io.weaviate.client6.v1.internal;

import java.time.OffsetDateTime;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;

import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.junit.runner.RunWith;

import com.jparams.junit4.JParamsTestRunner;
import com.jparams.junit4.data.DataMethod;
import com.jparams.junit4.description.Name;

import io.weaviate.client6.v1.api.collections.CollectionHandleDefaults;
import io.weaviate.client6.v1.api.collections.WeaviateObject;
import io.weaviate.client6.v1.api.collections.data.InsertManyRequest;
import io.weaviate.client6.v1.api.collections.query.Filter;
import io.weaviate.client6.v1.internal.grpc.protocol.WeaviateProtoBase;
import io.weaviate.client6.v1.internal.orm.CollectionDescriptor;

/**
* Timestamps must reach the server as RFC 3339, which requires the seconds.
*
* <p>
* {@link OffsetDateTime#toString()} omits them when the second and the
* nanosecond are both zero, so every timestamp on an exact minute boundary used
* to be rejected by the server. The existing date tests all seed from
* {@code OffsetDateTime.now()}, which practically never lands on one -- hence
* the literals here.
*/
@RunWith(JParamsTestRunner.class)
public class Rfc3339DateTest {
/** The value that used to serialize as "2024-03-01T00:00Z". */
private static final OffsetDateTime ROUND = OffsetDateTime.parse("2024-03-01T00:00:00Z");
private static final String ROUND_RFC3339 = "2024-03-01T00:00:00Z";

public static Object[][] timestamps() {
return new Object[][] {
{ "minute boundary", ROUND, ROUND_RFC3339 },
{ "zero nanos only", OffsetDateTime.parse("2024-03-01T00:00:30Z"), "2024-03-01T00:00:30Z" },
{ "millis", OffsetDateTime.parse("2024-03-01T12:34:56.789Z"), "2024-03-01T12:34:56.789Z" },
{ "nanos", OffsetDateTime.parse("2024-03-01T12:34:56.000000001Z"), "2024-03-01T12:34:56.000000001Z" },
{ "non-UTC offset", OffsetDateTime.parse("2024-03-01T00:00:00+02:00"), "2024-03-01T00:00:00+02:00" },
{ "negative offset", OffsetDateTime.parse("2024-03-01T00:00:00-05:30"), "2024-03-01T00:00:00-05:30" },
};
}

@Name("{0}")
@DataMethod(source = Rfc3339DateTest.class, method = "timestamps")
@Test
public void test_format(String __, OffsetDateTime value, String want) {
Assertions.assertThat(DateUtil.toRFC3339(value)).isEqualTo(want);
}

/** Whatever we write has to be readable again. */
@Name("{0}")
@DataMethod(source = Rfc3339DateTest.class, method = "timestamps")
@Test
public void test_roundTrip(String __, OffsetDateTime value, String ___) {
Assertions.assertThat(DateUtil.fromISO8601(DateUtil.toRFC3339(value))).isEqualTo(value);
}

/** The reader stays lenient, so dates written by older clients still parse. */
@Test
public void test_readsTheOldTruncatedForm() {
Assertions.assertThat(DateUtil.fromISO8601("2024-03-01T00:00Z")).isEqualTo(ROUND);
}

public static Object[][] comparisons() {
return new Object[][] {
{ "eq", (Function<OffsetDateTime, Filter>) v -> Filter.property("when").eq(v) },
{ "ne", (Function<OffsetDateTime, Filter>) v -> Filter.property("when").ne(v) },
{ "lt", (Function<OffsetDateTime, Filter>) v -> Filter.property("when").lt(v) },
{ "lte", (Function<OffsetDateTime, Filter>) v -> Filter.property("when").lte(v) },
{ "gt", (Function<OffsetDateTime, Filter>) v -> Filter.property("when").gt(v) },
{ "gte", (Function<OffsetDateTime, Filter>) v -> Filter.property("when").gte(v) },
// The metadata filters take OffsetDateTime only -- no String overload to fall
// back on, so these had no workaround at all.
{ "createdAt", (Function<OffsetDateTime, Filter>) v -> Filter.createdAt().gt(v) },
{ "lastUpdatedAt", (Function<OffsetDateTime, Filter>) v -> Filter.lastUpdatedAt().lt(v) },
};
}

@Name("{0}")
@DataMethod(source = Rfc3339DateTest.class, method = "comparisons")
@Test
public void test_filterOperandKeepsSeconds(String __, Function<OffsetDateTime, Filter> build) {
Assertions.assertThat(marshal(build.apply(ROUND)).getValueText()).isEqualTo(ROUND_RFC3339);
}

@Test
public void test_filterArrayOperandKeepsSeconds() {
var filter = Filter.property("when").containsAny(ROUND, OffsetDateTime.parse("2024-03-01T00:00:01Z"));

Assertions.assertThat(marshal(filter).getValueTextArray().getValuesList())
.containsExactly(ROUND_RFC3339, "2024-03-01T00:00:01Z");
}

@Test
public void test_insertManyKeepsSeconds() {
var properties = Map.<String, Object>of(
"scalar", ROUND,
"list", List.of(ROUND),
"array", new OffsetDateTime[] { ROUND });

var fields = InsertManyRequest.buildObject(
WeaviateObject.<Map<String, Object>>of(o -> o.properties(properties)),
CollectionDescriptor.ofMap("Things"),
new CollectionHandleDefaults(Optional.empty(), Optional.empty()))
.getProperties().getNonRefProperties().getFieldsMap();

Assertions.assertThat(fields.get("scalar").getStringValue()).as("scalar").isEqualTo(ROUND_RFC3339);
Assertions.assertThat(fields.get("list").getListValue().getValues(0).getStringValue())
.as("list").isEqualTo(ROUND_RFC3339);
Assertions.assertThat(fields.get("array").getListValue().getValues(0).getStringValue())
.as("array").isEqualTo(ROUND_RFC3339);
}

private static WeaviateProtoBase.Filters marshal(Filter filter) {
var builder = WeaviateProtoBase.Filters.newBuilder();
filter.appendTo(builder);
return builder.build();
}
}
35 changes: 35 additions & 0 deletions src/test/java/io/weaviate/client6/v1/internal/json/JSONTest.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.weaviate.client6.v1.internal.json;

import java.time.OffsetDateTime;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -2269,6 +2270,40 @@ public static Object[][] testCases() {
}
"""
},

// DateUtil.CustomTypeAdapterFactory
//
// Weaviate wants RFC 3339, which requires the seconds. OffsetDateTime's own
// toString() drops them when second and nano are both zero, so a timestamp on
// an exact minute boundary used to go out as "2024-03-01T00:00Z" and get
// rejected.
{
OffsetDateTime.class,
OffsetDateTime.parse("2024-03-01T00:00:00Z"),
"\"2024-03-01T00:00:00Z\"",
},
{
OffsetDateTime.class,
OffsetDateTime.parse("2024-03-01T00:00:01Z"),
"\"2024-03-01T00:00:01Z\"",
},
// The fraction is preserved as-is: neither invented nor truncated.
{
OffsetDateTime.class,
OffsetDateTime.parse("2024-03-01T12:34:56.789Z"),
"\"2024-03-01T12:34:56.789Z\"",
},
{
OffsetDateTime.class,
OffsetDateTime.parse("2024-03-01T00:00:00.000000001Z"),
"\"2024-03-01T00:00:00.000000001Z\"",
},
// A non-UTC offset keeps its offset rather than being normalised.
{
OffsetDateTime.class,
OffsetDateTime.parse("2024-03-01T00:00:00+02:00"),
"\"2024-03-01T00:00:00+02:00\"",
},
};
}

Expand Down
Loading