diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 614a2b9..aed8029 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -10,20 +10,16 @@ import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.*; -import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; -import ucar.ma2.MAMath; import java.io.BufferedReader; import java.io.IOException; @@ -40,20 +36,10 @@ import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; import static dev.zarr.zarrjava.core.Node.ZARR_JSON; -import static dev.zarr.zarrjava.utils.Utils.toLongArray; import static org.junit.Assert.assertThrows; public class ZarrV3Test extends ZarrTest { - static Stream> invalidCodecBuilder() { - return Stream.of( - c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), - c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) - ); - } - static Stream invalidChunkSizes() { return Stream.of( new int[]{1}, @@ -61,32 +47,6 @@ static Stream invalidChunkSizes() { ); } - static Stream invalidShardSizes() { - return Stream.of( - new int[]{4}, //wrong dims - new int[]{4, 4, 4}, //wrong dims - new int[]{1, 1}, //smaller than inner chunk shape - new int[]{5, 5}, //no exact multiple of inner chunk shape - new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension - new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension - ); - } - - static Stream invalidShardSizesWithNested() { - return invalidShardSizes().flatMap(shardSize -> - Stream.of(true, false).map(nested -> Arguments.of(shardSize, nested)) - ); - } - - static Stream invalidTransposeOrder() { - return Stream.of( - new int[]{1, 0, 0}, - new int[]{1, 2, 3}, - new int[]{1, 2, 3, 0}, - new int[]{1, 2} - ); - } - static Stream> codecBuilders() { return Stream.of( CodecBuilder::withBlosc, @@ -119,36 +79,6 @@ static Stream unalignedArrayAccessProvider() { return builder.build(); } - static Stream dataTypeAndEndianProvider() { - return Stream.of( - Arguments.of(DataType.INT16, BytesCodec.Endian.LITTLE), - Arguments.of(DataType.INT16, BytesCodec.Endian.BIG), - Arguments.of(DataType.UINT16, BytesCodec.Endian.LITTLE), - Arguments.of(DataType.UINT16, BytesCodec.Endian.BIG), - Arguments.of(DataType.INT32, BytesCodec.Endian.LITTLE), - Arguments.of(DataType.INT32, BytesCodec.Endian.BIG), - Arguments.of(DataType.UINT32, BytesCodec.Endian.LITTLE), - Arguments.of(DataType.UINT32, BytesCodec.Endian.BIG), - Arguments.of(DataType.FLOAT32, BytesCodec.Endian.LITTLE), - Arguments.of(DataType.FLOAT32, BytesCodec.Endian.BIG), - Arguments.of(DataType.FLOAT64, BytesCodec.Endian.LITTLE), - Arguments.of(DataType.FLOAT64, BytesCodec.Endian.BIG) - ); - } - - @ParameterizedTest - @MethodSource("invalidCodecBuilder") - public void testCheckInvalidCodecConfiguration(Function codecBuilder) { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(4, 4) - .withDataType(DataType.UINT32) - .withChunkShape(2, 2) - .withCodecs(codecBuilder); - - assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); - } - @Test public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { int[] testData = new int[16 * 16 * 16]; @@ -184,111 +114,6 @@ public void testCheckInvalidChunkDimensions(int[] chunkSize) { assertThrows(ZarrException.class, builder::build); } - @ParameterizedTest - @MethodSource("invalidShardSizesWithNested") - public void testCheckShardingBounds(int[] shardSize, boolean nested) { - long[] shape = new long[]{10, 10}; - int[] innerChunkSize = new int[]{2, 2}; - - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32).withChunkShape(shardSize); - - if (nested) { - int[] nestedChunkSize = new int[]{4, 4}; - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); - } - builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); - assertThrows(ZarrException.class, builder::build); - } - - @ParameterizedTest - @CsvSource({"0,true", "0,false", "5, true", "5, false"}) - public void testZstdCodecReadWrite(int level, boolean checksum) throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "level_" + level); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withCodecs(c -> c.withZstd(level, checksum)); - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - @Test - public void testShardingWithZstdCodecReadWrite() throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testShardingWithZstdCodecReadWrite"); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(8, 8, 8) - .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{2, 4, 8}, c1 -> c1.withZstd())); - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - @Test - public void testTransposeCodec() throws ZarrException { - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); - ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ - 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - new long[]{2, 3, 3}, - new int[]{2, 3, 3}, - DataType.UINT32, - null)); - - assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); - assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); - } - - @ParameterizedTest - @MethodSource("invalidTransposeOrder") - public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { - int[] shapeInt = new int[]{2, 3, 3}; - long[] shapeLong = new long[]{2, 3, 3}; - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - shapeLong, - shapeInt, - DataType.UINT32, - null)); - - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); - assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); - } - - @Test - public void testShardingReadCutout() throws IOException, ZarrException { - Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new long[]{1, 64, 64, 64}); - Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); - Assertions.assertEquals(-98, outArray.getByte(0)); - } - @Test public void testAccess() throws IOException, ZarrException { Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); @@ -306,57 +131,6 @@ public void testAccess() throws IOException, ZarrException { writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); } - @ParameterizedTest - @ValueSource(strings = {"start", "end"}) - public void testShardingReadWrite(String indexLocation) throws IOException, ZarrException { - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); - ucar.ma2.Array readArrayContent = readArray.read(); - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), - readArray.metadata() - ); - writeArray.write(readArrayContent); - ucar.ma2.Array outArray = writeArray.read(); - - assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); - } - - @Test - public void testCodecs() throws IOException, ZarrException { - long[] readShape = new long[]{1, 1, 1024, 1024}; - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); - ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); - { - Array gzipArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withGzip(5)).build() - ); - gzipArray.write(readArrayContent); - ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); - } - { - Array bloscArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withBlosc("zstd", 5)).build() - ); - bloscArray.write(readArrayContent); - ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); - } - { - Array zstdArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withZstd(10)).build() - ); - zstdArray.write(readArrayContent); - ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); - } - } - @Test public void testArrayMetadataBuilder() throws ZarrException { long[] shape = new long[]{1, 4096, 4096, 1536}; @@ -678,22 +452,6 @@ public void testAttributes() throws IOException, ZarrException { Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); } - @Test - public void testCodecWithoutConfiguration() throws ZarrException, IOException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecWithoutConfigurationV3"); - Array array = Array.create(storeHandle, Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - .withCodecs(CodecBuilder::withBytes) - .build() - ); - Assertions.assertTrue(storeHandle.resolve(ZARR_JSON).exists()); - Codec bytesCodec = array.metadata().codecs[0]; - Assertions.assertInstanceOf(BytesCodec.class, bytesCodec); - Assertions.assertNull(((BytesCodec) bytesCodec).configuration); - } - @ParameterizedTest @MethodSource("chunkKeyEncodingsAndCodecs") public void testZarrJsonFormat(Function chunkKeyEncodingsAndCodecs) throws ZarrException, IOException { @@ -1035,21 +793,4 @@ public void testLargeArrayWithOffsetBeyondMaxInt() throws IOException, ZarrExcep Assertions.assertEquals(largeSize, array.metadata().shape[1]); } - @ParameterizedTest - @MethodSource("dataTypeAndEndianProvider") - public void testEndianness(DataType dataType, BytesCodec.Endian endian) throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testEndiannessV3").resolve(dataType.name()).resolve(endian.name()); - ucar.ma2.Array testData = testdata(dataType); - - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(toLongArray(testData.getShape())) - .withDataType(dataType) - .withCodecs(c -> c.withBytes(endian)) - .build(); - Array array = Array.create(storeHandle, metadata); - array.write(testData); - Array reopenedArray = Array.open(storeHandle); - ucar.ma2.Array readData = reopenedArray.read(); - assertIsTestdata(readData, dataType); - } } diff --git a/src/test/java/dev/zarr/zarrjava/codec/BytesCodecTest.java b/src/test/java/dev/zarr/zarrjava/codec/BytesCodecTest.java new file mode 100644 index 0000000..c43027c --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/codec/BytesCodecTest.java @@ -0,0 +1,77 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.ZarrTest; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.util.stream.Stream; + +import static dev.zarr.zarrjava.core.Node.ZARR_JSON; +import static dev.zarr.zarrjava.utils.Utils.toLongArray; + +public class BytesCodecTest extends ZarrTest { + + static Stream dataTypeAndEndianProvider() { + return Stream.of( + Arguments.of(DataType.INT16, BytesCodec.Endian.LITTLE), + Arguments.of(DataType.INT16, BytesCodec.Endian.BIG), + Arguments.of(DataType.UINT16, BytesCodec.Endian.LITTLE), + Arguments.of(DataType.UINT16, BytesCodec.Endian.BIG), + Arguments.of(DataType.INT32, BytesCodec.Endian.LITTLE), + Arguments.of(DataType.INT32, BytesCodec.Endian.BIG), + Arguments.of(DataType.UINT32, BytesCodec.Endian.LITTLE), + Arguments.of(DataType.UINT32, BytesCodec.Endian.BIG), + Arguments.of(DataType.FLOAT32, BytesCodec.Endian.LITTLE), + Arguments.of(DataType.FLOAT32, BytesCodec.Endian.BIG), + Arguments.of(DataType.FLOAT64, BytesCodec.Endian.LITTLE), + Arguments.of(DataType.FLOAT64, BytesCodec.Endian.BIG) + ); + } + + @Test + public void testCodecWithoutConfiguration() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecWithoutConfigurationV3"); + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withCodecs(CodecBuilder::withBytes) + .build() + ); + Assertions.assertTrue(storeHandle.resolve(ZARR_JSON).exists()); + Codec bytesCodec = array.metadata().codecs[0]; + Assertions.assertInstanceOf(BytesCodec.class, bytesCodec); + Assertions.assertNull(((BytesCodec) bytesCodec).configuration); + } + + @ParameterizedTest + @MethodSource("dataTypeAndEndianProvider") + public void testEndianness(DataType dataType, BytesCodec.Endian endian) throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testEndiannessV3").resolve(dataType.name()).resolve(endian.name()); + ucar.ma2.Array testData = testdata(dataType); + + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(toLongArray(testData.getShape())) + .withDataType(dataType) + .withCodecs(c -> c.withBytes(endian)) + .build(); + Array array = Array.create(storeHandle, metadata); + array.write(testData); + Array reopenedArray = Array.open(storeHandle); + ucar.ma2.Array readData = reopenedArray.read(); + assertIsTestdata(readData, dataType); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/codec/CodecPipelineTest.java b/src/test/java/dev/zarr/zarrjava/codec/CodecPipelineTest.java new file mode 100644 index 0000000..d55e014 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/codec/CodecPipelineTest.java @@ -0,0 +1,43 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.ZarrTest; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadataBuilder; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.function.Function; +import java.util.stream.Stream; + +import static org.junit.Assert.assertThrows; + +public class CodecPipelineTest extends ZarrTest { + + static Stream> invalidCodecBuilder() { + return Stream.of( + c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), + c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) + ); + } + + @ParameterizedTest + @MethodSource("invalidCodecBuilder") + public void testCheckInvalidCodecConfiguration(Function codecBuilder) { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(4, 4) + .withDataType(DataType.UINT32) + .withChunkShape(2, 2) + .withCodecs(codecBuilder); + + assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/codec/CompressionCodecsTest.java b/src/test/java/dev/zarr/zarrjava/codec/CompressionCodecsTest.java new file mode 100644 index 0000000..12aec69 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/codec/CompressionCodecsTest.java @@ -0,0 +1,77 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.ZarrTest; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadataBuilder; +import dev.zarr.zarrjava.v3.DataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.IOException; +import java.util.Arrays; + +public class CompressionCodecsTest extends ZarrTest { + + @ParameterizedTest + @CsvSource({"0,true", "0,false", "5, true", "5, false"}) + public void testZstdCodecReadWrite(int level, boolean checksum) throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "level_" + level); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withCodecs(c -> c.withZstd(level, checksum)); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + @Test + public void testCodecs() throws IOException, ZarrException { + long[] readShape = new long[]{1, 1, 1024, 1024}; + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); + ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); + { + Array gzipArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withGzip(5)).build() + ); + gzipArray.write(readArrayContent); + ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); + } + { + Array bloscArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withBlosc("zstd", 5)).build() + ); + bloscArray.write(readArrayContent); + ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); + } + { + Array zstdArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withZstd(10)).build() + ); + zstdArray.write(readArrayContent); + ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/codec/ShardingCodecTest.java b/src/test/java/dev/zarr/zarrjava/codec/ShardingCodecTest.java new file mode 100644 index 0000000..e765cba --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/codec/ShardingCodecTest.java @@ -0,0 +1,106 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.ZarrTest; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadataBuilder; +import dev.zarr.zarrjava.v3.DataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.util.Arrays; +import java.util.stream.Stream; + +import static org.junit.Assert.assertThrows; + +public class ShardingCodecTest extends ZarrTest { + + static Stream invalidShardSizes() { + return Stream.of( + new int[]{4}, //wrong dims + new int[]{4, 4, 4}, //wrong dims + new int[]{1, 1}, //smaller than inner chunk shape + new int[]{5, 5}, //no exact multiple of inner chunk shape + new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension + new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension + ); + } + + static Stream invalidShardSizesWithNested() { + return invalidShardSizes().flatMap(shardSize -> + Stream.of(true, false).map(nested -> Arguments.of(shardSize, nested)) + ); + } + + @ParameterizedTest + @MethodSource("invalidShardSizesWithNested") + public void testCheckShardingBounds(int[] shardSize, boolean nested) { + long[] shape = new long[]{10, 10}; + int[] innerChunkSize = new int[]{2, 2}; + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32).withChunkShape(shardSize); + + if (nested) { + int[] nestedChunkSize = new int[]{4, 4}; + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); + } + builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); + assertThrows(ZarrException.class, builder::build); + } + + @Test + public void testShardingWithZstdCodecReadWrite() throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testShardingWithZstdCodecReadWrite"); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(8, 8, 8) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{2, 4, 8}, c1 -> c1.withZstd())); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + @Test + public void testShardingReadCutout() throws IOException, ZarrException { + Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new long[]{1, 64, 64, 64}); + Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); + Assertions.assertEquals(-98, outArray.getByte(0)); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "end"}) + public void testShardingReadWrite(String indexLocation) throws IOException, ZarrException { + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); + ucar.ma2.Array readArrayContent = readArray.read(); + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), + readArray.metadata() + ); + writeArray.write(readArrayContent); + ucar.ma2.Array outArray = writeArray.read(); + + assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/codec/TransposeCodecTest.java b/src/test/java/dev/zarr/zarrjava/codec/TransposeCodecTest.java new file mode 100644 index 0000000..c115513 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/codec/TransposeCodecTest.java @@ -0,0 +1,62 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.ZarrTest; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import ucar.ma2.MAMath; + +import java.util.stream.Stream; + +import static org.junit.Assert.assertThrows; + +public class TransposeCodecTest extends ZarrTest { + + static Stream invalidTransposeOrder() { + return Stream.of( + new int[]{1, 0, 0}, + new int[]{1, 2, 3}, + new int[]{1, 2, 3, 0}, + new int[]{1, 2} + ); + } + + @Test + public void testTransposeCodec() throws ZarrException { + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, + DataType.UINT32, + null)); + + assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); + assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); + } + + @ParameterizedTest + @MethodSource("invalidTransposeOrder") + public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { + int[] shapeInt = new int[]{2, 3, 3}; + long[] shapeLong = new long[]{2, 3, 3}; + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + shapeLong, + shapeInt, + DataType.UINT32, + null)); + + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); + assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); + } +}