From 9e813ee0dc4c49afda2a2ebcedd18a3f5ad49441 Mon Sep 17 00:00:00 2001 From: Sid Date: Thu, 2 Apr 2026 14:49:03 +0800 Subject: [PATCH] refactor(loader): move UDF0 contract from Loader to capability mix-ins - Remove UDF0> from Loader base interface - Add UDF0 to SupportsCopyToLocal - Add UDF0> to SupportsReadPartitions and SupportsScanPartitions - Add new SupportsInMemoryLoading interface with UDF0> --- .../piper/plugin/api/loader/Loader.java | 5 +-- .../api/loader/SupportsCopyToLocal.java | 3 +- .../api/loader/SupportsInMemoryLoading.java | 35 +++++++++++++++++++ .../api/loader/SupportsReadPartitions.java | 6 +++- .../api/loader/SupportsScanPartitions.java | 4 ++- 5 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsInMemoryLoading.java diff --git a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/Loader.java b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/Loader.java index 64f2f1c..9c16623 100644 --- a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/Loader.java +++ b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/Loader.java @@ -19,11 +19,8 @@ import com.atgenomix.seqslab.piper.plugin.api.DataSource; import com.atgenomix.seqslab.piper.plugin.api.Operator; import com.atgenomix.seqslab.piper.tags.DeveloperApi; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.api.java.UDF0; import org.apache.spark.sql.types.StructType; -import java.util.Iterator; /** * The operator responsible for loading (reading) a dataset into in-memory DataFrame or copying to local host @@ -48,7 +45,7 @@ * @see SupportsScanPartitions */ @DeveloperApi -public interface Loader extends Operator, UDF0> { +public interface Loader extends Operator { /** * Initializes this operator with a specific data source. diff --git a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsCopyToLocal.java b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsCopyToLocal.java index e235fb0..37e8076 100644 --- a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsCopyToLocal.java +++ b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsCopyToLocal.java @@ -18,6 +18,7 @@ import com.atgenomix.seqslab.piper.tags.DeveloperApi; import com.atgenomix.seqslab.piper.tags.FeatureBeforeCall; +import org.apache.spark.sql.api.java.UDF0; /** * A mix-in interface for {@link Loader}. Dataset loaders can implement this interface to support @@ -28,7 +29,7 @@ */ @DeveloperApi @FeatureBeforeCall -public interface SupportsCopyToLocal extends Loader { +public interface SupportsCopyToLocal extends Loader, UDF0 { /** * Sets the local destination path where the datasets will be saved. diff --git a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsInMemoryLoading.java b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsInMemoryLoading.java new file mode 100644 index 0000000..5d6d06c --- /dev/null +++ b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsInMemoryLoading.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021-2022, ATGENOMIX INCORPORATED. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.atgenomix.seqslab.piper.plugin.api.loader; + +import com.atgenomix.seqslab.piper.tags.DeveloperApi; +import com.atgenomix.seqslab.piper.tags.FeatureBeforeCall; +import org.apache.spark.sql.api.java.UDF0; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; + +/** + * A mix-in interface for {@link Loader}. Dataset loaders can implement this interface to support + * copying storage files or directories directly to local file system. + * Loaders supporting this interface typically localize datasets that do not require + * in-memory processing optimization, e.g. reference files. + * This feature is invoked before calling operator function. + */ +@DeveloperApi +@FeatureBeforeCall +public interface SupportsInMemoryLoading extends Loader, UDF0> { +} diff --git a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsReadPartitions.java b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsReadPartitions.java index 96fc7ae..1b6e8dc 100644 --- a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsReadPartitions.java +++ b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsReadPartitions.java @@ -18,6 +18,10 @@ import com.atgenomix.seqslab.piper.tags.DeveloperApi; import com.atgenomix.seqslab.piper.tags.FeatureBeforeCall; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.api.java.UDF0; + +import java.util.Iterator; /** * A mix-in interface for {@link Loader}. Dataset loaders can implement this interface to support @@ -27,7 +31,7 @@ */ @DeveloperApi @FeatureBeforeCall -public interface SupportsReadPartitions extends Loader { +public interface SupportsReadPartitions extends Loader, UDF0> { /** * Get the number of partitions in the data source. diff --git a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsScanPartitions.java b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsScanPartitions.java index 6fbb046..121b812 100644 --- a/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsScanPartitions.java +++ b/src/main/java/com/atgenomix/seqslab/piper/plugin/api/loader/SupportsScanPartitions.java @@ -19,16 +19,18 @@ import com.atgenomix.seqslab.piper.tags.DeveloperApi; import com.atgenomix.seqslab.piper.tags.FeatureBeforeCall; import org.apache.spark.sql.Row; +import org.apache.spark.sql.api.java.UDF0; import java.util.Iterator; + /** * A mix-in interface for {@link Loader}. Dataset loaders can implement this interface to support * transformation of partitioned datasets read by SeqsLab. */ @DeveloperApi @FeatureBeforeCall -public interface SupportsScanPartitions extends Loader { +public interface SupportsScanPartitions extends Loader, UDF0> { /** * Set the partition loaded by SeqsLab for applying Loader's call function.