[WIP][ISSUE #22]rocketmq-connect-redis adapt to the new connect api #31
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.rocketmq.connect.redis.connector; | ||
|
|
||
| import io.openmessaging.KeyValue; | ||
| import io.openmessaging.connector.api.Task; | ||
| import io.openmessaging.connector.api.sink.SinkConnector; | ||
| import org.apache.rocketmq.connect.redis.config.Config; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| /** | ||
| * author: doubleDimple | ||
| */ | ||
| public class RedisSinkConnector extends SinkConnector { | ||
|
|
||
| private static final Logger LOGGER = LoggerFactory.getLogger(RedisSinkConnector.class); | ||
|
|
||
| private volatile boolean configValid = false; | ||
| private volatile boolean adminStarted; | ||
| private KeyValue keyValue; | ||
|
|
||
| @Override | ||
|
There was a problem hiding this comment.
|
||
| public String verifyAndSetConfig(KeyValue config) { | ||
| this.keyValue = config; | ||
|
There was a problem hiding this comment.
|
||
| String msg = Config.checkConfig(keyValue); | ||
| if (msg != null) { | ||
| return msg; | ||
| } | ||
| this.configValid = true; | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public void start() { | ||
| LOGGER.info("the redisSinkConnector is start..."); | ||
| } | ||
|
There was a problem hiding this comment.
|
||
|
|
||
| @Override | ||
| public void stop() { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public void pause() { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public void resume() { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public Class<? extends Task> taskClass() { | ||
| return RedisSinkTask.class; | ||
|
There was a problem hiding this comment.
|
||
| } | ||
|
|
||
| @Override | ||
| public List<KeyValue> taskConfigs() { | ||
| List<KeyValue> keyValues = new ArrayList<>(); | ||
| keyValues.add(this.keyValue); | ||
| return keyValues; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.rocketmq.connect.redis.connector; | ||
|
|
||
| import com.alibaba.fastjson.JSONObject; | ||
| import io.openmessaging.KeyValue; | ||
| import io.openmessaging.connector.api.common.QueueMetaData; | ||
| import io.openmessaging.connector.api.data.EntryType; | ||
| import io.openmessaging.connector.api.data.Field; | ||
| import io.openmessaging.connector.api.data.Schema; | ||
| import io.openmessaging.connector.api.data.SinkDataEntry; | ||
| import io.openmessaging.connector.api.sink.SinkTask; | ||
| import org.apache.rocketmq.connect.redis.config.Config; | ||
| import org.apache.rocketmq.connect.redis.converter.KVEntryConverter; | ||
| import org.apache.rocketmq.connect.redis.converter.RedisEntryConverter; | ||
| import org.apache.rocketmq.connect.redis.handler.DefaultRedisEventHandler; | ||
| import org.apache.rocketmq.connect.redis.handler.RedisEventHandler; | ||
| import org.apache.rocketmq.connect.redis.processor.DefaultRedisEventProcessor; | ||
| import org.apache.rocketmq.connect.redis.processor.RedisEventProcessor; | ||
| import org.apache.rocketmq.connect.redis.sink.RedisUpdater; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import java.io.IOException; | ||
| import java.util.Collection; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| /** | ||
| * author doubleDimple | ||
| */ | ||
| public class RedisSinkTask extends SinkTask { | ||
|
|
||
|
There was a problem hiding this comment.
|
||
| private static final Logger LOGGER = LoggerFactory.getLogger(RedisSinkTask.class); | ||
|
|
||
| private RedisUpdater updater; | ||
|
|
||
| /** | ||
| * listening and handle Redis event. | ||
| */ | ||
| private RedisEventProcessor eventProcessor; | ||
| private Config config; | ||
| /** | ||
| * convert kVEntry to list of sourceDataEntry | ||
| */ | ||
| private KVEntryConverter kvEntryConverter; | ||
|
|
||
| public RedisEventProcessor getEventProcessor() { | ||
| return eventProcessor; | ||
| } | ||
|
|
||
| public void setEventProcessor(RedisEventProcessor eventProcessor) { | ||
| this.eventProcessor = eventProcessor; | ||
| } | ||
|
|
||
| public Config getConfig() { | ||
|
There was a problem hiding this comment.
|
||
| return config; | ||
| } | ||
|
|
||
| @Override | ||
|
There was a problem hiding this comment. The |
||
| public void put(Collection<SinkDataEntry> sinkDataEntries) { | ||
| //save data from MQ to redis | ||
| for (SinkDataEntry sinkDataEntry : sinkDataEntries) { | ||
| Map<Field, Object[]> fieldMap = new HashMap<>(); | ||
| Object[] payloads = sinkDataEntry.getPayload(); | ||
|
There was a problem hiding this comment.
|
||
|
|
||
| Schema schema = sinkDataEntry.getSchema(); | ||
| EntryType entryType = sinkDataEntry.getEntryType(); | ||
|
|
||
| List<Field> fields = schema.getFields(); | ||
|
There was a problem hiding this comment.
|
||
| Boolean parseError = false; | ||
| if (!fields.isEmpty()) { | ||
| for (Field field : fields) { | ||
|
There was a problem hiding this comment.
|
||
| Object fieldValue = payloads[field.getIndex()]; | ||
|
There was a problem hiding this comment.
|
||
| Object[] value = JSONObject.parseArray((String)fieldValue).toArray(); | ||
| if (value.length == 2) { | ||
| fieldMap.put(field, value); | ||
| } else { | ||
|
There was a problem hiding this comment. Unsafe cast: |
||
| LOGGER.error("parseArray error, fieldValue:{}", fieldValue); | ||
| parseError = true; | ||
| } | ||
|
There was a problem hiding this comment. When |
||
| } | ||
| } | ||
| if (!parseError) { | ||
| Boolean isSuccess = updater.push(fieldMap, entryType); | ||
| if (!isSuccess) { | ||
| LOGGER.error("push data error, entryType:{}, fieldMap:{}", fieldMap, entryType); | ||
| } | ||
|
There was a problem hiding this comment.
|
||
| } | ||
| } | ||
| } | ||
|
|
||
|
There was a problem hiding this comment. The |
||
| @Override | ||
| public void commit(Map<QueueMetaData, Long> offsets) { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
|
There was a problem hiding this comment. The sink task's |
||
| public void start(KeyValue keyValue) { | ||
| this.kvEntryConverter = new RedisEntryConverter(); | ||
|
|
||
|
There was a problem hiding this comment.
|
||
| this.config = new Config(); | ||
| this.config.load(keyValue); | ||
| LOGGER.info("task config msg: {}", this.config.toString()); | ||
|
|
||
| this.eventProcessor = new DefaultRedisEventProcessor(config); | ||
|
There was a problem hiding this comment.
|
||
| RedisEventHandler eventHandler = new DefaultRedisEventHandler(this.config); | ||
| this.eventProcessor.registEventHandler(eventHandler); | ||
| try { | ||
| this.eventProcessor.start(); | ||
| LOGGER.info("Redis task start."); | ||
| } catch (IOException e) { | ||
| e.printStackTrace(); | ||
| LOGGER.error("processor start error: [{}]", e.getMessage()); | ||
| this.stop(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void stop() { | ||
| if (this.eventProcessor != null) { | ||
| try { | ||
| this.eventProcessor.stop(); | ||
| LOGGER.info("Redis task is stopped."); | ||
| } catch (IOException e) { | ||
| LOGGER.error("processor stop error: {}", e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void pause() { | ||
|
|
||
| } | ||
|
|
||
| @Override | ||
| public void resume() { | ||
|
|
||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| package org.apache.rocketmq.connect.redis.sink; | ||
|
There was a problem hiding this comment. Missing Apache License header. All other new/modified files in this PR have the ASF license header added, but RedisUpdater.java does not, which will fail the apache-rat license check added in pom.xml. There was a problem hiding this comment. Missing Apache License header. All other new files in this PR include the standard ASF license block. This will also cause |
||
|
|
||
| import io.openmessaging.connector.api.data.EntryType; | ||
| import io.openmessaging.connector.api.data.Field; | ||
|
|
||
| import java.util.Map; | ||
|
|
||
| public class RedisUpdater { | ||
|
|
||
|
|
||
| public Boolean push(Map<Field, Object[]> fieldMap, EntryType entryType) { | ||
|
|
||
|
There was a problem hiding this comment. RedisUpdater.push() is a stub that always returns null. RedisSinkTask.put() calls this method and checks the Boolean return value without null-safety, which will cause a NullPointerException at There was a problem hiding this comment.
|
||
| return null; | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Large diff (752 lines). Consider breaking into smaller, focused PRs for easier review.