Implement Kafka forwarding

5 files changed, 148 insertions, 13 deletions

diff --git a/build.gradle b/build.gradle
index 0e7d77bce..a5daf5924 100644
--- a/build.gradle
+++ b/build.gradle
@@ -83,6 +83,7 @@ dependencies {
     implementation "com.sun.xml.bind:jaxb-impl:3.0.2" // needs upgrade
     implementation "javax.activation:activation:1.1.1"
     implementation "com.amazonaws:aws-java-sdk-sns:1.12.314"
+    implementation "org.apache.kafka:kafka-clients:3.3.1"
     implementation ("com.google.firebase:firebase-admin:9.0.0") {
         exclude group: 'com.google.cloud', module: 'google-cloud-firestore'
     }
diff --git a/src/main/java/org/traccar/MainModule.java b/src/main/java/org/traccar/MainModule.java
index 9d450fef7..6e59527bc 100644
--- a/src/main/java/org/traccar/MainModule.java
+++ b/src/main/java/org/traccar/MainModule.java
@@ -37,8 +37,10 @@ import org.traccar.database.LdapProvider;
 import org.traccar.database.StatisticsManager;
 import org.traccar.forward.EventForwarder;
 import org.traccar.forward.EventForwarderJson;
+import org.traccar.forward.EventForwarderKafka;
 import org.traccar.forward.PositionForwarder;
 import org.traccar.forward.PositionForwarderJson;
+import org.traccar.forward.PositionForwarderKafka;
 import org.traccar.forward.PositionForwarderUrl;
 import org.traccar.geocoder.AddressFormat;
 import org.traccar.geocoder.BanGeocoder;
@@ -314,9 +316,13 @@ public class MainModule extends AbstractModule {
 
     @Singleton
     @Provides
-    public static EventForwarder provideEventForwarder(Config config, Client client) {
+    public static EventForwarder provideEventForwarder(Config config, Client client, ObjectMapper objectMapper) {
         if (config.hasKey(Keys.EVENT_FORWARD_URL)) {
-            return new EventForwarderJson(config, client);
+            if (config.getString(Keys.EVENT_FORWARD_TYPE).equals("kafka")) {
+                return new EventForwarderKafka(config, objectMapper);
+            } else {
+                return new EventForwarderJson(config, client);
+            }
         }
         return null;
     }
@@ -325,10 +331,13 @@ public class MainModule extends AbstractModule {
     @Provides
     public static PositionForwarder providePositionForwarder(Config config, Client client, ObjectMapper objectMapper) {
         if (config.hasKey(Keys.FORWARD_URL)) {
-            if (config.getBoolean(Keys.FORWARD_JSON)) {
-                return new PositionForwarderJson(config, client, objectMapper);
-            } else {
-                return new PositionForwarderUrl(config, client, objectMapper);
+            switch (config.getString(Keys.FORWARD_TYPE)) {
+                case "json":
+                    return new PositionForwarderJson(config, client, objectMapper);
+                case "kafka":
+                    return new PositionForwarderKafka(config, objectMapper);
+                default:
+                    return new PositionForwarderUrl(config, client, objectMapper);
             }
         }
         return null;
diff --git a/src/main/java/org/traccar/config/Keys.java b/src/main/java/org/traccar/config/Keys.java
index b60cd82a0..2224192d9 100644
--- a/src/main/java/org/traccar/config/Keys.java
+++ b/src/main/java/org/traccar/config/Keys.java
@@ -699,6 +699,14 @@ public final class Keys {
             "max-age=3600,public");
 
     /**
+     * Position forwarding format. Available options are "url", "json" and "kafka". Default is "url".
+     */
+    public static final ConfigKey<String> FORWARD_TYPE = new StringConfigKey(
+            "forward.type",
+            List.of(KeyType.CONFIG),
+            "url");
+
+    /**
      * URL to forward positions. Data is passed through URL parameters. For example, {uniqueId} for device identifier,
      * {latitude} and {longitude} for coordinates.
      */
@@ -714,13 +722,6 @@ public final class Keys {
             List.of(KeyType.CONFIG));
 
     /**
-     * Boolean value to enable forwarding in JSON format.
-     */
-    public static final ConfigKey<Boolean> FORWARD_JSON = new BooleanConfigKey(
-            "forward.json",
-            List.of(KeyType.CONFIG));
-
-    /**
      * Position forwarding retrying enable. When enabled, additional attempts are made to deliver positions. If initial
      * delivery fails, because of an unreachable server or an HTTP response different from '2xx', the software waits
      * for 'forward.retry.delay' milliseconds to retry delivery. On subsequent failures, this delay is duplicated.
@@ -759,6 +760,14 @@ public final class Keys {
             100);
 
     /**
+     * Events forwarding format. Available options are "json" and "kafka". Default is "json".
+     */
+    public static final ConfigKey<String> EVENT_FORWARD_TYPE = new StringConfigKey(
+            "event.forward.type",
+            List.of(KeyType.CONFIG),
+            "json");
+
+    /**
      * Events forwarding URL.
      */
     public static final ConfigKey<String> EVENT_FORWARD_URL = new StringConfigKey(
diff --git a/src/main/java/org/traccar/forward/EventForwarderKafka.java b/src/main/java/org/traccar/forward/EventForwarderKafka.java
new file mode 100644
index 000000000..71e06ddd1
--- /dev/null
+++ b/src/main/java/org/traccar/forward/EventForwarderKafka.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2022 Anton Tananaev (anton@traccar.org)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.traccar.forward;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.traccar.config.Config;
+import org.traccar.config.Keys;
+
+import java.util.Properties;
+
+public class EventForwarderKafka implements EventForwarder {
+
+    private final Producer<String, String> producer;
+    private final ObjectMapper objectMapper;
+
+    public EventForwarderKafka(Config config, ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
+        Properties properties = new Properties();
+        properties.put("bootstrap.servers", config.getString(Keys.EVENT_FORWARD_URL));
+        properties.put("acks", "all");
+        properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+        producer = new KafkaProducer<>(properties);
+    }
+
+    @SuppressWarnings("deprecation")
+    @Override
+    protected void finalize() {
+        producer.close();
+    }
+
+    @Override
+    public void forward(EventData eventData, ResultHandler resultHandler) {
+        try {
+            String key = Long.toString(eventData.getDevice().getId());
+            String value = objectMapper.writeValueAsString(eventData);
+            producer.send(new ProducerRecord<>("events", key, value));
+            resultHandler.onResult(true, null);
+        } catch (JsonProcessingException e) {
+            resultHandler.onResult(false, e);
+        }
+    }
+
+}
diff --git a/src/main/java/org/traccar/forward/PositionForwarderKafka.java b/src/main/java/org/traccar/forward/PositionForwarderKafka.java
new file mode 100644
index 000000000..3921539ac
--- /dev/null
+++ b/src/main/java/org/traccar/forward/PositionForwarderKafka.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2022 Anton Tananaev (anton@traccar.org)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.traccar.forward;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.traccar.config.Config;
+import org.traccar.config.Keys;
+
+import java.util.Properties;
+
+public class PositionForwarderKafka implements PositionForwarder {
+
+    private final Producer<String, String> producer;
+    private final ObjectMapper objectMapper;
+
+    public PositionForwarderKafka(Config config, ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
+        Properties properties = new Properties();
+        properties.put("bootstrap.servers", config.getString(Keys.EVENT_FORWARD_URL));
+        properties.put("acks", "all");
+        properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
+        producer = new KafkaProducer<>(properties);
+    }
+
+    @Override
+    public void forward(PositionData positionData, ResultHandler resultHandler) {
+        try {
+            String key = Long.toString(positionData.getDevice().getId());
+            String value = objectMapper.writeValueAsString(positionData);
+            producer.send(new ProducerRecord<>("positions", key, value));
+            resultHandler.onResult(true, null);
+        } catch (JsonProcessingException e) {
+            resultHandler.onResult(false, e);
+        }
+    }
+
+}