Health check for stored messages

3 files changed, 43 insertions, 4 deletions

diff --git a/src/main/java/org/traccar/config/Keys.java b/src/main/java/org/traccar/config/Keys.java
index e51ba07c4..ac2adca8a 100644
--- a/src/main/java/org/traccar/config/Keys.java
+++ b/src/main/java/org/traccar/config/Keys.java
@@ -1811,6 +1811,15 @@ public final class Keys {
             List.of(KeyType.CONFIG));
 
     /**
+     * If this parameter is set, Traccar will monitor drops in the number of stored messages. If it drops more than
+     * the threshold, it will mark service as failing for systemd. Threshold is a value from 0.0 to 1.0. For example,
+     * value 0.7 means that the number of messages in the last period is only 70% of what it was in the previous.
+     */
+    public static final ConfigKey<Double> WEB_HEALTH_CHECK_DROP_THRESHOLD = new DoubleConfigKey(
+            "web.healthCheck.dropThreshold",
+            List.of(KeyType.CONFIG));
+
+    /**
      * Sets SameSite cookie attribute value.
      * Supported options: Lax, Strict, None.
      */
diff --git a/src/main/java/org/traccar/database/StatisticsManager.java b/src/main/java/org/traccar/database/StatisticsManager.java
index 8711289a0..d13f72793 100644
--- a/src/main/java/org/traccar/database/StatisticsManager.java
+++ b/src/main/java/org/traccar/database/StatisticsManager.java
@@ -178,6 +178,10 @@ public class StatisticsManager {
         }
     }
 
+    public synchronized int messageStoredCount() {
+        return messagesStored;
+    }
+
     public synchronized int messageStoredCount(long deviceId) {
         return deviceMessages.getOrDefault(deviceId, 0);
     }
diff --git a/src/main/java/org/traccar/schedule/TaskHealthCheck.java b/src/main/java/org/traccar/schedule/TaskHealthCheck.java
index a60935f18..b3387eca7 100644
--- a/src/main/java/org/traccar/schedule/TaskHealthCheck.java
+++ b/src/main/java/org/traccar/schedule/TaskHealthCheck.java
@@ -24,6 +24,8 @@ import org.traccar.config.Keys;
 
 import jakarta.inject.Inject;
 import jakarta.ws.rs.client.Client;
+import org.traccar.database.StatisticsManager;
+
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
@@ -33,6 +35,7 @@ public class TaskHealthCheck implements ScheduleTask {
 
     private final Config config;
     private final Client client;
+    private final StatisticsManager statisticsManager;
 
     private final long gracePeriod = System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1);
 
@@ -40,11 +43,16 @@ public class TaskHealthCheck implements ScheduleTask {
 
     private boolean enabled;
     private long period;
+    private double dropThreshold;
+
+    private int messageLastTotal;
+    private int messageLastPeriod;
 
     @Inject
-    public TaskHealthCheck(Config config, Client client) {
+    public TaskHealthCheck(Config config, Client client, StatisticsManager statisticsManager) {
         this.config = config;
         this.client = client;
+        this.statisticsManager = statisticsManager;
         if (!config.getBoolean(Keys.WEB_DISABLE_HEALTH_CHECK)
                 && System.getProperty("os.name").toLowerCase().startsWith("linux")) {
             try {
@@ -55,6 +63,7 @@ public class TaskHealthCheck implements ScheduleTask {
                 }
                 if (period > 0) {
                     LOGGER.info("Health check enabled with period {}", period);
+                    dropThreshold = config.getDouble(Keys.WEB_HEALTH_CHECK_DROP_THRESHOLD);
                     enabled = true;
                 }
             } catch (UnsatisfiedLinkError e) {
@@ -80,11 +89,28 @@ public class TaskHealthCheck implements ScheduleTask {
     public void run() {
         LOGGER.debug("Health check running");
         if (System.currentTimeMillis() > gracePeriod) {
+            boolean success = true;
+
             int status = client.target(getUrl()).request().get().getStatus();
-            if (status == 200) {
+            if (status != 200) {
+                success = false;
+                LOGGER.warn("Web health check failed with status {}", status);
+            }
+
+            if (dropThreshold > 0) {
+                int messageCurrentTotal = statisticsManager.messageStoredCount();
+                int messageCurrentPeriod = messageCurrentTotal - messageLastTotal;
+                double drop = messageCurrentPeriod / (double) messageLastPeriod;
+                if (drop < dropThreshold) {
+                    success = false;
+                    LOGGER.warn("Message health check failed with drop {}", drop);
+                }
+                messageLastTotal = messageCurrentTotal;
+                messageLastPeriod = messageCurrentPeriod;
+            }
+
+            if (success) {
                 notifyWatchdog();
-            } else {
-                LOGGER.warn("Health check failed with status {}", status);
             }
         } else {
             notifyWatchdog();