Merge pull request #4740 from soshial/relative_filtering

Add support for relative filtering

4 files changed, 78 insertions, 26 deletions

diff --git a/setup/default.xml b/setup/default.xml
index 6c0eef111..a592bbbed 100644
--- a/setup/default.xml
+++ b/setup/default.xml
@@ -51,6 +51,10 @@
         SELECT * FROM tc_positions WHERE deviceId = :deviceId AND fixTime BETWEEN :from AND :to ORDER BY fixTime
     </entry>
 
+    <entry key='database.selectPrecedingPosition'>
+        SELECT * FROM tc_positions WHERE deviceId = :deviceId AND fixTime &lt;= :time ORDER BY fixTime DESC LIMIT 1
+    </entry>
+
     <entry key='database.selectLatestPositions'>
         SELECT tc_positions.* FROM tc_positions INNER JOIN tc_devices ON tc_positions.id = tc_devices.positionid;
     </entry>
diff --git a/src/main/java/org/traccar/config/Keys.java b/src/main/java/org/traccar/config/Keys.java
index 1411e8a13..e8e0ff207 100644
--- a/src/main/java/org/traccar/config/Keys.java
+++ b/src/main/java/org/traccar/config/Keys.java
@@ -893,8 +893,11 @@ public final class Keys {
             Collections.singletonList(KeyType.GLOBAL));
 
     /**
-     * Filter records by Maximum Speed value in knots. Can be used to filter jumps to far locations even if they're
-     * marked as valid. Shouldn't be too low. Start testing with values at about 25000.
+     * Filter records by Maximum Speed value in knots. Can be used to filter jumps to far locations even if Position
+     * appears valid or if Position `speed` field reported by the device is also within limits. Calculates speed from
+     * the distance to the previous position and the elapsed time.
+     *
+     * Tip: Shouldn't be too low. Start testing with values at about 25000.
      */
     public static final ConfigKey<Integer> FILTER_MAX_SPEED = new ConfigKey<>(
             "filter.maxSpeed",
@@ -908,8 +911,20 @@ public final class Keys {
             Collections.singletonList(KeyType.GLOBAL));
 
     /**
-     * Time limit for the filtering in seconds. If the time difference between last position and a new one is more than
-     * this limit, the new position will not be filtered out.
+     * If false, the server expects all locations to come sequentially (for each device). Filter checks for duplicates,
+     * distance, speed, or time period only against the location that was last received by server.
+     *
+     * If true, the server expects locations to come at random order (since tracking device might go offline).
+     * Filter checks for duplicates, distance, speed, or time period against the preceding Position's.
+     * Important: setting to true can cause potential performance issues.
+     */
+    public static final ConfigKey<Boolean> FILTER_RELATIVE = new ConfigKey<>(
+            "filter.relative",
+            Collections.singletonList(KeyType.GLOBAL));
+
+    /**
+     * Time limit for the filtering in seconds. If the time difference between the last position was received by server
+     * and a new position is received by server is more than this limit, the new position will not be filtered out.
      */
     public static final ConfigKey<Long> FILTER_SKIP_LIMIT = new ConfigKey<>(
             "filter.skipLimit",
@@ -917,6 +932,7 @@ public final class Keys {
 
     /**
      * Enable attributes skipping. Attribute skipping can be enabled in the config or device attributes.
+     * If position contains any attribute mentioned in "filter.skipAttributes" config key, position is not filtered out.
      */
     public static final ConfigKey<Boolean> FILTER_SKIP_ATTRIBUTES_ENABLE = new ConfigKey<>(
             "filter.skipAttributes.enable",
diff --git a/src/main/java/org/traccar/database/DataManager.java b/src/main/java/org/traccar/database/DataManager.java
index 00c802fde..ebd0dcade 100644
--- a/src/main/java/org/traccar/database/DataManager.java
+++ b/src/main/java/org/traccar/database/DataManager.java
@@ -333,6 +333,13 @@ public class DataManager {
                 .executeQuery(Position.class);
     }
 
+    public Position getPrecedingPosition(long deviceId, Date date) throws SQLException {
+        return QueryBuilder.create(dataSource, getQuery("database.selectPrecedingPosition"))
+                .setLong("deviceId", deviceId)
+                .setDate("time", date)
+                .executeQuerySingle(Position.class);
+    }
+
     public void updateLatestPosition(Position position) throws SQLException {
         QueryBuilder.create(dataSource, getQuery("database.updateLatestPosition"))
                 .setDate("now", new Date())
diff --git a/src/main/java/org/traccar/handler/FilterHandler.java b/src/main/java/org/traccar/handler/FilterHandler.java
index 7cd9153c1..049512765 100644
--- a/src/main/java/org/traccar/handler/FilterHandler.java
+++ b/src/main/java/org/traccar/handler/FilterHandler.java
@@ -25,6 +25,9 @@ import org.traccar.config.Keys;
 import org.traccar.helper.UnitsConverter;
 import org.traccar.model.Position;
 
+import java.sql.SQLException;
+import java.util.Date;
+
 @ChannelHandler.Sharable
 public class FilterHandler extends BaseDataHandler {
 
@@ -40,6 +43,7 @@ public class FilterHandler extends BaseDataHandler {
     private int filterDistance;
     private int filterMaxSpeed;
     private long filterMinPeriod;
+    private boolean filterRelative;
     private long skipLimit;
     private boolean skipAttributes;
 
@@ -54,14 +58,15 @@ public class FilterHandler extends BaseDataHandler {
         filterDistance = config.getInteger(Keys.FILTER_DISTANCE);
         filterMaxSpeed = config.getInteger(Keys.FILTER_MAX_SPEED);
         filterMinPeriod = config.getInteger(Keys.FILTER_MIN_PERIOD) * 1000;
+        filterRelative = config.getBoolean(Keys.FILTER_RELATIVE);
         skipLimit = config.getLong(Keys.FILTER_SKIP_LIMIT) * 1000;
         skipAttributes = config.getBoolean(Keys.FILTER_SKIP_ATTRIBUTES_ENABLE);
     }
 
     private boolean filterInvalid(Position position) {
         return filterInvalid && (!position.getValid()
-           || position.getLatitude() > 90 || position.getLongitude() > 180
-           || position.getLatitude() < -90 || position.getLongitude() < -180);
+                || position.getLatitude() > 90 || position.getLongitude() > 180
+                || position.getLatitude() < -90 || position.getLongitude() < -180);
     }
 
     private boolean filterZero(Position position) {
@@ -144,20 +149,13 @@ public class FilterHandler extends BaseDataHandler {
 
         StringBuilder filterType = new StringBuilder();
 
-        Position last = null;
-        if (Context.getIdentityManager() != null) {
-            last = Context.getIdentityManager().getLastPosition(position.getDeviceId());
-        }
-
+        // filter out invalid data
         if (filterInvalid(position)) {
             filterType.append("Invalid ");
         }
         if (filterZero(position)) {
             filterType.append("Zero ");
         }
-        if (filterDuplicate(position, last) && !skipLimit(position, last) && !skipAttributes(position)) {
-            filterType.append("Duplicate ");
-        }
         if (filterFuture(position)) {
             filterType.append("Future ");
         }
@@ -167,17 +165,37 @@ public class FilterHandler extends BaseDataHandler {
         if (filterApproximate(position)) {
             filterType.append("Approximate ");
         }
-        if (filterStatic(position) && !skipLimit(position, last) && !skipAttributes(position)) {
-            filterType.append("Static ");
-        }
-        if (filterDistance(position, last) && !skipLimit(position, last) && !skipAttributes(position)) {
-            filterType.append("Distance ");
-        }
-        if (filterMaxSpeed(position, last)) {
-            filterType.append("MaxSpeed ");
-        }
-        if (filterMinPeriod(position, last)) {
-            filterType.append("MinPeriod ");
+
+        // filter out excessive data
+        long deviceId = position.getDeviceId();
+        if (filterDuplicate || filterStatic || filterDistance > 0 || filterMaxSpeed > 0 || filterMinPeriod > 0) {
+            Position preceding = null;
+            if (filterRelative) {
+                try {
+                    Date newFixTime = position.getFixTime();
+                    preceding = Context.getDataManager().getPrecedingPosition(deviceId, newFixTime);
+                } catch (SQLException e) {
+                    LOGGER.warn("Error retrieving preceding position; fallbacking to last received position.", e);
+                    preceding = getLastReceivedPosition(deviceId);
+                }
+            } else {
+                preceding = getLastReceivedPosition(deviceId);
+            }
+            if (filterDuplicate(position, preceding)) {
+                filterType.append("Duplicate ");
+            }
+            if (filterStatic(position) && !skipLimit(position, preceding) && !skipAttributes(position)) {
+                filterType.append("Static ");
+            }
+            if (filterDistance(position, preceding) && !skipLimit(position, preceding) && !skipAttributes(position)) {
+                filterType.append("Distance ");
+            }
+            if (filterMaxSpeed(position, preceding)) {
+                filterType.append("MaxSpeed ");
+            }
+            if (filterMinPeriod(position, preceding)) {
+                filterType.append("MinPeriod ");
+            }
         }
 
         if (filterType.length() > 0) {
@@ -186,7 +204,7 @@ public class FilterHandler extends BaseDataHandler {
             message.append("Position filtered by ");
             message.append(filterType.toString());
             message.append("filters from device: ");
-            message.append(Context.getIdentityManager().getById(position.getDeviceId()).getUniqueId());
+            message.append(Context.getIdentityManager().getById(deviceId).getUniqueId());
 
             LOGGER.info(message.toString());
             return true;
@@ -195,6 +213,13 @@ public class FilterHandler extends BaseDataHandler {
         return false;
     }
 
+    private Position getLastReceivedPosition(long deviceId) {
+        if (Context.getIdentityManager() != null) {
+            return Context.getIdentityManager().getLastPosition(deviceId);
+        }
+        return null;
+    }
+
     @Override
     protected Position handlePosition(Position position) {
         if (filter(position)) {