add support for relative filtering

4 files changed, 120 insertions, 24 deletions

diff --git a/setup/default.xml b/setup/default.xml
index 0564182c7..e798fcf88 100644
--- a/setup/default.xml
+++ b/setup/default.xml
@@ -22,6 +22,7 @@
 
     <entry key='filter.enable'>true</entry>
     <entry key='filter.future'>86400</entry>
+    <entry key='filter.relative'>false</entry>
 
     <entry key='event.ignoreDuplicateAlerts'>true</entry>
     <entry key='processing.computedAttributes.enable'>true</entry>
@@ -48,6 +49,18 @@
         SELECT * FROM tc_positions WHERE deviceId = :deviceId AND fixTime BETWEEN :from AND :to ORDER BY fixTime
     </entry>
 
+    <entry key='database.selectPositionByTime'>
+        SELECT * FROM tc_positions WHERE deviceId = :deviceId AND fixTime = :time LIMIT 1
+    </entry>
+
+    <entry key='database.selectPrevPosition'>
+        SELECT * FROM tc_positions WHERE deviceId = :deviceId AND NOT fixTime lt :time LIMIT 1 ORDER BY fixTime DESC
+    </entry>
+
+    <entry key='database.selectNextPosition'>
+        SELECT * FROM tc_positions WHERE deviceId = :deviceId AND fixTime gt :time LIMIT 1 ORDER BY fixTime ASC
+    </entry>
+
     <entry key='database.selectLatestPositions'>
         SELECT tc_positions.* FROM tc_positions INNER JOIN tc_devices ON tc_positions.id = tc_devices.positionid;
     </entry>
diff --git a/src/main/java/org/traccar/config/Keys.java b/src/main/java/org/traccar/config/Keys.java
index 6d2296fb1..f7d61a993 100644
--- a/src/main/java/org/traccar/config/Keys.java
+++ b/src/main/java/org/traccar/config/Keys.java
@@ -887,8 +887,19 @@ public final class Keys {
             Collections.singletonList(KeyType.GLOBAL));
 
     /**
-     * Time limit for the filtering in seconds. If the time difference between last position and a new one is more than
-     * this limit, the new position will not be filtered out.
+     * If FALSE, then filter compares duplicates, distance or period only with the last received location.
+     * Server expects all locations to come sequentially.
+     *
+     * If TRUE, then filter compares duplicates, distance or period with Position's fixTime strictly before and after.
+     * Server expects locations to come at random order, since tracking device might go offline.
+     */
+    public static final ConfigKey<Boolean> FILTER_RELATIVE = new ConfigKey<>(
+            "filter.relative",
+            Collections.singletonList(KeyType.GLOBAL));
+
+    /**
+     * Time limit for the filtering in seconds. If the time difference between the last position was received by server
+     * and a new position is received by server is more than this limit, the new position will not be filtered out.
      */
     public static final ConfigKey<Long> FILTER_SKIP_LIMIT = new ConfigKey<>(
             "filter.skipLimit",
@@ -896,6 +907,7 @@ public final class Keys {
 
     /**
      * Enable attributes skipping. Attribute skipping can be enabled in the config or device attributes.
+     * If position contains any attribute mentioned in "filter.skipAttributes" config key, position is not filtered out.
      */
     public static final ConfigKey<Boolean> FILTER_SKIP_ATTRIBUTES_ENABLE = new ConfigKey<>(
             "filter.skipAttributes.enable",
diff --git a/src/main/java/org/traccar/database/DataManager.java b/src/main/java/org/traccar/database/DataManager.java
index 15137ad91..3f702f78a 100644
--- a/src/main/java/org/traccar/database/DataManager.java
+++ b/src/main/java/org/traccar/database/DataManager.java
@@ -328,6 +328,42 @@ public class DataManager {
                 .executeQuery(Position.class);
     }
 
+    public Position getPositionByTime(long deviceId, Date date) throws SQLException {
+        Collection<Position> positions = QueryBuilder.create(dataSource, getQuery("database.selectPositionByTime"))
+                .setLong("deviceId", deviceId)
+                .setDate("time", date)
+                .executeQuery(Position.class);
+        if (positions.isEmpty()) {
+            return null;
+        } else {
+            return positions.iterator().next();
+        }
+    }
+
+    public Position getPrevPosition(long deviceId, Date date) throws SQLException {
+        Collection<Position> positions = QueryBuilder.create(dataSource, getQuery("database.selectPrevPosition"))
+                .setLong("deviceId", deviceId)
+                .setDate("time", date)
+                .executeQuery(Position.class);
+        if (positions.isEmpty()) {
+            return null;
+        } else {
+            return positions.iterator().next();
+        }
+    }
+
+    public Position getNextPosition(long deviceId, Date date) throws SQLException {
+        Collection<Position> positions = QueryBuilder.create(dataSource, getQuery("database.selectNextPosition"))
+                .setLong("deviceId", deviceId)
+                .setDate("time", date)
+                .executeQuery(Position.class);
+        if (positions.isEmpty()) {
+            return null;
+        } else {
+            return positions.iterator().next();
+        }
+    }
+
     public void updateLatestPosition(Position position) throws SQLException {
         QueryBuilder.create(dataSource, getQuery("database.updateLatestPosition"))
                 .setDate("now", new Date())
diff --git a/src/main/java/org/traccar/handler/FilterHandler.java b/src/main/java/org/traccar/handler/FilterHandler.java
index 7cd9153c1..817d6c8ba 100644
--- a/src/main/java/org/traccar/handler/FilterHandler.java
+++ b/src/main/java/org/traccar/handler/FilterHandler.java
@@ -25,6 +25,9 @@ import org.traccar.config.Keys;
 import org.traccar.helper.UnitsConverter;
 import org.traccar.model.Position;
 
+import java.sql.SQLException;
+import java.util.Date;
+
 @ChannelHandler.Sharable
 public class FilterHandler extends BaseDataHandler {
 
@@ -40,6 +43,7 @@ public class FilterHandler extends BaseDataHandler {
     private int filterDistance;
     private int filterMaxSpeed;
     private long filterMinPeriod;
+    private boolean filterRelative;
     private long skipLimit;
     private boolean skipAttributes;
 
@@ -54,14 +58,15 @@ public class FilterHandler extends BaseDataHandler {
         filterDistance = config.getInteger(Keys.FILTER_DISTANCE);
         filterMaxSpeed = config.getInteger(Keys.FILTER_MAX_SPEED);
         filterMinPeriod = config.getInteger(Keys.FILTER_MIN_PERIOD) * 1000;
+        filterRelative = config.getBoolean(Keys.FILTER_RELATIVE);
         skipLimit = config.getLong(Keys.FILTER_SKIP_LIMIT) * 1000;
         skipAttributes = config.getBoolean(Keys.FILTER_SKIP_ATTRIBUTES_ENABLE);
     }
 
     private boolean filterInvalid(Position position) {
         return filterInvalid && (!position.getValid()
-           || position.getLatitude() > 90 || position.getLongitude() > 180
-           || position.getLatitude() < -90 || position.getLongitude() < -180);
+                || position.getLatitude() > 90 || position.getLongitude() > 180
+                || position.getLatitude() < -90 || position.getLongitude() < -180);
     }
 
     private boolean filterZero(Position position) {
@@ -144,20 +149,13 @@ public class FilterHandler extends BaseDataHandler {
 
         StringBuilder filterType = new StringBuilder();
 
-        Position last = null;
-        if (Context.getIdentityManager() != null) {
-            last = Context.getIdentityManager().getLastPosition(position.getDeviceId());
-        }
-
+        // general filtering
         if (filterInvalid(position)) {
             filterType.append("Invalid ");
         }
         if (filterZero(position)) {
             filterType.append("Zero ");
         }
-        if (filterDuplicate(position, last) && !skipLimit(position, last) && !skipAttributes(position)) {
-            filterType.append("Duplicate ");
-        }
         if (filterFuture(position)) {
             filterType.append("Future ");
         }
@@ -167,19 +165,56 @@ public class FilterHandler extends BaseDataHandler {
         if (filterApproximate(position)) {
             filterType.append("Approximate ");
         }
-        if (filterStatic(position) && !skipLimit(position, last) && !skipAttributes(position)) {
-            filterType.append("Static ");
-        }
-        if (filterDistance(position, last) && !skipLimit(position, last) && !skipAttributes(position)) {
-            filterType.append("Distance ");
-        }
-        if (filterMaxSpeed(position, last)) {
-            filterType.append("MaxSpeed ");
-        }
-        if (filterMinPeriod(position, last)) {
-            filterType.append("MinPeriod ");
-        }
+        if (filterRelative) {
+            try {
+                if (filterStatic(position) && !skipAttributes(position)) {
+                    filterType.append("Static ");
+                }
 
+                Date newfixTime = position.getFixTime();
+                Position duplicate = Context.getDataManager().getPositionByTime(position.getDeviceId(), newfixTime);
+                if (filterDuplicate(position, duplicate) && !skipLimit(position, duplicate)
+                        && !skipAttributes(position)) {
+                    filterType.append("Duplicate ");
+                }
+
+                Position previous = Context.getDataManager().getPrevPosition(position.getDeviceId(), newfixTime);
+                Position next = Context.getDataManager().getNextPosition(position.getDeviceId(), newfixTime);
+                if ((filterDistance(position, previous) || filterDistance(next, position))
+                        && !skipAttributes(position)) {
+                    filterType.append("Distance ");
+                }
+                if (filterMaxSpeed(position, previous) || filterMaxSpeed(next, position)) {
+                    filterType.append("MaxSpeed ");
+                }
+                if (filterMinPeriod(position, previous) || filterMinPeriod(next, position)) {
+                    filterType.append("MinPeriod ");
+                }
+            } catch (SQLException e) {
+                LOGGER.warn("Error filtering position", e);
+            }
+        } else {
+            Position last = null;
+            if (Context.getIdentityManager() != null) {
+                last = Context.getIdentityManager().getLastPosition(position.getDeviceId());
+            }
+
+            if (filterDuplicate(position, last) && !skipLimit(position, last) && !skipAttributes(position)) {
+                filterType.append("Duplicate ");
+            }
+            if (filterStatic(position) && !skipLimit(position, last) && !skipAttributes(position)) {
+                filterType.append("Static ");
+            }
+            if (filterDistance(position, last) && !skipLimit(position, last) && !skipAttributes(position)) {
+                filterType.append("Distance ");
+            }
+            if (filterMaxSpeed(position, last)) {
+                filterType.append("MaxSpeed ");
+            }
+            if (filterMinPeriod(position, last)) {
+                filterType.append("MinPeriod ");
+            }
+        }
         if (filterType.length() > 0) {
 
             StringBuilder message = new StringBuilder();