From 7e239583698a169971f5bd817adbabdacba8dc56 Mon Sep 17 00:00:00 2001 From: Anton Tananaev Date: Sat, 23 Feb 2019 12:12:31 -0800 Subject: Update filter handler --- src/org/traccar/BasePipelineFactory.java | 19 +- src/org/traccar/FilterHandler.java | 258 -------------------------- src/org/traccar/MainModule.java | 11 ++ src/org/traccar/config/Keys.java | 75 ++++++++ src/org/traccar/processing/FilterHandler.java | 210 +++++++++++++++++++++ 5 files changed, 304 insertions(+), 269 deletions(-) delete mode 100644 src/org/traccar/FilterHandler.java create mode 100644 src/org/traccar/processing/FilterHandler.java (limited to 'src') diff --git a/src/org/traccar/BasePipelineFactory.java b/src/org/traccar/BasePipelineFactory.java index 4bc41bd55..022eeeffa 100644 --- a/src/org/traccar/BasePipelineFactory.java +++ b/src/org/traccar/BasePipelineFactory.java @@ -31,6 +31,7 @@ import io.netty.channel.socket.DatagramPacket; import io.netty.handler.timeout.IdleStateHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.traccar.config.Keys; import org.traccar.events.CommandResultEventHandler; import org.traccar.events.DriverEventHandler; import org.traccar.events.FuelDropEventHandler; @@ -42,6 +43,7 @@ import org.traccar.events.OverspeedEventHandler; import org.traccar.events.AlertEventHandler; import org.traccar.processing.ComputedAttributesHandler; import org.traccar.processing.CopyAttributesHandler; +import org.traccar.processing.FilterHandler; import java.net.InetSocketAddress; import java.net.SocketAddress; @@ -54,7 +56,6 @@ public abstract class BasePipelineFactory extends ChannelInitializer { private final TrackerServer server; private int timeout; - private FilterHandler filterHandler; private DistanceHandler distanceHandler; private EngineHoursHandler engineHoursHandler; private RemoteAddressHandler remoteAddressHandler; @@ -199,10 +200,6 @@ public abstract class BasePipelineFactory extends ChannelInitializer { remoteAddressHandler = new RemoteAddressHandler(); } - if (Context.getConfig().getBoolean("filter.enable")) { - filterHandler = new FilterHandler(); - } - if (Context.getGeocoder() != null && !Context.getConfig().getBoolean("geocoder.ignorePositions")) { geocoderHandler = new GeocoderHandler( Context.getGeocoder(), @@ -304,7 +301,7 @@ public abstract class BasePipelineFactory extends ChannelInitializer { addHandlers( pipeline, - filterHandler, + Main.getInjector().getInstance(FilterHandler.class), geocoderHandler, motionHandler, engineHoursHandler, @@ -332,12 +329,12 @@ public abstract class BasePipelineFactory extends ChannelInitializer { } private void addDynamicHandlers(ChannelPipeline pipeline) { - if (Context.getConfig().hasKey("extra.handlers")) { - String[] handlers = Context.getConfig().getString("extra.handlers").split(","); - for (String handler : handlers) { + String handlers = Context.getConfig().getString(Keys.EXTRA_HANDLERS); + if (handlers != null) { + for (String handler : handlers.split(",")) { try { - pipeline.addLast((ChannelHandler) Class.forName(handler).newInstance()); - } catch (ClassNotFoundException | InstantiationException | IllegalAccessException error) { + pipeline.addLast((ChannelHandler) Class.forName(handler).getDeclaredConstructor().newInstance()); + } catch (ReflectiveOperationException error) { LOGGER.warn("Dynamic handler error", error); } } diff --git a/src/org/traccar/FilterHandler.java b/src/org/traccar/FilterHandler.java deleted file mode 100644 index 6f2bb0d2e..000000000 --- a/src/org/traccar/FilterHandler.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright 2014 - 2018 Anton Tananaev (anton@traccar.org) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.traccar; - -import io.netty.channel.ChannelHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.traccar.config.Config; -import org.traccar.helper.UnitsConverter; -import org.traccar.model.Position; - -@ChannelHandler.Sharable -public class FilterHandler extends BaseDataHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(FilterHandler.class); - - private boolean filterInvalid; - private boolean filterZero; - private boolean filterDuplicate; - private long filterFuture; - private boolean filterApproximate; - private int filterAccuracy; - private boolean filterStatic; - private int filterDistance; - private int filterMaxSpeed; - private long filterMinPeriod; - private long skipLimit; - private boolean skipAttributes; - - public void setFilterInvalid(boolean filterInvalid) { - this.filterInvalid = filterInvalid; - } - - public void setFilterZero(boolean filterZero) { - this.filterZero = filterZero; - } - - public void setFilterDuplicate(boolean filterDuplicate) { - this.filterDuplicate = filterDuplicate; - } - - public void setFilterFuture(long filterFuture) { - this.filterFuture = filterFuture; - } - - public void setFilterAccuracy(int filterAccuracy) { - this.filterAccuracy = filterAccuracy; - } - - public void setFilterApproximate(boolean filterApproximate) { - this.filterApproximate = filterApproximate; - } - - public void setFilterStatic(boolean filterStatic) { - this.filterStatic = filterStatic; - } - - public void setFilterDistance(int filterDistance) { - this.filterDistance = filterDistance; - } - - public void setFilterMaxSpeed(int filterMaxSpeed) { - this.filterMaxSpeed = filterMaxSpeed; - } - - public void setFilterMinPeriod(int filterMinPeriod) { - this.filterMinPeriod = filterMinPeriod; - } - - public void setSkipLimit(long skipLimit) { - this.skipLimit = skipLimit; - } - - public void setSkipAttributes(boolean skipAttributes) { - this.skipAttributes = skipAttributes; - } - - public FilterHandler() { - Config config = Context.getConfig(); - if (config != null) { - filterInvalid = config.getBoolean("filter.invalid"); - filterZero = config.getBoolean("filter.zero"); - filterDuplicate = config.getBoolean("filter.duplicate"); - filterFuture = config.getLong("filter.future") * 1000; - filterAccuracy = config.getInteger("filter.accuracy"); - filterApproximate = config.getBoolean("filter.approximate"); - filterStatic = config.getBoolean("filter.static"); - filterDistance = config.getInteger("filter.distance"); - filterMaxSpeed = config.getInteger("filter.maxSpeed"); - filterMinPeriod = config.getInteger("filter.minPeriod") * 1000; - skipLimit = config.getLong("filter.skipLimit") * 1000; - skipAttributes = config.getBoolean("filter.skipAttributes.enable"); - } - } - - private boolean filterInvalid(Position position) { - return filterInvalid && (!position.getValid() - || position.getLatitude() > 90 || position.getLongitude() > 180 - || position.getLatitude() < -90 || position.getLongitude() < -180); - } - - private boolean filterZero(Position position) { - return filterZero && position.getLatitude() == 0.0 && position.getLongitude() == 0.0; - } - - private boolean filterDuplicate(Position position, Position last) { - if (filterDuplicate && last != null && position.getFixTime().equals(last.getFixTime())) { - for (String key : position.getAttributes().keySet()) { - if (!last.getAttributes().containsKey(key)) { - return false; - } - } - return true; - } - return false; - } - - private boolean filterFuture(Position position) { - return filterFuture != 0 && position.getFixTime().getTime() > System.currentTimeMillis() + filterFuture; - } - - private boolean filterAccuracy(Position position) { - return filterAccuracy != 0 && position.getAccuracy() > filterAccuracy; - } - - private boolean filterApproximate(Position position) { - return filterApproximate && position.getBoolean(Position.KEY_APPROXIMATE); - } - - private boolean filterStatic(Position position) { - return filterStatic && position.getSpeed() == 0.0; - } - - private boolean filterDistance(Position position, Position last) { - if (filterDistance != 0 && last != null) { - return position.getDouble(Position.KEY_DISTANCE) < filterDistance; - } - return false; - } - - private boolean filterMaxSpeed(Position position, Position last) { - if (filterMaxSpeed != 0 && last != null) { - double distance = position.getDouble(Position.KEY_DISTANCE); - double time = position.getFixTime().getTime() - last.getFixTime().getTime(); - return UnitsConverter.knotsFromMps(distance / (time / 1000)) > filterMaxSpeed; - } - return false; - } - - private boolean filterMinPeriod(Position position, Position last) { - if (filterMinPeriod != 0 && last != null) { - long time = position.getFixTime().getTime() - last.getFixTime().getTime(); - return time > 0 && time < filterMinPeriod; - } - return false; - } - - private boolean skipLimit(Position position, Position last) { - if (skipLimit != 0 && last != null) { - return (position.getServerTime().getTime() - last.getServerTime().getTime()) > skipLimit; - } - return false; - } - - private boolean skipAttributes(Position position) { - if (skipAttributes) { - String attributesString = Context.getIdentityManager().lookupAttributeString( - position.getDeviceId(), "filter.skipAttributes", "", true); - for (String attribute : attributesString.split("[ ,]")) { - if (position.getAttributes().containsKey(attribute)) { - return true; - } - } - } - return false; - } - - private boolean filter(Position position) { - - StringBuilder filterType = new StringBuilder(); - - Position last = null; - if (Context.getIdentityManager() != null) { - last = Context.getIdentityManager().getLastPosition(position.getDeviceId()); - } - - if (skipLimit(position, last) || skipAttributes(position)) { - return false; - } - - if (filterInvalid(position)) { - filterType.append("Invalid "); - } - if (filterZero(position)) { - filterType.append("Zero "); - } - if (filterDuplicate(position, last)) { - filterType.append("Duplicate "); - } - if (filterFuture(position)) { - filterType.append("Future "); - } - if (filterAccuracy(position)) { - filterType.append("Accuracy "); - } - if (filterApproximate(position)) { - filterType.append("Approximate "); - } - if (filterStatic(position)) { - filterType.append("Static "); - } - if (filterDistance(position, last)) { - filterType.append("Distance "); - } - if (filterMaxSpeed(position, last)) { - filterType.append("MaxSpeed "); - } - if (filterMinPeriod(position, last)) { - filterType.append("MinPeriod "); - } - - if (filterType.length() > 0) { - - StringBuilder message = new StringBuilder(); - message.append("Position filtered by "); - message.append(filterType.toString()); - message.append("filters from device: "); - message.append(Context.getIdentityManager().getById(position.getDeviceId()).getUniqueId()); - - LOGGER.info(message.toString()); - return true; - } - - return false; - } - - @Override - protected Position handlePosition(Position position) { - if (filter(position)) { - return null; - } - return position; - } - -} diff --git a/src/org/traccar/MainModule.java b/src/org/traccar/MainModule.java index 26bdfcd60..60f774544 100644 --- a/src/org/traccar/MainModule.java +++ b/src/org/traccar/MainModule.java @@ -22,6 +22,7 @@ import com.google.inject.Singleton; import org.traccar.config.Config; import org.traccar.config.Keys; import org.traccar.database.IdentityManager; +import org.traccar.processing.FilterHandler; import javax.ws.rs.client.Client; @@ -47,6 +48,16 @@ public class MainModule extends AbstractModule { return Context.getClient(); } + @Singleton + @Provides + public static FilterHandler provideFilterHandler(Config config) { + if (config.getBoolean(Keys.FILTER_ENABLE)) { + return new FilterHandler(config); + } else { + return null; + } + } + @Singleton @Provides public static WebDataHandler provideWebDataHandler( diff --git a/src/org/traccar/config/Keys.java b/src/org/traccar/config/Keys.java index 51adfbc13..5b26854ed 100644 --- a/src/org/traccar/config/Keys.java +++ b/src/org/traccar/config/Keys.java @@ -17,6 +17,11 @@ package org.traccar.config; public final class Keys { + public static final ConfigKey EXTRA_HANDLERS = new ConfigKey( + "extra.handlers", + String.class, + "List of external handler classes to use in Netty pipeline."); + public static final ConfigKey FORWARD_ENABLE = new ConfigKey( "forward.enable", Boolean.class, @@ -38,6 +43,76 @@ public final class Keys { Boolean.class, "Boolean value to enable forwarding in JSON format."); + public static final ConfigKey FILTER_ENABLE = new ConfigKey( + "filter.enable", + Boolean.class, + "Boolean flag to enable or disable position filtering."); + + public static final ConfigKey FILTER_INVALID = new ConfigKey( + "filter.invalid", + Boolean.class, + "Filter invalid (valid field is set to false) positions."); + + public static final ConfigKey FILTER_ZERO = new ConfigKey( + "filter.zero", + Boolean.class, + "Filter zero coordinates. Zero latitude and longitude are theoretically valid values, but it practice it " + + "usually indicates invalid GPS data."); + + public static final ConfigKey FILTER_DUPLICATE = new ConfigKey( + "filter.duplicate", + Boolean.class, + "Filter duplicate records (duplicates are detected by time value)."); + + public static final ConfigKey FILTER_FUTURE = new ConfigKey( + "filter.future", + Long.class, + "Filter records with fix time in future. The values is specified in seconds. Records that have fix time " + + "more than specified number of seconds later than current server time would be filtered out."); + + public static final ConfigKey FILTER_ACCURACY = new ConfigKey( + "filter.accuracy", + Integer.class, + "Filter positions with accuracy less than specified value in meters."); + + public static final ConfigKey FILTER_APPROXIMATE = new ConfigKey( + "filter.approximate", + Boolean.class, + "Filter cell and wifi locations that are coming from geolocation provider."); + + public static final ConfigKey FILTER_STATIC = new ConfigKey( + "filter.static", + Boolean.class, + "Filter positions with exactly zero speed values."); + + public static final ConfigKey FILTER_DISTANCE = new ConfigKey( + "filter.distance", + Integer.class, + "Filter records by distance. The values is specified in meters. If the new position is less far than this " + + "value from the last one it gets filtered out."); + + public static final ConfigKey FILTER_MAX_SPEED = new ConfigKey( + "filter.maxSpeed", + Integer.class, + "Filter records by Maximum Speed value in knots. Can be used to filter jumps to far locations even if " + + "they're marked as valid. Shouldn't be too low. Start testing with values at about 25000."); + + public static final ConfigKey FILTER_MIN_PERIOD = new ConfigKey( + "filter.minPeriod", + Integer.class, + "Filter position if time from previous position is less than specified value in seconds."); + + public static final ConfigKey FILTER_SKIP_LIMIT = new ConfigKey( + "filter.skipLimit", + Long.class, + "Time limit for the filtering in seconds. If the time difference between last position and a new one is " + + "more than this limit, the new position will not be filtered out."); + + public static final ConfigKey FILTER_SKIP_ATTRIBUTES_ENABLE = new ConfigKey( + "filter.skipAttributes.enable", + Boolean.class, + "Enable attributes skipping. Attribute skipping can be enabled in the config or device attributes"); + private Keys() { } diff --git a/src/org/traccar/processing/FilterHandler.java b/src/org/traccar/processing/FilterHandler.java new file mode 100644 index 000000000..df62b1e6d --- /dev/null +++ b/src/org/traccar/processing/FilterHandler.java @@ -0,0 +1,210 @@ +/* + * Copyright 2014 - 2018 Anton Tananaev (anton@traccar.org) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.traccar.processing; + +import io.netty.channel.ChannelHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.traccar.BaseDataHandler; +import org.traccar.Context; +import org.traccar.config.Config; +import org.traccar.config.Keys; +import org.traccar.helper.UnitsConverter; +import org.traccar.model.Position; + +@ChannelHandler.Sharable +public class FilterHandler extends BaseDataHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(FilterHandler.class); + + private boolean filterInvalid; + private boolean filterZero; + private boolean filterDuplicate; + private long filterFuture; + private boolean filterApproximate; + private int filterAccuracy; + private boolean filterStatic; + private int filterDistance; + private int filterMaxSpeed; + private long filterMinPeriod; + private long skipLimit; + private boolean skipAttributes; + + public FilterHandler(Config config) { + filterInvalid = config.getBoolean(Keys.FILTER_INVALID); + filterZero = config.getBoolean(Keys.FILTER_ZERO); + filterDuplicate = config.getBoolean(Keys.FILTER_DUPLICATE); + filterFuture = config.getLong(Keys.FILTER_FUTURE) * 1000; + filterAccuracy = config.getInteger(Keys.FILTER_ACCURACY); + filterApproximate = config.getBoolean(Keys.FILTER_APPROXIMATE); + filterStatic = config.getBoolean(Keys.FILTER_STATIC); + filterDistance = config.getInteger(Keys.FILTER_DISTANCE); + filterMaxSpeed = config.getInteger(Keys.FILTER_MAX_SPEED); + filterMinPeriod = config.getInteger(Keys.FILTER_MIN_PERIOD) * 1000; + skipLimit = config.getLong(Keys.FILTER_SKIP_LIMIT) * 1000; + skipAttributes = config.getBoolean(Keys.FILTER_SKIP_ATTRIBUTES_ENABLE); + } + + private boolean filterInvalid(Position position) { + return filterInvalid && (!position.getValid() + || position.getLatitude() > 90 || position.getLongitude() > 180 + || position.getLatitude() < -90 || position.getLongitude() < -180); + } + + private boolean filterZero(Position position) { + return filterZero && position.getLatitude() == 0.0 && position.getLongitude() == 0.0; + } + + private boolean filterDuplicate(Position position, Position last) { + if (filterDuplicate && last != null && position.getFixTime().equals(last.getFixTime())) { + for (String key : position.getAttributes().keySet()) { + if (!last.getAttributes().containsKey(key)) { + return false; + } + } + return true; + } + return false; + } + + private boolean filterFuture(Position position) { + return filterFuture != 0 && position.getFixTime().getTime() > System.currentTimeMillis() + filterFuture; + } + + private boolean filterAccuracy(Position position) { + return filterAccuracy != 0 && position.getAccuracy() > filterAccuracy; + } + + private boolean filterApproximate(Position position) { + return filterApproximate && position.getBoolean(Position.KEY_APPROXIMATE); + } + + private boolean filterStatic(Position position) { + return filterStatic && position.getSpeed() == 0.0; + } + + private boolean filterDistance(Position position, Position last) { + if (filterDistance != 0 && last != null) { + return position.getDouble(Position.KEY_DISTANCE) < filterDistance; + } + return false; + } + + private boolean filterMaxSpeed(Position position, Position last) { + if (filterMaxSpeed != 0 && last != null) { + double distance = position.getDouble(Position.KEY_DISTANCE); + double time = position.getFixTime().getTime() - last.getFixTime().getTime(); + return UnitsConverter.knotsFromMps(distance / (time / 1000)) > filterMaxSpeed; + } + return false; + } + + private boolean filterMinPeriod(Position position, Position last) { + if (filterMinPeriod != 0 && last != null) { + long time = position.getFixTime().getTime() - last.getFixTime().getTime(); + return time > 0 && time < filterMinPeriod; + } + return false; + } + + private boolean skipLimit(Position position, Position last) { + if (skipLimit != 0 && last != null) { + return (position.getServerTime().getTime() - last.getServerTime().getTime()) > skipLimit; + } + return false; + } + + private boolean skipAttributes(Position position) { + if (skipAttributes) { + String attributesString = Context.getIdentityManager().lookupAttributeString( + position.getDeviceId(), "filter.skipAttributes", "", true); + for (String attribute : attributesString.split("[ ,]")) { + if (position.getAttributes().containsKey(attribute)) { + return true; + } + } + } + return false; + } + + private boolean filter(Position position) { + + StringBuilder filterType = new StringBuilder(); + + Position last = null; + if (Context.getIdentityManager() != null) { + last = Context.getIdentityManager().getLastPosition(position.getDeviceId()); + } + + if (skipLimit(position, last) || skipAttributes(position)) { + return false; + } + + if (filterInvalid(position)) { + filterType.append("Invalid "); + } + if (filterZero(position)) { + filterType.append("Zero "); + } + if (filterDuplicate(position, last)) { + filterType.append("Duplicate "); + } + if (filterFuture(position)) { + filterType.append("Future "); + } + if (filterAccuracy(position)) { + filterType.append("Accuracy "); + } + if (filterApproximate(position)) { + filterType.append("Approximate "); + } + if (filterStatic(position)) { + filterType.append("Static "); + } + if (filterDistance(position, last)) { + filterType.append("Distance "); + } + if (filterMaxSpeed(position, last)) { + filterType.append("MaxSpeed "); + } + if (filterMinPeriod(position, last)) { + filterType.append("MinPeriod "); + } + + if (filterType.length() > 0) { + + StringBuilder message = new StringBuilder(); + message.append("Position filtered by "); + message.append(filterType.toString()); + message.append("filters from device: "); + message.append(Context.getIdentityManager().getById(position.getDeviceId()).getUniqueId()); + + LOGGER.info(message.toString()); + return true; + } + + return false; + } + + @Override + protected Position handlePosition(Position position) { + if (filter(position)) { + return null; + } + return position; + } + +} -- cgit v1.2.3