Merge remote-tracking branch 'refs/remotes/tananaev/master'

# Conflicts:
#	tools/test-generator.py

4 files changed, 183 insertions, 229 deletions

diff --git a/src/org/traccar/database/DataManager.java b/src/org/traccar/database/DataManager.java
index 710eceebb..04d0d44ea 100644
--- a/src/org/traccar/database/DataManager.java
+++ b/src/org/traccar/database/DataManager.java
@@ -20,16 +20,9 @@ import java.lang.reflect.Method;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.sql.SQLException;
-import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collection;
 import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import javax.naming.InitialContext;
 import javax.sql.DataSource;
@@ -43,7 +36,6 @@ import liquibase.resource.FileSystemResourceAccessor;
 import liquibase.resource.ResourceAccessor;
 
 import org.traccar.Config;
-import org.traccar.Context;
 import org.traccar.helper.Log;
 import org.traccar.model.Device;
 import org.traccar.model.DevicePermission;
@@ -64,25 +56,15 @@ import com.zaxxer.hikari.HikariDataSource;
 
 public class DataManager {
 
-    public static final long DEFAULT_REFRESH_DELAY = 300;
-
     private final Config config;
 
     private DataSource dataSource;
 
-    private final long dataRefreshDelay;
-
-    private final ReadWriteLock groupsLock = new ReentrantReadWriteLock();
-    private final Map<Long, Group> groupsById = new HashMap<>();
-    private long groupsLastUpdate;
-
     public DataManager(Config config) throws Exception {
         this.config = config;
 
         initDatabase();
         initDatabaseSchema();
-
-        dataRefreshDelay = config.getLong("database.refreshDelay", DEFAULT_REFRESH_DELAY) * 1000;
     }
 
     public DataSource getDataSource() {
@@ -131,54 +113,6 @@ public class DataManager {
         }
     }
 
-    private void updateGroupCache(boolean force) throws SQLException {
-        boolean needWrite;
-        groupsLock.readLock().lock();
-        try {
-            needWrite = force || System.currentTimeMillis() - groupsLastUpdate > dataRefreshDelay;
-        } finally {
-            groupsLock.readLock().unlock();
-        }
-
-        if (needWrite) {
-            groupsLock.writeLock().lock();
-            try {
-                if (force || System.currentTimeMillis() - groupsLastUpdate > dataRefreshDelay) {
-                    groupsById.clear();
-                    for (Group group : getAllGroups()) {
-                        groupsById.put(group.getId(), group);
-                    }
-                    groupsLastUpdate = System.currentTimeMillis();
-                }
-            } finally {
-                groupsLock.writeLock().unlock();
-            }
-        }
-    }
-
-    public Group getGroupById(long id) {
-        boolean forceUpdate;
-        groupsLock.readLock().lock();
-        try {
-            forceUpdate = !groupsById.containsKey(id);
-        } finally {
-            groupsLock.readLock().unlock();
-        }
-
-        try {
-            updateGroupCache(forceUpdate);
-        } catch (SQLException e) {
-            Log.warning(e);
-        }
-
-        groupsLock.readLock().lock();
-        try {
-            return groupsById.get(id);
-        } finally {
-            groupsLock.readLock().unlock();
-        }
-    }
-
     private String getQuery(String key) {
         String query = config.getString(key);
         if (query == null) {
@@ -311,44 +245,16 @@ public class DataManager {
                 .executeQuery(Group.class);
     }
 
-    public Collection<Group> getGroups(long userId) throws SQLException {
-        Collection<Group> groups = new ArrayList<>();
-        for (long id : Context.getPermissionsManager().getGroupPermissions(userId)) {
-            groups.add(getGroupById(id));
-        }
-        return groups;
-    }
-
-    private void checkGroupCycles(Group group) {
-        groupsLock.readLock().lock();
-        try {
-            Set<Long> groups = new HashSet<>();
-            while (group != null) {
-                if (groups.contains(group.getId())) {
-                    throw new IllegalArgumentException("Cycle in group hierarchy");
-                }
-                groups.add(group.getId());
-                group = groupsById.get(group.getGroupId());
-            }
-        } finally {
-            groupsLock.readLock().unlock();
-        }
-    }
-
     public void addGroup(Group group) throws SQLException {
-        checkGroupCycles(group);
         group.setId(QueryBuilder.create(dataSource, getQuery("database.insertGroup"), true)
                 .setObject(group)
                 .executeUpdate());
-        updateGroupCache(true);
     }
 
     public void updateGroup(Group group) throws SQLException {
-        checkGroupCycles(group);
         QueryBuilder.create(dataSource, getQuery("database.updateGroup"))
                 .setObject(group)
                 .executeUpdate();
-        updateGroupCache(true);
     }
 
     public void removeGroup(long groupId) throws SQLException {
diff --git a/src/org/traccar/database/DeviceManager.java b/src/org/traccar/database/DeviceManager.java
index 173e68062..4dd7b41cb 100644
--- a/src/org/traccar/database/DeviceManager.java
+++ b/src/org/traccar/database/DeviceManager.java
@@ -18,39 +18,46 @@ package org.traccar.database;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.traccar.Config;
 import org.traccar.Context;
 import org.traccar.helper.Log;
 import org.traccar.model.Device;
+import org.traccar.model.Group;
 import org.traccar.model.Position;
 
 public class DeviceManager implements IdentityManager {
 
+    public static final long DEFAULT_REFRESH_DELAY = 300;
+
     private final Config config;
     private final DataManager dataManager;
     private final long dataRefreshDelay;
 
-    private final ReadWriteLock devicesLock = new ReentrantReadWriteLock();
-    private final Map<Long, Device> devicesById = new HashMap<>();
-    private final Map<String, Device> devicesByUniqueId = new HashMap<>();
-    private long devicesLastUpdate;
+    private Map<Long, Device> devicesById;
+    private Map<String, Device> devicesByUniqueId;
+    private AtomicLong devicesLastUpdate = new AtomicLong();
+
+    private Map<Long, Group> groupsById;
+    private AtomicLong groupsLastUpdate = new AtomicLong();
 
     private final Map<Long, Position> positions = new ConcurrentHashMap<>();
 
     public DeviceManager(DataManager dataManager) {
         this.dataManager = dataManager;
         this.config = Context.getConfig();
-        dataRefreshDelay = config.getLong("database.refreshDelay", DataManager.DEFAULT_REFRESH_DELAY) * 1000;
+        dataRefreshDelay = config.getLong("database.refreshDelay", DEFAULT_REFRESH_DELAY) * 1000;
         if (dataManager != null) {
             try {
+                updateGroupCache(true);
+                updateDeviceCache(true);
                 for (Position position : dataManager.getLatestPositions()) {
                     positions.put(position.getDeviceId(), position);
                 }
@@ -61,91 +68,77 @@ public class DeviceManager implements IdentityManager {
     }
 
     private void updateDeviceCache(boolean force) throws SQLException {
-        boolean needWrite;
-        devicesLock.readLock().lock();
-        try {
-            needWrite = force || System.currentTimeMillis() - devicesLastUpdate > dataRefreshDelay;
-        } finally {
-            devicesLock.readLock().unlock();
-        }
 
-        if (needWrite) {
-            devicesLock.writeLock().lock();
-            try {
-                if (force || System.currentTimeMillis() - devicesLastUpdate > dataRefreshDelay) {
-                    devicesById.clear();
-                    devicesByUniqueId.clear();
-                    GeofenceManager geofenceManager = Context.getGeofenceManager();
-                    for (Device device : dataManager.getAllDevices()) {
-                        devicesById.put(device.getId(), device);
-                        devicesByUniqueId.put(device.getUniqueId(), device);
-                        if (geofenceManager != null) {
-                            Position lastPosition = getLastPosition(device.getId());
-                            if (lastPosition != null) {
-                                device.setGeofenceIds(geofenceManager.getCurrentDeviceGeofences(lastPosition));
-                            }
+        long lastUpdate = devicesLastUpdate.get();
+        if ((force || System.currentTimeMillis() - lastUpdate > dataRefreshDelay)
+                && devicesLastUpdate.compareAndSet(lastUpdate, System.currentTimeMillis())) {
+            GeofenceManager geofenceManager = Context.getGeofenceManager();
+            Collection<Device> databaseDevices = dataManager.getAllDevices();
+            if (devicesById == null) {
+                devicesById = new ConcurrentHashMap<>(databaseDevices.size());
+            }
+            if (devicesByUniqueId == null) {
+                devicesByUniqueId = new ConcurrentHashMap<>(databaseDevices.size());
+            }
+            Set<Long> databaseDevicesIds = new HashSet<>();
+            Set<String> databaseDevicesUniqueIds = new HashSet<>();
+            for (Device device : databaseDevices) {
+                databaseDevicesIds.add(device.getId());
+                databaseDevicesUniqueIds.add(device.getUniqueId());
+                if (devicesById.containsKey(device.getId())) {
+                    Device cachedDevice = devicesById.get(device.getId());
+                    cachedDevice.setName(device.getName());
+                    cachedDevice.setGroupId(device.getGroupId());
+                    cachedDevice.setAttributes(device.getAttributes());
+                    if (!device.getUniqueId().equals(cachedDevice.getUniqueId())) {
+                        devicesByUniqueId.remove(cachedDevice.getUniqueId());
+                        devicesByUniqueId.put(device.getUniqueId(), cachedDevice);
+                    }
+                    cachedDevice.setUniqueId(device.getUniqueId());
+                } else {
+                    devicesById.put(device.getId(), device);
+                    devicesByUniqueId.put(device.getUniqueId(), device);
+                    if (geofenceManager != null) {
+                        Position lastPosition = getLastPosition(device.getId());
+                        if (lastPosition != null) {
+                            device.setGeofenceIds(geofenceManager.getCurrentDeviceGeofences(lastPosition));
                         }
                     }
-                    devicesLastUpdate = System.currentTimeMillis();
+                    device.setStatus(Device.STATUS_OFFLINE);
+                    device.setMotion(Device.STATUS_STOPPED);
+                }
+            }
+            for (Long cachedDeviceId : devicesById.keySet()) {
+                if (!databaseDevicesIds.contains(cachedDeviceId)) {
+                    devicesById.remove(cachedDeviceId);
                 }
-            } finally {
-                devicesLock.writeLock().unlock();
             }
+            for (String cachedDeviceUniqId : devicesByUniqueId.keySet()) {
+                if (!databaseDevicesUniqueIds.contains(cachedDeviceUniqId)) {
+                    devicesByUniqueId.remove(cachedDeviceUniqId);
+                }
+            }
+            databaseDevicesIds.clear();
+            databaseDevicesUniqueIds.clear();
         }
     }
 
     @Override
     public Device getDeviceById(long id) {
-        boolean forceUpdate;
-        devicesLock.readLock().lock();
-        try {
-            forceUpdate = !devicesById.containsKey(id);
-        } finally {
-            devicesLock.readLock().unlock();
-        }
-
-        try {
-            updateDeviceCache(forceUpdate);
-        } catch (SQLException e) {
-            Log.warning(e);
-        }
-
-        devicesLock.readLock().lock();
-        try {
-            return devicesById.get(id);
-        } finally {
-            devicesLock.readLock().unlock();
-        }
+        return devicesById.get(id);
     }
 
     @Override
     public Device getDeviceByUniqueId(String uniqueId) throws SQLException {
-        boolean forceUpdate;
-        devicesLock.readLock().lock();
-        try {
-            forceUpdate = !devicesByUniqueId.containsKey(uniqueId) && !config.getBoolean("database.ignoreUnknown");
-        } finally {
-            devicesLock.readLock().unlock();
-        }
+        boolean forceUpdate = !devicesByUniqueId.containsKey(uniqueId) && !config.getBoolean("database.ignoreUnknown");
 
         updateDeviceCache(forceUpdate);
 
-        devicesLock.readLock().lock();
-        try {
-            return devicesByUniqueId.get(uniqueId);
-        } finally {
-            devicesLock.readLock().unlock();
-        }
+        return devicesByUniqueId.get(uniqueId);
     }
 
     public Collection<Device> getAllDevices() {
-        boolean forceUpdate;
-        devicesLock.readLock().lock();
-        try {
-            forceUpdate = devicesById.isEmpty();
-        } finally {
-            devicesLock.readLock().unlock();
-        }
+        boolean forceUpdate = devicesById.isEmpty();
 
         try {
             updateDeviceCache(forceUpdate);
@@ -153,23 +146,13 @@ public class DeviceManager implements IdentityManager {
             Log.warning(e);
         }
 
-        devicesLock.readLock().lock();
-        try {
-            return devicesById.values();
-        } finally {
-            devicesLock.readLock().unlock();
-        }
+        return devicesById.values();
     }
 
     public Collection<Device> getDevices(long userId) throws SQLException {
         Collection<Device> devices = new ArrayList<>();
-        devicesLock.readLock().lock();
-        try {
-            for (long id : Context.getPermissionsManager().getDevicePermissions(userId)) {
-                    devices.add(devicesById.get(id));
-            }
-        } finally {
-            devicesLock.readLock().unlock();
+        for (long id : Context.getPermissionsManager().getDevicePermissions(userId)) {
+            devices.add(devicesById.get(id));
         }
         return devices;
     }
@@ -177,56 +160,34 @@ public class DeviceManager implements IdentityManager {
     public void addDevice(Device device) throws SQLException {
         dataManager.addDevice(device);
 
-        devicesLock.writeLock().lock();
-        try {
-            devicesById.put(device.getId(), device);
-            devicesByUniqueId.put(device.getUniqueId(), device);
-        } finally {
-            devicesLock.writeLock().unlock();
-        }
+        devicesById.put(device.getId(), device);
+        devicesByUniqueId.put(device.getUniqueId(), device);
     }
 
     public void updateDevice(Device device) throws SQLException {
         dataManager.updateDevice(device);
 
-        devicesLock.writeLock().lock();
-        try {
-            devicesById.put(device.getId(), device);
-            devicesByUniqueId.put(device.getUniqueId(), device);
-        } finally {
-            devicesLock.writeLock().unlock();
-        }
+        devicesById.put(device.getId(), device);
+        devicesByUniqueId.put(device.getUniqueId(), device);
     }
 
     public void updateDeviceStatus(Device device) throws SQLException {
         dataManager.updateDeviceStatus(device);
-
-        devicesLock.writeLock().lock();
-        try {
-            if (devicesById.containsKey(device.getId())) {
-                Device cachedDevice = devicesById.get(device.getId());
-                cachedDevice.setStatus(device.getStatus());
-                cachedDevice.setMotion(device.getMotion());
-            }
-        } finally {
-            devicesLock.writeLock().unlock();
+        if (devicesById.containsKey(device.getId())) {
+            Device cachedDevice = devicesById.get(device.getId());
+            cachedDevice.setStatus(device.getStatus());
+            cachedDevice.setMotion(device.getMotion());
         }
     }
 
     public void removeDevice(long deviceId) throws SQLException {
         dataManager.removeDevice(deviceId);
 
-        devicesLock.writeLock().lock();
-        try {
-            if (devicesById.containsKey(deviceId)) {
-                String deviceUniqueId = devicesById.get(deviceId).getUniqueId();
-                devicesById.remove(deviceId);
-                devicesByUniqueId.remove(deviceUniqueId);
-            }
-        } finally {
-            devicesLock.writeLock().unlock();
+        if (devicesById.containsKey(deviceId)) {
+            String deviceUniqueId = devicesById.get(deviceId).getUniqueId();
+            devicesById.remove(deviceId);
+            devicesByUniqueId.remove(deviceUniqueId);
         }
-
         positions.remove(deviceId);
     }
 
@@ -237,13 +198,8 @@ public class DeviceManager implements IdentityManager {
 
             dataManager.updateLatestPosition(position);
 
-            devicesLock.writeLock().lock();
-            try {
-                if (devicesById.containsKey(position.getDeviceId())) {
-                    devicesById.get(position.getDeviceId()).setPositionId(position.getId());
-                }
-            } finally {
-                devicesLock.writeLock().unlock();
+            if (devicesById.containsKey(position.getDeviceId())) {
+                devicesById.get(position.getDeviceId()).setPositionId(position.getId());
             }
 
             positions.put(position.getDeviceId(), position);
@@ -273,4 +229,85 @@ public class DeviceManager implements IdentityManager {
 
         return result;
     }
+
+    private void updateGroupCache(boolean force) throws SQLException {
+
+        long lastUpdate = groupsLastUpdate.get();
+        if ((force || System.currentTimeMillis() - lastUpdate > dataRefreshDelay)
+                && groupsLastUpdate.compareAndSet(lastUpdate, System.currentTimeMillis())) {
+            Collection<Group> databaseGroups = dataManager.getAllGroups();
+            if (groupsById == null) {
+                groupsById = new ConcurrentHashMap<>(databaseGroups.size());
+            }
+            Set<Long> databaseGroupsIds = new HashSet<>();
+            for (Group group : databaseGroups) {
+                databaseGroupsIds.add(group.getId());
+                if (groupsById.containsKey(group.getId())) {
+                    Group cachedGroup = groupsById.get(group.getId());
+                    cachedGroup.setName(group.getName());
+                    cachedGroup.setGroupId(group.getGroupId());
+                } else {
+                    groupsById.put(group.getId(), group);
+                }
+            }
+            for (Long cachedGroupId : groupsById.keySet()) {
+                if (!databaseGroupsIds.contains(cachedGroupId)) {
+                    devicesById.remove(cachedGroupId);
+                }
+            }
+            databaseGroupsIds.clear();
+        }
+    }
+
+    public Group getGroupById(long id) {
+        return groupsById.get(id);
+    }
+
+    public Collection<Group> getAllGroups() {
+        boolean forceUpdate = groupsById.isEmpty();
+
+        try {
+            updateGroupCache(forceUpdate);
+        } catch (SQLException e) {
+            Log.warning(e);
+        }
+
+        return groupsById.values();
+    }
+
+    public Collection<Group> getGroups(long userId) throws SQLException {
+        Collection<Group> groups = new ArrayList<>();
+        for (long id : Context.getPermissionsManager().getGroupPermissions(userId)) {
+            groups.add(getGroupById(id));
+        }
+        return groups;
+    }
+
+    private void checkGroupCycles(Group group) {
+        Set<Long> groups = new HashSet<>();
+        while (group != null) {
+            if (groups.contains(group.getId())) {
+                throw new IllegalArgumentException("Cycle in group hierarchy");
+            }
+            groups.add(group.getId());
+            group = groupsById.get(group.getGroupId());
+        }
+    }
+
+    public void addGroup(Group group) throws SQLException {
+        checkGroupCycles(group);
+        dataManager.addGroup(group);
+        groupsById.put(group.getId(), group);
+    }
+
+    public void updateGroup(Group group) throws SQLException {
+        checkGroupCycles(group);
+        dataManager.updateGroup(group);
+        groupsById.put(group.getId(), group);
+    }
+
+    public void removeGroup(long groupId) throws SQLException {
+        dataManager.removeGroup(groupId);
+        groupsById.remove(groupId);
+    }
 }
diff --git a/src/org/traccar/database/GeofenceManager.java b/src/org/traccar/database/GeofenceManager.java
index dc31172b9..74dff70f4 100644
--- a/src/org/traccar/database/GeofenceManager.java
+++ b/src/org/traccar/database/GeofenceManager.java
@@ -155,31 +155,41 @@ public class GeofenceManager {
     public final void refresh() {
         if (dataManager != null) {
             try {
+
+                Collection<GroupGeofence> databaseGroupGeofences = dataManager.getGroupGeofences();
                 groupGeofencesLock.writeLock().lock();
-                deviceGeofencesLock.writeLock().lock();
                 try {
                     groupGeofences.clear();
-                    for (GroupGeofence groupGeofence : dataManager.getGroupGeofences()) {
+                    for (GroupGeofence groupGeofence : databaseGroupGeofences) {
                         getGroupGeofences(groupGeofence.getGroupId()).add(groupGeofence.getGeofenceId());
                     }
+                } finally {
+                    groupGeofencesLock.writeLock().unlock();
+                }
 
+                Collection<DeviceGeofence> databaseDeviceGeofences = dataManager.getDeviceGeofences();
+                Collection<Device> allDevices = Context.getDeviceManager().getAllDevices();
+
+                groupGeofencesLock.readLock().lock();
+                deviceGeofencesLock.writeLock().lock();
+                try {
                     deviceGeofences.clear();
                     deviceGeofencesWithGroups.clear();
 
-                    for (DeviceGeofence deviceGeofence : dataManager.getDeviceGeofences()) {
+                    for (DeviceGeofence deviceGeofence : databaseDeviceGeofences) {
                         getDeviceGeofences(deviceGeofences, deviceGeofence.getDeviceId())
                             .add(deviceGeofence.getGeofenceId());
                         getDeviceGeofences(deviceGeofencesWithGroups, deviceGeofence.getDeviceId())
                             .add(deviceGeofence.getGeofenceId());
                     }
 
-                    for (Device device : Context.getDeviceManager().getAllDevices()) {
+                    for (Device device : allDevices) {
                         long groupId = device.getGroupId();
                         while (groupId != 0) {
                             getDeviceGeofences(deviceGeofencesWithGroups,
                                     device.getId()).addAll(getGroupGeofences(groupId));
-                            if (dataManager.getGroupById(groupId) != null) {
-                                groupId = dataManager.getGroupById(groupId).getGroupId();
+                            if (Context.getDeviceManager().getGroupById(groupId) != null) {
+                                groupId = Context.getDeviceManager().getGroupById(groupId).getGroupId();
                             } else {
                                 groupId = 0;
                             }
@@ -204,8 +214,8 @@ public class GeofenceManager {
                     }
 
                 } finally {
-                    groupGeofencesLock.writeLock().unlock();
                     deviceGeofencesLock.writeLock().unlock();
+                    groupGeofencesLock.readLock().unlock();
                 }
 
             } catch (SQLException error) {
diff --git a/src/org/traccar/database/PermissionsManager.java b/src/org/traccar/database/PermissionsManager.java
index 5a15375b4..d786dcc4e 100644
--- a/src/org/traccar/database/PermissionsManager.java
+++ b/src/org/traccar/database/PermissionsManager.java
@@ -78,7 +78,8 @@ public class PermissionsManager {
                 users.put(user.getId(), user);
             }
 
-            GroupTree groupTree = new GroupTree(dataManager.getAllGroups(), Context.getDeviceManager().getAllDevices());
+            GroupTree groupTree = new GroupTree(Context.getDeviceManager().getAllGroups(),
+                    Context.getDeviceManager().getAllDevices());
             for (GroupPermission permission : dataManager.getGroupPermissions()) {
                 Set<Long> userGroupPermissions = getGroupPermissions(permission.getUserId());
                 Set<Long> userDevicePermissions = getDevicePermissions(permission.getUserId());