From 57672047108c0a41361feb1bce40602ca270a720 Mon Sep 17 00:00:00 2001 From: easbar Date: Wed, 29 Sep 2021 12:49:56 +0200 Subject: [PATCH 01/53] Set new version 5.0-SNAPSHOT --- CHANGELOG.md | 2 ++ client-hc/pom.xml | 4 ++-- core/pom.xml | 4 ++-- example/pom.xml | 4 ++-- hmm-lib/pom.xml | 4 ++-- map-matching/pom.xml | 4 ++-- navigation/pom.xml | 4 ++-- pom.xml | 2 +- reader-gtfs/pom.xml | 2 +- tools/pom.xml | 2 +- web-api/pom.xml | 4 ++-- web-bundle/pom.xml | 4 ++-- web-bundle/src/main/resources/com/graphhopper/maps/index.html | 2 +- web/pom.xml | 4 ++-- 14 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eda9dfa8b1..82f72125b27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +### 5.0 [not yet released] + ### 4.0 [29 Sep 2021] - faster node-based CH preparation (~20%), (#2390) diff --git a/client-hc/pom.xml b/client-hc/pom.xml index 6a4d04a9acf..df819da0a4e 100644 --- a/client-hc/pom.xml +++ b/client-hc/pom.xml @@ -22,14 +22,14 @@ 4.0.0 directions-api-client-hc - 4.0 + 5.0-SNAPSHOT jar GraphHopper Directions API hand-crafted Java Client. com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/core/pom.xml b/core/pom.xml index d4d1fdb78e1..afb234df4d4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -5,7 +5,7 @@ graphhopper-core GraphHopper Core - 4.0 + 5.0-SNAPSHOT jar GraphHopper is a fast and memory efficient Java road routing engine @@ -14,7 +14,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/example/pom.xml b/example/pom.xml index cd2546e5dc1..317fe0aac46 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-example - 4.0 + 5.0-SNAPSHOT jar GraphHopper Example com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/hmm-lib/pom.xml b/hmm-lib/pom.xml index c81478cb6ad..0f3f4b919fb 100644 --- a/hmm-lib/pom.xml +++ b/hmm-lib/pom.xml @@ -19,7 +19,7 @@ 4.0.0 hmm-lib-external - 4.0 + 5.0-SNAPSHOT jar hmm-lib @@ -29,7 +29,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/map-matching/pom.xml b/map-matching/pom.xml index f51bb17052a..83eaa17c25c 100644 --- a/map-matching/pom.xml +++ b/map-matching/pom.xml @@ -3,14 +3,14 @@ 4.0.0 com.graphhopper graphhopper-map-matching - 4.0 + 5.0-SNAPSHOT jar GraphHopper Map Matching com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/navigation/pom.xml b/navigation/pom.xml index 1bfb0120b72..655c01bea9c 100644 --- a/navigation/pom.xml +++ b/navigation/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-nav - 4.0 + 5.0-SNAPSHOT jar GraphHopper Navigation com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 02fce502b3d..b8d82eb02c7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.graphhopper graphhopper-parent GraphHopper Parent Project - 4.0 + 5.0-SNAPSHOT pom https://www.graphhopper.com 2012 diff --git a/reader-gtfs/pom.xml b/reader-gtfs/pom.xml index 5028d42a072..b7b611c0fb0 100644 --- a/reader-gtfs/pom.xml +++ b/reader-gtfs/pom.xml @@ -10,7 +10,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/tools/pom.xml b/tools/pom.xml index b425ac19b7c..47099203835 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -10,7 +10,7 @@ com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT package diff --git a/web-api/pom.xml b/web-api/pom.xml index a5e8f1919c9..5350eb5c7c9 100644 --- a/web-api/pom.xml +++ b/web-api/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-web-api jar - 4.0 + 5.0-SNAPSHOT GraphHopper Web API JSON Representation of the API classes com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/web-bundle/pom.xml b/web-bundle/pom.xml index a59281060b0..389299dbc41 100644 --- a/web-bundle/pom.xml +++ b/web-bundle/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-web-bundle jar - 4.0 + 5.0-SNAPSHOT GraphHopper Dropwizard Bundle Use the GraphHopper routing engine as a web-service com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT diff --git a/web-bundle/src/main/resources/com/graphhopper/maps/index.html b/web-bundle/src/main/resources/com/graphhopper/maps/index.html index 011f6d92920..bc0519f073a 100644 --- a/web-bundle/src/main/resources/com/graphhopper/maps/index.html +++ b/web-bundle/src/main/resources/com/graphhopper/maps/index.html @@ -36,7 +36,7 @@ - + diff --git a/web/pom.xml b/web/pom.xml index f66aa34c0b0..67aa7862306 100644 --- a/web/pom.xml +++ b/web/pom.xml @@ -5,14 +5,14 @@ 4.0.0 graphhopper-web jar - 4.0 + 5.0-SNAPSHOT GraphHopper Web Use the GraphHopper routing engine as a web-service com.graphhopper graphhopper-parent - 4.0 + 5.0-SNAPSHOT package From ce3a207efd3406cbac064a574a388ce26a392974 Mon Sep 17 00:00:00 2001 From: Andi Date: Wed, 29 Sep 2021 13:24:11 +0200 Subject: [PATCH 02/53] Move nodes and edges storage out of BaseGraph (#2399) --- .../com/graphhopper/storage/BaseGraph.java | 524 +++--------------- .../storage/BaseGraphNodesAndEdges.java | 399 +++++++++++++ .../com/graphhopper/storage/CHStorage.java | 4 +- .../com/graphhopper/storage/GHNodeAccess.java | 58 +- .../storage/GraphHopperStorage.java | 2 +- .../storage/RoutingCHEdgeIteratorImpl.java | 8 +- .../RoutingCHEdgeIteratorStateImpl.java | 8 +- .../graphhopper/storage/TurnCostStorage.java | 2 +- .../storage/AbstractGraphStorageTester.java | 3 +- .../storage/GraphHopperStorageTest.java | 19 +- 10 files changed, 524 insertions(+), 503 deletions(-) create mode 100644 core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 97273a63757..6eda12fd12c 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -26,7 +26,6 @@ import com.graphhopper.util.shapes.BBox; import java.util.Collections; -import java.util.Locale; import static com.graphhopper.util.Helper.nf; @@ -42,114 +41,47 @@ * loadExisting, (4) usage, (5) flush, (6) close */ class BaseGraph implements Graph { - // Currently distances are stored as 4 byte integers. using a conversion factor of 1000 the minimum distance - // that is not considered zero is 0.0005m (=0.5mm) and the maximum distance per edge is about 2.147.483m=2147km. - // See OSMReader.addEdge and #1871. - private static final double INT_DIST_FACTOR = 1000d; - static double MAX_DIST = Integer.MAX_VALUE / INT_DIST_FACTOR; - - final DataAccess edges; - final DataAccess nodes; - final BBox bounds; - final NodeAccess nodeAccess; private final static String STRING_IDX_NAME_KEY = "name"; + final BaseGraphNodesAndEdges store; + final NodeAccess nodeAccess; final StringIndex stringIndex; // can be null if turn costs are not supported final TurnCostStorage turnCostStorage; final BitUtil bitUtil; - private final int intsForFlags; // length | nodeA | nextNode | ... | nodeB - // as we use integer index in 'egdes' area => 'geometry' area is limited to 4GB (we use pos&neg values!) + // as we use integer index in 'edges' area => 'geometry' area is limited to 4GB (we use pos&neg values!) private final DataAccess wayGeometry; private final Directory dir; - /** - * interval [0,n) - */ - protected int edgeCount; - // node memory layout: - protected int N_EDGE_REF, N_LAT, N_LON, N_ELE, N_TC; - // edge memory layout: - int E_NODEA, E_NODEB, E_LINKA, E_LINKB, E_FLAGS, E_DIST, E_GEO, E_NAME; - /** - * Specifies how many entries (integers) are used per edge. - */ - int edgeEntryBytes; - /** - * Specifies how many entries (integers) are used per node - */ - int nodeEntryBytes; private boolean initialized = false; - /** - * interval [0,n) - */ - private int nodeCount; - private int edgeEntryIndex, nodeEntryIndex; private long maxGeoRef; - private boolean frozen = false; public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { this.dir = dir; - this.intsForFlags = intsForFlags; this.bitUtil = BitUtil.get(dir.getByteOrder()); this.wayGeometry = dir.find("geometry"); this.stringIndex = new StringIndex(dir); - this.nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType())); - this.edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType())); - this.bounds = BBox.createInverse(withElevation); - this.nodeAccess = new GHNodeAccess(this, withElevation); + this.store = new BaseGraphNodesAndEdges(dir, intsForFlags, withElevation, withTurnCosts, segmentSize); + this.nodeAccess = new GHNodeAccess(store); if (withTurnCosts) { turnCostStorage = new TurnCostStorage(this, dir.find("turn_costs")); + if (segmentSize >= 0) + turnCostStorage.setSegmentSize(segmentSize); } else { turnCostStorage = null; } if (segmentSize >= 0) { - setSegmentSize(segmentSize); + wayGeometry.setSegmentSize(segmentSize); + stringIndex.setSegmentSize(segmentSize); } } - private void setEdgeRef(int nodeId, int edgeId) { - nodes.setInt(toNodePointer(nodeId) + N_EDGE_REF, edgeId); - } - - int getEdgeRef(int nodeId) { - return nodes.getInt(toNodePointer(nodeId) + N_EDGE_REF); - } - - private int getNodeA(long edgePointer) { - return edges.getInt(edgePointer + E_NODEA); - } - - private int getNodeB(long edgePointer) { - return edges.getInt(edgePointer + E_NODEB); - } - - private int getLinkA(long edgePointer) { - return edges.getInt(edgePointer + E_LINKA); - } - - private int getLinkB(long edgePointer) { - return edges.getInt(edgePointer + E_LINKB); - } - - long toNodePointer(int node) { - if (node < 0 || node >= nodeCount) - throw new IllegalArgumentException("node: " + node + " out of bounds [0," + nodeCount + "["); - return (long) node * nodeEntryBytes; - } - - private long toEdgePointer(int edge) { - if (edge < 0 || edge >= edgeCount) - throw new IllegalArgumentException("edge: " + edge + " out of bounds [0," + edgeCount + "["); - return (long) edge * edgeEntryBytes; - } - private int getOtherNode(int nodeThis, long edgePointer) { - int nodeA = getNodeA(edgePointer); - return nodeThis == nodeA ? getNodeB(edgePointer) : nodeA; + int nodeA = store.getNodeA(edgePointer); + return nodeThis == nodeA ? store.getNodeB(edgePointer) : nodeA; } private boolean isAdjacentToNode(int node, long edgePointer) { - return getNodeA(edgePointer) == node || getNodeB(edgePointer) == node; + return store.getNodeA(edgePointer) == node || store.getNodeB(edgePointer) == node; } private static boolean isTestingEnabled() { @@ -169,90 +101,16 @@ void checkNotInitialized() { + "after calling create or loadExisting. Calling one of the methods twice is also not allowed."); } - void checkInitialized() { - if (!initialized) - throw new IllegalStateException("The graph has not yet been initialized."); - } - - private void loadNodesHeader() { - nodeEntryBytes = nodes.getHeader(0 * 4); - nodeCount = nodes.getHeader(1 * 4); - bounds.minLon = Helper.intToDegree(nodes.getHeader(2 * 4)); - bounds.maxLon = Helper.intToDegree(nodes.getHeader(3 * 4)); - bounds.minLat = Helper.intToDegree(nodes.getHeader(4 * 4)); - bounds.maxLat = Helper.intToDegree(nodes.getHeader(5 * 4)); - - if (bounds.hasElevation()) { - bounds.minEle = Helper.intToEle(nodes.getHeader(6 * 4)); - bounds.maxEle = Helper.intToEle(nodes.getHeader(7 * 4)); - } - - frozen = nodes.getHeader(8 * 4) == 1; - } - - private void setNodesHeader() { - nodes.setHeader(0 * 4, nodeEntryBytes); - nodes.setHeader(1 * 4, nodeCount); - nodes.setHeader(2 * 4, Helper.degreeToInt(bounds.minLon)); - nodes.setHeader(3 * 4, Helper.degreeToInt(bounds.maxLon)); - nodes.setHeader(4 * 4, Helper.degreeToInt(bounds.minLat)); - nodes.setHeader(5 * 4, Helper.degreeToInt(bounds.maxLat)); - if (bounds.hasElevation()) { - nodes.setHeader(6 * 4, Helper.eleToInt(bounds.minEle)); - nodes.setHeader(7 * 4, Helper.eleToInt(bounds.maxEle)); - } - - nodes.setHeader(8 * 4, isFrozen() ? 1 : 0); - } - - protected void loadEdgesHeader() { - edgeEntryBytes = edges.getHeader(0 * 4); - edgeCount = edges.getHeader(1 * 4); - } - - protected void setEdgesHeader() { - edges.setHeader(0, edgeEntryBytes); - edges.setHeader(1 * 4, edgeCount); - } - - protected int loadWayGeometryHeader() { + private void loadWayGeometryHeader() { maxGeoRef = bitUtil.combineIntsToLong(wayGeometry.getHeader(0), wayGeometry.getHeader(4)); - return 1; } - protected int setWayGeometryHeader() { + private void setWayGeometryHeader() { wayGeometry.setHeader(0, bitUtil.getIntLow(maxGeoRef)); wayGeometry.setHeader(4, bitUtil.getIntHigh(maxGeoRef)); - return 1; } - void initStorage() { - edgeEntryIndex = 0; - nodeEntryIndex = 0; - E_NODEA = nextEdgeEntryIndex(4); - E_NODEB = nextEdgeEntryIndex(4); - E_LINKA = nextEdgeEntryIndex(4); - E_LINKB = nextEdgeEntryIndex(4); - E_FLAGS = nextEdgeEntryIndex(intsForFlags * 4); - - E_DIST = nextEdgeEntryIndex(4); - E_GEO = nextEdgeEntryIndex(4); - E_NAME = nextEdgeEntryIndex(4); - - N_EDGE_REF = nextNodeEntryIndex(4); - N_LAT = nextNodeEntryIndex(4); - N_LON = nextNodeEntryIndex(4); - if (nodeAccess.is3D()) - N_ELE = nextNodeEntryIndex(4); - else - N_ELE = -1; - - if (supportsTurnCosts()) - N_TC = nextNodeEntryIndex(4); - else - N_TC = -1; - - initNodeAndEdgeEntrySize(); + private void setInitialized() { initialized = true; } @@ -260,64 +118,14 @@ boolean supportsTurnCosts() { return turnCostStorage != null; } - /** - * Initializes the node storage such that each node has no edge and no turn cost entry - */ - void initNodeRefs(long oldCapacity, long newCapacity) { - for (long pointer = oldCapacity + N_EDGE_REF; pointer < newCapacity; pointer += nodeEntryBytes) { - nodes.setInt(pointer, EdgeIterator.NO_EDGE); - } - if (supportsTurnCosts()) { - for (long pointer = oldCapacity + N_TC; pointer < newCapacity; pointer += nodeEntryBytes) { - nodes.setInt(pointer, TurnCostStorage.NO_TURN_ENTRY); - } - } - } - - protected final int nextEdgeEntryIndex(int sizeInBytes) { - int tmp = edgeEntryIndex; - edgeEntryIndex += sizeInBytes; - return tmp; - } - - protected final int nextNodeEntryIndex(int sizeInBytes) { - int tmp = nodeEntryIndex; - nodeEntryIndex += sizeInBytes; - return tmp; - } - - protected final void initNodeAndEdgeEntrySize() { - nodeEntryBytes = nodeEntryIndex; - edgeEntryBytes = edgeEntryIndex; - } - - /** - * Check if byte capacity of DataAcess nodes object is sufficient to include node index, else - * extend byte capacity - */ - final void ensureNodeIndex(int nodeIndex) { - checkInitialized(); - - if (nodeIndex < nodeCount) - return; - - long oldNodes = nodeCount; - nodeCount = nodeIndex + 1; - boolean capacityIncreased = nodes.ensureCapacity((long) nodeCount * nodeEntryBytes); - if (capacityIncreased) { - long newBytesCapacity = nodes.getCapacity(); - initNodeRefs(oldNodes * nodeEntryBytes, newBytesCapacity); - } - } - @Override public int getNodes() { - return nodeCount; + return store.getNodes(); } @Override public int getEdges() { - return edgeCount; + return store.getEdges(); } @Override @@ -327,34 +135,21 @@ public NodeAccess getNodeAccess() { @Override public BBox getBounds() { - return bounds; - } - - private void setSegmentSize(int bytes) { - checkNotInitialized(); - nodes.setSegmentSize(bytes); - edges.setSegmentSize(bytes); - wayGeometry.setSegmentSize(bytes); - stringIndex.setSegmentSize(bytes); - if (supportsTurnCosts()) { - turnCostStorage.setSegmentSize(bytes); - } + return store.getBounds(); } synchronized void freeze() { if (isFrozen()) throw new IllegalStateException("base graph already frozen"); - - frozen = true; + store.setFrozen(true); } synchronized boolean isFrozen() { - return frozen; + return store.getFrozen(); } void create(long initSize) { - nodes.create(initSize); - edges.create(initSize); + store.create(initSize); initSize = Math.min(initSize, 2000); wayGeometry.create(initSize); @@ -362,51 +157,15 @@ void create(long initSize) { if (supportsTurnCosts()) { turnCostStorage.create(initSize); } - initStorage(); + setInitialized(); // 0 stands for no separate geoRef maxGeoRef = 4; - - initNodeRefs(0, nodes.getCapacity()); } String toDetailsString() { - return "edges:" + nf(edgeCount) + "(" + edges.getCapacity() / Helper.MB + "MB), " - + "nodes:" + nf(getNodes()) + "(" + nodes.getCapacity() / Helper.MB + "MB), " + return store.toDetailsString() + ", " + "name:(" + stringIndex.getCapacity() / Helper.MB + "MB), " - + "geo:" + nf(maxGeoRef) + "(" + wayGeometry.getCapacity() / Helper.MB + "MB), " - + "bounds:" + bounds; - } - - public void debugPrint() { - final int printMax = 100; - System.out.println("nodes:"); - String formatNodes = "%12s | %12s | %12s | %12s \n"; - System.out.format(Locale.ROOT, formatNodes, "#", "N_EDGE_REF", "N_LAT", "N_LON"); - NodeAccess nodeAccess = getNodeAccess(); - for (int i = 0; i < Math.min(nodeCount, printMax); ++i) { - System.out.format(Locale.ROOT, formatNodes, i, getEdgeRef(i), nodeAccess.getLat(i), nodeAccess.getLon(i)); - } - if (nodeCount > printMax) { - System.out.format(Locale.ROOT, " ... %d more nodes\n", nodeCount - printMax); - } - System.out.println("edges:"); - String formatEdges = "%12s | %12s | %12s | %12s | %12s | %12s | %12s \n"; - System.out.format(Locale.ROOT, formatEdges, "#", "E_NODEA", "E_NODEB", "E_LINKA", "E_LINKB", "E_FLAGS", "E_DIST"); - IntsRef intsRef = new IntsRef(intsForFlags); - for (int i = 0; i < Math.min(edgeCount, printMax); ++i) { - long edgePointer = toEdgePointer(i); - readFlags(edgePointer, intsRef); - System.out.format(Locale.ROOT, formatEdges, i, - getNodeA(edgePointer), - getNodeB(edgePointer), - getLinkA(edgePointer), - getLinkB(edgePointer), - intsRef, - getDist(edgePointer)); - } - if (edgeCount > printMax) { - System.out.printf(Locale.ROOT, " ... %d more edges", edgeCount - printMax); - } + + "geo:" + nf(maxGeoRef) + "(" + wayGeometry.getCapacity() / Helper.MB + "MB)"; } /** @@ -431,10 +190,7 @@ public void flush() { if (!stringIndex.isClosed()) stringIndex.flush(); - setNodesHeader(); - setEdgesHeader(); - edges.flush(); - nodes.flush(); + store.flush(); if (supportsTurnCosts()) { turnCostStorage.flush(); } @@ -445,15 +201,14 @@ public void close() { wayGeometry.close(); if (!stringIndex.isClosed()) stringIndex.close(); - edges.close(); - nodes.close(); + store.close(); if (supportsTurnCosts()) { turnCostStorage.close(); } } long getCapacity() { - return edges.getCapacity() + nodes.getCapacity() + stringIndex.getCapacity() + return store.getCapacity() + stringIndex.getCapacity() + wayGeometry.getCapacity() + (supportsTurnCosts() ? turnCostStorage.getCapacity() : 0); } @@ -462,16 +217,13 @@ long getMaxGeoRef() { } void loadExisting(String dim) { - if (!nodes.loadExisting()) - throw new IllegalStateException("Cannot load nodes. corrupt file or directory? " + dir); + if (!store.loadExisting()) + throw new IllegalStateException("Cannot load edges or nodes. corrupt file or directory? " + dir); if (!dim.equalsIgnoreCase("" + nodeAccess.getDimension())) throw new IllegalStateException("Configured dimension (" + nodeAccess.getDimension() + ") is not equal " + "to dimension of loaded graph (" + dim + ")"); - if (!edges.loadExisting()) - throw new IllegalStateException("Cannot load edges. corrupt file or directory? " + dir); - if (!wayGeometry.loadExisting()) throw new IllegalStateException("Cannot load geometry. corrupt file or directory? " + dir); @@ -481,12 +233,7 @@ void loadExisting(String dim) { if (supportsTurnCosts() && !turnCostStorage.loadExisting()) throw new IllegalStateException("Cannot load turn cost storage. corrupt file or directory? " + dir); - // first define header indices of this storage - initStorage(); - - // now load some properties from stored data - loadNodesHeader(); - loadEdgesHeader(); + setInitialized(); loadWayGeometryHeader(); } @@ -496,8 +243,8 @@ void loadExisting(String dim) { * @return the updated iterator the properties where copied to. */ EdgeIteratorState copyProperties(EdgeIteratorState from, EdgeIteratorStateImpl to) { - long edgePointer = toEdgePointer(to.getEdge()); - writeFlags(edgePointer, from.getFlags()); + long edgePointer = store.toEdgePointer(to.getEdge()); + store.writeFlags(edgePointer, from.getFlags()); // copy the rest with higher level API to.setDistance(from.getDistance()). @@ -516,71 +263,13 @@ EdgeIteratorState copyProperties(EdgeIteratorState from, EdgeIteratorStateImpl t public EdgeIteratorState edge(int nodeA, int nodeB) { if (isFrozen()) throw new IllegalStateException("Cannot create edge if graph is already frozen"); - - ensureNodeIndex(Math.max(nodeA, nodeB)); - int edgeId = internalEdgeAdd(nextEdgeId(), nodeA, nodeB); + int edgeId = store.edge(nodeA, nodeB); EdgeIteratorStateImpl edge = new EdgeIteratorStateImpl(this); boolean valid = edge.init(edgeId, nodeB); assert valid; return edge; } - /** - * Writes a new edge to the array of edges and adds it to the linked list of edges at nodeA and nodeB - */ - final int internalEdgeAdd(int newEdgeId, int nodeA, int nodeB) { - writeEdge(newEdgeId, nodeA, nodeB); - long edgePointer = toEdgePointer(newEdgeId); - - int edge = getEdgeRef(nodeA); - if (edge > EdgeIterator.NO_EDGE) - edges.setInt(E_LINKA + edgePointer, edge); - setEdgeRef(nodeA, newEdgeId); - - if (nodeA != nodeB) { - edge = getEdgeRef(nodeB); - if (edge > EdgeIterator.NO_EDGE) - edges.setInt(E_LINKB + edgePointer, edge); - setEdgeRef(nodeB, newEdgeId); - } - return newEdgeId; - } - - /** - * Writes plain edge information to the edges index - */ - private long writeEdge(int edgeId, int nodeA, int nodeB) { - if (!EdgeIterator.Edge.isValid(edgeId)) - throw new IllegalStateException("Cannot write edge with illegal ID:" + edgeId + "; nodeA:" + nodeA + ", nodeB:" + nodeB); - - long edgePointer = toEdgePointer(edgeId); - edges.setInt(edgePointer + E_NODEA, nodeA); - edges.setInt(edgePointer + E_NODEB, nodeB); - edges.setInt(edgePointer + E_LINKA, EdgeIterator.NO_EDGE); - edges.setInt(edgePointer + E_LINKB, EdgeIterator.NO_EDGE); - return edgePointer; - } - - // for test only - void setEdgeCount(int cnt) { - edgeCount = cnt; - } - - /** - * Determine next free edgeId and ensure byte capacity to store edge - * - * @return next free edgeId - */ - protected int nextEdgeId() { - int nextEdge = edgeCount; - edgeCount++; - if (edgeCount < 0) - throw new IllegalStateException("too many edges. new edge id would be negative. " + toString()); - - edges.ensureCapacity(((long) edgeCount + 1) * edgeEntryBytes); - return nextEdge; - } - @Override public EdgeIteratorState getEdgeIteratorState(int edgeId, int adjNode) { EdgeIteratorStateImpl edge = new EdgeIteratorStateImpl(this); @@ -597,11 +286,6 @@ public EdgeIteratorState getEdgeIteratorStateForKey(int edgeKey) { return edge; } - final void checkAdjNodeBounds(int adjNode) { - if (adjNode < 0 && adjNode != Integer.MIN_VALUE || adjNode >= nodeCount) - throw new IllegalStateException("adjNode " + adjNode + " out of bounds [0," + nf(nodeCount) + ")"); - } - @Override public EdgeExplorer createEdgeExplorer(EdgeFilter filter) { return new EdgeIteratorImpl(this, filter); @@ -629,64 +313,23 @@ public Weighting wrapWeighting(Weighting weighting) { @Override public int getOtherNode(int edge, int node) { - long edgePointer = toEdgePointer(edge); + long edgePointer = store.toEdgePointer(edge); return getOtherNode(node, edgePointer); } @Override public boolean isAdjacentToNode(int edge, int node) { - long edgePointer = toEdgePointer(edge); + long edgePointer = store.toEdgePointer(edge); return isAdjacentToNode(node, edgePointer); } - private void readFlags(long edgePointer, IntsRef edgeFlags) { - int size = edgeFlags.ints.length; - for (int i = 0; i < size; i++) { - edgeFlags.ints[i] = edges.getInt(edgePointer + E_FLAGS + i * 4); - } - } - - private void writeFlags(long edgePointer, IntsRef edgeFlags) { - int size = edgeFlags.ints.length; - for (int i = 0; i < size; i++) { - edges.setInt(edgePointer + E_FLAGS + i * 4, edgeFlags.ints[i]); - } - } - - private void setDist(long edgePointer, double distance) { - edges.setInt(edgePointer + E_DIST, distToInt(distance)); - } - - /** - * Translates double distance to integer in order to save it in a DataAccess object - */ - private int distToInt(double distance) { - if (distance < 0) - throw new IllegalArgumentException("Distance cannot be negative: " + distance); - if (distance > MAX_DIST) { - distance = MAX_DIST; - } - int integ = (int) Math.round(distance * INT_DIST_FACTOR); - assert integ >= 0 : "distance out of range"; - return integ; - } - - /** - * returns distance (already translated from integer to double) - */ - private double getDist(long pointer) { - int val = edges.getInt(pointer + E_DIST); - // do never return infinity even if INT MAX, see #435 - return val / INT_DIST_FACTOR; - } - private void setWayGeometry_(PointList pillarNodes, long edgePointer, boolean reverse) { if (pillarNodes != null && !pillarNodes.isEmpty()) { if (pillarNodes.getDimension() != nodeAccess.getDimension()) throw new IllegalArgumentException("Cannot use pointlist which is " + pillarNodes.getDimension() + "D for graph which is " + nodeAccess.getDimension() + "D"); - long existingGeoRef = Helper.toUnsignedLong(edges.getInt(edgePointer + E_GEO)); + long existingGeoRef = Helper.toUnsignedLong(store.getGeoRef(edgePointer)); int len = pillarNodes.size(); int dim = nodeAccess.getDimension(); @@ -701,7 +344,7 @@ private void setWayGeometry_(PointList pillarNodes, long edgePointer, boolean re long nextGeoRef = nextGeoRef(len * dim); setWayGeometryAtGeoRef(pillarNodes, edgePointer, reverse, nextGeoRef); } else { - edges.setInt(edgePointer + E_GEO, 0); + store.setGeoRef(edgePointer, 0); } } @@ -713,7 +356,7 @@ private void setWayGeometryAtGeoRef(PointList pillarNodes, long edgePointer, boo ensureGeometry(geoRefPosition, totalLen); byte[] wayGeometryBytes = createWayGeometryBytes(pillarNodes, reverse); wayGeometry.setBytes(geoRefPosition, wayGeometryBytes, wayGeometryBytes.length); - edges.setInt(edgePointer + E_GEO, Helper.toSignedInt(geoRef)); + store.setGeoRef(edgePointer, Helper.toSignedInt(geoRef)); } private byte[] createWayGeometryBytes(PointList pillarNodes, boolean reverse) { @@ -750,7 +393,7 @@ private PointList fetchWayGeometry_(long edgePointer, boolean reverse, FetchMode pillarNodes.add(nodeAccess, adjNode); return pillarNodes; } - long geoRef = Helper.toUnsignedLong(edges.getInt(edgePointer + E_GEO)); + long geoRef = Helper.toUnsignedLong(store.getGeoRef(edgePointer)); int count = 0; byte[] bytes = null; if (geoRef > 0) { @@ -814,8 +457,7 @@ private void setName(long edgePointer, String name) { int stringIndexRef = (int) stringIndex.add(Collections.singletonMap(STRING_IDX_NAME_KEY, name)); if (stringIndexRef < 0) throw new IllegalStateException("Too many names are stored, currently limited to int pointer"); - - edges.setInt(edgePointer + E_NAME, stringIndexRef); + store.setNameRef(edgePointer, stringIndexRef); } private void ensureGeometry(long bytePos, int byteLength) { @@ -831,6 +473,10 @@ private long nextGeoRef(int arrayLength) { return tmp; } + public boolean isClosed() { + return store.isClosed(); + } + protected static class EdgeIteratorImpl extends EdgeIteratorStateImpl implements EdgeExplorer, EdgeIterator { final EdgeFilter filter; int nextEdgeId; @@ -845,7 +491,7 @@ public EdgeIteratorImpl(BaseGraph baseGraph, EdgeFilter filter) { @Override public EdgeIterator setBaseNode(int baseNode) { - nextEdgeId = edgeId = baseGraph.getEdgeRef(baseNode); + nextEdgeId = edgeId = store.getEdgeRef(store.toNodePointer(baseNode)); this.baseNode = baseNode; return this; } @@ -861,16 +507,16 @@ public final boolean next() { } void goToNext() { - edgePointer = baseGraph.toEdgePointer(nextEdgeId); + edgePointer = store.toEdgePointer(nextEdgeId); edgeId = nextEdgeId; - int nodeA = baseGraph.getNodeA(edgePointer); + int nodeA = store.getNodeA(edgePointer); boolean baseNodeIsNodeA = baseNode == nodeA; - adjNode = baseNodeIsNodeA ? baseGraph.getNodeB(edgePointer) : nodeA; + adjNode = baseNodeIsNodeA ? store.getNodeB(edgePointer) : nodeA; reverse = !baseNodeIsNodeA; freshFlags = false; // position to next edge - nextEdgeId = baseNodeIsNodeA ? baseGraph.getLinkA(edgePointer) : baseGraph.getLinkB(edgePointer); + nextEdgeId = baseNodeIsNodeA ? store.getLinkA(edgePointer) : store.getLinkB(edgePointer); assert nextEdgeId != edgeId : ("endless loop detected for base node: " + baseNode + ", adj node: " + adjNode + ", edge pointer: " + edgePointer + ", edge: " + edgeId); } @@ -893,17 +539,17 @@ public AllEdgeIterator(BaseGraph baseGraph) { @Override public int length() { - return baseGraph.edgeCount; + return store.getEdges(); } @Override public boolean next() { edgeId++; - if (edgeId >= baseGraph.edgeCount) + if (edgeId >= store.getEdges()) return false; - edgePointer = baseGraph.toEdgePointer(edgeId); - baseNode = baseGraph.getNodeA(edgePointer); - adjNode = baseGraph.getNodeB(edgePointer); + edgePointer = store.toEdgePointer(edgeId); + baseNode = store.getNodeA(edgePointer); + adjNode = store.getNodeB(edgePointer); freshFlags = false; reverse = false; return true; @@ -932,6 +578,7 @@ public final EdgeIteratorState detach(boolean reverseArg) { static class EdgeIteratorStateImpl implements EdgeIteratorState { final BaseGraph baseGraph; + final BaseGraphNodesAndEdges store; long edgePointer = -1; int baseNode; int adjNode; @@ -943,19 +590,20 @@ static class EdgeIteratorStateImpl implements EdgeIteratorState { public EdgeIteratorStateImpl(BaseGraph baseGraph) { this.baseGraph = baseGraph; - this.edgeFlags = new IntsRef(baseGraph.intsForFlags); + this.edgeFlags = new IntsRef(baseGraph.store.getIntsForFlags()); + store = baseGraph.store; } /** * @return false if the edge has not a node equal to expectedAdjNode */ final boolean init(int edgeId, int expectedAdjNode) { - if (edgeId < 0 || edgeId >= baseGraph.edgeCount) - throw new IllegalArgumentException("edge: " + edgeId + " out of bounds: [0," + baseGraph.edgeCount + "["); + if (edgeId < 0 || edgeId >= store.getEdges()) + throw new IllegalArgumentException("edge: " + edgeId + " out of bounds: [0," + store.getEdges() + "["); this.edgeId = edgeId; - edgePointer = baseGraph.toEdgePointer(edgeId); - baseNode = baseGraph.getNodeA(edgePointer); - adjNode = baseGraph.getNodeB(edgePointer); + edgePointer = store.toEdgePointer(edgeId); + baseNode = store.getNodeA(edgePointer); + adjNode = store.getNodeB(edgePointer); freshFlags = false; if (expectedAdjNode == adjNode || expectedAdjNode == Integer.MIN_VALUE) { @@ -978,9 +626,9 @@ final void init(int edgeKey) { if (edgeKey < 0) throw new IllegalArgumentException("edge keys must not be negative, given: " + edgeKey); this.edgeId = GHUtility.getEdgeFromEdgeKey(edgeKey); - edgePointer = baseGraph.toEdgePointer(edgeId); - baseNode = baseGraph.getNodeA(edgePointer); - adjNode = baseGraph.getNodeB(edgePointer); + edgePointer = store.toEdgePointer(edgeId); + baseNode = store.getNodeA(edgePointer); + adjNode = store.getNodeB(edgePointer); freshFlags = false; if (edgeKey % 2 == 0 || baseNode == adjNode) { @@ -1005,19 +653,19 @@ public final int getAdjNode() { @Override public double getDistance() { - return baseGraph.getDist(edgePointer); + return store.getDist(edgePointer); } @Override public EdgeIteratorState setDistance(double dist) { - baseGraph.setDist(edgePointer, dist); + store.setDist(edgePointer, dist); return this; } @Override public IntsRef getFlags() { if (!freshFlags) { - baseGraph.readFlags(edgePointer, edgeFlags); + store.readFlags(edgePointer, edgeFlags); freshFlags = true; } return edgeFlags; @@ -1025,8 +673,8 @@ public IntsRef getFlags() { @Override public final EdgeIteratorState setFlags(IntsRef edgeFlags) { - assert edgeId < baseGraph.edgeCount : "must be edge but was shortcut: " + edgeId + " >= " + baseGraph.edgeCount + ". Use setFlagsAndWeight"; - baseGraph.writeFlags(edgePointer, edgeFlags); + assert edgeId < store.getEdges() : "must be edge but was shortcut: " + edgeId + " >= " + store.getEdges() + ". Use setFlagsAndWeight"; + store.writeFlags(edgePointer, edgeFlags); for (int i = 0; i < edgeFlags.ints.length; i++) { this.edgeFlags.ints[i] = edgeFlags.ints[i]; } @@ -1042,7 +690,7 @@ public boolean get(BooleanEncodedValue property) { @Override public EdgeIteratorState set(BooleanEncodedValue property, boolean value) { property.setBool(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1054,7 +702,7 @@ public boolean getReverse(BooleanEncodedValue property) { @Override public EdgeIteratorState setReverse(BooleanEncodedValue property, boolean value) { property.setBool(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1064,7 +712,7 @@ public EdgeIteratorState set(BooleanEncodedValue property, boolean fwd, boolean throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setBool(reverse, getFlags(), fwd); property.setBool(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1076,7 +724,7 @@ public int get(IntEncodedValue property) { @Override public EdgeIteratorState set(IntEncodedValue property, int value) { property.setInt(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1088,7 +736,7 @@ public int getReverse(IntEncodedValue property) { @Override public EdgeIteratorState setReverse(IntEncodedValue property, int value) { property.setInt(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1098,7 +746,7 @@ public EdgeIteratorState set(IntEncodedValue property, int fwd, int bwd) { throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setInt(reverse, getFlags(), fwd); property.setInt(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1110,7 +758,7 @@ public double get(DecimalEncodedValue property) { @Override public EdgeIteratorState set(DecimalEncodedValue property, double value) { property.setDecimal(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1122,7 +770,7 @@ public double getReverse(DecimalEncodedValue property) { @Override public EdgeIteratorState setReverse(DecimalEncodedValue property, double value) { property.setDecimal(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1132,7 +780,7 @@ public EdgeIteratorState set(DecimalEncodedValue property, double fwd, double bw throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setDecimal(reverse, getFlags(), fwd); property.setDecimal(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1144,7 +792,7 @@ public > T get(EnumEncodedValue property) { @Override public > EdgeIteratorState set(EnumEncodedValue property, T value) { property.setEnum(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1156,7 +804,7 @@ public > T getReverse(EnumEncodedValue property) { @Override public > EdgeIteratorState setReverse(EnumEncodedValue property, T value) { property.setEnum(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1166,7 +814,7 @@ public > EdgeIteratorState set(EnumEncodedValue property, T throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setEnum(reverse, getFlags(), fwd); property.setEnum(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1178,7 +826,7 @@ public String get(StringEncodedValue property) { @Override public EdgeIteratorState set(StringEncodedValue property, String value) { property.setString(reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1190,7 +838,7 @@ public String getReverse(StringEncodedValue property) { @Override public EdgeIteratorState setReverse(StringEncodedValue property, String value) { property.setString(!reverse, getFlags(), value); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1200,7 +848,7 @@ public EdgeIteratorState set(StringEncodedValue property, String fwd, String bwd throw new IllegalArgumentException("EncodedValue " + property.getName() + " supports only one direction"); property.setString(reverse, getFlags(), fwd); property.setString(!reverse, getFlags(), bwd); - baseGraph.writeFlags(edgePointer, getFlags()); + store.writeFlags(edgePointer, getFlags()); return this; } @@ -1242,7 +890,7 @@ public int getOrigEdgeLast() { @Override public String getName() { - int stringIndexRef = baseGraph.edges.getInt(edgePointer + baseGraph.E_NAME); + int stringIndexRef = store.getNameRef(edgePointer); String name = baseGraph.stringIndex.get(stringIndexRef, STRING_IDX_NAME_KEY); // preserve backward compatibility (returns null if not explicitly set) return name == null ? "" : name; diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java new file mode 100644 index 00000000000..f2e647c5921 --- /dev/null +++ b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java @@ -0,0 +1,399 @@ +/* + * Licensed to GraphHopper GmbH under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + * + * GraphHopper GmbH licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphhopper.storage; + +import com.graphhopper.util.EdgeIterator; +import com.graphhopper.util.Helper; +import com.graphhopper.util.shapes.BBox; + +import java.util.Locale; + +import static com.graphhopper.util.EdgeIterator.NO_EDGE; +import static com.graphhopper.util.Helper.nf; + +/** + * Underlying storage for nodes and edges of {@link BaseGraph}. Nodes and edges are stored using two {@link DataAccess} + * instances. Nodes and edges are simply stored sequentially, see the memory layout in the constructor. + */ +class BaseGraphNodesAndEdges { + // Currently distances are stored as 4 byte integers. using a conversion factor of 1000 the minimum distance + // that is not considered zero is 0.0005m (=0.5mm) and the maximum distance per edge is about 2.147.483m=2147km. + // See OSMReader.addEdge and #1871. + private static final double INT_DIST_FACTOR = 1000d; + static double MAX_DIST = Integer.MAX_VALUE / INT_DIST_FACTOR; + + // nodes + private final DataAccess nodes; + private final int N_EDGE_REF, N_LAT, N_LON, N_ELE, N_TC; + private int nodeEntryBytes; + private int nodeCount; + + // edges + private final DataAccess edges; + private final int E_NODEA, E_NODEB, E_LINKA, E_LINKB, E_FLAGS, E_DIST, E_GEO, E_NAME; + private final int intsForFlags; + private int edgeEntryBytes; + private int edgeCount; + + private final boolean withTurnCosts; + private final boolean withElevation; + + // we do not write the bounding box directly to storage, but rather to this bbox object. we only write to storage + // when flushing. why? just because we did it like this in the past, and otherwise we run into rounding errors, + // because of: #2393 + public final BBox bounds; + private boolean frozen; + + public BaseGraphNodesAndEdges(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { + nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType())); + edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType())); + if (segmentSize >= 0) { + nodes.setSegmentSize(segmentSize); + edges.setSegmentSize(segmentSize); + } + this.intsForFlags = intsForFlags; + this.withTurnCosts = withTurnCosts; + this.withElevation = withElevation; + bounds = BBox.createInverse(withElevation); + + // memory layout for nodes + N_EDGE_REF = 0; + N_LAT = 4; + N_LON = 8; + N_ELE = N_LON + (withElevation ? 4 : 0); + N_TC = N_ELE + (withTurnCosts ? 4 : 0); + nodeEntryBytes = N_TC + 4; + + // memory layout for edges + E_NODEA = 0; + E_NODEB = 4; + E_LINKA = 8; + E_LINKB = 12; + E_FLAGS = 16; + E_DIST = E_FLAGS + intsForFlags * 4; + E_GEO = E_DIST + 4; + E_NAME = E_GEO + 4; + edgeEntryBytes = E_NAME + 4; + } + + public void create(long initSize) { + nodes.create(initSize); + edges.create(initSize); + } + + public boolean loadExisting() { + if (!nodes.loadExisting() || !edges.loadExisting()) + return false; + + // now load some properties from stored data + nodeEntryBytes = nodes.getHeader(0 * 4); + nodeCount = nodes.getHeader(1 * 4); + bounds.minLon = Helper.intToDegree(nodes.getHeader(2 * 4)); + bounds.maxLon = Helper.intToDegree(nodes.getHeader(3 * 4)); + bounds.minLat = Helper.intToDegree(nodes.getHeader(4 * 4)); + bounds.maxLat = Helper.intToDegree(nodes.getHeader(5 * 4)); + if (withElevation) { + bounds.minEle = Helper.intToEle(nodes.getHeader(6 * 4)); + bounds.maxEle = Helper.intToEle(nodes.getHeader(7 * 4)); + } + frozen = nodes.getHeader(8 * 4) == 1; + + edgeEntryBytes = edges.getHeader(0 * 4); + edgeCount = edges.getHeader(1 * 4); + return true; + } + + public void flush() { + nodes.setHeader(0 * 4, nodeEntryBytes); + nodes.setHeader(1 * 4, nodeCount); + nodes.setHeader(2 * 4, Helper.degreeToInt(bounds.minLon)); + nodes.setHeader(3 * 4, Helper.degreeToInt(bounds.maxLon)); + nodes.setHeader(4 * 4, Helper.degreeToInt(bounds.minLat)); + nodes.setHeader(5 * 4, Helper.degreeToInt(bounds.maxLat)); + if (withElevation) { + nodes.setHeader(6 * 4, Helper.eleToInt(bounds.minEle)); + nodes.setHeader(7 * 4, Helper.eleToInt(bounds.maxEle)); + } + nodes.setHeader(8 * 4, frozen ? 1 : 0); + + edges.setHeader(0, edgeEntryBytes); + edges.setHeader(1 * 4, edgeCount); + + edges.flush(); + nodes.flush(); + } + + public void close() { + edges.close(); + nodes.close(); + } + + public int getNodes() { + return nodeCount; + } + + public int getEdges() { + return edgeCount; + } + + public int getIntsForFlags() { + return intsForFlags; + } + + public boolean withElevation() { + return withElevation; + } + + public boolean withTurnCosts() { + return withTurnCosts; + } + + public BBox getBounds() { + return bounds; + } + + public long getCapacity() { + return nodes.getCapacity() + edges.getCapacity(); + } + + public boolean isClosed() { + assert nodes.isClosed() == edges.isClosed(); + return nodes.isClosed(); + } + + public int edge(int nodeA, int nodeB) { + if (edgeCount == Integer.MAX_VALUE) + throw new IllegalStateException("Maximum edge count exceeded: " + edgeCount); + ensureNodeCapacity(Math.max(nodeA, nodeB)); + final int edge = edgeCount; + final long edgePointer = (long) edgeCount * edgeEntryBytes; + edgeCount++; + edges.ensureCapacity((long) edgeCount * edgeEntryBytes); + + setNodeA(edgePointer, nodeA); + setNodeB(edgePointer, nodeB); + // we keep a linked list of edges at each node. here we prepend the new edge at the already existing linked + // list of edges. + long nodePointerA = toNodePointer(nodeA); + int edgeRefA = getEdgeRef(nodePointerA); + setLinkA(edgePointer, EdgeIterator.Edge.isValid(edgeRefA) ? edgeRefA : NO_EDGE); + setEdgeRef(nodePointerA, edge); + + if (nodeA != nodeB) { + long nodePointerB = toNodePointer(nodeB); + int edgeRefB = getEdgeRef(nodePointerB); + setLinkB(edgePointer, EdgeIterator.Edge.isValid(edgeRefB) ? edgeRefB : NO_EDGE); + setEdgeRef(nodePointerB, edge); + } + return edge; + } + + public void ensureNodeCapacity(int node) { + if (node < nodeCount) + return; + + int oldNodes = nodeCount; + nodeCount = node + 1; + nodes.ensureCapacity((long) nodeCount * nodeEntryBytes); + for (int n = oldNodes; n < nodeCount; ++n) { + setEdgeRef(toNodePointer(n), NO_EDGE); + if (withTurnCosts) + setTurnCostRef(toNodePointer(n), TurnCostStorage.NO_TURN_ENTRY); + } + } + + public long toNodePointer(int node) { + if (node < 0 || node >= nodeCount) + throw new IllegalArgumentException("node: " + node + " out of bounds [0," + nodeCount + "["); + return (long) node * nodeEntryBytes; + } + + public long toEdgePointer(int edge) { + if (edge < 0 || edge >= edgeCount) + throw new IllegalArgumentException("edge: " + edge + " out of bounds [0," + edgeCount + "["); + return (long) edge * edgeEntryBytes; + } + + public void readFlags(long edgePointer, IntsRef edgeFlags) { + int size = edgeFlags.ints.length; + for (int i = 0; i < size; ++i) + edgeFlags.ints[i] = edges.getInt(edgePointer + E_FLAGS + i * 4); + } + + public void writeFlags(long edgePointer, IntsRef edgeFlags) { + int size = edgeFlags.ints.length; + for (int i = 0; i < size; ++i) + edges.setInt(edgePointer + E_FLAGS + i * 4, edgeFlags.ints[i]); + } + + public void setNodeA(long edgePointer, int nodeA) { + edges.setInt(edgePointer + E_NODEA, nodeA); + } + + public void setNodeB(long edgePointer, int nodeB) { + edges.setInt(edgePointer + E_NODEB, nodeB); + } + + public void setLinkA(long edgePointer, int linkA) { + edges.setInt(edgePointer + E_LINKA, linkA); + } + + public void setLinkB(long edgePointer, int linkB) { + edges.setInt(edgePointer + E_LINKB, linkB); + } + + public void setDist(long edgePointer, double distance) { + edges.setInt(edgePointer + E_DIST, distToInt(distance)); + } + + public void setGeoRef(long edgePointer, int geoRef) { + edges.setInt(edgePointer + E_GEO, geoRef); + } + + public void setNameRef(long edgePointer, int nameRef) { + edges.setInt(edgePointer + E_NAME, nameRef); + } + + public int getNodeA(long edgePointer) { + return edges.getInt(edgePointer + E_NODEA); + } + + public int getNodeB(long edgePointer) { + return edges.getInt(edgePointer + E_NODEB); + } + + public int getLinkA(long edgePointer) { + return edges.getInt(edgePointer + E_LINKA); + } + + public int getLinkB(long edgePointer) { + return edges.getInt(edgePointer + E_LINKB); + } + + public double getDist(long pointer) { + int val = edges.getInt(pointer + E_DIST); + // do never return infinity even if INT MAX, see #435 + return val / INT_DIST_FACTOR; + } + + public int getGeoRef(long edgePointer) { + return edges.getInt(edgePointer + E_GEO); + } + + public int getNameRef(long edgePointer) { + return edges.getInt(edgePointer + E_NAME); + } + + public void setEdgeRef(long nodePointer, int edgeRef) { + nodes.setInt(nodePointer + N_EDGE_REF, edgeRef); + } + + public void setLat(long nodePointer, double lat) { + nodes.setInt(nodePointer + N_LAT, Helper.degreeToInt(lat)); + } + + public void setLon(long nodePointer, double lon) { + nodes.setInt(nodePointer + N_LON, Helper.degreeToInt(lon)); + } + + public void setEle(long elePointer, double ele) { + nodes.setInt(elePointer + N_ELE, Helper.eleToInt(ele)); + } + + public void setTurnCostRef(long nodePointer, int tcRef) { + nodes.setInt(nodePointer + N_TC, tcRef); + } + + public int getEdgeRef(long nodePointer) { + return nodes.getInt(nodePointer + N_EDGE_REF); + } + + public double getLat(long nodePointer) { + return Helper.intToDegree(nodes.getInt(nodePointer + N_LAT)); + } + + public double getLon(long nodePointer) { + return Helper.intToDegree(nodes.getInt(nodePointer + N_LON)); + } + + public double getEle(long nodePointer) { + return Helper.intToEle(nodes.getInt(nodePointer + N_ELE)); + } + + public int getTurnCostRef(long nodePointer) { + return nodes.getInt(nodePointer + N_TC); + } + + public void setFrozen(boolean frozen) { + this.frozen = frozen; + } + + public boolean getFrozen() { + return frozen; + } + + public void debugPrint() { + final int printMax = 100; + System.out.println("nodes:"); + String formatNodes = "%12s | %12s | %12s | %12s \n"; + System.out.format(Locale.ROOT, formatNodes, "#", "N_EDGE_REF", "N_LAT", "N_LON"); + for (int i = 0; i < Math.min(nodeCount, printMax); ++i) { + long nodePointer = toNodePointer(i); + System.out.format(Locale.ROOT, formatNodes, i, getEdgeRef(nodePointer), getLat(nodePointer), getLon(nodePointer)); + } + if (nodeCount > printMax) { + System.out.format(Locale.ROOT, " ... %d more nodes\n", nodeCount - printMax); + } + System.out.println("edges:"); + String formatEdges = "%12s | %12s | %12s | %12s | %12s | %12s | %12s \n"; + System.out.format(Locale.ROOT, formatEdges, "#", "E_NODEA", "E_NODEB", "E_LINKA", "E_LINKB", "E_FLAGS", "E_DIST"); + IntsRef intsRef = new IntsRef(intsForFlags); + for (int i = 0; i < Math.min(edgeCount, printMax); ++i) { + long edgePointer = toEdgePointer(i); + readFlags(edgePointer, intsRef); + System.out.format(Locale.ROOT, formatEdges, i, + getNodeA(edgePointer), + getNodeB(edgePointer), + getLinkA(edgePointer), + getLinkB(edgePointer), + intsRef, + getDist(edgePointer)); + } + if (edgeCount > printMax) { + System.out.printf(Locale.ROOT, " ... %d more edges", edgeCount - printMax); + } + } + + private int distToInt(double distance) { + if (distance < 0) + throw new IllegalArgumentException("Distance cannot be negative: " + distance); + if (distance > MAX_DIST) { + distance = MAX_DIST; + } + int intDist = (int) Math.round(distance * INT_DIST_FACTOR); + assert intDist >= 0 : "distance out of range"; + return intDist; + } + + public String toDetailsString() { + return "edges: " + nf(edgeCount) + "(" + edges.getCapacity() / Helper.MB + "MB), " + + "nodes: " + nf(nodeCount) + "(" + nodes.getCapacity() / Helper.MB + "MB), " + + "bounds: " + bounds; + } +} diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java index 74324f90f67..b59ad6849f1 100644 --- a/core/src/main/java/com/graphhopper/storage/CHStorage.java +++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java @@ -31,7 +31,7 @@ * DataAccess-based storage for CH shortcuts. Stores shortcuts and CH levels sequentially using two DataAccess objects * and gives read/write access to the different shortcut and node fields. *

- * This can be seen as an extension to a base graph: We assign a CH level to each nodes and add additional edges to + * This can be seen as an extension to a base graph: We assign a CH level to each node and add additional edges to * the graph ('shortcuts'). The shortcuts need to be ordered in a certain way, but this is not enforced here. * * @see CHStorageBuilder to build a valid storage that can be used for routing @@ -58,7 +58,7 @@ public class CHStorage { private int nodeCount = -1; private boolean edgeBased; - // some shortcuts exceed the maximum storable weight and we count them here + // some shortcuts exceed the maximum storable weight, and we count them here private int numShortcutsExceedingWeight; // use this to report shortcuts with too small weights diff --git a/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java b/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java index 6ad8cdaa3e9..9d0d396e4a0 100644 --- a/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java +++ b/core/src/main/java/com/graphhopper/storage/GHNodeAccess.java @@ -17,8 +17,6 @@ */ package com.graphhopper.storage; -import com.graphhopper.util.Helper; - /** * A helper class for GraphHopperStorage for its node access. *

@@ -26,59 +24,55 @@ * @author Peter Karich */ class GHNodeAccess implements NodeAccess { - private final BaseGraph baseGraph; - private final boolean elevation; + private final BaseGraphNodesAndEdges store; - public GHNodeAccess(BaseGraph baseGraph, boolean withElevation) { - this.baseGraph = baseGraph; - this.elevation = withElevation; + public GHNodeAccess(BaseGraphNodesAndEdges store) { + this.store = store; } @Override public void ensureNode(int nodeId) { - baseGraph.ensureNodeIndex(nodeId); + store.ensureNodeCapacity(nodeId); } @Override public final void setNode(int nodeId, double lat, double lon, double ele) { - baseGraph.ensureNodeIndex(nodeId); - long tmp = baseGraph.toNodePointer(nodeId); - baseGraph.nodes.setInt(tmp + baseGraph.N_LAT, Helper.degreeToInt(lat)); - baseGraph.nodes.setInt(tmp + baseGraph.N_LON, Helper.degreeToInt(lon)); + store.ensureNodeCapacity(nodeId); + store.setLat(store.toNodePointer(nodeId), lat); + store.setLon(store.toNodePointer(nodeId), lon); - if (is3D()) { + if (store.withElevation()) { // meter precision is sufficient for now - baseGraph.nodes.setInt(tmp + baseGraph.N_ELE, Helper.eleToInt(ele)); - baseGraph.bounds.update(lat, lon, ele); - + store.setEle(store.toNodePointer(nodeId), ele); + store.bounds.update(lat, lon, ele); } else { - baseGraph.bounds.update(lat, lon); + store.bounds.update(lat, lon); } } @Override public final double getLat(int nodeId) { - return Helper.intToDegree(baseGraph.nodes.getInt(baseGraph.toNodePointer(nodeId) + baseGraph.N_LAT)); + return store.getLat(store.toNodePointer(nodeId)); } @Override public final double getLon(int nodeId) { - return Helper.intToDegree(baseGraph.nodes.getInt(baseGraph.toNodePointer(nodeId) + baseGraph.N_LON)); + return store.getLon(store.toNodePointer(nodeId)); } @Override public final double getEle(int nodeId) { - if (!elevation) - throw new IllegalStateException("Cannot access elevation - 3D is not enabled"); - - return Helper.intToEle(baseGraph.nodes.getInt(baseGraph.toNodePointer(nodeId) + baseGraph.N_ELE)); + if (!store.withElevation()) + throw new IllegalStateException("elevation is disabled"); + return store.getEle(store.toNodePointer(nodeId)); } + @Override public final void setTurnCostIndex(int index, int turnCostIndex) { - if (baseGraph.supportsTurnCosts()) { - baseGraph.ensureNodeIndex(index); - long tmp = baseGraph.toNodePointer(index); - baseGraph.nodes.setInt(tmp + baseGraph.N_TC, turnCostIndex); + if (store.withTurnCosts()) { + // todo: remove ensure? + store.ensureNodeCapacity(index); + store.setTurnCostRef(store.toNodePointer(index), turnCostIndex); } else { throw new AssertionError("This graph does not support turn costs"); } @@ -86,21 +80,19 @@ public final void setTurnCostIndex(int index, int turnCostIndex) { @Override public final int getTurnCostIndex(int index) { - if (baseGraph.supportsTurnCosts()) - return baseGraph.nodes.getInt(baseGraph.toNodePointer(index) + baseGraph.N_TC); + if (store.withTurnCosts()) + return store.getTurnCostRef(store.toNodePointer(index)); else throw new AssertionError("This graph does not support turn costs"); } @Override public final boolean is3D() { - return elevation; + return store.withElevation(); } @Override public int getDimension() { - if (elevation) - return 3; - return 2; + return store.withElevation() ? 3 : 2; } } diff --git a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java index ba38078c6c8..bd03a44d47e 100644 --- a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java +++ b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java @@ -314,7 +314,7 @@ public void close() { } public boolean isClosed() { - return baseGraph.nodes.isClosed(); + return baseGraph.isClosed(); } public long getCapacity() { diff --git a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java index 0180f96a0b8..f0245508d97 100644 --- a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java +++ b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorImpl.java @@ -54,7 +54,7 @@ public RoutingCHEdgeIterator setBaseNode(int baseNode) { assert baseGraph.isFrozen(); baseIterator.setBaseNode(baseNode); int lastShortcut = store.getLastShortcut(store.toNodePointer(baseNode)); - nextEdgeId = edgeId = lastShortcut < 0 ? baseIterator.edgeId : baseGraph.edgeCount + lastShortcut; + nextEdgeId = edgeId = lastShortcut < 0 ? baseIterator.edgeId : baseGraph.getEdges() + lastShortcut; return this; } @@ -63,13 +63,13 @@ public boolean next() { // we first traverse shortcuts (in decreasing order) and when we are done we use the base iterator to traverse // the base edges as well. shortcuts are filtered using shortcutFilter, but base edges are only filtered by // access/finite weight. - while (nextEdgeId >= baseGraph.edgeCount) { - shortcutPointer = store.toShortcutPointer(nextEdgeId - baseGraph.edgeCount); + while (nextEdgeId >= baseGraph.getEdges()) { + shortcutPointer = store.toShortcutPointer(nextEdgeId - baseGraph.getEdges()); baseNode = store.getNodeA(shortcutPointer); adjNode = store.getNodeB(shortcutPointer); edgeId = nextEdgeId; nextEdgeId--; - if (nextEdgeId < baseGraph.edgeCount || store.getNodeA(store.toShortcutPointer(nextEdgeId - baseGraph.edgeCount)) != baseNode) + if (nextEdgeId < baseGraph.getEdges() || store.getNodeA(store.toShortcutPointer(nextEdgeId - baseGraph.getEdges())) != baseNode) nextEdgeId = baseIterator.edgeId; // todo: note that it would be more efficient (but cost more memory) to separate in/out edges, // especially for edge-based where we do not use bidirectional shortcuts diff --git a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java index b5a45ac7e1c..54626fc0d24 100644 --- a/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java +++ b/core/src/main/java/com/graphhopper/storage/RoutingCHEdgeIteratorStateImpl.java @@ -45,11 +45,11 @@ public RoutingCHEdgeIteratorStateImpl(CHStorage store, BaseGraph baseGraph, Base } boolean init(int edge, int expectedAdjNode) { - if (edge < 0 || edge >= baseGraph.edgeCount + store.getShortcuts()) - throw new IllegalArgumentException("edge must be in bounds: [0," + (baseGraph.edgeCount + store.getShortcuts()) + "["); + if (edge < 0 || edge >= baseGraph.getEdges() + store.getShortcuts()) + throw new IllegalArgumentException("edge must be in bounds: [0," + (baseGraph.getEdges() + store.getShortcuts()) + "["); edgeId = edge; if (isShortcut()) { - shortcutPointer = store.toShortcutPointer(edge - baseGraph.edgeCount); + shortcutPointer = store.toShortcutPointer(edge - baseGraph.getEdges()); baseNode = store.getNodeA(shortcutPointer); adjNode = store.getNodeB(shortcutPointer); @@ -103,7 +103,7 @@ public int getAdjNode() { @Override public boolean isShortcut() { - return edgeId >= baseGraph.edgeCount; + return edgeId >= baseGraph.getEdges(); } @Override diff --git a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java index a362a02f120..8347d116663 100644 --- a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java +++ b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java @@ -243,7 +243,7 @@ public interface TurnRelationIterator { private class Itr implements TurnRelationIterator { private int viaNode = -1; private int turnCostIndex = -1; - private IntsRef intsRef = TurnCost.createFlags(); + private final IntsRef intsRef = TurnCost.createFlags(); private long turnCostPtr() { return (long) turnCostIndex * BYTES_PER_ENTRY; diff --git a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java index ab039dedd96..257fd25ad98 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java @@ -26,7 +26,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.Closeable; import java.io.File; import java.util.ArrayList; import java.util.List; @@ -117,7 +116,7 @@ public void tearDown() { public void testSetTooBigDistance_435() { graph = createGHStorage(); - double maxDist = BaseGraph.MAX_DIST; + double maxDist = BaseGraphNodesAndEdges.MAX_DIST; EdgeIteratorState edge1 = GHUtility.setSpeed(60, true, true, carEncoder, graph.edge(0, 1).setDistance(maxDist)); assertEquals(maxDist, edge1.getDistance(), 1); diff --git a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java index 13956f7b8d1..d2dab7fbd48 100644 --- a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java +++ b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java @@ -57,14 +57,7 @@ protected GraphHopperStorage newGHStorage(Directory dir, boolean enabled3D, int @Test public void testNoCreateCalled() { - try (GraphHopperStorage gs = GraphBuilder.start(encodingManager).build()) { - ((BaseGraph) gs.getBaseGraph()).ensureNodeIndex(123); - fail("IllegalStateException should be raised"); - } catch (IllegalStateException err) { - // ok - } catch (Exception ex) { - fail("IllegalStateException should be raised, but was " + ex.toString()); - } + assertThrows(Throwable.class, () -> GraphBuilder.start(encodingManager).build().edge(0, 1)); } @Test @@ -160,16 +153,6 @@ protected void checkGraph(Graph g) { assertEquals(GHUtility.asSet(0), GHUtility.getNeighbors(explorer.setBaseNode(2))); } - @Test - public void testBigDataEdge() { - Directory dir = new RAMDirectory(); - GraphHopperStorage graph = new GraphHopperStorage(dir, encodingManager, false); - graph.create(defaultSize); - ((BaseGraph) graph.getBaseGraph()).setEdgeCount(Integer.MAX_VALUE / 2); - assertTrue(graph.getAllEdges().next()); - graph.close(); - } - @Test public void testDoThrowExceptionIfDimDoesNotMatch() { graph = newGHStorage(new RAMDirectory(defaultGraphLoc, true), false); From e98be3b800e932654e2e02f55adec6058fd7c610 Mon Sep 17 00:00:00 2001 From: easbar Date: Wed, 29 Sep 2021 13:32:55 +0200 Subject: [PATCH 03/53] Mention turn restrictions in profiles.md --- docs/core/profiles.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/core/profiles.md b/docs/core/profiles.md index d5410409493..780f844dcb1 100644 --- a/docs/core/profiles.md +++ b/docs/core/profiles.md @@ -54,6 +54,9 @@ weightings: - curvature (prefers routes with lots of curves for enjoyable motorcycle rides) - custom (enables custom profiles, see the next section) +Another important profile setting is `turn_costs: true/false`. Use this to enable turn restrictions for each profile. +You can learn more about this setting [here](./turn-restrictions.md) + The profile name is used to select the profile when executing routing queries. To do this use the `profile` request parameter, for example `/route?point=49.5,11.1&profile=car` or `/route?point=49.5,11.1&profile=some_other_profile`. From 023378185018eb842a1947edc3afc884b2c3f228 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 30 Sep 2021 19:39:30 +0200 Subject: [PATCH 04/53] country_rules: fix documentation in example config (#2414) --- config-example.yml | 6 +++--- core/src/main/java/com/graphhopper/GraphHopper.java | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/config-example.yml b/config-example.yml index 0432d559945..a933e485550 100644 --- a/config-example.yml +++ b/config-example.yml @@ -164,9 +164,9 @@ graphhopper: # custom_areas.directory: path/to/custom_areas ##### Country Rules ##### - # GraphHopper applies country-specific routing rules (enabled by default) during import. - # Use this flag to disable these rules (you need to repeat the import for changes to take effect) - # country_rules.enable: false + # GraphHopper applies country-specific routing rules during import (not enabled by default). + # You need to redo the import for changes to take effect. + # country_rules.enabled: true # Dropwizard server configuration server: diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index 9e368495078..ff5e0ab4f1d 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -460,10 +460,7 @@ public GraphHopper init(GraphHopperConfig ghConfig) { graphHopperFolder = pruneFileEnd(osmFile) + "-gh"; } - if (ghConfig.has("country_rules.enabled")) { - boolean countryRulesEnabled = ghConfig.getBool("country_rules.enabled", false); - countryRuleFactory = countryRulesEnabled ? new CountryRuleFactory() : null; - } + countryRuleFactory = ghConfig.getBool("country_rules.enabled", false) ? new CountryRuleFactory() : null; customAreasDirectory = ghConfig.getString("custom_areas.directory", customAreasDirectory); // graph From fb59390c927f94999bc38593a039dbc2867c1647 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 1 Oct 2021 07:13:12 +0200 Subject: [PATCH 05/53] HMM-Lib: Bump Java version to 8 (#2413) --- hmm-lib/pom.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/hmm-lib/pom.xml b/hmm-lib/pom.xml index 0f3f4b919fb..cb420216004 100644 --- a/hmm-lib/pom.xml +++ b/hmm-lib/pom.xml @@ -57,17 +57,4 @@ UTF-8 UTF-8 - - - - - maven-compiler-plugin - 3.1 - - 1.7 - 1.7 - - - - From ff8d658c6537251d42ad9d7827067863b5a43484 Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 1 Oct 2021 11:19:07 +0200 Subject: [PATCH 06/53] Consider heading when snapping (#2411) * current tests pass * minor clarifications * fix test and heading for only one point is fine * changed comment * make test stable * no need to check edge twice as it is the same segment for both directions; do not forget to check end pillar node in distance calc * only calc heading of geometry first (#2415) * only calc heading of geometry first * Fix test (?!) * minor * Flip heading edge filter condition * move method and add comments * minor Co-authored-by: Andi --- .../graphhopper/routing/HeadingResolver.java | 11 +- .../java/com/graphhopper/routing/Router.java | 23 ++- .../com/graphhopper/routing/ViaRouting.java | 23 ++- .../routing/util/HeadingEdgeFilter.java | 68 +++++++++ .../java/com/graphhopper/GraphHopperTest.java | 2 +- .../routing/HeadingRoutingTest.java | 139 +++++++++++++++++- 6 files changed, 238 insertions(+), 28 deletions(-) create mode 100644 core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java diff --git a/core/src/main/java/com/graphhopper/routing/HeadingResolver.java b/core/src/main/java/com/graphhopper/routing/HeadingResolver.java index 1548ffabba5..db22126a898 100644 --- a/core/src/main/java/com/graphhopper/routing/HeadingResolver.java +++ b/core/src/main/java/com/graphhopper/routing/HeadingResolver.java @@ -19,12 +19,14 @@ package com.graphhopper.routing; import com.carrotsearch.hppc.IntArrayList; +import com.graphhopper.routing.util.EdgeFilter; import com.graphhopper.storage.Graph; import com.graphhopper.util.*; +import com.graphhopper.util.shapes.GHPoint; public class HeadingResolver { private final EdgeExplorer edgeExplorer; - private double toleranceRad = deg2Rad(100); + private double toleranceRad = Math.toRadians(100); public HeadingResolver(Graph graph) { this.edgeExplorer = graph.createEdgeExplorer(); @@ -63,12 +65,7 @@ public IntArrayList getEdgesWithDifferentHeading(int baseNode, double heading) { * Sets the tolerance for {@link #getEdgesWithDifferentHeading} in degrees. */ public HeadingResolver setTolerance(double tolerance) { - this.toleranceRad = deg2Rad(tolerance); + this.toleranceRad = Math.toRadians(tolerance); return this; } - - private static double deg2Rad(double deg) { - return Math.toRadians(deg); - } - } diff --git a/core/src/main/java/com/graphhopper/routing/Router.java b/core/src/main/java/com/graphhopper/routing/Router.java index 26d9ce07b0d..26b23a746b5 100644 --- a/core/src/main/java/com/graphhopper/routing/Router.java +++ b/core/src/main/java/com/graphhopper/routing/Router.java @@ -24,6 +24,7 @@ import com.graphhopper.ResponsePath; import com.graphhopper.config.Profile; import com.graphhopper.routing.ch.CHRoutingAlgorithmFactory; +import com.graphhopper.routing.ev.BooleanEncodedValue; import com.graphhopper.routing.ev.EncodedValueLookup; import com.graphhopper.routing.ev.Subnetwork; import com.graphhopper.routing.lm.LMRoutingAlgorithmFactory; @@ -165,12 +166,12 @@ private void checkHeadings(GHRequest request) { } private void checkPointHints(GHRequest request) { - if (request.getPointHints().size() > 0 && request.getPointHints().size() != request.getPoints().size()) + if (!request.getPointHints().isEmpty() && request.getPointHints().size() != request.getPoints().size()) throw new IllegalArgumentException("If you pass " + POINT_HINT + ", you need to pass exactly one hint for every point, empty hints will be ignored"); } private void checkCurbsides(GHRequest request) { - if (request.getCurbsides().size() > 0 && request.getCurbsides().size() != request.getPoints().size()) + if (!request.getCurbsides().isEmpty() && request.getCurbsides().size() != request.getPoints().size()) throw new IllegalArgumentException("If you pass " + CURBSIDE + ", you need to pass exactly one curbside for every point, empty curbsides will be ignored"); } @@ -196,7 +197,7 @@ protected GHResponse routeRoundTrip(GHRequest request, FlexSolver solver) { StopWatch sw = new StopWatch().start(); double startHeading = request.getHeadings().isEmpty() ? Double.NaN : request.getHeadings().get(0); RoundTripRouting.Params params = new RoundTripRouting.Params(request.getHints(), startHeading, routerConfig.getMaxRoundTripRetries()); - List snaps = RoundTripRouting.lookup(request.getPoints(), solver.getSnapFilter(), locationIndex, params); + List snaps = RoundTripRouting.lookup(request.getPoints(), solver.createSnapFilter(), locationIndex, params); ghRsp.addDebugInfo("idLookup:" + sw.stop().getSeconds() + "s"); QueryGraph queryGraph = QueryGraph.create(ghStorage, snaps); @@ -216,7 +217,8 @@ protected GHResponse routeAlt(GHRequest request, Solver solver) { throw new IllegalArgumentException("Currently alternative routes work only with start and end point. You tried to use: " + request.getPoints().size() + " points"); GHResponse ghRsp = new GHResponse(); StopWatch sw = new StopWatch().start(); - List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.getSnapFilter(), locationIndex, request.getSnapPreventions(), request.getPointHints()); + List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.createSnapFilter(), locationIndex, + request.getSnapPreventions(), request.getPointHints(), solver.createDirectedSnapFilter(), request.getHeadings()); ghRsp.addDebugInfo("idLookup:" + sw.stop().getSeconds() + "s"); QueryGraph queryGraph = QueryGraph.create(ghStorage, snaps); PathCalculator pathCalculator = solver.createPathCalculator(queryGraph); @@ -246,7 +248,8 @@ protected GHResponse routeAlt(GHRequest request, Solver solver) { protected GHResponse routeVia(GHRequest request, Solver solver) { GHResponse ghRsp = new GHResponse(); StopWatch sw = new StopWatch().start(); - List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.getSnapFilter(), locationIndex, request.getSnapPreventions(), request.getPointHints()); + List snaps = ViaRouting.lookup(encodingManager, request.getPoints(), solver.createSnapFilter(), locationIndex, + request.getSnapPreventions(), request.getPointHints(), solver.createDirectedSnapFilter(), request.getHeadings()); ghRsp.addDebugInfo("idLookup:" + sw.stop().getSeconds() + "s"); // (base) query graph used to resolve headings, curbsides etc. this is not necessarily the same thing as // the (possibly implementation specific) query graph used by PathCalculator @@ -254,7 +257,8 @@ protected GHResponse routeVia(GHRequest request, Solver solver) { PathCalculator pathCalculator = solver.createPathCalculator(queryGraph); boolean passThrough = getPassThrough(request.getHints()); boolean forceCurbsides = getForceCurbsides(request.getHints()); - ViaRouting.Result result = ViaRouting.calcPaths(request.getPoints(), queryGraph, snaps, solver.weighting, pathCalculator, request.getCurbsides(), forceCurbsides, request.getHeadings(), passThrough); + ViaRouting.Result result = ViaRouting.calcPaths(request.getPoints(), queryGraph, snaps, solver.weighting, + pathCalculator, request.getCurbsides(), forceCurbsides, request.getHeadings(), passThrough); if (request.getPoints().size() != result.paths.size() + 1) throw new RuntimeException("There should be exactly one more point than paths. points:" + request.getPoints().size() + ", paths:" + result.paths.size()); @@ -377,10 +381,15 @@ protected void checkProfileCompatibility() { protected abstract Weighting createWeighting(); - protected EdgeFilter getSnapFilter() { + protected EdgeFilter createSnapFilter() { return new DefaultSnapFilter(weighting, lookup.getBooleanEncodedValue(Subnetwork.key(profile.getName()))); } + protected EdgeFilter createDirectedSnapFilter() { + BooleanEncodedValue inSubnetworkEnc = lookup.getBooleanEncodedValue(Subnetwork.key(profile.getName())); + return edgeState -> !edgeState.get(inSubnetworkEnc) && Double.isFinite(weighting.calcEdgeWeightWithAccess(edgeState, false)); + } + protected abstract PathCalculator createPathCalculator(QueryGraph queryGraph); private List getTurnCostProfiles() { diff --git a/core/src/main/java/com/graphhopper/routing/ViaRouting.java b/core/src/main/java/com/graphhopper/routing/ViaRouting.java index 6966dd1e5b8..2cc67a77e14 100644 --- a/core/src/main/java/com/graphhopper/routing/ViaRouting.java +++ b/core/src/main/java/com/graphhopper/routing/ViaRouting.java @@ -24,7 +24,7 @@ import com.graphhopper.routing.ev.RoadEnvironment; import com.graphhopper.routing.querygraph.QueryGraph; import com.graphhopper.routing.util.EdgeFilter; -import com.graphhopper.routing.util.FiniteWeightFilter; +import com.graphhopper.routing.util.HeadingEdgeFilter; import com.graphhopper.routing.util.NameSimilarityEdgeFilter; import com.graphhopper.routing.util.SnapPreventionEdgeFilter; import com.graphhopper.routing.weighting.Weighting; @@ -54,27 +54,36 @@ public class ViaRouting { /** * @throws MultiplePointsNotFoundException in case one or more points could not be resolved */ - public static List lookup(EncodedValueLookup lookup, List points, EdgeFilter edgeFilter, LocationIndex locationIndex, List snapPreventions, List pointHints) { + public static List lookup(EncodedValueLookup lookup, List points, EdgeFilter snapFilter, + LocationIndex locationIndex, List snapPreventions, List pointHints, + EdgeFilter directedSnapFilter, List headings) { if (points.size() < 2) throw new IllegalArgumentException("At least 2 points have to be specified, but was:" + points.size()); final EnumEncodedValue roadClassEnc = lookup.getEnumEncodedValue(RoadClass.KEY, RoadClass.class); final EnumEncodedValue roadEnvEnc = lookup.getEnumEncodedValue(RoadEnvironment.KEY, RoadEnvironment.class); EdgeFilter strictEdgeFilter = snapPreventions.isEmpty() - ? edgeFilter - : new SnapPreventionEdgeFilter(edgeFilter, roadClassEnc, roadEnvEnc, snapPreventions); + ? snapFilter + : new SnapPreventionEdgeFilter(snapFilter, roadClassEnc, roadEnvEnc, snapPreventions); List snaps = new ArrayList<>(points.size()); IntArrayList pointsNotFound = new IntArrayList(); for (int placeIndex = 0; placeIndex < points.size(); placeIndex++) { GHPoint point = points.get(placeIndex); Snap snap = null; - if (!pointHints.isEmpty()) + if (placeIndex < headings.size() && !Double.isNaN(headings.get(placeIndex))) { + if (!pointHints.isEmpty() && !Helper.isEmpty(pointHints.get(placeIndex))) + throw new IllegalArgumentException("Cannot specify heading and point_hint at the same time. " + + "Make sure you specify either an empty point_hint (String) or a NaN heading (double) for point " + placeIndex); + snap = locationIndex.findClosest(point.lat, point.lon, new HeadingEdgeFilter(directedSnapFilter, headings.get(placeIndex), point)); + } else if (!pointHints.isEmpty()) { snap = locationIndex.findClosest(point.lat, point.lon, new NameSimilarityEdgeFilter(strictEdgeFilter, pointHints.get(placeIndex), point, 100)); - else if (!snapPreventions.isEmpty()) + } else if (!snapPreventions.isEmpty()) { snap = locationIndex.findClosest(point.lat, point.lon, strictEdgeFilter); + } + if (snap == null || !snap.isValid()) - snap = locationIndex.findClosest(point.lat, point.lon, edgeFilter); + snap = locationIndex.findClosest(point.lat, point.lon, snapFilter); if (!snap.isValid()) pointsNotFound.add(placeIndex); diff --git a/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java new file mode 100644 index 00000000000..a6b986fe5ad --- /dev/null +++ b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java @@ -0,0 +1,68 @@ +package com.graphhopper.routing.util; + +import com.graphhopper.util.*; +import com.graphhopper.util.shapes.GHPoint; + +public class HeadingEdgeFilter implements EdgeFilter { + + private final double heading; + private final EdgeFilter directedEdgeFilter; + private final GHPoint pointNearHeading; + + public HeadingEdgeFilter(EdgeFilter directedEdgeFilter, double heading, GHPoint pointNearHeading) { + this.directedEdgeFilter = directedEdgeFilter; + this.heading = heading; + this.pointNearHeading = pointNearHeading; + } + + @Override + public boolean accept(EdgeIteratorState edgeState) { + final double tolerance = 30; + // we only accept edges that are not too far away. It might happen that only far away edges match the heading + // in which case we rather rely on the fallback snapping than return a match here. + final double maxDistance = 20; + double headingOfEdge = getHeadingOfGeometryNearPoint(edgeState, pointNearHeading, maxDistance); + if (Double.isNaN(headingOfEdge)) + // this edge is too far away. we do not accept it. + return false; + // we accept the edge if either of the two directions roughly has the right heading + return Math.abs(headingOfEdge - heading) < tolerance && directedEdgeFilter.accept(edgeState) || + Math.abs((headingOfEdge + 180) % 360 - heading) < tolerance && directedEdgeFilter.accept(edgeState.detach(true)); + } + + /** + * Calculates the heading (in degrees) of the given edge in fwd direction near the given point. If the point is + * too far away from the edge (according to the maxDistance parameter) it returns Double.NaN. + */ + static double getHeadingOfGeometryNearPoint(EdgeIteratorState edgeState, GHPoint point, double maxDistance) { + final DistanceCalc calcDist = DistanceCalcEarth.DIST_EARTH; + double closestDistance = Double.POSITIVE_INFINITY; + PointList points = edgeState.fetchWayGeometry(FetchMode.ALL); + int closestPoint = -1; + for (int i = 1; i < points.size(); i++) { + double fromLat = points.getLat(i - 1), fromLon = points.getLon(i - 1); + double toLat = points.getLat(i), toLon = points.getLon(i); + // the 'distance' between the point and an edge segment is either the vertical distance to the segment or + // the distance to the closer one of the two endpoints. here we save one call to calcDist per segment, + // because each endpoint appears in two segments (except the first and last). + double distance = calcDist.validEdgeDistance(point.lat, point.lon, fromLat, fromLon, toLat, toLon) + ? calcDist.calcDenormalizedDist(calcDist.calcNormalizedEdgeDistance(point.lat, point.lon, fromLat, fromLon, toLat, toLon)) + : calcDist.calcDist(fromLat, fromLon, point.lat, point.lon); + if (i == points.size() - 1) + distance = Math.min(distance, calcDist.calcDist(toLat, toLon, point.lat, point.lon)); + if (distance > maxDistance) + continue; + if (distance < closestDistance) { + closestDistance = distance; + closestPoint = i; + } + } + if (closestPoint < 0) + return Double.NaN; + + double fromLat = points.getLat(closestPoint - 1), fromLon = points.getLon(closestPoint - 1); + double toLat = points.getLat(closestPoint), toLon = points.getLon(closestPoint); + // calcOrientation returns value relative to East, but heading is relative to North + return (Math.toDegrees(AngleCalc.ANGLE_CALC.calcOrientation(fromLat, fromLon, toLat, toLon)) + 90) % 360; + } +} diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index 6bcc4759762..f68fe4cc2c6 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -230,7 +230,7 @@ public void testUTurn() { request.addPoint(new GHPoint(43.743887, 7.431151)); request.addPoint(new GHPoint(43.744007, 7.431076)); //Force initial U-Turn - request.setHeadings(Arrays.asList(200., Double.NaN)); + request.setHeadings(Arrays.asList(200.)); request.setAlgorithm(ASTAR).setProfile(profile); GHResponse rsp = hopper.route(request); diff --git a/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java b/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java index 37b037a10c5..23d911f108a 100644 --- a/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java +++ b/core/src/test/java/com/graphhopper/routing/HeadingRoutingTest.java @@ -155,7 +155,7 @@ public void headingTest5() { GHPoint via = new GHPoint(0.000, 0.0015); GHRequest req = new GHRequest(). setPoints(Arrays.asList(start, via, end)). - setHeadings(Arrays.asList(0., 3.14 / 2, Double.NaN)). + setHeadings(Arrays.asList(0., 90., Double.NaN)). setProfile("profile"). setPathDetails(Collections.singletonList("edge_key")); req.putHint(Parameters.Routing.PASS_THROUGH, true); @@ -164,6 +164,96 @@ public void headingTest5() { assertArrayEquals(new int[]{5, 4, 3, 8, 7, 6, 5, 4, 3, 2}, calcNodes(graph, response.getAll().get(0))); } + @Test + public void testHeadingWithSnapFilter() { + GraphHopperStorage graph = createSquareGraphWithTunnel(); + Router router = createRouter(graph); + // Start at 8 (slightly north to make it independent on some edge ordering and always use 8-3 or 3-8 as fallback) + GHPoint start = new GHPoint(0.0011, 0.001); + // End at middle of edge 2-3 + GHPoint end = new GHPoint(0.002, 0.0005); + + // no heading + GHRequest req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + GHResponse response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + // same start + end but heading=0, parallel to 3-8-7 + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setHeadings(Arrays.asList(0.)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + // heading=90 parallel to 1->5 + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setHeadings(Arrays.asList(90., Double.NaN)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{1, 5, 4, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + for (double angle = 0; angle < 360; angle += 10) { + // Ignore angles nearly parallel to 1->5. I.e. it should fallback to results with 8-3.. or 3-8.. + if (angle >= 60 && angle <= 120) continue; + + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setHeadings(Arrays.asList(angle, Double.NaN)). + setProfile("profile"). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + + int[] expectedNodes = (angle >= 130 && angle <= 250) ? new int[]{3, 8, 7, 0, 1, 2, 3} : new int[]{8, 3, 2}; + // System.out.println(Arrays.toString(calcNodes(graph, response.getAll().get(0))) + " angle:" + angle); + assertArrayEquals(expectedNodes, calcNodes(graph, response.getAll().get(0)), "angle: " + angle); + } + } + + @Test + public void testHeadingWithSnapFilter2() { + GraphHopperStorage graph = createSquareGraphWithTunnel(); + Router router = createRouter(graph); + // Start at 8 (slightly east to snap to edge 1->5 per default) + GHPoint start = new GHPoint(0.001, 0.0011); + // End at middle of edge 2-3 + GHPoint end = new GHPoint(0.002, 0.0005); + + GHRequest req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setProfile("profile"). + setHeadings(Arrays.asList(0.)). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + GHResponse response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + + req = new GHRequest(). + setPoints(Arrays.asList(start, end)). + setProfile("profile"). + setHeadings(Arrays.asList(180.)). + setPathDetails(Collections.singletonList("edge_key")); + req.putHint(Parameters.Routing.PASS_THROUGH, true); + response = router.route(req); + assertFalse(response.hasErrors()); + assertArrayEquals(new int[]{8, 3, 2}, calcNodes(graph, response.getAll().get(0))); + } + @Test public void headingTest6() { // Test if snaps at tower nodes are ignored @@ -199,11 +289,11 @@ private GraphHopperStorage createSquareGraph() { EncodingManager encodingManager = new EncodingManager.Builder().add(carEncoder).add(Subnetwork.create("profile")).build(); GraphHopperStorage g = new GraphBuilder(encodingManager).create(); - // 2---3---4 - // / | \ - // 1----8----5 - // / | / - // 0----7---6 + // 2---3---4 + // | | | + // 1---8---5 + // | | | + // 0---7---6 NodeAccess na = g.getNodeAccess(); na.setNode(0, 0.000, 0.000); na.setNode(1, 0.001, 0.000); @@ -232,6 +322,43 @@ private GraphHopperStorage createSquareGraph() { return g; } + private GraphHopperStorage createSquareGraphWithTunnel() { + CarFlagEncoder carEncoder = new CarFlagEncoder(); + EncodingManager encodingManager = new EncodingManager.Builder().add(carEncoder).add(Subnetwork.create("profile")).build(); + GraphHopperStorage g = new GraphBuilder(encodingManager).create(); + + // 2----3---4 + // | | | + // 1->- 8 >-5 (edge 1->5 is not connected to 8) + // | | | + // 0----7---6 + NodeAccess na = g.getNodeAccess(); + na.setNode(0, 0.000, 0.000); + na.setNode(1, 0.001, 0.000); + na.setNode(2, 0.002, 0.000); + na.setNode(3, 0.002, 0.001); + na.setNode(4, 0.002, 0.002); + na.setNode(5, 0.001, 0.002); + na.setNode(6, 0.000, 0.002); + na.setNode(7, 0.000, 0.001); + na.setNode(8, 0.001, 0.001); + + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(0, 1).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(1, 2).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(2, 3).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(3, 4).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(4, 5).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(5, 6).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(6, 7).setDistance(100)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(7, 0).setDistance(100)); + + GHUtility.setSpeed(60, true, false, carEncoder, g.edge(1, 5).setDistance(110)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(3, 8).setDistance(110)); + GHUtility.setSpeed(60, true, true, carEncoder, g.edge(7, 8).setDistance(110)); + + return g; + } + private int[] calcNodes(Graph graph, ResponsePath responsePath) { List edgeKeys = responsePath.getPathDetails().get("edge_key"); int[] result = new int[edgeKeys.size() + 1]; From 2c4351ce07c821ce688516e6ebbfa81975e0b6de Mon Sep 17 00:00:00 2001 From: Andi Date: Fri, 1 Oct 2021 12:10:13 +0200 Subject: [PATCH 07/53] Convert `Directory#find` to `Directory#create` (#2412) * Make sure GHDirectory#find only uses each name once * Rename Directory#find -> Directory#create --- .../java/com/graphhopper/coll/OSMIDMap.java | 4 +- .../com/graphhopper/reader/PillarInfo.java | 2 +- .../dem/AbstractSRTMElevationProvider.java | 2 +- .../dem/AbstractTiffElevationProvider.java | 2 +- .../routing/lm/LandmarkStorage.java | 9 ++++- .../routing/subnetwork/SubnetworkStorage.java | 2 +- .../com/graphhopper/search/StringIndex.java | 4 +- .../com/graphhopper/storage/BaseGraph.java | 4 +- .../storage/BaseGraphNodesAndEdges.java | 4 +- .../com/graphhopper/storage/CHStorage.java | 4 +- .../com/graphhopper/storage/Directory.java | 9 +---- .../com/graphhopper/storage/GHDirectory.java | 37 ++++--------------- .../storage/StorableProperties.java | 2 +- .../storage/index/LineIntIndex.java | 2 +- .../storage/index/LocationIndexTree.java | 25 ++++++------- .../com/graphhopper/coll/OSMIDMapTest.java | 2 +- .../reader/dem/HeightTileTest.java | 6 +-- .../routing/lm/LandmarkStorageTest.java | 6 +-- .../storage/AbstractDirectoryTester.java | 10 ++--- .../storage/AbstractGraphStorageTester.java | 1 + .../com/graphhopper/gtfs/GraphHopperGtfs.java | 24 +++++++----- 21 files changed, 71 insertions(+), 90 deletions(-) diff --git a/core/src/main/java/com/graphhopper/coll/OSMIDMap.java b/core/src/main/java/com/graphhopper/coll/OSMIDMap.java index 079bf2c99a5..08f8868def5 100644 --- a/core/src/main/java/com/graphhopper/coll/OSMIDMap.java +++ b/core/src/main/java/com/graphhopper/coll/OSMIDMap.java @@ -45,9 +45,9 @@ public OSMIDMap(Directory dir) { public OSMIDMap(Directory dir, int noNumber) { this.dir = dir; this.noEntryValue = noNumber; - keys = dir.find("osmid_map_keys"); + keys = dir.create("osmid_map_keys"); keys.create(2000); - values = dir.find("osmid_map_values"); + values = dir.create("osmid_map_values"); values.create(1000); } diff --git a/core/src/main/java/com/graphhopper/reader/PillarInfo.java b/core/src/main/java/com/graphhopper/reader/PillarInfo.java index 81ec1f8c893..fd2e3eb221d 100644 --- a/core/src/main/java/com/graphhopper/reader/PillarInfo.java +++ b/core/src/main/java/com/graphhopper/reader/PillarInfo.java @@ -38,7 +38,7 @@ public class PillarInfo implements PointAccess { public PillarInfo(boolean enabled3D, Directory dir) { this.enabled3D = enabled3D; this.dir = dir; - this.da = dir.find("tmp_pillar_info").create(100); + this.da = dir.create("tmp_pillar_info").create(100); this.rowSizeInBytes = getDimension() * 4; } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 3a5e1075755..106d586893f 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -102,7 +102,7 @@ public double getEle(double lat, double lon) { if (fileName == null) return 0; - DataAccess heights = getDirectory().find("dem" + intKey); + DataAccess heights = getDirectory().create("dem" + intKey); boolean loadExisting = false; try { loadExisting = heights.loadExisting(); diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index bf0a491a71b..2d16b527c11 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -117,7 +117,7 @@ public double getEle(double lat, double lon) { demProvider.setInterpolate(interpolate); cacheData.put(name, demProvider); - DataAccess heights = getDirectory().find(name + ".gh"); + DataAccess heights = getDirectory().create(name + ".gh"); demProvider.setHeights(heights); boolean loadExisting = false; try { diff --git a/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java b/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java index 17f4e1c0fa3..b0ef05e67af 100644 --- a/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java +++ b/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java @@ -132,7 +132,7 @@ public String toString() { // use the node based traversal as this is a smaller weight approximation and will still produce correct results // In this sense its even 'better' to use node-based. this.traversalMode = TraversalMode.NODE_BASED; - this.landmarkWeightDA = dir.find("landmarks_" + lmConfig.getName()); + this.landmarkWeightDA = dir.create("landmarks_" + lmConfig.getName()); this.landmarks = landmarks; // one short per landmark and two directions => 2*2 byte @@ -759,6 +759,13 @@ private LandmarkExplorer findLandmarks(int[] landmarkNodeIdsToReturn, int startN return explorer; } + /** + * For testing only + */ + DataAccess _getInternalDA() { + return landmarkWeightDA; + } + /** * This class is used to calculate landmark location (equally distributed). * It derives from DijkstraBidirectionRef, but is only used as forward or backward search. diff --git a/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java b/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java index 18e107f6e6a..703bba5d36e 100644 --- a/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java +++ b/core/src/main/java/com/graphhopper/routing/subnetwork/SubnetworkStorage.java @@ -32,7 +32,7 @@ public class SubnetworkStorage { public SubnetworkStorage(Directory dir, String postfix) { DAType type = dir.getDefaultType(); - da = dir.find("subnetwork_" + postfix, type.isMMap() ? DAType.MMAP : (type.isStoring() ? DAType.RAM_STORE : DAType.RAM)); + da = dir.create("subnetwork_" + postfix, type.isMMap() ? DAType.MMAP : (type.isStoring() ? DAType.RAM_STORE : DAType.RAM)); } /** diff --git a/core/src/main/java/com/graphhopper/search/StringIndex.java b/core/src/main/java/com/graphhopper/search/StringIndex.java index 50fab782462..381082bd07b 100644 --- a/core/src/main/java/com/graphhopper/search/StringIndex.java +++ b/core/src/main/java/com/graphhopper/search/StringIndex.java @@ -58,9 +58,9 @@ public StringIndex(Directory dir) { * Specify a larger cacheSize to reduce disk usage. Note that this increases the memory usage of this object. */ public StringIndex(Directory dir, final int cacheSize) { - keys = dir.find("string_index_keys"); + keys = dir.create("string_index_keys"); keys.setSegmentSize(10 * 1024); - vals = dir.find("string_index_vals"); + vals = dir.create("string_index_vals"); smallCache = new LinkedHashMap(cacheSize, 0.75f, true) { @Override protected boolean removeEldestEntry(Map.Entry entry) { diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 6eda12fd12c..0456f4181e4 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -58,12 +58,12 @@ class BaseGraph implements Graph { public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { this.dir = dir; this.bitUtil = BitUtil.get(dir.getByteOrder()); - this.wayGeometry = dir.find("geometry"); + this.wayGeometry = dir.create("geometry"); this.stringIndex = new StringIndex(dir); this.store = new BaseGraphNodesAndEdges(dir, intsForFlags, withElevation, withTurnCosts, segmentSize); this.nodeAccess = new GHNodeAccess(store); if (withTurnCosts) { - turnCostStorage = new TurnCostStorage(this, dir.find("turn_costs")); + turnCostStorage = new TurnCostStorage(this, dir.create("turn_costs")); if (segmentSize >= 0) turnCostStorage.setSegmentSize(segmentSize); } else { diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java index f2e647c5921..178bfbcaa83 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java @@ -61,8 +61,8 @@ class BaseGraphNodesAndEdges { private boolean frozen; public BaseGraphNodesAndEdges(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { - nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType())); - edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType())); + nodes = dir.create("nodes", DAType.getPreferredInt(dir.getDefaultType())); + edges = dir.create("edges", DAType.getPreferredInt(dir.getDefaultType())); if (segmentSize >= 0) { nodes.setSegmentSize(segmentSize); edges.setSegmentSize(segmentSize); diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java index b59ad6849f1..7f30c18480f 100644 --- a/core/src/main/java/com/graphhopper/storage/CHStorage.java +++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java @@ -66,8 +66,8 @@ public class CHStorage { public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased) { this.edgeBased = edgeBased; - this.nodesCH = dir.find("nodes_ch_" + name, DAType.getPreferredInt(dir.getDefaultType())); - this.shortcuts = dir.find("shortcuts_" + name, DAType.getPreferredInt(dir.getDefaultType())); + this.nodesCH = dir.create("nodes_ch_" + name, DAType.getPreferredInt(dir.getDefaultType())); + this.shortcuts = dir.create("shortcuts_" + name, DAType.getPreferredInt(dir.getDefaultType())); if (segmentSize >= 0) { nodesCH.setSegmentSize(segmentSize); shortcuts.setSegmentSize(segmentSize); diff --git a/core/src/main/java/com/graphhopper/storage/Directory.java b/core/src/main/java/com/graphhopper/storage/Directory.java index 6ae6d633755..8811378ede0 100644 --- a/core/src/main/java/com/graphhopper/storage/Directory.java +++ b/core/src/main/java/com/graphhopper/storage/Directory.java @@ -43,9 +43,9 @@ public interface Directory { * Tries to find the object with that name if not existent it creates one and associates the * location with it. A name is unique in one Directory. */ - DataAccess find(String name); + DataAccess create(String name); - DataAccess find(String name, DAType type); + DataAccess create(String name, DAType type); /** * Removes the specified object from the directory. @@ -67,10 +67,5 @@ public interface Directory { */ void close(); - /** - * Returns all created directories. - */ - Collection getAll(); - Directory create(); } diff --git a/core/src/main/java/com/graphhopper/storage/GHDirectory.java b/core/src/main/java/com/graphhopper/storage/GHDirectory.java index 7934a629bfe..9c0c051f486 100644 --- a/core/src/main/java/com/graphhopper/storage/GHDirectory.java +++ b/core/src/main/java/com/graphhopper/storage/GHDirectory.java @@ -19,7 +19,6 @@ import java.io.File; import java.nio.ByteOrder; -import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -35,7 +34,6 @@ public class GHDirectory implements Directory { private final DAType defaultType; private final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; protected Map map = new HashMap<>(); - protected Map types = new HashMap<>(); public GHDirectory(String _location, DAType defaultType) { this.defaultType = defaultType; @@ -56,36 +54,22 @@ public ByteOrder getByteOrder() { return byteOrder; } - public Directory put(String name, DAType type) { - if (!name.equals(toLowerCase(name))) - throw new IllegalArgumentException("Since 0.7 DataAccess objects does no longer accept upper case names"); - - types.put(name, type); - return this; - } - @Override - public DataAccess find(String name) { - DAType type = types.get(name); - if (type == null) - type = defaultType; - - return find(name, type); + public DataAccess create(String name) { + return create(name, defaultType); } @Override - public DataAccess find(String name, DAType type) { + public DataAccess create(String name, DAType type) { if (!name.equals(toLowerCase(name))) throw new IllegalArgumentException("Since 0.7 DataAccess objects does no longer accept upper case names"); - DataAccess da = map.get(name); - if (da != null) { - if (!type.equals(da.getType())) - throw new IllegalStateException("Found existing DataAccess object '" + name - + "' but types did not match. Requested:" + type + ", was:" + da.getType()); - return da; - } + if (map.containsKey(name)) + // we do not allow creating two DataAccess with the same name, because on disk there can only be one DA + // per file name + throw new IllegalStateException("DataAccess " + name + " has already been created"); + DataAccess da; if (type.isInMemory()) { if (type.isInteg()) { if (type.isStoring()) @@ -154,11 +138,6 @@ public Directory create() { return this; } - @Override - public Collection getAll() { - return map.values(); - } - @Override public String toString() { return getLocation(); diff --git a/core/src/main/java/com/graphhopper/storage/StorableProperties.java b/core/src/main/java/com/graphhopper/storage/StorableProperties.java index e798ca75b49..39c3a93c291 100644 --- a/core/src/main/java/com/graphhopper/storage/StorableProperties.java +++ b/core/src/main/java/com/graphhopper/storage/StorableProperties.java @@ -41,7 +41,7 @@ public class StorableProperties { private final DataAccess da; public StorableProperties(Directory dir) { - this.da = dir.find("properties"); + this.da = dir.create("properties"); // reduce size da.setSegmentSize(1 << 15); } diff --git a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java index 4237dc38fe6..6e0e3a8657f 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java +++ b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java @@ -49,7 +49,7 @@ public class LineIntIndex { public LineIntIndex(BBox bBox, Directory dir, String name) { this.bounds = bBox; - this.dataAccess = dir.find(name, DAType.getPreferredInt(dir.getDefaultType())); + this.dataAccess = dir.create(name, DAType.getPreferredInt(dir.getDefaultType())); } public boolean loadExisting() { diff --git a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java index ef1fe4f7f4d..7a19325d715 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java +++ b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java @@ -76,6 +76,18 @@ public LocationIndexTree(Graph g, Directory dir) { this.graph = g; this.nodeAccess = g.getNodeAccess(); this.directory = dir; + + // Clone this defensively -- In case something funny happens and things get added to the Graph after + // this index is built. Reason is that the expected structure of the index is a function of the bbox, so we + // need it to be immutable. + BBox bounds = graph.getBounds().clone(); + + // I want to be able to create a location index for the empty graph without error, but for that + // I need valid bounds so that the initialization logic works. + if (!bounds.isValid()) + bounds = new BBox(-10.0, 10.0, -10.0, 10.0); + + lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); } public int getMinResolutionInMeter() { @@ -116,18 +128,6 @@ public LocationIndex setResolution(int minResolutionInMeter) { } public boolean loadExisting() { - // Clone this defensively -- In case something funny happens and things get added to the Graph after - // this index is built. Reason is that the expected structure of the index is a function of the bbox, so we - // need it to be immutable. - BBox bounds = graph.getBounds().clone(); - - // I want to be able to create a location index for the empty graph without error, but for that - // I need valid bounds so that the initialization logic works. - if (!bounds.isValid()) - bounds = new BBox(-10.0,10.0,-10.0,10.0); - - lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); - if (!lineIntIndex.loadExisting()) return false; @@ -166,7 +166,6 @@ public LocationIndex prepareIndex(EdgeFilter edgeFilter) { InMemConstructionIndex inMemConstructionIndex = prepareInMemConstructionIndex(bounds, edgeFilter); - lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); lineIntIndex.setMinResolutionInMeter(minResolutionInMeter); lineIntIndex.store(inMemConstructionIndex); lineIntIndex.setChecksum(checksum()); diff --git a/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java b/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java index fa7128f4f8f..d035a0de0bc 100644 --- a/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java +++ b/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java @@ -59,7 +59,7 @@ public void testGet() { @Test public void testBinSearch() { - DataAccess da = new RAMDirectory().find(""); + DataAccess da = new RAMDirectory().create(""); da.create(100); da.setInt(0 * 4, 1); diff --git a/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java b/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java index a6bb09b5bd7..02a70c42f99 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/HeightTileTest.java @@ -34,7 +34,7 @@ public void testGetHeight() { int width = 10; int height = 20; HeightTile instance = new HeightTile(0, 0, width, height, 1e-6, 10, 20); - DataAccess heights = new RAMDirectory().find("tmp"); + DataAccess heights = new RAMDirectory().create("tmp"); heights.create(2 * width * height); instance.setHeights(heights); init(heights, width, height, 1); @@ -77,7 +77,7 @@ public void testGetHeight() { public void testGetHeightForNegativeTile() { int width = 10; HeightTile instance = new HeightTile(-20, -20, width, width, 1e-6, 10, 10); - DataAccess heights = new RAMDirectory().find("tmp"); + DataAccess heights = new RAMDirectory().create("tmp"); heights.create(2 * 10 * 10); instance.setHeights(heights); init(heights, width, width, 1); @@ -98,7 +98,7 @@ public void testGetHeightForNegativeTile() { @Test public void testInterpolate() { HeightTile instance = new HeightTile(0, 0, 2, 2, 1e-6, 10, 10).setInterpolate(true); - DataAccess heights = new RAMDirectory().find("tmp"); + DataAccess heights = new RAMDirectory().create("tmp"); heights.create(2 * 2 * 2); instance.setHeights(heights); double topLeft = 0; diff --git a/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java b/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java index bcdf8fb544c..89fbc7130da 100644 --- a/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java +++ b/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java @@ -95,11 +95,9 @@ public double calcEdgeWeight(EdgeIteratorState edgeState, boolean reverse) { public void testSetGetWeight() { GHUtility.setSpeed(60, true, true, encoder, graph.edge(0, 1).setDistance(40.1)); Directory dir = new RAMDirectory(); - DataAccess da = dir.find("landmarks_c1"); - da.create(2000); - LandmarkStorage lms = new LandmarkStorage(graph, dir, new LMConfig("c1", new FastestWeighting(encoder)), 4). setMaximumWeight(LandmarkStorage.PRECISION); + lms._getInternalDA().create(2000); // 2^16=65536, use -1 for infinity and -2 for maximum lms.setWeight(0, 65536); // reached maximum value but do not reset to 0 instead use 2^16-2 @@ -109,7 +107,7 @@ public void testSetGetWeight() { lms.setWeight(0, 79999); assertEquals(65534, lms.getFromWeight(0, 0)); - da.setInt(0, Integer.MAX_VALUE); + lms._getInternalDA().setInt(0, Integer.MAX_VALUE); assertTrue(lms.isInfinity(0)); // for infinity return much bigger value // assertEquals(Integer.MAX_VALUE, lms.getFromWeight(0, 0)); diff --git a/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java b/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java index 345225df6b9..cc2d6eda0dd 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java @@ -24,7 +24,7 @@ import java.io.File; -import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * @author Peter Karich @@ -50,17 +50,15 @@ public void setUp() { @Test public void testNoDuplicates() { Directory dir = createDir(); - DataAccess da1 = dir.find("testing"); - DataAccess da2 = dir.find("testing"); - assertSame(da1, da2); + DataAccess da1 = dir.create("testing"); + assertThrows(IllegalStateException.class, () -> dir.create("testing")); da1.close(); - da2.close(); } @Test public void testNoErrorForDACreate() { Directory dir = createDir(); - da = dir.find("testing"); + da = dir.create("testing"); da.create(100); da.flush(); } diff --git a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java index 257fd25ad98..60b1f6ad127 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java @@ -677,6 +677,7 @@ public String toString() { assertEquals(Integer.MAX_VALUE / 3, edge.getFlags().ints[0]); graph.close(); + dir = new RAMDirectory(); graph = new GraphHopperStorage(dir, manager, false).create(defaultSize); DecimalEncodedValue avSpeed0Enc = manager.getDecimalEncodedValue(getKey("car0", "average_speed")); diff --git a/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java b/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java index 0b108b21cb6..4bce6490c92 100644 --- a/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java +++ b/reader-gtfs/src/main/java/com/graphhopper/gtfs/GraphHopperGtfs.java @@ -37,6 +37,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.*; @@ -66,16 +68,15 @@ protected void importOSM() { @Override protected LocationIndex createLocationIndex(Directory dir) { - LocationIndexTree tmpIndex = new LocationIndexTree(getGraphHopperStorage(), dir); - if (tmpIndex.loadExisting()) { - return tmpIndex; - } else { - LocationIndexTree locationIndexTree = new LocationIndexTree(getGraphHopperStorage(), new RAMDirectory()); - if (!locationIndexTree.loadExisting()) { - locationIndexTree.prepareIndex(); - } - return locationIndexTree; + // if the location index was already created (we are 'loading') we use it. but we must not create the location + // index object in case the index does not exist yet, because we only can create it once. we are not ready yet, + // because first we need to import PT. + if (Files.exists(Paths.get(getGraphHopperLocation()).resolve("location_index"))) { + LocationIndexTree index = new LocationIndexTree(getGraphHopperStorage(), dir); + index.loadExisting(); + return index; } + return null; } static class TransferWithTime { @@ -91,7 +92,9 @@ protected void importPublicTransit() { ensureWriteAccess(); getGtfsStorage().create(); GraphHopperStorage graphHopperStorage = getGraphHopperStorage(); - LocationIndex streetNetworkIndex = getLocationIndex(); + // temporary location index for the street network that we only use during import + LocationIndexTree streetNetworkIndex = new LocationIndexTree(getGraphHopperStorage(), new RAMDirectory()); + streetNetworkIndex.prepareIndex(); try { int idx = 0; List gtfsFiles = ghConfig.has("gtfs.file") ? Arrays.asList(ghConfig.getString("gtfs.file", "").split(",")) : Collections.emptyList(); @@ -125,6 +128,7 @@ protected void importPublicTransit() { throw new RuntimeException("Error while constructing transit network. Is your GTFS file valid? Please check log for possible causes.", e); } streetNetworkIndex.close(); + // now we build the final location index LocationIndexTree locationIndex = new LocationIndexTree(getGraphHopperStorage(), getGraphHopperStorage().getDirectory()); PtEncodedValues ptEncodedValues = PtEncodedValues.fromEncodingManager(getEncodingManager()); EnumEncodedValue typeEnc = ptEncodedValues.getTypeEnc(); From 9c184bb42fc15ccdfe202100e9d21167393948b9 Mon Sep 17 00:00:00 2001 From: easbar Date: Fri, 1 Oct 2021 12:24:38 +0200 Subject: [PATCH 08/53] Remove small segment size optimization for sea level elevation tiles --- .../graphhopper/reader/dem/AbstractSRTMElevationProvider.java | 4 ++-- .../graphhopper/reader/dem/AbstractTiffElevationProvider.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 106d586893f..68f46b0552d 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -119,8 +119,8 @@ public double getEle(double lat, double lon) { demProvider.setHeights(heights); demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10) + .flush(); return 0; } } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index 2d16b527c11..8bdc8c84973 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -135,8 +135,7 @@ public double getEle(double lat, double lon) { } catch (IOException e) { demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10).flush(); return 0; } From 0b703cc75ebaa6a5269fd69b061f697cf7510715 Mon Sep 17 00:00:00 2001 From: easbar Date: Fri, 1 Oct 2021 12:25:17 +0200 Subject: [PATCH 09/53] Revert "Remove small segment size optimization for sea level elevation tiles" This reverts commit 9c184bb42fc15ccdfe202100e9d21167393948b9. --- .../graphhopper/reader/dem/AbstractSRTMElevationProvider.java | 4 ++-- .../graphhopper/reader/dem/AbstractTiffElevationProvider.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 68f46b0552d..106d586893f 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -119,8 +119,8 @@ public double getEle(double lat, double lon) { demProvider.setHeights(heights); demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.create(10) - .flush(); + heights.setSegmentSize(100).create(10). + flush(); return 0; } } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index 8bdc8c84973..2d16b527c11 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -135,7 +135,8 @@ public double getEle(double lat, double lon) { } catch (IOException e) { demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.create(10).flush(); + heights.setSegmentSize(100).create(10). + flush(); return 0; } From 44ff428e100e441ac3fea83c18ea2c3d0d69c378 Mon Sep 17 00:00:00 2001 From: otbutz Date: Fri, 1 Oct 2021 12:50:03 +0200 Subject: [PATCH 10/53] Register country rules in a map (#2418) Co-authored-by: Thomas Butz --- .../util/countryrules/AustriaCountryRule.java | 1 - .../util/countryrules/CountryRuleFactory.java | 26 +++++++++++++------ .../util/countryrules/GermanyCountryRule.java | 1 - 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java index ae51cfb1970..e214ac0f8ad 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java @@ -24,7 +24,6 @@ import com.graphhopper.routing.util.TransportationMode; public class AustriaCountryRule implements CountryRule { - public final static AustriaCountryRule RULE = new AustriaCountryRule(); @Override public double getMaxSpeed(ReaderWay readerWay, TransportationMode transportationMode, double currentMaxSpeed) { diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java index 2ccd8a2ccbc..7a68a6dbfd1 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRuleFactory.java @@ -18,18 +18,28 @@ package com.graphhopper.routing.util.countryrules; + +import static com.graphhopper.routing.ev.Country.*; + +import java.util.EnumMap; +import java.util.Map; + import com.graphhopper.routing.ev.Country; public class CountryRuleFactory { + + private final Map rules = new EnumMap<>(Country.class); + + public CountryRuleFactory() { + rules.put(AUT, new AustriaCountryRule()); + rules.put(DEU, new GermanyCountryRule()); + } public CountryRule getCountryRule(Country country) { - switch (country) { - case DEU: - return GermanyCountryRule.RULE; - case AUT: - return AustriaCountryRule.RULE; - default: - return null; - } + return rules.get(country); + } + + public Map getCountryToRuleMap() { + return rules; } } diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java index 8c4adcee1e4..d78c800da9d 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java @@ -28,7 +28,6 @@ * @author Robin Boldt */ public class GermanyCountryRule implements CountryRule { - public final static GermanyCountryRule RULE = new GermanyCountryRule(); /** * In Germany there are roads without a speed limit. For these roads, this method From db214409e72857b7ee6a72e98f1b085483269ef3 Mon Sep 17 00:00:00 2001 From: easbar Date: Fri, 1 Oct 2021 14:04:50 +0200 Subject: [PATCH 11/53] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82f72125b27..97005f16e10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ### 5.0 [not yet released] +- consider heading when snapping coordinates to the road network, this is especially important for navigation (#2411) + ### 4.0 [29 Sep 2021] - faster node-based CH preparation (~20%), (#2390) From f7397fdceb82bdfa679318e2447c7cd6032d074e Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 4 Oct 2021 10:40:36 +0200 Subject: [PATCH 12/53] use jdk17 ga and jdk18 ea (#2419) * use jdk17 ga and jdk18 ea * adoptopenjdk is adoptium and I cannot find ea versions, so use the default ones (sormuras/bach -> install-jdk.properties) * since jdk17 locale IDs changed --- .travis.yml | 6 +++--- .../com/graphhopper/util/TranslationMap.java | 20 +++++++++++-------- .../graphhopper/util/TranslationMapTest.java | 8 ++++++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98fe5bdd04d..afa1b626fff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,10 @@ env: matrix: include: - jdk: openjdk8 - - env: JDK='OpenJDK 16' - install: . ./install-jdk.sh -F 16 -C --url 'https://api.adoptopenjdk.net/v3/binary/latest/16/ga/linux/x64/jdk/hotspot/normal/adoptopenjdk' - env: JDK='OpenJDK 17' - install: . ./install-jdk.sh -F 17 -C --url 'https://api.adoptopenjdk.net/v3/binary/latest/17/ea/linux/x64/jdk/hotspot/normal/adoptopenjdk' + install: . ./install-jdk.sh -F 17 -C + - env: JDK='OpenJDK 18' + install: . ./install-jdk.sh -F ea -C # avoid default dependency command for maven, 'true' means 'return true' and continue install: true diff --git a/core/src/main/java/com/graphhopper/util/TranslationMap.java b/core/src/main/java/com/graphhopper/util/TranslationMap.java index bd5052ae9e9..8fb172603d6 100644 --- a/core/src/main/java/com/graphhopper/util/TranslationMap.java +++ b/core/src/main/java/com/graphhopper/util/TranslationMap.java @@ -85,14 +85,18 @@ public void add(Translation tr) { if (!locale.getCountry().isEmpty() && !translations.containsKey(tr.getLanguage())) translations.put(tr.getLanguage(), tr); - // Map old Java 'standard' to latest, Java is a bit ugly here: http://stackoverflow.com/q/13974169/194609 - // Hebrew - if ("iw".equals(locale.getLanguage())) - translations.put("he", tr); - - // Indonesia - if ("in".equals(locale.getLanguage())) - translations.put("id", tr); + // Hebrew locale was "iw" in old JDKs but is now he + // required in old JDKs: + if ("iw".equals(locale.getLanguage())) translations.put("he", tr); + // required since jdk17 to still provide translation for "iw": + if ("he".equals(locale.getLanguage())) translations.put("iw", tr); + + // Indonesia locale was "in_ID" in old JDKs but is now id_ID + // required in old JDKs: + if ("in".equals(locale.getLanguage())) translations.put("id", tr); + // required since jdk17 to still provide translation for "in": + if ("id".equals(locale.getLanguage())) translations.put("in", tr); + // Indian locales are: en-IN and hi-IN and are not overwritten by that } /** diff --git a/core/src/test/java/com/graphhopper/util/TranslationMapTest.java b/core/src/test/java/com/graphhopper/util/TranslationMapTest.java index 4ce2d78b6f2..1922c08f878 100644 --- a/core/src/test/java/com/graphhopper/util/TranslationMapTest.java +++ b/core/src/test/java/com/graphhopper/util/TranslationMapTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Locale; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -60,8 +61,11 @@ public void testToString() { assertEquals("רגל", trMap.tr("web.FOOT")); // Indonesian - assertEquals("in", SINGLETON.get("in").getLanguage()); - assertEquals("in", SINGLETON.get("in_ID").getLanguage()); + // for jdk17 and later "id" is returned, before "in" was returned + String lang = SINGLETON.get("id").getLanguage(); + assertTrue(Arrays.asList("id", "in").contains(lang)); + assertEquals(lang, SINGLETON.get("in").getLanguage()); + assertEquals(lang, SINGLETON.get("in_ID").getLanguage()); // Vietnamese assertEquals("vi", SINGLETON.get("vi").getLanguage()); From 3073c0adf1e194f12b29f5b6336af14fae24ada9 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 4 Oct 2021 10:32:05 +0200 Subject: [PATCH 13/53] fix orientation calculation for #2411 #1463 --- .../routing/util/HeadingEdgeFilter.java | 3 +-- .../java/com/graphhopper/GraphHopperTest.java | 6 ++--- .../routing/util/HeadingEdgeFilterTest.java | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java diff --git a/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java index a6b986fe5ad..74ff0aed256 100644 --- a/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java +++ b/core/src/main/java/com/graphhopper/routing/util/HeadingEdgeFilter.java @@ -62,7 +62,6 @@ static double getHeadingOfGeometryNearPoint(EdgeIteratorState edgeState, GHPoint double fromLat = points.getLat(closestPoint - 1), fromLon = points.getLon(closestPoint - 1); double toLat = points.getLat(closestPoint), toLon = points.getLon(closestPoint); - // calcOrientation returns value relative to East, but heading is relative to North - return (Math.toDegrees(AngleCalc.ANGLE_CALC.calcOrientation(fromLat, fromLon, toLat, toLon)) + 90) % 360; + return AngleCalc.ANGLE_CALC.calcAzimuth(fromLat, fromLon, toLat, toLon); } } diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index f68fe4cc2c6..214109c2877 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -238,12 +238,12 @@ public void testUTurn() { assertFalse(rsp.hasErrors()); ResponsePath res = rsp.getBest(); InstructionList il = res.getInstructions(); - assertEquals(3, il.size()); + assertEquals(4, il.size()); // Initial U-turn - assertEquals("make a U-turn onto Avenue Princesse Grace", il.get(0).getTurnDescription(tr)); - // Second U-turn to get to destination assertEquals("make a U-turn onto Avenue Princesse Grace", il.get(1).getTurnDescription(tr)); + // Second U-turn to get to destination + assertEquals("make a U-turn onto Avenue Princesse Grace", il.get(2).getTurnDescription(tr)); } private void testImportCloseAndLoad(boolean ch, boolean lm, boolean sort, boolean custom) { diff --git a/core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java b/core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java new file mode 100644 index 00000000000..95045e55f8f --- /dev/null +++ b/core/src/test/java/com/graphhopper/routing/util/HeadingEdgeFilterTest.java @@ -0,0 +1,26 @@ +package com.graphhopper.routing.util; + +import com.graphhopper.storage.GraphBuilder; +import com.graphhopper.storage.GraphHopperStorage; +import com.graphhopper.util.EdgeIteratorState; +import com.graphhopper.util.shapes.GHPoint; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class HeadingEdgeFilterTest { + + @Test + public void getHeading() { + GHPoint point = new GHPoint(55.67093, 12.577294); + CarFlagEncoder carEncoder = new CarFlagEncoder(); + EncodingManager encodingManager = new EncodingManager.Builder().add(carEncoder).build(); + GraphHopperStorage g = new GraphBuilder(encodingManager).create(); + EdgeIteratorState edge = g.edge(0, 1); + g.getNodeAccess().setNode(0, 55.671044, 12.5771583); + g.getNodeAccess().setNode(1, 55.6704136, 12.5784324); + // GHUtility.setSpeed(50, 0, carEncoder, edge.getFlags()); + + assertEquals(131.2, HeadingEdgeFilter.getHeadingOfGeometryNearPoint(edge, point, 20), .1); + } +} \ No newline at end of file From 3be6bdd873b2ac7bef436780326751b8700527df Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 4 Oct 2021 12:22:01 +0200 Subject: [PATCH 14/53] Remove small segment size optimization for sea level elevation tiles (#2417) --- .../graphhopper/reader/dem/AbstractSRTMElevationProvider.java | 4 ++-- .../graphhopper/reader/dem/AbstractTiffElevationProvider.java | 3 +-- .../java/com/graphhopper/reader/dem/CGIARProviderTest.java | 4 ++-- .../java/com/graphhopper/reader/dem/GMTEDProviderTest.java | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 106d586893f..68f46b0552d 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -119,8 +119,8 @@ public double getEle(double lat, double lon) { demProvider.setHeights(heights); demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10) + .flush(); return 0; } } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index 2d16b527c11..8bdc8c84973 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -135,8 +135,7 @@ public double getEle(double lat, double lon) { } catch (IOException e) { demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). - flush(); + heights.create(10).flush(); return 0; } diff --git a/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java b/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java index 6e161418b56..e62213bf86b 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java @@ -89,9 +89,9 @@ public void downloadFile(String url, String toFile) throws IOException { }); assertEquals(0, instance.getEle(46, -20), 1); - // file not found => small! + // file not found assertTrue(file.exists()); - assertEquals(228, file.length()); + assertEquals(1048676, file.length()); instance.setDownloader(new Downloader("test GH") { @Override diff --git a/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java b/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java index 1bc732a0d07..64eb6c1c005 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java @@ -96,9 +96,9 @@ public void downloadFile(String url, String toFile) throws IOException { }); assertEquals(0, instance.getEle(46, -20), 1); - // file not found => small! + // file not found assertTrue(file.exists()); - assertEquals(228, file.length()); + assertEquals(1048676, file.length()); instance.setDownloader(new Downloader("test GH") { @Override From 8de5b4b6381fc1f245f9c68330c736c1a97daed4 Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 4 Oct 2021 12:53:10 +0200 Subject: [PATCH 15/53] Remove DataAccess#setSegmentSize (#2421) --- .../com/graphhopper/search/StringIndex.java | 16 +++------------ .../storage/AbstractDataAccess.java | 12 ++++++----- .../com/graphhopper/storage/BaseGraph.java | 16 +++------------ .../storage/BaseGraphNodesAndEdges.java | 8 ++------ .../com/graphhopper/storage/CHStorage.java | 8 ++------ .../com/graphhopper/storage/DataAccess.java | 6 ------ .../com/graphhopper/storage/Directory.java | 11 ++++++++-- .../com/graphhopper/storage/GHDirectory.java | 20 ++++++++++++++----- .../graphhopper/storage/MMapDataAccess.java | 5 ++--- .../graphhopper/storage/RAMDataAccess.java | 19 +++++++++--------- .../graphhopper/storage/RAMIntDataAccess.java | 15 +++++++------- .../storage/StorableProperties.java | 4 ++-- .../graphhopper/storage/TurnCostStorage.java | 4 ---- .../graphhopper/search/StringIndexTest.java | 10 +++++----- .../graphhopper/storage/CHStorageTest.java | 2 +- .../graphhopper/storage/DataAccessTest.java | 17 ++++++++-------- .../storage/GraphHopperStorageTest.java | 5 ----- .../storage/MMapDataAccessTest.java | 8 ++++---- .../storage/RAMDataAccessTest.java | 4 ++-- .../storage/RAMIntDataAccessTest.java | 4 ++-- 20 files changed, 84 insertions(+), 110 deletions(-) diff --git a/core/src/main/java/com/graphhopper/search/StringIndex.java b/core/src/main/java/com/graphhopper/search/StringIndex.java index 381082bd07b..98707e04adc 100644 --- a/core/src/main/java/com/graphhopper/search/StringIndex.java +++ b/core/src/main/java/com/graphhopper/search/StringIndex.java @@ -50,17 +50,12 @@ public class StringIndex { private long lastEntryPointer = -1; private Map lastEntryMap; - public StringIndex(Directory dir) { - this(dir, 1000); - } - /** * Specify a larger cacheSize to reduce disk usage. Note that this increases the memory usage of this object. */ - public StringIndex(Directory dir, final int cacheSize) { - keys = dir.create("string_index_keys"); - keys.setSegmentSize(10 * 1024); - vals = dir.create("string_index_vals"); + public StringIndex(Directory dir, final int cacheSize, final int segmentSize) { + keys = dir.create("string_index_keys", segmentSize); + vals = dir.create("string_index_vals", segmentSize); smallCache = new LinkedHashMap(cacheSize, 0.75f, true) { @Override protected boolean removeEldestEntry(Map.Entry entry) { @@ -327,11 +322,6 @@ public boolean isClosed() { return vals.isClosed() && keys.isClosed(); } - public void setSegmentSize(int segments) { - keys.setSegmentSize(segments); - vals.setSegmentSize(segments); - } - public long getCapacity() { return vals.getCapacity() + keys.getCapacity(); } diff --git a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java index 081302f2d33..5c03f4edeb0 100644 --- a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java @@ -37,14 +37,14 @@ public abstract class AbstractDataAccess implements DataAccess { protected final ByteOrder byteOrder; protected final BitUtil bitUtil; private final String location; - protected int header[] = new int[(HEADER_OFFSET - 20) / 4]; + protected int[] header = new int[(HEADER_OFFSET - 20) / 4]; protected String name; - protected int segmentSizeInBytes = SEGMENT_SIZE_DEFAULT; + protected int segmentSizeInBytes; protected int segmentSizePower; protected int indexDivisor; protected boolean closed = false; - public AbstractDataAccess(String name, String location, ByteOrder order) { + public AbstractDataAccess(String name, String location, ByteOrder order, int segmentSize) { byteOrder = order; bitUtil = BitUtil.get(order); this.name = name; @@ -52,6 +52,9 @@ public AbstractDataAccess(String name, String location, ByteOrder order) { throw new IllegalArgumentException("Create DataAccess object via its corresponding Directory!"); this.location = location; + if (segmentSize < 0) + segmentSize = SEGMENT_SIZE_DEFAULT; + setSegmentSize(segmentSize); } @Override @@ -121,8 +124,7 @@ protected void copyHeader(DataAccess da) { } } - @Override - public DataAccess setSegmentSize(int bytes) { + DataAccess setSegmentSize(int bytes) { if (bytes > 0) { // segment size should be a power of 2 int tmp = (int) (Math.log(bytes) / Math.log(2)); diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 0456f4181e4..0153688f874 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -58,21 +58,11 @@ class BaseGraph implements Graph { public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { this.dir = dir; this.bitUtil = BitUtil.get(dir.getByteOrder()); - this.wayGeometry = dir.create("geometry"); - this.stringIndex = new StringIndex(dir); + this.wayGeometry = dir.create("geometry", segmentSize); + this.stringIndex = new StringIndex(dir, 1000, segmentSize); this.store = new BaseGraphNodesAndEdges(dir, intsForFlags, withElevation, withTurnCosts, segmentSize); this.nodeAccess = new GHNodeAccess(store); - if (withTurnCosts) { - turnCostStorage = new TurnCostStorage(this, dir.create("turn_costs")); - if (segmentSize >= 0) - turnCostStorage.setSegmentSize(segmentSize); - } else { - turnCostStorage = null; - } - if (segmentSize >= 0) { - wayGeometry.setSegmentSize(segmentSize); - stringIndex.setSegmentSize(segmentSize); - } + turnCostStorage = withTurnCosts ? new TurnCostStorage(this, dir.create("turn_costs", segmentSize)) : null; } private int getOtherNode(int nodeThis, long edgePointer) { diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java index 178bfbcaa83..2593452eaa1 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java @@ -61,12 +61,8 @@ class BaseGraphNodesAndEdges { private boolean frozen; public BaseGraphNodesAndEdges(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { - nodes = dir.create("nodes", DAType.getPreferredInt(dir.getDefaultType())); - edges = dir.create("edges", DAType.getPreferredInt(dir.getDefaultType())); - if (segmentSize >= 0) { - nodes.setSegmentSize(segmentSize); - edges.setSegmentSize(segmentSize); - } + nodes = dir.create("nodes", DAType.getPreferredInt(dir.getDefaultType()), segmentSize); + edges = dir.create("edges", DAType.getPreferredInt(dir.getDefaultType()), segmentSize); this.intsForFlags = intsForFlags; this.withTurnCosts = withTurnCosts; this.withElevation = withElevation; diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java index 7f30c18480f..c948a70ad06 100644 --- a/core/src/main/java/com/graphhopper/storage/CHStorage.java +++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java @@ -66,12 +66,8 @@ public class CHStorage { public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased) { this.edgeBased = edgeBased; - this.nodesCH = dir.create("nodes_ch_" + name, DAType.getPreferredInt(dir.getDefaultType())); - this.shortcuts = dir.create("shortcuts_" + name, DAType.getPreferredInt(dir.getDefaultType())); - if (segmentSize >= 0) { - nodesCH.setSegmentSize(segmentSize); - shortcuts.setSegmentSize(segmentSize); - } + this.nodesCH = dir.create("nodes_ch_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize); + this.shortcuts = dir.create("shortcuts_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize); // shortcuts are stored consecutively using this layout (the last two entries only exist for edge-based): // NODEA | NODEB | WEIGHT | SKIP_EDGE1 | SKIP_EDGE2 | S_ORIG_FIRST | S_ORIG_LAST S_NODEA = 0; diff --git a/core/src/main/java/com/graphhopper/storage/DataAccess.java b/core/src/main/java/com/graphhopper/storage/DataAccess.java index 7b1f7962124..5bb8158ac07 100644 --- a/core/src/main/java/com/graphhopper/storage/DataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/DataAccess.java @@ -124,12 +124,6 @@ public interface DataAccess extends Closeable { */ int getSegmentSize(); - /** - * In order to increase allocated space one needs to layout the underlying storage in segments. - * This is how you can customize the size. - */ - DataAccess setSegmentSize(int bytes); - /** * @return the number of segments. */ diff --git a/core/src/main/java/com/graphhopper/storage/Directory.java b/core/src/main/java/com/graphhopper/storage/Directory.java index 8811378ede0..91e5f730794 100644 --- a/core/src/main/java/com/graphhopper/storage/Directory.java +++ b/core/src/main/java/com/graphhopper/storage/Directory.java @@ -40,13 +40,20 @@ public interface Directory { ByteOrder getByteOrder(); /** - * Tries to find the object with that name if not existent it creates one and associates the - * location with it. A name is unique in one Directory. + * Creates a new DataAccess object with the given name in the location of this Directory. Each name can only + * be used once. */ DataAccess create(String name); + /** + * @param segmentSize segment size in bytes or -1 to use the default of the corresponding DataAccess implementation + */ + DataAccess create(String name, int segmentSize); + DataAccess create(String name, DAType type); + DataAccess create(String name, DAType type, int segmentSize); + /** * Removes the specified object from the directory. */ diff --git a/core/src/main/java/com/graphhopper/storage/GHDirectory.java b/core/src/main/java/com/graphhopper/storage/GHDirectory.java index 9c0c051f486..a266d9c8336 100644 --- a/core/src/main/java/com/graphhopper/storage/GHDirectory.java +++ b/core/src/main/java/com/graphhopper/storage/GHDirectory.java @@ -59,8 +59,18 @@ public DataAccess create(String name) { return create(name, defaultType); } + @Override + public DataAccess create(String name, int segmentSize) { + return create(name, defaultType, segmentSize); + } + @Override public DataAccess create(String name, DAType type) { + return create(name, type, -1); + } + + @Override + public DataAccess create(String name, DAType type, int segmentSize) { if (!name.equals(toLowerCase(name))) throw new IllegalArgumentException("Since 0.7 DataAccess objects does no longer accept upper case names"); @@ -73,15 +83,15 @@ public DataAccess create(String name, DAType type) { if (type.isInMemory()) { if (type.isInteg()) { if (type.isStoring()) - da = new RAMIntDataAccess(name, location, true, byteOrder); + da = new RAMIntDataAccess(name, location, true, byteOrder, segmentSize); else - da = new RAMIntDataAccess(name, location, false, byteOrder); + da = new RAMIntDataAccess(name, location, false, byteOrder, segmentSize); } else if (type.isStoring()) - da = new RAMDataAccess(name, location, true, byteOrder); + da = new RAMDataAccess(name, location, true, byteOrder, segmentSize); else - da = new RAMDataAccess(name, location, false, byteOrder); + da = new RAMDataAccess(name, location, false, byteOrder, segmentSize); } else if (type.isMMap()) { - da = new MMapDataAccess(name, location, byteOrder, type.isAllowWrites()); + da = new MMapDataAccess(name, location, byteOrder, type.isAllowWrites(), segmentSize); } else { throw new IllegalArgumentException("DAType not supported " + type); } diff --git a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java index 25eadfd2235..6d7873862db 100644 --- a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java @@ -63,8 +63,8 @@ public final class MMapDataAccess extends AbstractDataAccess { private RandomAccessFile raFile; private List segments = new ArrayList<>(); - MMapDataAccess(String name, String location, ByteOrder order, boolean allowWrites) { - super(name, location, order); + MMapDataAccess(String name, String location, ByteOrder order, boolean allowWrites, int segmentSize) { + super(name, location, order, segmentSize); this.allowWrites = allowWrites; } @@ -174,7 +174,6 @@ public MMapDataAccess create(long bytes) { } initRandomAccessFile(); bytes = Math.max(10 * 4, bytes); - setSegmentSize(segmentSizeInBytes); ensureCapacity(bytes); return this; } diff --git a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java index 124aaedeebb..1549c2130a9 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java @@ -34,8 +34,8 @@ public class RAMDataAccess extends AbstractDataAccess { private byte[][] segments = new byte[0][]; private boolean store; - RAMDataAccess(String name, String location, boolean store, ByteOrder order) { - super(name, location, order); + RAMDataAccess(String name, String location, boolean store, ByteOrder order, int segmentSize) { + super(name, location, order, segmentSize); this.store = store; } @@ -57,7 +57,6 @@ public RAMDataAccess create(long bytes) { if (segments.length > 0) throw new IllegalThreadStateException("already created"); - setSegmentSize(segmentSizeInBytes); ensureCapacity(Math.max(10 * 4, bytes)); return this; } @@ -175,7 +174,7 @@ public final void setInt(long bytePos, int value) { @Override public final int getInt(long bytePos) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 4 > segmentSizeInBytes) @@ -185,7 +184,7 @@ public final int getInt(long bytePos) { @Override public final void setShort(long bytePos, short value) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 2 > segmentSizeInBytes) { @@ -199,7 +198,7 @@ public final void setShort(long bytePos, short value) { @Override public final short getShort(long bytePos) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 2 > segmentSizeInBytes) @@ -211,7 +210,7 @@ public final short getShort(long bytePos) { @Override public void setBytes(long bytePos, byte[] values, int length) { assert length <= segmentSizeInBytes : "the length has to be smaller or equal to the segment size: " + length + " vs. " + segmentSizeInBytes; - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); byte[] seg = segments[bufferIndex]; @@ -229,7 +228,7 @@ public void setBytes(long bytePos, byte[] values, int length) { @Override public void getBytes(long bytePos, byte[] values, int length) { assert length <= segmentSizeInBytes : "the length has to be smaller or equal to the segment size: " + length + " vs. " + segmentSizeInBytes; - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); byte[] seg = segments[bufferIndex]; @@ -246,7 +245,7 @@ public void getBytes(long bytePos, byte[] values, int length) { @Override public final void setByte(long bytePos, byte value) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); segments[bufferIndex][index] = value; @@ -254,7 +253,7 @@ public final void setByte(long bytePos, byte value) { @Override public final byte getByte(long bytePos) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); return segments[bufferIndex][index]; diff --git a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java index 5e8bfa1b36a..e899d75204c 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java @@ -36,8 +36,8 @@ class RAMIntDataAccess extends AbstractDataAccess { private boolean store; private int segmentSizeIntsPower; - RAMIntDataAccess(String name, String location, boolean store, ByteOrder order) { - super(name, location, order); + RAMIntDataAccess(String name, String location, boolean store, ByteOrder order, int segmentSize) { + super(name, location, order, segmentSize); this.store = store; } @@ -59,7 +59,6 @@ public RAMIntDataAccess create(long bytes) { if (segments.length > 0) throw new IllegalThreadStateException("already created"); - setSegmentSize(segmentSizeInBytes); ensureCapacity(Math.max(10 * 4, bytes)); return this; } @@ -173,7 +172,7 @@ public void flush() { @Override public final void setInt(long bytePos, int value) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; bytePos >>>= 2; int bufferIndex = (int) (bytePos >>> segmentSizeIntsPower); int index = (int) (bytePos & indexDivisor); @@ -182,7 +181,7 @@ public final void setInt(long bytePos, int value) { @Override public final int getInt(long bytePos) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; bytePos >>>= 2; int bufferIndex = (int) (bytePos >>> segmentSizeIntsPower); int index = (int) (bytePos & indexDivisor); @@ -191,7 +190,7 @@ public final int getInt(long bytePos) { @Override public final void setShort(long bytePos, short value) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; if (bytePos % 4 != 0 && bytePos % 4 != 2) throw new IllegalMonitorStateException("bytePos of wrong multiple for RAMInt " + bytePos); @@ -207,7 +206,7 @@ public final void setShort(long bytePos, short value) { @Override public final short getShort(long bytePos) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; if (bytePos % 4 != 0 && bytePos % 4 != 2) throw new IllegalMonitorStateException("bytePos of wrong multiple for RAMInt " + bytePos); @@ -258,7 +257,7 @@ public int getSegments() { } @Override - public DataAccess setSegmentSize(int bytes) { + DataAccess setSegmentSize(int bytes) { super.setSegmentSize(bytes); segmentSizeIntsPower = (int) (Math.log(segmentSizeInBytes / 4) / Math.log(2)); indexDivisor = segmentSizeInBytes / 4 - 1; diff --git a/core/src/main/java/com/graphhopper/storage/StorableProperties.java b/core/src/main/java/com/graphhopper/storage/StorableProperties.java index 39c3a93c291..056a21873c2 100644 --- a/core/src/main/java/com/graphhopper/storage/StorableProperties.java +++ b/core/src/main/java/com/graphhopper/storage/StorableProperties.java @@ -41,9 +41,9 @@ public class StorableProperties { private final DataAccess da; public StorableProperties(Directory dir) { - this.da = dir.create("properties"); // reduce size - da.setSegmentSize(1 << 15); + int segmentSize = 1 << 15; + this.da = dir.create("properties", segmentSize); } public synchronized boolean loadExisting() { diff --git a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java index 8347d116663..8464898290b 100644 --- a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java +++ b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java @@ -54,10 +54,6 @@ public TurnCostStorage(BaseGraph baseGraph, DataAccess turnCosts) { this.turnCosts = turnCosts; } - public void setSegmentSize(int bytes) { - turnCosts.setSegmentSize(bytes); - } - public TurnCostStorage create(long initBytes) { turnCosts.create(initBytes); return this; diff --git a/core/src/test/java/com/graphhopper/search/StringIndexTest.java b/core/src/test/java/com/graphhopper/search/StringIndexTest.java index c13adbce2ed..7c983a436e5 100644 --- a/core/src/test/java/com/graphhopper/search/StringIndexTest.java +++ b/core/src/test/java/com/graphhopper/search/StringIndexTest.java @@ -15,7 +15,7 @@ public class StringIndexTest { private StringIndex create() { - return new StringIndex(new RAMDirectory()).create(1000); + return new StringIndex(new RAMDirectory(), 1000, -1).create(1000); } Map createMap(String... strings) { @@ -164,12 +164,12 @@ public void testFlush() { String location = "./target/stringindex-store"; Helper.removeDir(new File(location)); - StringIndex index = new StringIndex(new RAMDirectory(location, true).create()).create(1000); + StringIndex index = new StringIndex(new RAMDirectory(location, true).create(), 1000, -1).create(1000); long pointer = index.add(createMap("", "test")); index.flush(); index.close(); - index = new StringIndex(new RAMDirectory(location, true)); + index = new StringIndex(new RAMDirectory(location, true), 1000, -1); assertTrue(index.loadExisting()); assertEquals("test", index.get(pointer, "")); // make sure bytePointer is correctly set after loadExisting @@ -185,7 +185,7 @@ public void testLoadKeys() { String location = "./target/stringindex-store"; Helper.removeDir(new File(location)); - StringIndex index = new StringIndex(new RAMDirectory(location, true).create()).create(1000); + StringIndex index = new StringIndex(new RAMDirectory(location, true).create(), 1000, -1).create(1000); long pointerA = index.add(createMap("c", "test value")); assertEquals(2, index.getKeys().size()); long pointerB = index.add(createMap("a", "value", "b", "another value")); @@ -194,7 +194,7 @@ public void testLoadKeys() { index.flush(); index.close(); - index = new StringIndex(new RAMDirectory(location, true)); + index = new StringIndex(new RAMDirectory(location, true), 1000, -1); assertTrue(index.loadExisting()); assertEquals("[, c, a, b]", index.getKeys().toString()); assertEquals("test value", index.get(pointerA, "c")); diff --git a/core/src/test/java/com/graphhopper/storage/CHStorageTest.java b/core/src/test/java/com/graphhopper/storage/CHStorageTest.java index e14d8d65790..6569cf364fd 100644 --- a/core/src/test/java/com/graphhopper/storage/CHStorageTest.java +++ b/core/src/test/java/com/graphhopper/storage/CHStorageTest.java @@ -71,7 +71,7 @@ public void testBigWeight() { @Test public void testLargeNodeA() { int nodeA = Integer.MAX_VALUE; - RAMIntDataAccess access = new RAMIntDataAccess("", "", false, ByteOrder.LITTLE_ENDIAN); + RAMIntDataAccess access = new RAMIntDataAccess("", "", false, ByteOrder.LITTLE_ENDIAN, -1); access.create(1000); access.setInt(0, nodeA << 1 | 1 & PrepareEncoder.getScFwdDir()); assertTrue(access.getInt(0) < 0); diff --git a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java index f82b79e1689..dd842de36c3 100644 --- a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java @@ -37,7 +37,11 @@ public abstract class DataAccessTest { protected String directory; protected String name = "dataacess"; - public abstract DataAccess createDataAccess(String location); + public DataAccess createDataAccess(String location) { + return createDataAccess(location, 128); + } + + public abstract DataAccess createDataAccess(String location, int segmentSize); @BeforeEach public void setUp() { @@ -89,11 +93,10 @@ public void testLoadFlush() { public void testExceptionIfNoEnsureCapacityWasCalled() { DataAccess da = createDataAccess(name); assertFalse(da.loadExisting()); - // throw some undefined exception if no ensureCapacity was called try { da.setInt(2 * 4, 321); - assertTrue(false); - } catch (Exception ex) { + fail(); + } catch (Throwable t) { } } @@ -154,8 +157,7 @@ public void testEnsureCapacity() { @Test public void testSegments() { - DataAccess da = createDataAccess(name); - da.setSegmentSize(128); + DataAccess da = createDataAccess(name, 128); da.create(10); assertEquals(1, da.getSegments()); da.ensureCapacity(500); @@ -175,8 +177,7 @@ public void testSegments() { @Test public void testSegmentSize() { - DataAccess da = createDataAccess(name); - da.setSegmentSize(20); + DataAccess da = createDataAccess(name, 20); assertEquals(128, da.getSegmentSize()); da.close(); } diff --git a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java index d2dab7fbd48..a9c10df705c 100644 --- a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java +++ b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java @@ -55,11 +55,6 @@ protected GraphHopperStorage newGHStorage(Directory dir, boolean enabled3D, int return GraphBuilder.start(encodingManager).setDir(dir).set3D(enabled3D).setSegmentSize(segmentSize).build(); } - @Test - public void testNoCreateCalled() { - assertThrows(Throwable.class, () -> GraphBuilder.start(encodingManager).build().edge(0, 1)); - } - @Test public void testSave_and_fileFormat() { graph = newGHStorage(new RAMDirectory(defaultGraphLoc, true), true).create(defaultSize); diff --git a/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java index 2559a84cffd..a6db8a011b4 100644 --- a/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java @@ -26,13 +26,13 @@ */ public class MMapDataAccessTest extends DataAccessTest { @Override - public DataAccess createDataAccess(String name) { - return new MMapDataAccess(name, directory, defaultOrder, true).setSegmentSize(128); + public DataAccess createDataAccess(String name, int segmentSize) { + return new MMapDataAccess(name, directory, defaultOrder, true, segmentSize); } @Test public void textMixRAM2MMAP() { - DataAccess da = new RAMDataAccess(name, directory, true, defaultOrder); + DataAccess da = new RAMDataAccess(name, directory, true, defaultOrder, -1); assertFalse(da.loadExisting()); da.create(100); da.setInt(7 * 4, 123); @@ -54,7 +54,7 @@ public void textMixMMAP2RAM() { // TODO "memory mapped flush" is expensive and not required. only writing the header is required. da.flush(); da.close(); - da = new RAMDataAccess(name, directory, true, defaultOrder); + da = new RAMDataAccess(name, directory, true, defaultOrder, -1); assertTrue(da.loadExisting()); assertEquals(123, da.getInt(7 * 4)); da.close(); diff --git a/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java index 897901f855e..cb5ca3c867e 100644 --- a/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java @@ -22,7 +22,7 @@ */ public class RAMDataAccessTest extends DataAccessTest { @Override - public DataAccess createDataAccess(String name) { - return new RAMDataAccess(name, directory, true, defaultOrder).setSegmentSize(128); + public DataAccess createDataAccess(String name, int segmentSize) { + return new RAMDataAccess(name, directory, true, defaultOrder, segmentSize); } } diff --git a/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java index 4f72fac4c79..4610d9369c9 100644 --- a/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java @@ -22,8 +22,8 @@ */ public class RAMIntDataAccessTest extends DataAccessTest { @Override - public DataAccess createDataAccess(String name) { - return new RAMIntDataAccess(name, directory, true, defaultOrder).setSegmentSize(128); + public DataAccess createDataAccess(String name, int segmentSize) { + return new RAMIntDataAccess(name, directory, true, defaultOrder, segmentSize); } @Override From 2d4567eea95027a739058bf641183552c25bd6fd Mon Sep 17 00:00:00 2001 From: easbar Date: Mon, 4 Oct 2021 17:40:57 +0200 Subject: [PATCH 16/53] Extract ReaderConfig from GraphHopper class --- .../java/com/graphhopper/GraphHopper.java | 74 +++------------- .../com/graphhopper/reader/ReaderNode.java | 2 +- .../graphhopper/reader/osm/OSMInputFile.java | 4 +- .../com/graphhopper/reader/osm/OSMReader.java | 48 +++------- .../graphhopper/routing/OSMReaderConfig.java | 88 +++++++++++++++++++ .../java/com/graphhopper/GraphHopperTest.java | 6 +- .../graphhopper/reader/osm/OSMReaderTest.java | 7 +- .../routing/RoutingAlgorithmWithOSMTest.java | 4 +- ...fficChangeWithNodeOrderingReusingTest.java | 2 +- 9 files changed, 129 insertions(+), 106 deletions(-) create mode 100644 core/src/main/java/com/graphhopper/routing/OSMReaderConfig.java diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index ff5e0ab4f1d..c1d50eb9db8 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -25,10 +25,7 @@ import com.graphhopper.reader.dem.*; import com.graphhopper.reader.osm.OSMReader; import com.graphhopper.reader.osm.conditional.DateRangeParser; -import com.graphhopper.routing.DefaultWeightingFactory; -import com.graphhopper.routing.Router; -import com.graphhopper.routing.RouterConfig; -import com.graphhopper.routing.WeightingFactory; +import com.graphhopper.routing.*; import com.graphhopper.routing.ch.CHPreparationHandler; import com.graphhopper.routing.ev.*; import com.graphhopper.routing.lm.LMConfig; @@ -100,8 +97,7 @@ public class GraphHopper { private LockFactory lockFactory = new NativeFSLockFactory(); private boolean allowWrites = true; private boolean fullyLoaded = false; - private boolean smoothElevation = false; - private double longEdgeSamplingDistance = Double.MAX_VALUE; + private final OSMReaderConfig osmReaderConfig = new OSMReaderConfig(); // for routing private final RouterConfig routerConfig = new RouterConfig(); // for index @@ -117,8 +113,6 @@ public class GraphHopper { // for data reader private String osmFile; - private double dataReaderWayPointMaxDistance = 1; - private int dataReaderWorkerThreads = 2; private ElevationProvider eleProvider = ElevationProvider.NOOP; private FlagEncoderFactory flagEncoderFactory = new DefaultFlagEncoderFactory(); private EncodedValueFactory encodedValueFactory = new DefaultEncodedValueFactory(); @@ -148,29 +142,6 @@ public GraphHopper setElevationProvider(ElevationProvider eleProvider) { return this; } - /** - * Threads for data reading. - */ - protected int getWorkerThreads() { - return dataReaderWorkerThreads; - } - - /** - * Return maximum distance (in meter) to reduce points via douglas peucker while OSM import. - */ - protected double getWayPointMaxDistance() { - return dataReaderWayPointMaxDistance; - } - - /** - * This parameter specifies how to reduce points via douglas peucker while OSM import. Higher - * value means more details, unit is meter. Default is 1. Disable via 0. - */ - public GraphHopper setWayPointMaxDistance(double wayPointMaxDistance) { - this.dataReaderWayPointMaxDistance = wayPointMaxDistance; - return this; - } - public GraphHopper setPathDetailsBuilderFactory(PathDetailsBuilderFactory pathBuilderFactory) { this.pathBuilderFactory = pathBuilderFactory; return this; @@ -283,22 +254,6 @@ public GraphHopper setElevation(boolean includeElevation) { return this; } - /** - * Sets the distance distance between elevation samples on long edges - */ - public GraphHopper setLongEdgeSamplingDistance(double longEdgeSamplingDistance) { - this.longEdgeSamplingDistance = longEdgeSamplingDistance; - return this; - } - - /** - * Sets the max elevation discrepancy between way points and the simplified polyline in meters - */ - public GraphHopper setElevationWayPointMaxDistance(double elevationWayPointMaxDistance) { - this.routerConfig.setElevationWayPointMaxDistance(elevationWayPointMaxDistance); - return this; - } - public String getGraphHopperLocation() { return ghLocation; } @@ -496,13 +451,14 @@ public GraphHopper init(GraphHopperConfig ghConfig) { lockFactory = new NativeFSLockFactory(); // elevation - this.smoothElevation = ghConfig.getBool("graph.elevation.smoothing", false); - this.longEdgeSamplingDistance = ghConfig.getDouble("graph.elevation.long_edge_sampling_distance", Double.MAX_VALUE); - setElevationWayPointMaxDistance(ghConfig.getDouble("graph.elevation.way_point_max_distance", Double.MAX_VALUE)); + osmReaderConfig.setSmoothElevation(ghConfig.getBool("graph.elevation.smoothing", osmReaderConfig.isSmoothElevation())); + osmReaderConfig.setLongEdgeSamplingDistance(ghConfig.getDouble("graph.elevation.long_edge_sampling_distance", osmReaderConfig.getLongEdgeSamplingDistance())); + osmReaderConfig.setElevationMaxWayPointDistance(ghConfig.getDouble("graph.elevation.way_point_max_distance", osmReaderConfig.getElevationMaxWayPointDistance())); + routerConfig.setElevationWayPointMaxDistance(ghConfig.getDouble("graph.elevation.way_point_max_distance", routerConfig.getElevationWayPointMaxDistance())); ElevationProvider elevationProvider = createElevationProvider(ghConfig); setElevationProvider(elevationProvider); - if (longEdgeSamplingDistance < Double.MAX_VALUE && !elevationProvider.canInterpolate()) + if (osmReaderConfig.getLongEdgeSamplingDistance() < Double.MAX_VALUE && !elevationProvider.canInterpolate()) logger.warn("Long edge sampling enabled, but bilinear interpolation disabled. See #1953"); // optimizable prepare @@ -513,9 +469,8 @@ public GraphHopper init(GraphHopperConfig ghConfig) { lmPreparationHandler.init(ghConfig); // osm import - dataReaderWayPointMaxDistance = ghConfig.getDouble(Routing.INIT_WAY_POINT_MAX_DISTANCE, dataReaderWayPointMaxDistance); - - dataReaderWorkerThreads = ghConfig.getInt("datareader.worker_threads", dataReaderWorkerThreads); + osmReaderConfig.setMaxWayPointDistance(ghConfig.getDouble(Routing.INIT_WAY_POINT_MAX_DISTANCE, osmReaderConfig.getMaxWayPointDistance())); + osmReaderConfig.setWorkerThreads(ghConfig.getInt("datareader.worker_threads", osmReaderConfig.getWorkerThreads())); // index preciseIndexResolution = ghConfig.getInt("index.high_resolution", preciseIndexResolution); @@ -691,14 +646,9 @@ protected void importOSM() { AreaIndex areaIndex = new AreaIndex<>(customAreas); logger.info("start creating graph from " + osmFile); - OSMReader reader = new OSMReader(ghStorage).setFile(_getOSMFile()). + OSMReader reader = new OSMReader(ghStorage, osmReaderConfig).setFile(_getOSMFile()). setAreaIndex(areaIndex). setElevationProvider(eleProvider). - setWorkerThreads(dataReaderWorkerThreads). - setWayPointMaxDistance(dataReaderWayPointMaxDistance). - setWayPointElevationMaxDistance(routerConfig.getElevationWayPointMaxDistance()). - setSmoothElevation(smoothElevation). - setLongEdgeSamplingDistance(longEdgeSamplingDistance). setCountryRuleFactory(countryRuleFactory); logger.info("using " + ghStorage.toString() + ", memory:" + getMemInfo()); try { @@ -1220,4 +1170,8 @@ public boolean getFullyLoaded() { public RouterConfig getRouterConfig() { return routerConfig; } + + public OSMReaderConfig getReaderConfig() { + return osmReaderConfig; + } } \ No newline at end of file diff --git a/core/src/main/java/com/graphhopper/reader/ReaderNode.java b/core/src/main/java/com/graphhopper/reader/ReaderNode.java index 65930ef59d0..6d322335c36 100644 --- a/core/src/main/java/com/graphhopper/reader/ReaderNode.java +++ b/core/src/main/java/com/graphhopper/reader/ReaderNode.java @@ -50,7 +50,7 @@ public String toString() { txt.append(getLat()); txt.append(" lon="); txt.append(getLon()); - if (!getTags().isEmpty()) { + if (hasTags()) { txt.append("\n"); txt.append(tagsToString()); } diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMInputFile.java b/core/src/main/java/com/graphhopper/reader/osm/OSMInputFile.java index 394ede97dc0..296d39f07ec 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMInputFile.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMInputFile.java @@ -75,8 +75,8 @@ public OSMInputFile open() throws XMLStreamException { /** * Currently on for pbf format. Default is number of cores. */ - public OSMInputFile setWorkerThreads(int num) { - workerThreads = num; + public OSMInputFile setWorkerThreads(int threads) { + workerThreads = threads; return this; } diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 2f551008c9d..491880e778a 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -24,6 +24,7 @@ import com.graphhopper.reader.dem.EdgeSampling; import com.graphhopper.reader.dem.ElevationProvider; import com.graphhopper.reader.dem.GraphElevationSmoothing; +import com.graphhopper.routing.OSMReaderConfig; import com.graphhopper.routing.ev.BooleanEncodedValue; import com.graphhopper.routing.ev.Country; import com.graphhopper.routing.util.AreaIndex; @@ -78,19 +79,17 @@ public class OSMReader implements TurnCostParser.ExternalInternalMap { protected static final int TOWER_NODE = -2; private static final Logger LOGGER = LoggerFactory.getLogger(OSMReader.class); private final GraphHopperStorage ghStorage; + private final OSMReaderConfig config; private final Graph graph; private final NodeAccess nodeAccess; private final LongIndexedContainer barrierNodeIds = new LongArrayList(); private final DistanceCalc distCalc = DistanceCalcEarth.DIST_EARTH; private final DouglasPeucker simplifyAlgo = new DouglasPeucker(); private CountryRuleFactory countryRuleFactory = null; - private boolean smoothElevation = false; - private double longEdgeSamplingDistance = 0; protected long zeroCounter = 0; protected PillarInfo pillarInfo; private long locations; private final EncodingManager encodingManager; - private int workerThreads = 2; // Choosing the best Map is hard. We need a memory efficient and fast solution for big data sets! // // very slow: new SparseLongLongArray @@ -106,7 +105,7 @@ public class OSMReader implements TurnCostParser.ExternalInternalMap { // stores osm way ids used by relations to identify which edge ids needs to be mapped later private GHLongHashSet osmWayIdSet = new GHLongHashSet(); private IntLongMap edgeIdToOsmWayIdMap; - private boolean doSimplify = true; + private final boolean doSimplify; private int nextTowerId = 0; private int nextPillarId = 0; // negative but increasing to avoid clash with custom created OSM files @@ -118,12 +117,17 @@ public class OSMReader implements TurnCostParser.ExternalInternalMap { private final IntsRef tempRelFlags; private final TurnCostStorage tcs; - public OSMReader(GraphHopperStorage ghStorage) { + public OSMReader(GraphHopperStorage ghStorage, OSMReaderConfig config) { this.ghStorage = ghStorage; + this.config = config; this.graph = ghStorage; this.nodeAccess = graph.getNodeAccess(); this.encodingManager = ghStorage.getEncodingManager(); + doSimplify = config.getMaxWayPointDistance() > 0; + simplifyAlgo.setMaxDistance(config.getMaxWayPointDistance()); + simplifyAlgo.setElevationMaxDistance(config.getElevationMaxWayPointDistance()); + osmNodeIdToInternalNodeMap = new GHLongIntBTree(200); osmNodeIdToNodeFlagsMap = new GHLongLongHashMap(200, .5f); osmWayIdToRouteWeightMap = new GHLongLongHashMap(200, .5f); @@ -308,7 +312,7 @@ private void writeOsmToGraph(File osmFile) { } protected OSMInput openOsmInputFile(File osmFile) throws XMLStreamException, IOException { - return new OSMInputFile(osmFile).setWorkerThreads(workerThreads).open(); + return new OSMInputFile(osmFile).setWorkerThreads(config.getWorkerThreads()).open(); } /** @@ -708,12 +712,12 @@ EdgeIteratorState addEdge(int fromIndex, int toIndex, PointList pointList, IntsR throw new AssertionError("Dimension does not match for pointList vs. nodeAccess " + pointList.getDimension() + " <-> " + nodeAccess.getDimension()); // Smooth the elevation before calculating the distance because the distance will be incorrect if calculated afterwards - if (this.smoothElevation) + if (config.isSmoothElevation()) pointList = GraphElevationSmoothing.smoothElevation(pointList); // sample points along long edges - if (this.longEdgeSamplingDistance < Double.MAX_VALUE && pointList.is3D()) - pointList = EdgeSampling.sample(pointList, longEdgeSamplingDistance, distCalc, eleProvider); + if (config.getLongEdgeSamplingDistance() < Double.MAX_VALUE && pointList.is3D()) + pointList = EdgeSampling.sample(pointList, config.getLongEdgeSamplingDistance(), distCalc, eleProvider); if (doSimplify && pointList.size() > 2) simplifyAlgo.simplify(pointList); @@ -955,37 +959,11 @@ public OSMReader setAreaIndex(AreaIndex areaIndex) { return this; } - public OSMReader setWayPointMaxDistance(double maxDist) { - doSimplify = maxDist > 0; - simplifyAlgo.setMaxDistance(maxDist); - return this; - } - - public OSMReader setWayPointElevationMaxDistance(double elevationWayPointMaxDistance) { - simplifyAlgo.setElevationMaxDistance(elevationWayPointMaxDistance); - return this; - } - - public OSMReader setSmoothElevation(boolean smoothElevation) { - this.smoothElevation = smoothElevation; - return this; - } - public OSMReader setCountryRuleFactory(CountryRuleFactory countryRuleFactory) { this.countryRuleFactory = countryRuleFactory; return this; } - public OSMReader setLongEdgeSamplingDistance(double longEdgeSamplingDistance) { - this.longEdgeSamplingDistance = longEdgeSamplingDistance; - return this; - } - - public OSMReader setWorkerThreads(int numOfWorkers) { - this.workerThreads = numOfWorkers; - return this; - } - public OSMReader setElevationProvider(ElevationProvider eleProvider) { if (eleProvider == null) throw new IllegalStateException("Use the NOOP elevation provider instead of null or don't call setElevationProvider"); diff --git a/core/src/main/java/com/graphhopper/routing/OSMReaderConfig.java b/core/src/main/java/com/graphhopper/routing/OSMReaderConfig.java new file mode 100644 index 00000000000..62a4fa07a4e --- /dev/null +++ b/core/src/main/java/com/graphhopper/routing/OSMReaderConfig.java @@ -0,0 +1,88 @@ +/* + * Licensed to GraphHopper GmbH under one or more contributor + * license agreements. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. + * + * GraphHopper GmbH licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.graphhopper.routing; + +public class OSMReaderConfig { + private double maxWayPointDistance = 1; + private double elevationMaxWayPointDistance = Double.MAX_VALUE; + private boolean smoothElevation = false; + private double longEdgeSamplingDistance = Double.MAX_VALUE; + private int workerThreads = 2; + + public double getMaxWayPointDistance() { + return maxWayPointDistance; + } + + /** + * This parameter affects the routine used to simplify the edge geometries (Douglas-Peucker). Higher values mean + * more details are preserved. The default is 1 (meter). Simplification can be disabled by setting it to 0. + */ + public OSMReaderConfig setMaxWayPointDistance(double maxWayPointDistance) { + this.maxWayPointDistance = maxWayPointDistance; + return this; + } + + public double getElevationMaxWayPointDistance() { + return elevationMaxWayPointDistance; + } + + /** + * Sets the max elevation discrepancy between way points and the simplified polyline in meters + */ + public OSMReaderConfig setElevationMaxWayPointDistance(double elevationMaxWayPointDistance) { + this.elevationMaxWayPointDistance = elevationMaxWayPointDistance; + return this; + } + + public boolean isSmoothElevation() { + return smoothElevation; + } + + /** + * Enables/disables elevation smoothing + */ + public OSMReaderConfig setSmoothElevation(boolean smoothElevation) { + this.smoothElevation = smoothElevation; + return this; + } + + public double getLongEdgeSamplingDistance() { + return longEdgeSamplingDistance; + } + + /** + * Sets the distance between elevation samples on long edges + */ + public OSMReaderConfig setLongEdgeSamplingDistance(double longEdgeSamplingDistance) { + this.longEdgeSamplingDistance = longEdgeSamplingDistance; + return this; + } + + public int getWorkerThreads() { + return workerThreads; + } + + /** + * Sets the number of threads used for the OSM import + */ + public OSMReaderConfig setWorkerThreads(int workerThreads) { + this.workerThreads = workerThreads; + return this; + } +} diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index 214109c2877..2e1b1898283 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -1107,8 +1107,10 @@ public void testSRTMWithLongEdgeSampling() { setGraphHopperLocation(GH_LOCATION). setOSMFile(MONACO). setStoreOnFlush(true). - setElevationWayPointMaxDistance(1). - setProfiles(new Profile("profile").setVehicle(vehicle).setWeighting(weighting)). + setProfiles(new Profile("profile").setVehicle(vehicle).setWeighting(weighting)); + hopper.getRouterConfig().setElevationWayPointMaxDistance(1.); + hopper.getReaderConfig(). + setElevationMaxWayPointDistance(1.). setLongEdgeSamplingDistance(30); SRTMProvider elevationProvider = new SRTMProvider(DIR); diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index 24ce7afde5d..98a9441d80a 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -27,6 +27,7 @@ import com.graphhopper.reader.ReaderWay; import com.graphhopper.reader.dem.ElevationProvider; import com.graphhopper.reader.dem.SRTMProvider; +import com.graphhopper.routing.OSMReaderConfig; import com.graphhopper.routing.ev.*; import com.graphhopper.routing.util.*; import com.graphhopper.routing.util.countryrules.CountryRuleFactory; @@ -452,7 +453,7 @@ public void testBarriersOnTowerNodes() { public void testRelation() { EncodingManager manager = EncodingManager.create("bike"); GraphHopperStorage ghStorage = new GraphHopperStorage(new RAMDirectory(), manager, false); - OSMReader reader = new OSMReader(ghStorage); + OSMReader reader = new OSMReader(ghStorage, new OSMReaderConfig()); ReaderRelation osmRel = new ReaderRelation(1); osmRel.add(new ReaderRelation.Member(ReaderRelation.WAY, 1, "")); osmRel.add(new ReaderRelation.Member(ReaderRelation.WAY, 2, "")); @@ -605,7 +606,7 @@ public void testEstimatedCenter() { lonMap.put(1, 1.0d); lonMap.put(2, 1.0d); - OSMReader osmreader = new OSMReader(ghStorage) { + OSMReader osmreader = new OSMReader(ghStorage, new OSMReaderConfig()) { // mock data access @Override double getTmpLatitude(int id) { @@ -925,7 +926,7 @@ public void testCountries() throws IOException { EncodingManager em = EncodingManager.create("car"); EnumEncodedValue roadAccessEnc = em.getEnumEncodedValue(RoadAccess.KEY, RoadAccess.class); GraphHopperStorage graph = new GraphBuilder(em).build(); - OSMReader reader = new OSMReader(graph); + OSMReader reader = new OSMReader(graph, new OSMReaderConfig()); reader.setCountryRuleFactory(new CountryRuleFactory()); reader.setAreaIndex(createCountryIndex()); // there are two edges, both with highway=track, one in Berlin, one in Paris diff --git a/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java b/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java index 401b905e05a..1cee2453c74 100644 --- a/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java +++ b/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java @@ -599,7 +599,7 @@ public void testDisconnectedAreaAndMultiplePoints() { @Test public void testMonacoParallel() throws InterruptedException { GraphHopper hopper = createHopper(MONACO, new Profile("car").setVehicle("car").setWeighting("shortest")); - hopper.setWayPointMaxDistance(0); + hopper.getReaderConfig().setMaxWayPointDistance(0); hopper.getRouterConfig().setSimplifyResponse(false); hopper.importOrLoad(); final List queries = createMonacoCarQueries(); @@ -687,7 +687,7 @@ private GraphHopper createHopper(String osmFile, Profile... profiles) { setGraphHopperLocation(GH_LOCATION); hopper.getRouterConfig().setSimplifyResponse(false); hopper.setMinNetworkSize(0); - hopper.setWayPointMaxDistance(0); + hopper.getReaderConfig().setMaxWayPointDistance(0); hopper.getLMPreparationHandler().setLMProfiles(new LMProfile(profiles[0].getName())); hopper.getCHPreparationHandler().setCHProfiles(new CHProfile(profiles[0].getName())); return hopper; diff --git a/core/src/test/java/com/graphhopper/routing/TrafficChangeWithNodeOrderingReusingTest.java b/core/src/test/java/com/graphhopper/routing/TrafficChangeWithNodeOrderingReusingTest.java index 1bb5bd74590..a398ef53b8a 100644 --- a/core/src/test/java/com/graphhopper/routing/TrafficChangeWithNodeOrderingReusingTest.java +++ b/core/src/test/java/com/graphhopper/routing/TrafficChangeWithNodeOrderingReusingTest.java @@ -87,7 +87,7 @@ public void testPerformanceForRandomTrafficChange(Fixture f) throws IOException LOGGER.info("Running performance test, max deviation percentage: " + f.maxDeviationPercentage); // read osm - OSMReader reader = new OSMReader(f.ghStorage); + OSMReader reader = new OSMReader(f.ghStorage, new OSMReaderConfig()); reader.setFile(new File(OSM_FILE)); reader.readGraph(); f.ghStorage.freeze(); From 5a394491da2e6ea0a1eeb46c4b9595fbfaf8e9d3 Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 5 Oct 2021 08:30:21 +0200 Subject: [PATCH 17/53] More OSMReader cleanup --- .../com/graphhopper/reader/osm/OSMReader.java | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 491880e778a..7bda261dda5 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -56,7 +56,7 @@ * {@link #osmNodeIdToInternalNodeMap} returns EMPTY. *

* 1. b) Reads relations from OSM file. In case that the relation is a route relation, it stores - * specific relation attributes required for routing into {@link #osmWayIdToRouteWeightMap} for all the ways + * specific relation attributes required for routing into {@link #osmWayIdToRelationFlagsMap} for all the ways * of the relation. *

* 2.a) Reads nodes from OSM file and stores lat+lon information either into the intermediate @@ -101,7 +101,7 @@ public class OSMReader implements TurnCostParser.ExternalInternalMap { // remember how many times a node was used to identify tower nodes private LongIntMap osmNodeIdToInternalNodeMap; private GHLongLongHashMap osmNodeIdToNodeFlagsMap; - private GHLongLongHashMap osmWayIdToRouteWeightMap; + private GHLongLongHashMap osmWayIdToRelationFlagsMap; // stores osm way ids used by relations to identify which edge ids needs to be mapped later private GHLongHashSet osmWayIdSet = new GHLongHashSet(); private IntLongMap edgeIdToOsmWayIdMap; @@ -130,7 +130,7 @@ public OSMReader(GraphHopperStorage ghStorage, OSMReaderConfig config) { osmNodeIdToInternalNodeMap = new GHLongIntBTree(200); osmNodeIdToNodeFlagsMap = new GHLongLongHashMap(200, .5f); - osmWayIdToRouteWeightMap = new GHLongLongHashMap(200, .5f); + osmWayIdToRelationFlagsMap = new GHLongLongHashMap(200, .5f); pillarInfo = new PillarInfo(nodeAccess.is3D(), ghStorage.getDirectory()); tempRelFlags = encodingManager.createRelationFlags(); if (tempRelFlags.length != 2) @@ -332,8 +332,6 @@ protected void processWay(ReaderWay way) { if (!encodingManager.acceptWay(way, acceptWay)) return; - IntsRef relationFlags = getRelFlagsMap(way.getId()); - // TODO move this after we have created the edge and know the coordinates => encodingManager.applyWayTags LongArrayList osmNodeIds = way.getNodes(); // Estimate length of ways containing a route tag e.g. for ferry speed calculation @@ -383,6 +381,7 @@ protected void processWay(ReaderWay way) { // also add all custom areas as artificial tag way.setTag("custom_areas", customAreas); + IntsRef relationFlags = getRelFlagsMap(way.getId()); IntsRef edgeFlags = encodingManager.handleWayTags(way, acceptWay, relationFlags); if (edgeFlags.isEmpty()) return; @@ -529,19 +528,20 @@ boolean addNode(ReaderNode node) { if (nodeType == EMPTY_NODE) return false; - double lat = node.getLat(); - double lon = node.getLon(); - double ele = eleProvider.getEle(node); if (nodeType == TOWER_NODE) { - addTowerNode(node.getId(), lat, lon, ele); + addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); } else if (nodeType == PILLAR_NODE) { - pillarInfo.setNode(nextPillarId, lat, lon, ele); - getNodeMap().put(node.getId(), nextPillarId + 3); - nextPillarId++; + addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); } return true; } + private void addPillarNode(long osmId, double lat, double lon, double ele) { + pillarInfo.setNode(nextPillarId, lat, lon, ele); + getNodeMap().put(osmId, nextPillarId + 3); + nextPillarId++; + } + /** * The nodeFlags store the encoders to check for accessibility in edgeFlags. E.g. if nodeFlags==3, then the * accessibility of the first two encoders will be check in edgeFlags @@ -817,7 +817,7 @@ protected void finishedReading() { eleProvider.release(); osmNodeIdToInternalNodeMap = null; osmNodeIdToNodeFlagsMap = null; - osmWayIdToRouteWeightMap = null; + osmWayIdToRelationFlagsMap = null; osmWayIdSet = null; edgeIdToOsmWayIdMap = null; } @@ -830,19 +830,23 @@ long addBarrierNode(long nodeId) { int graphIndex = getNodeMap().get(nodeId); if (graphIndex < TOWER_NODE) { graphIndex = -graphIndex - 3; - newNode = new ReaderNode(createNewNodeId(), nodeAccess.getLat(graphIndex), nodeAccess.getLon(graphIndex)); - } else { + newNode = new ReaderNode(createArtificialOSMNodeId(), nodeAccess.getLat(graphIndex), nodeAccess.getLon(graphIndex)); + } else if (graphIndex > -TOWER_NODE) { graphIndex = graphIndex - 3; - newNode = new ReaderNode(createNewNodeId(), pillarInfo.getLat(graphIndex), pillarInfo.getLon(graphIndex)); + newNode = new ReaderNode(createArtificialOSMNodeId(), pillarInfo.getLat(graphIndex), pillarInfo.getLon(graphIndex)); + } else { + throw new IllegalStateException("Cannot add barrier nodes for osm node ids that do not appear in ways or nodes"); } - final long id = newNode.getId(); - prepareHighwayNode(id); - addNode(newNode); - return id; + final long osmId = newNode.getId(); + if (getNodeMap().get(osmId) != -1) + throw new IllegalStateException("Artificial osm node id already exists: " + osmId); + getNodeMap().put(osmId, PILLAR_NODE); + addPillarNode(osmId, newNode.getLat(), newNode.getLon(), eleProvider.getEle(newNode)); + return osmId; } - private long createNewNodeId() { + private long createArtificialOSMNodeId() { return newUniqueOsmId++; } @@ -939,11 +943,11 @@ protected LongLongMap getNodeFlagsMap() { } int getRelFlagsMapSize() { - return osmWayIdToRouteWeightMap.size(); + return osmWayIdToRelationFlagsMap.size(); } IntsRef getRelFlagsMap(long osmId) { - long relFlagsAsLong = osmWayIdToRouteWeightMap.get(osmId); + long relFlagsAsLong = osmWayIdToRelationFlagsMap.get(osmId); tempRelFlags.ints[0] = (int) relFlagsAsLong; tempRelFlags.ints[1] = (int) (relFlagsAsLong >> 32); return tempRelFlags; @@ -951,7 +955,7 @@ IntsRef getRelFlagsMap(long osmId) { void putRelFlagsMap(long osmId, IntsRef relFlags) { long relFlagsAsLong = ((long) relFlags.ints[1] << 32) | (relFlags.ints[0] & 0xFFFFFFFFL); - osmWayIdToRouteWeightMap.put(osmId, relFlagsAsLong); + osmWayIdToRelationFlagsMap.put(osmId, relFlagsAsLong); } public OSMReader setAreaIndex(AreaIndex areaIndex) { From fe62a2d2a7e7792581fb32cebf589849bc568c27 Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 5 Oct 2021 08:38:19 +0200 Subject: [PATCH 18/53] More OSMReader cleanup: extract setArtificialWayTags --- .../com/graphhopper/reader/osm/OSMReader.java | 102 +++++++++--------- 1 file changed, 54 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 7bda261dda5..ec23faae9e0 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -332,60 +332,14 @@ protected void processWay(ReaderWay way) { if (!encodingManager.acceptWay(way, acceptWay)) return; - // TODO move this after we have created the edge and know the coordinates => encodingManager.applyWayTags - LongArrayList osmNodeIds = way.getNodes(); - // Estimate length of ways containing a route tag e.g. for ferry speed calculation - int first = getNodeMap().get(osmNodeIds.get(0)); - int last = getNodeMap().get(osmNodeIds.get(osmNodeIds.size() - 1)); - double firstLat = getTmpLatitude(first), firstLon = getTmpLongitude(first); - double lastLat = getTmpLatitude(last), lastLon = getTmpLongitude(last); - GHPoint estimatedCenter = null; - if (!Double.isNaN(firstLat) && !Double.isNaN(firstLon) && !Double.isNaN(lastLat) && !Double.isNaN(lastLon)) { - double estimatedDist = distCalc.calcDist(firstLat, firstLon, lastLat, lastLon); - // Add artificial tag for the estimated distance and center - way.setTag("estimated_distance", estimatedDist); - estimatedCenter = new GHPoint((firstLat + lastLat) / 2, (firstLon + lastLon) / 2); - way.setTag("estimated_center", estimatedCenter); - } - - if (way.getTag("duration") != null) { - try { - long dur = OSMReaderUtility.parseDuration(way.getTag("duration")); - // Provide the duration value in seconds in an artificial graphhopper specific tag: - way.setTag("duration:seconds", Long.toString(dur)); - } catch (Exception ex) { - LOGGER.warn("Parsing error in way with OSMID=" + way.getId() + " : " + ex.getMessage()); - } - } + setArtificialWayTags(way); - List customAreas = estimatedCenter == null || areaIndex == null - ? emptyList() - : areaIndex.query(estimatedCenter.lat, estimatedCenter.lon); - // special handling for countries: since they are built-in with GraphHopper they are always fed to the encodingmanager - Country country = Country.MISSING; - for (CustomArea customArea : customAreas) { - Object countryCode = customArea.getProperties().get("ISO3166-1:alpha3"); - if (countryCode == null) - continue; - if (country != Country.MISSING) - LOGGER.warn("Multiple countries found for way {}: {}, {}", way.getId(), country, countryCode); - country = Country.valueOf(countryCode.toString()); - } - way.setTag("country", country); - - if (countryRuleFactory != null) { - CountryRule countryRule = countryRuleFactory.getCountryRule(country); - if (countryRule != null) - way.setTag("country_rule", countryRule); - } - - // also add all custom areas as artificial tag - way.setTag("custom_areas", customAreas); IntsRef relationFlags = getRelFlagsMap(way.getId()); IntsRef edgeFlags = encodingManager.handleWayTags(way, acceptWay, relationFlags); if (edgeFlags.isEmpty()) return; + LongArrayList osmNodeIds = way.getNodes(); List createdEdges = new ArrayList<>(); // look for barriers along the way final int size = osmNodeIds.size(); @@ -445,6 +399,58 @@ protected void processWay(ReaderWay way) { } } + private void setArtificialWayTags(ReaderWay way) { + // TODO move this after we have created the edge and know the coordinates => encodingManager.applyWayTags + LongArrayList osmNodeIds = way.getNodes(); + // Estimate length of ways containing a route tag e.g. for ferry speed calculation + int first = getNodeMap().get(osmNodeIds.get(0)); + int last = getNodeMap().get(osmNodeIds.get(osmNodeIds.size() - 1)); + double firstLat = getTmpLatitude(first), firstLon = getTmpLongitude(first); + double lastLat = getTmpLatitude(last), lastLon = getTmpLongitude(last); + GHPoint estimatedCenter = null; + if (!Double.isNaN(firstLat) && !Double.isNaN(firstLon) && !Double.isNaN(lastLat) && !Double.isNaN(lastLon)) { + double estimatedDist = distCalc.calcDist(firstLat, firstLon, lastLat, lastLon); + // Add artificial tag for the estimated distance and center + way.setTag("estimated_distance", estimatedDist); + estimatedCenter = new GHPoint((firstLat + lastLat) / 2, (firstLon + lastLon) / 2); + way.setTag("estimated_center", estimatedCenter); + } + + if (way.getTag("duration") != null) { + try { + long dur = OSMReaderUtility.parseDuration(way.getTag("duration")); + // Provide the duration value in seconds in an artificial graphhopper specific tag: + way.setTag("duration:seconds", Long.toString(dur)); + } catch (Exception ex) { + LOGGER.warn("Parsing error in way with OSMID=" + way.getId() + " : " + ex.getMessage()); + } + } + + List customAreas = estimatedCenter == null || areaIndex == null + ? emptyList() + : areaIndex.query(estimatedCenter.lat, estimatedCenter.lon); + // special handling for countries: since they are built-in with GraphHopper they are always fed to the encodingmanager + Country country = Country.MISSING; + for (CustomArea customArea : customAreas) { + Object countryCode = customArea.getProperties().get("ISO3166-1:alpha3"); + if (countryCode == null) + continue; + if (country != Country.MISSING) + LOGGER.warn("Multiple countries found for way {}: {}, {}", way.getId(), country, countryCode); + country = Country.valueOf(countryCode.toString()); + } + way.setTag("country", country); + + if (countryRuleFactory != null) { + CountryRule countryRule = countryRuleFactory.getCountryRule(country); + if (countryRule != null) + way.setTag("country_rule", countryRule); + } + + // also add all custom areas as artificial tag + way.setTag("custom_areas", customAreas); + } + protected void processRelation(ReaderRelation relation) { if (tcs != null && relation.hasTag("type", "restriction")) storeTurnRelation(createTurnRelations(relation)); From 4d087f97a734a21f2fc1a0a75d70d9e01a930a1b Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 5 Oct 2021 13:43:26 +0200 Subject: [PATCH 19/53] More OSMReader cleanup: set edge flags where edge is created --- .../com/graphhopper/reader/osm/OSMReader.java | 47 ++++++++----------- .../graphhopper/reader/osm/OSMReaderTest.java | 8 ++-- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index ec23faae9e0..84b30f12432 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -326,8 +326,6 @@ protected void processWay(ReaderWay way) { if (!way.hasTags()) return; - long wayOsmId = way.getId(); - EncodingManager.AcceptWay acceptWay = new EncodingManager.AcceptWay(); if (!encodingManager.acceptWay(way, acceptWay)) return; @@ -340,7 +338,6 @@ protected void processWay(ReaderWay way) { return; LongArrayList osmNodeIds = way.getNodes(); - List createdEdges = new ArrayList<>(); // look for barriers along the way final int size = osmNodeIds.size(); int lastBarrier = -1; @@ -365,13 +362,13 @@ protected void processWay(ReaderWay way) { LongArrayList partNodeIds = new LongArrayList(); partNodeIds.add(osmNodeIds.buffer, lastBarrier, length); partNodeIds.set(length - 1, newNodeId); - createdEdges.addAll(addOSMWay(partNodeIds, edgeFlags, wayOsmId)); + addOSMWay(partNodeIds, edgeFlags, way); // create zero length edge for barrier - createdEdges.addAll(addBarrierEdge(newNodeId, nodeId, edgeFlags, nodeFlags, wayOsmId)); + addBarrierEdge(newNodeId, nodeId, edgeFlags, nodeFlags, way); } else { // run edge from real first node to shadow node - createdEdges.addAll(addBarrierEdge(nodeId, newNodeId, edgeFlags, nodeFlags, wayOsmId)); + addBarrierEdge(nodeId, newNodeId, edgeFlags, nodeFlags, way); // exchange first node for created barrier node osmNodeIds.set(0, newNodeId); @@ -387,15 +384,11 @@ protected void processWay(ReaderWay way) { if (lastBarrier < size - 1) { LongArrayList partNodeIds = new LongArrayList(); partNodeIds.add(osmNodeIds.buffer, lastBarrier, size - lastBarrier); - createdEdges.addAll(addOSMWay(partNodeIds, edgeFlags, wayOsmId)); + addOSMWay(partNodeIds, edgeFlags, way); } } else { // no barriers - simply add the whole way - createdEdges.addAll(addOSMWay(way.getNodes(), edgeFlags, wayOsmId)); - } - - for (EdgeIteratorState edge : createdEdges) { - encodingManager.applyWayTags(way, edge); + addOSMWay(way.getNodes(), edgeFlags, way); } } @@ -602,9 +595,8 @@ int addTowerNode(long osmId, double lat, double lon, double ele) { /** * This method creates from an OSM way (via the osm ids) one or more edges in the graph. */ - Collection addOSMWay(final LongIndexedContainer osmNodeIds, final IntsRef flags, final long wayOsmId) { + void addOSMWay(final LongIndexedContainer osmNodeIds, final IntsRef flags, final ReaderWay way) { final PointList pointList = new PointList(osmNodeIds.size(), nodeAccess.is3D()); - final List newEdges = new ArrayList<>(5); int firstNode = -1; int lastInBoundsPillarNode = -1; try { @@ -627,7 +619,7 @@ Collection addOSMWay(final LongIndexedContainer osmNodeIds, f } if (firstExisting < 0) { assert lastExisting < 0; - return newEdges; + return; } for (int i = firstExisting; i <= lastExisting; i++) { final long osmNodeId = osmNodeIds.get(i); @@ -650,7 +642,7 @@ Collection addOSMWay(final LongIndexedContainer osmNodeIds, f tmpNode = -tmpNode - 3; if (pointList.size() > 1 && firstNode >= 0) { // TOWER node - newEdges.add(addEdge(firstNode, tmpNode, pointList, flags, wayOsmId)); + addEdge(firstNode, tmpNode, pointList, flags, way); pointList.clear(); pointList.add(nodeAccess, tmpNode); } @@ -683,12 +675,12 @@ Collection addOSMWay(final LongIndexedContainer osmNodeIds, f int lastGHNodeId = getNodeMap().get(lastOsmNodeId); if (lastGHNodeId < TOWER_NODE) { LOGGER.warn("Pillar node " + lastOsmNodeId + " is already a tower node and used in loop, see #1533. " + - "Fix mapping for way " + wayOsmId + ", nodes:" + osmNodeIds); + "Fix mapping for way " + way.getId() + ", nodes:" + osmNodeIds); break; } int newEndNode = -handlePillarNode(lastGHNodeId, lastOsmNodeId, pointList, true) - 3; - newEdges.add(addEdge(firstNode, newEndNode, pointList, flags, wayOsmId)); + addEdge(firstNode, newEndNode, pointList, flags, way); pointList.clear(); pointList.add(nodeAccess, newEndNode); firstNode = newEndNode; @@ -696,7 +688,7 @@ Collection addOSMWay(final LongIndexedContainer osmNodeIds, f pointList.add(nodeAccess, tmpNode); if (firstNode >= 0) { - newEdges.add(addEdge(firstNode, tmpNode, pointList, flags, wayOsmId)); + addEdge(firstNode, tmpNode, pointList, flags, way); pointList.clear(); pointList.add(nodeAccess, tmpNode); } @@ -707,10 +699,9 @@ Collection addOSMWay(final LongIndexedContainer osmNodeIds, f LOGGER.error("Couldn't properly add edge with osm ids:" + osmNodeIds, ex); throw ex; } - return newEdges; } - EdgeIteratorState addEdge(int fromIndex, int toIndex, PointList pointList, IntsRef flags, long wayOsmId) { + void addEdge(int fromIndex, int toIndex, PointList pointList, IntsRef flags, ReaderWay way) { // sanity checks if (fromIndex < 0 || toIndex < 0) throw new AssertionError("to or from index is invalid for this edge " + fromIndex + "->" + toIndex + ", points:" + pointList); @@ -719,7 +710,7 @@ EdgeIteratorState addEdge(int fromIndex, int toIndex, PointList pointList, IntsR // Smooth the elevation before calculating the distance because the distance will be incorrect if calculated afterwards if (config.isSmoothElevation()) - pointList = GraphElevationSmoothing.smoothElevation(pointList); + GraphElevationSmoothing.smoothElevation(pointList); // sample points along long edges if (config.getLongEdgeSamplingDistance() < Double.MAX_VALUE && pointList.is3D()) @@ -739,14 +730,14 @@ EdgeIteratorState addEdge(int fromIndex, int toIndex, PointList pointList, IntsR double maxDistance = (Integer.MAX_VALUE - 1) / 1000d; if (Double.isNaN(towerNodeDistance)) { - LOGGER.warn("Bug in OSM or GraphHopper. Illegal tower node distance " + towerNodeDistance + " reset to 1m, osm way " + wayOsmId); + LOGGER.warn("Bug in OSM or GraphHopper. Illegal tower node distance " + towerNodeDistance + " reset to 1m, osm way " + way.getId()); towerNodeDistance = 1; } if (Double.isInfinite(towerNodeDistance) || towerNodeDistance > maxDistance) { // Too large is very rare and often the wrong tagging. See #435 // so we can avoid the complexity of splitting the way for now (new towernodes would be required, splitting up geometry etc) - LOGGER.warn("Bug in OSM or GraphHopper. Too big tower node distance " + towerNodeDistance + " reset to large value, osm way " + wayOsmId); + LOGGER.warn("Bug in OSM or GraphHopper. Too big tower node distance " + towerNodeDistance + " reset to large value, osm way " + way.getId()); towerNodeDistance = maxDistance; } @@ -760,10 +751,10 @@ EdgeIteratorState addEdge(int fromIndex, int toIndex, PointList pointList, IntsR checkCoordinates(toIndex, pointList.get(pointList.size() - 1)); iter.setWayGeometry(pointList.shallowCopy(1, pointList.size() - 1, false)); } + encodingManager.applyWayTags(way, iter); checkDistance(iter); - storeOsmWayID(iter.getEdge(), wayOsmId); - return iter; + storeOsmWayID(iter.getEdge(), way.getId()); } private void checkCoordinates(int nodeIndex, GHPoint point) { @@ -859,7 +850,7 @@ private long createArtificialOSMNodeId() { /** * Add a zero length edge with reduced routing options to the graph. */ - Collection addBarrierEdge(long fromId, long toId, IntsRef inEdgeFlags, long nodeFlags, long wayOsmId) { + void addBarrierEdge(long fromId, long toId, IntsRef inEdgeFlags, long nodeFlags, ReaderWay way) { IntsRef edgeFlags = IntsRef.deepCopyOf(inEdgeFlags); // clear blocked directions from flags for (BooleanEncodedValue accessEnc : encodingManager.getAccessEncFromNodeFlags(nodeFlags)) { @@ -870,7 +861,7 @@ Collection addBarrierEdge(long fromId, long toId, IntsRef inE barrierNodeIds.clear(); barrierNodeIds.add(fromId); barrierNodeIds.add(toId); - return addOSMWay(barrierNodeIds, edgeFlags, wayOsmId); + addOSMWay(barrierNodeIds, edgeFlags, way); } /** diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index 98a9441d80a..e2b8052bbff 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -46,7 +46,10 @@ import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static com.graphhopper.util.GHUtility.readCountries; import static org.junit.jupiter.api.Assertions.*; @@ -619,8 +622,7 @@ public void testEstimatedCenter() { } @Override - Collection addOSMWay(LongIndexedContainer osmNodeIds, IntsRef wayFlags, long osmId) { - return Collections.emptyList(); + void addOSMWay(LongIndexedContainer osmNodeIds, IntsRef wayFlags, ReaderWay way) { } }; From 224e67241aa39cebddf3654829b3991577dbff85 Mon Sep 17 00:00:00 2001 From: otbutz Date: Tue, 5 Oct 2021 15:07:53 +0200 Subject: [PATCH 20/53] Pass current access to the rule (#2420) Co-authored-by: Thomas Butz --- .../graphhopper/routing/util/parsers/OSMRoadAccessParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/graphhopper/routing/util/parsers/OSMRoadAccessParser.java b/core/src/main/java/com/graphhopper/routing/util/parsers/OSMRoadAccessParser.java index 41e83851dd3..b98e6660070 100644 --- a/core/src/main/java/com/graphhopper/routing/util/parsers/OSMRoadAccessParser.java +++ b/core/src/main/java/com/graphhopper/routing/util/parsers/OSMRoadAccessParser.java @@ -59,7 +59,7 @@ public IntsRef handleWayTags(IntsRef edgeFlags, ReaderWay readerWay, boolean fer CountryRule countryRule = readerWay.getTag("country_rule", null); if (countryRule != null) - accessValue = countryRule.getAccess(readerWay, TransportationMode.CAR, YES); + accessValue = countryRule.getAccess(readerWay, TransportationMode.CAR, accessValue); roadAccessEnc.setEnum(false, edgeFlags, accessValue); return edgeFlags; From 963fb8d157dcc04559844eb12d6a96b976b52a0b Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 8 Oct 2021 11:48:22 +0200 Subject: [PATCH 21/53] Minor improvement to avoid changing the url (#2428) --- .../src/main/resources/com/graphhopper/maps/js/main-template.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-bundle/src/main/resources/com/graphhopper/maps/js/main-template.js b/web-bundle/src/main/resources/com/graphhopper/maps/js/main-template.js index e387bed0bc8..4576fef8023 100644 --- a/web-bundle/src/main/resources/com/graphhopper/maps/js/main-template.js +++ b/web-bundle/src/main/resources/com/graphhopper/maps/js/main-template.js @@ -302,7 +302,7 @@ $(document).ready(function (e) { checkInput(); }, function (err) { console.log(err); - $('#error').html('GraphHopper API offline? Refresh' + '
Status: ' + err.statusText + '
' + host); + $('#error').html('GraphHopper API offline? Refresh' + '
Status: ' + err.statusText + '
' + host); bounds = { "minLon": -180, From 6ec41e0b44087af91cff65f32dc2f0184e8acae8 Mon Sep 17 00:00:00 2001 From: Michael Zilske Date: Mon, 11 Oct 2021 17:03:36 +0200 Subject: [PATCH 22/53] update example: vbb gtfs url --- reader-gtfs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reader-gtfs/README.md b/reader-gtfs/README.md index c66d7b75929..dc01f957dd9 100644 --- a/reader-gtfs/README.md +++ b/reader-gtfs/README.md @@ -12,7 +12,7 @@ git clone https://github.com/graphhopper/graphhopper cd graphhopper # download GTFS from Berlin & Brandenburg in Germany (VBB) and the 'surrounding' OpenStreetMap data for the walk network -wget -O gtfs-vbb.zip https://www.vbb.de/media/download/2029 +wget -O gtfs-vbb.zip https://www.vbb.de/fileadmin/user_upload/VBB/Dokumente/API-Datensaetze/GTFS.zip wget http://download.geofabrik.de/europe/germany/brandenburg-latest.osm.pbf mvn clean package -DskipTests From feab1d92c474cd2df5ded83cf5a10dbe13f3f6ab Mon Sep 17 00:00:00 2001 From: Michael Zilske Date: Mon, 11 Oct 2021 18:20:15 +0200 Subject: [PATCH 23/53] whatever this fixed, this fix is wrong --- .../com/graphhopper/resources/IsochroneResource.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/web-bundle/src/main/java/com/graphhopper/resources/IsochroneResource.java b/web-bundle/src/main/java/com/graphhopper/resources/IsochroneResource.java index 12a9b471e56..b2910476f88 100644 --- a/web-bundle/src/main/java/com/graphhopper/resources/IsochroneResource.java +++ b/web-bundle/src/main/java/com/graphhopper/resources/IsochroneResource.java @@ -143,13 +143,11 @@ public Response doGet( for (Double z : zs) { logger.info("Building contour z={}", z); MultiPolygon isochrone = contourBuilder.computeIsoline(z, result.seedEdges); - if (!isochrone.isEmpty()) { - if (fullGeometry) { - isochrones.add(isochrone); - } else { - Polygon maxPolygon = heuristicallyFindMainConnectedComponent(isochrone, isochrone.getFactory().createPoint(new Coordinate(point.get().lon, point.get().lat))); - isochrones.add(isochrone.getFactory().createPolygon(((LinearRing) maxPolygon.getExteriorRing()))); - } + if (fullGeometry) { + isochrones.add(isochrone); + } else { + Polygon maxPolygon = heuristicallyFindMainConnectedComponent(isochrone, isochrone.getFactory().createPoint(new Coordinate(point.get().lon, point.get().lat))); + isochrones.add(isochrone.getFactory().createPolygon(((LinearRing) maxPolygon.getExteriorRing()))); } } ArrayList features = new ArrayList<>(); From fba26ce0144e7cc0e7e8126801a452a346765509 Mon Sep 17 00:00:00 2001 From: Michael Zilske Date: Mon, 11 Oct 2021 20:14:17 +0200 Subject: [PATCH 24/53] "reverse flow" pt isochrones; fixes #2429 --- .../graphhopper/resources/PtIsochroneResource.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/web-bundle/src/main/java/com/graphhopper/resources/PtIsochroneResource.java b/web-bundle/src/main/java/com/graphhopper/resources/PtIsochroneResource.java index 6e70aae5cef..5ce7475950f 100644 --- a/web-bundle/src/main/java/com/graphhopper/resources/PtIsochroneResource.java +++ b/web-bundle/src/main/java/com/graphhopper/resources/PtIsochroneResource.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.function.Function; import java.util.function.Predicate; import javax.inject.Inject; import javax.ws.rs.DefaultValue; @@ -86,8 +85,6 @@ public class PtIsochroneResource { private final GraphHopperStorage graphHopperStorage; private final LocationIndex locationIndex; - private final Function z = label -> (double) label.currentTime; - @Inject public PtIsochroneResource(GtfsStorage gtfsStorage, EncodingManager encodingManager, GraphHopperStorage graphHopperStorage, LocationIndex locationIndex) { this.gtfsStorage = gtfsStorage; @@ -121,7 +118,7 @@ public Response doGet( throw new IllegalArgumentException(String.format(Locale.ROOT, "Illegal value for required parameter %s: [%s]", "pt.earliest_departure_time", departureTimeString)); } - double targetZ = initialTime.toEpochMilli() + seconds * 1000; + double targetZ = seconds * 1000; GeometryFactory geometryFactory = new GeometryFactory(); final FlagEncoder footEncoder = encodingManager.getEncoder("foot"); @@ -142,15 +139,15 @@ public Response doGet( MultiCriteriaLabelSetting.SPTVisitor sptVisitor = nodeLabel -> { Coordinate nodeCoordinate = new Coordinate(nodeAccess.getLon(nodeLabel.adjNode), nodeAccess.getLat(nodeLabel.adjNode)); - z1.merge(nodeCoordinate, this.z.apply(nodeLabel), Math::min); + z1.merge(nodeCoordinate, (double) (nodeLabel.currentTime - initialTime.toEpochMilli()) * (reverseFlow ? -1 : 1), Math::min); }; if (format.equals("multipoint")) { - calcLabels(router, snap.getClosestNode(), initialTime, sptVisitor, label -> label.currentTime <= targetZ); + calcLabels(router, snap.getClosestNode(), initialTime, sptVisitor, label -> (label.currentTime - initialTime.toEpochMilli()) * (reverseFlow ? -1 : 1) <= targetZ); MultiPoint exploredPoints = geometryFactory.createMultiPointFromCoords(z1.keySet().toArray(new Coordinate[0])); return wrap(exploredPoints); } else { - calcLabels(router, snap.getClosestNode(), initialTime, sptVisitor, label -> label.currentTime <= targetZ); + calcLabels(router, snap.getClosestNode(), initialTime, sptVisitor, label -> (label.currentTime - initialTime.toEpochMilli()) * (reverseFlow ? -1 : 1) <= targetZ); MultiPoint exploredPoints = geometryFactory.createMultiPointFromCoords(z1.keySet().toArray(new Coordinate[0])); // Get at least all nodes within our bounding box (I think convex hull would be enough.) From 03399bc2733293d9df17fcf8858a7cb73ed6bf1b Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 12 Oct 2021 07:34:17 +0200 Subject: [PATCH 25/53] Skip duplicate node lookup in OSM reader --- .../com/graphhopper/reader/osm/OSMReader.java | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 84b30f12432..98d5684fe57 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -42,7 +42,10 @@ import javax.xml.stream.XMLStreamException; import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; import static com.graphhopper.util.Helper.nf; import static java.util.Collections.emptyList; @@ -263,15 +266,11 @@ private void writeOsmToGraph(File osmFile) { long relationStart = -1; long counter = 1; try (OSMInput in = openOsmInputFile(osmFile)) { - LongIntMap nodeFilter = getNodeMap(); - ReaderElement item; while ((item = in.getNext()) != null) { switch (item.getType()) { case ReaderElement.NODE: - if (nodeFilter.get(item.getId()) != EMPTY_NODE) { - processNode((ReaderNode) item); - } + processNode((ReaderNode) item); break; case ReaderElement.WAY: @@ -510,7 +509,14 @@ public int getInternalNodeIdOfOsmNode(long nodeOsmId) { } protected void processNode(ReaderNode node) { - addNode(node); + int nodeType = getNodeMap().get(node.getId()); + if (nodeType == EMPTY_NODE) { + return; + } else if (nodeType == TOWER_NODE) { + addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); + } else if (nodeType == PILLAR_NODE) { + addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); + } // analyze node tags for barriers if (node.hasTags()) { @@ -522,19 +528,6 @@ protected void processNode(ReaderNode node) { locations++; } - boolean addNode(ReaderNode node) { - int nodeType = getNodeMap().get(node.getId()); - if (nodeType == EMPTY_NODE) - return false; - - if (nodeType == TOWER_NODE) { - addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); - } else if (nodeType == PILLAR_NODE) { - addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); - } - return true; - } - private void addPillarNode(long osmId, double lat, double lon, double ele) { pillarInfo.setNode(nextPillarId, lat, lon, ele); getNodeMap().put(osmId, nextPillarId + 3); From 971d686daa4c31633fdb25600972f836ab53d434 Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 12 Oct 2021 08:28:34 +0200 Subject: [PATCH 26/53] Remove no longer needed artificial tag estimated_center in OSMReader --- core/src/main/java/com/graphhopper/reader/osm/OSMReader.java | 1 - .../test/java/com/graphhopper/reader/osm/OSMReaderTest.java | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 98d5684fe57..fa9923250e1 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -405,7 +405,6 @@ private void setArtificialWayTags(ReaderWay way) { // Add artificial tag for the estimated distance and center way.setTag("estimated_distance", estimatedDist); estimatedCenter = new GHPoint((firstLat + lastLat) / 2, (firstLon + lastLon) / 2); - way.setTag("estimated_center", estimatedCenter); } if (way.getTag("duration") != null) { diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index e2b8052bbff..3a838883139 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -597,7 +597,7 @@ public void testRoadAttributes() { } @Test - public void testEstimatedCenter() { + public void testEstimatedDistance() { final CarFlagEncoder encoder = new CarFlagEncoder(); EncodingManager manager = EncodingManager.create(encoder); GraphHopperStorage ghStorage = newGraph(dir, manager, false, false); @@ -634,9 +634,6 @@ void addOSMWay(LongIndexedContainer osmNodeIds, IntsRef wayFlags, ReaderWay way) osmreader.getNodeMap().put(2, 2); osmreader.processWay(way); - GHPoint p = way.getTag("estimated_center", null); - assertEquals(1.15, p.lat, 1e-3); - assertEquals(1.0, p.lon, 1e-3); Double d = way.getTag("estimated_distance", null); assertEquals(11119.5, d, 1e-1); } From 2b7c2c54f49c1b41e2ea3ccb5a6f8ecc68caa882 Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 12 Oct 2021 10:11:43 +0200 Subject: [PATCH 27/53] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97005f16e10..cf3c7ae09e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ### 5.0 [not yet released] - consider heading when snapping coordinates to the road network, this is especially important for navigation (#2411) +- OSMReader no longer sets the artificial 'estimated_center' tag and processNode also receives EMPTY_NODEs (971d686) ### 4.0 [29 Sep 2021] From f438c37c985691d8a1bdfcd20e8d493b87ae5bf8 Mon Sep 17 00:00:00 2001 From: easbar Date: Thu, 14 Oct 2021 07:17:05 +0200 Subject: [PATCH 28/53] Add some routing tests that involve barrier edges, including some barriers at junctions --- .../java/com/graphhopper/GraphHopperTest.java | 68 ++++++++++++++ .../graphhopper/reader/osm/OSMReaderTest.java | 21 +++++ .../graphhopper/reader/osm/test-barriers.xml | 9 +- .../graphhopper/reader/osm/test-barriers2.xml | 88 +++++++++++++++++++ 4 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 core/src/test/resources/com/graphhopper/reader/osm/test-barriers2.xml diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index 2e1b1898283..a5ec147854e 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -2310,6 +2310,74 @@ public void issue2306_2() { } } + @Test + public void testBarriers() { + GraphHopper hopper = new GraphHopper(). + setGraphHopperLocation(GH_LOCATION). + setOSMFile("../map-matching/files/leipzig_germany.osm.pbf"). + setProfiles( + new Profile("car").setVehicle("car").setWeighting("fastest"), + new Profile("bike").setVehicle("bike").setWeighting("fastest"), + new Profile("foot").setVehicle("foot").setWeighting("fastest") + ). + setMinNetworkSize(0); + hopper.importOrLoad(); + + { + // the bollard blocks the road for bikes, and we need to take a big detour. note that this bollard connects + // two ways + GHResponse bikeRsp = hopper.route(new GHRequest(51.257709, 12.309269, 51.257594, 12.308882).setProfile("bike")); + assertEquals(1185, bikeRsp.getBest().getDistance(), 1); + // pedestrians can just pass the bollard + GHResponse footRsp = hopper.route(new GHRequest(51.257709, 12.309269, 51.257594, 12.308882).setProfile("foot")); + assertEquals(28, footRsp.getBest().getDistance(), 1); + } + + { + // here the bollard blocks the road for cars + GHResponse carRsp = hopper.route(new GHRequest(51.301113, 12.432168, 51.30123, 12.431728).setProfile("car")); + assertEquals(368, carRsp.getBest().getDistance(), 1); + // ... but not for bikes + GHResponse bikeRsp = hopper.route(new GHRequest(51.301113, 12.432168, 51.30123, 12.431728).setProfile("bike")); + assertEquals(48, bikeRsp.getBest().getDistance(), 1); + } + + { + // cars need to take a detour to the south (on newer maps an even bigger detour going north is necessary) + GHResponse carRsp = hopper.route(new GHRequest(51.350105, 12.289968, 51.350246, 12.287779).setProfile("car")); + assertEquals(285, carRsp.getBest().getDistance(), 1); + // ... bikes can just pass the bollard + GHResponse bikeRsp = hopper.route(new GHRequest(51.350105, 12.289968, 51.350246, 12.287779).setProfile("bike")); + assertEquals(152, bikeRsp.getBest().getDistance(), 1); + } + + { + // these are bollards that are located right on a junction. this should never happen according to OSM mapping + // rules, but it still does. the problem with such barriers is that we can only block one direction and it + // is unclear which one is right + + // here the barrier node actually disconnects a dead-end road that should rather be connected! + GHResponse carRsp = hopper.route(new GHRequest(51.327121, 12.572396, 51.327173, 12.574038).setProfile("car")); + assertTrue(carRsp.hasErrors() && carRsp.getErrors().toString().contains("Connection between locations not found"), carRsp.getErrors().toString()); + GHResponse bikeRsp = hopper.route(new GHRequest(51.327121, 12.572396, 51.327173, 12.574038).setProfile("bike")); + assertEquals(124, bikeRsp.getBest().getDistance(), 1); + + // Here the barrier node prevents us from travelling straight along Pufendorfstraße. Entering Pufendorfstraße + // from 'An der Streuobstwiese' is allowed though, but this seems wrong. We probably add a wrong barrier + // edge here (the mapping was fixed in newer OSM versions, so the barrier is no longer at the junction) + carRsp = hopper.route(new GHRequest(51.344134, 12.317986, 51.344231, 12.317482).setProfile("car")); + assertEquals(393, carRsp.getBest().getDistance(), 1); + bikeRsp = hopper.route(new GHRequest(51.344134, 12.317986, 51.344231, 12.317482).setProfile("bike")); + assertEquals(36, bikeRsp.getBest().getDistance(), 1); + + // Here we have to go all the way around, but not sure if this is the intended effect of this bollard node + carRsp = hopper.route(new GHRequest(51.355455, 12.40202, 51.355318, 12.401741).setProfile("car")); + assertEquals(1217, carRsp.getBest().getDistance(), 1); + bikeRsp = hopper.route(new GHRequest(51.355455, 12.40202, 51.355318, 12.401741).setProfile("bike")); + assertEquals(24, bikeRsp.getBest().getDistance(), 1); + } + } + @Test public void germanyCountryRuleAvoidsTracks() { final String profile = "profile"; diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index 3a838883139..19434bc6f9e 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -380,6 +380,27 @@ public void testBarriers() { assertFalse(iter.next()); } + @Test + public void testBarrierBetweenWays() { + GraphHopper hopper = new GraphHopperFacade("test-barriers2.xml"). + setMinNetworkSize(0). + importOrLoad(); + + Graph graph = hopper.getGraphHopperStorage(); + // there are seven ways, but there should also be five barrier edges + // note that because of the extra edge at the loop way we do not split the loop + assertEquals(12, graph.getEdges()); + int loops = 0; + AllEdgesIterator iter = graph.getAllEdges(); + while (iter.next()) { + // there are 'loop' edges, but only between different nodes + assertNotEquals(iter.getBaseNode(), iter.getAdjNode()); + if (graph.getNodeAccess().getLat(iter.getBaseNode()) == graph.getNodeAccess().getLat(iter.getAdjNode())) + loops++; + } + assertEquals(5 + 1, loops); + } + @Test public void avoidsLoopEdges_1525() { // loops in OSM should be avoided by adding additional tower node (see #1525, #1531) diff --git a/core/src/test/resources/com/graphhopper/reader/osm/test-barriers.xml b/core/src/test/resources/com/graphhopper/reader/osm/test-barriers.xml index b525437cfe1..1465a4a77bb 100644 --- a/core/src/test/resources/com/graphhopper/reader/osm/test-barriers.xml +++ b/core/src/test/resources/com/graphhopper/reader/osm/test-barriers.xml @@ -22,9 +22,12 @@ - + diff --git a/core/src/test/resources/com/graphhopper/reader/osm/test-barriers2.xml b/core/src/test/resources/com/graphhopper/reader/osm/test-barriers2.xml new file mode 100644 index 00000000000..02d97b2cffc --- /dev/null +++ b/core/src/test/resources/com/graphhopper/reader/osm/test-barriers2.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 9c80d6f105d17bb810a47511f56ec677d52d00aa Mon Sep 17 00:00:00 2001 From: easbar Date: Thu, 14 Oct 2021 08:35:39 +0200 Subject: [PATCH 29/53] More OSMReader cleanup: Use isTower/PillarNode --- .../com/graphhopper/reader/osm/OSMReader.java | 64 ++++++++++--------- .../routing/util/AbstractFlagEncoder.java | 4 +- .../routing/util/parsers/TurnCostParser.java | 2 +- .../graphhopper/reader/osm/OSMReaderTest.java | 2 + .../parsers/OSMTurnRelationParserTest.java | 2 +- 5 files changed, 40 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index fa9923250e1..56fd9165dc8 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -76,9 +76,7 @@ */ public class OSMReader implements TurnCostParser.ExternalInternalMap { protected static final int EMPTY_NODE = -1; - // pillar node is >= 3 protected static final int PILLAR_NODE = 1; - // tower node is <= -3 protected static final int TOWER_NODE = -2; private static final Logger LOGGER = LoggerFactory.getLogger(OSMReader.class); private final GraphHopperStorage ghStorage; @@ -451,7 +449,7 @@ void storeTurnRelation(List turnRelations) { for (OSMTurnRelation turnRelation : turnRelations) { int viaNode = getInternalNodeIdOfOsmNode(turnRelation.getViaOsmNodeId()); // street with restriction was not included (access or tag limits etc) - if (viaNode != EMPTY_NODE) + if (viaNode >= 0) encodingManager.handleTurnRelationTags(turnRelation, this, graph); } } @@ -468,21 +466,20 @@ public long getOsmIdOfInternalEdge(int edgeId) { @Override public int getInternalNodeIdOfOsmNode(long nodeOsmId) { int id = getNodeMap().get(nodeOsmId); - if (id < TOWER_NODE) + if (isTowerNode(id)) return -id - 3; - return EMPTY_NODE; + return -1; } // TODO remove this ugly stuff via better preprocessing phase! E.g. putting every tags etc into a helper file! double getTmpLatitude(int id) { if (id == EMPTY_NODE) return Double.NaN; - if (id < TOWER_NODE) { - // tower node + if (isTowerNode(id)) { id = -id - 3; return nodeAccess.getLat(id); - } else if (id > -TOWER_NODE) { + } else if (isPillarNode(id)) { // pillar node id = id - 3; return pillarInfo.getLat(id); @@ -494,12 +491,10 @@ public int getInternalNodeIdOfOsmNode(long nodeOsmId) { double getTmpLongitude(int id) { if (id == EMPTY_NODE) return Double.NaN; - if (id < TOWER_NODE) { - // tower node + if (isTowerNode(id)) { id = -id - 3; return nodeAccess.getLon(id); - } else if (id > -TOWER_NODE) { - // pillar node + } else if (isPillarNode(id)) { id = id - 3; return pillarInfo.getLon(id); } else @@ -596,15 +591,15 @@ void addOSMWay(final LongIndexedContainer osmNodeIds, final IntsRef flags, final int firstExisting = -1; int lastExisting = -1; for (int i = 0; i < osmNodeIds.size(); ++i) { - final long tmpNode = getNodeMap().get(osmNodeIds.get(i)); - if (tmpNode > -TOWER_NODE || tmpNode < TOWER_NODE) { + final int tmpNode = getNodeMap().get(osmNodeIds.get(i)); + if (isNodeId(tmpNode)) { firstExisting = i; break; } } for (int i = osmNodeIds.size() - 1; i >= 0; --i) { - final long tmpNode = getNodeMap().get(osmNodeIds.get(i)); - if (tmpNode > -TOWER_NODE || tmpNode < TOWER_NODE) { + final int tmpNode = getNodeMap().get(osmNodeIds.get(i)); + if (isNodeId(tmpNode)) { lastExisting = i; break; } @@ -625,9 +620,9 @@ void addOSMWay(final LongIndexedContainer osmNodeIds, final IntsRef flags, final if (tmpNode == PILLAR_NODE) { // In some cases no node information is saved for the specified osmId. - // ie. a way references a which does not exist in the current file. + // i.e. a way references a which does not exist in the current file. // => if the node before was a pillar node then convert into to tower node (as it is also end-standing). - if (!pointList.isEmpty() && lastInBoundsPillarNode > -TOWER_NODE) { + if (!pointList.isEmpty() && isPillarNode(lastInBoundsPillarNode)) { // transform the pillar node to a tower node tmpNode = lastInBoundsPillarNode; tmpNode = handlePillarNode(tmpNode, osmNodeId, null, true); @@ -644,28 +639,25 @@ void addOSMWay(final LongIndexedContainer osmNodeIds, final IntsRef flags, final continue; } - if (tmpNode <= -TOWER_NODE && tmpNode >= TOWER_NODE) + if (!isNodeId(tmpNode)) throw new AssertionError("Mapped index not in correct bounds " + tmpNode + ", " + osmNodeId); - if (tmpNode > -TOWER_NODE) { + if (isPillarNode(tmpNode)) { + // PILLAR node, but convert to towerNode if end-standing boolean convertToTowerNode = i == firstExisting || i == lastExisting; - if (!convertToTowerNode) { + if (!convertToTowerNode) lastInBoundsPillarNode = tmpNode; - } - - // PILLAR node, but convert to towerNode if end-standing tmpNode = handlePillarNode(tmpNode, osmNodeId, pointList, convertToTowerNode); } - if (tmpNode < TOWER_NODE) { - // TOWER node + if (isTowerNode(tmpNode)) { tmpNode = -tmpNode - 3; if (firstNode >= 0 && firstNode == tmpNode) { // loop detected. See #1525 and #1533. Insert last OSM ID as tower node. Do this for all loops so that users can manipulate loops later arbitrarily. long lastOsmNodeId = osmNodeIds.get(i - 1); int lastGHNodeId = getNodeMap().get(lastOsmNodeId); - if (lastGHNodeId < TOWER_NODE) { + if (isTowerNode(lastGHNodeId)) { LOGGER.warn("Pillar node " + lastOsmNodeId + " is already a tower node and used in loop, see #1533. " + "Fix mapping for way " + way.getId() + ", nodes:" + osmNodeIds); break; @@ -817,10 +809,10 @@ protected void finishedReading() { long addBarrierNode(long nodeId) { ReaderNode newNode; int graphIndex = getNodeMap().get(nodeId); - if (graphIndex < TOWER_NODE) { + if (isTowerNode(graphIndex)) { graphIndex = -graphIndex - 3; newNode = new ReaderNode(createArtificialOSMNodeId(), nodeAccess.getLat(graphIndex), nodeAccess.getLon(graphIndex)); - } else if (graphIndex > -TOWER_NODE) { + } else if (isPillarNode(graphIndex)) { graphIndex = graphIndex - 3; newNode = new ReaderNode(createArtificialOSMNodeId(), pillarInfo.getLat(graphIndex), pillarInfo.getLon(graphIndex)); } else { @@ -992,4 +984,18 @@ public Date getDataDate() { public String toString() { return getClass().getSimpleName(); } + + private boolean isTowerNode(int id) { + // tower nodes are indexed -3, -4, -5, ... + return id < TOWER_NODE; + } + + private boolean isPillarNode(int id) { + // pillar nodes are indexed 3, 4, 5, .. + return id > -TOWER_NODE; + } + + private boolean isNodeId(int id) { + return id > -TOWER_NODE || id < TOWER_NODE; + } } diff --git a/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java b/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java index f5cf9579396..72a1903c96b 100644 --- a/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java +++ b/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java @@ -165,9 +165,7 @@ public int getMaxTurnCosts() { public long handleNodeTags(ReaderNode node) { boolean blockByDefault = node.hasTag("barrier", blockByDefaultBarriers); if (blockByDefault || node.hasTag("barrier", passByDefaultBarriers)) { - boolean locked = false; - if (node.hasTag("locked", "yes")) - locked = true; + boolean locked = node.hasTag("locked", "yes"); for (String res : restrictions) { if (!locked && node.hasTag(res, intendedValues)) diff --git a/core/src/main/java/com/graphhopper/routing/util/parsers/TurnCostParser.java b/core/src/main/java/com/graphhopper/routing/util/parsers/TurnCostParser.java index b39e947548d..cd9e4035f1b 100644 --- a/core/src/main/java/com/graphhopper/routing/util/parsers/TurnCostParser.java +++ b/core/src/main/java/com/graphhopper/routing/util/parsers/TurnCostParser.java @@ -37,7 +37,7 @@ public interface TurnCostParser { /** * This map associates the internal GraphHopper nodes IDs with external IDs (OSM) and similarly for the edge IDs - * required to write the turn costs. + * required to write the turn costs. Returns -1 if there is no entry for the given OSM ID. */ interface ExternalInternalMap { int getInternalNodeIdOfOsmNode(long nodeOsmId); diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index 19434bc6f9e..bde2fbd53aa 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -253,6 +253,8 @@ public void testWayReferencesNotExistingAdjNode_issue19() { Graph graph = hopper.getGraphHopperStorage(); assertEquals(2, graph.getNodes()); + // the missing node is ignored, but the separated nodes are still connected + assertEquals(1, graph.getEdges()); int n10 = AbstractGraphStorageTester.getIdOf(graph, 51.2492152); int n30 = AbstractGraphStorageTester.getIdOf(graph, 51.2); diff --git a/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTurnRelationParserTest.java b/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTurnRelationParserTest.java index 056f20dda90..cd9c1eb03fc 100644 --- a/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTurnRelationParserTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTurnRelationParserTest.java @@ -37,7 +37,7 @@ public void testGetRestrictionAsEntries() { @Override public int getInternalNodeIdOfOsmNode(long nodeOsmId) { - return osmNodeToInternal.get(nodeOsmId); + return osmNodeToInternal.getOrDefault(nodeOsmId, -1); } @Override From 598a6a9868c45a56b5718c4d4d214f548e3996bb Mon Sep 17 00:00:00 2001 From: easbar Date: Sat, 16 Oct 2021 08:01:56 +0200 Subject: [PATCH 30/53] Remove OSMIDMap --- .../java/com/graphhopper/coll/OSMIDMap.java | 138 ------------------ .../com/graphhopper/coll/OSMIDMapTest.java | 138 ------------------ 2 files changed, 276 deletions(-) delete mode 100644 core/src/main/java/com/graphhopper/coll/OSMIDMap.java delete mode 100644 core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java diff --git a/core/src/main/java/com/graphhopper/coll/OSMIDMap.java b/core/src/main/java/com/graphhopper/coll/OSMIDMap.java deleted file mode 100644 index 08f8868def5..00000000000 --- a/core/src/main/java/com/graphhopper/coll/OSMIDMap.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to GraphHopper GmbH under one or more contributor - * license agreements. See the NOTICE file distributed with this work for - * additional information regarding copyright ownership. - * - * GraphHopper GmbH licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except in - * compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.graphhopper.coll; - -import com.graphhopper.storage.DataAccess; -import com.graphhopper.storage.Directory; -import com.graphhopper.util.BitUtil; -import com.graphhopper.util.Helper; - -/** - * This is a special purpose map for writing increasing OSM IDs with consecutive values. It stores - * a map from long to int in a memory friendly way and but does NOT provide O(1) access. - *

- * - * @author Peter Karich - */ -public class OSMIDMap implements LongIntMap { - private static final BitUtil bitUtil = BitUtil.LITTLE; - private final DataAccess keys; - private final DataAccess values; - private final int noEntryValue; - private final Directory dir; - private long lastKey = Long.MIN_VALUE; - private long size; - - public OSMIDMap(Directory dir) { - this(dir, -1); - } - - public OSMIDMap(Directory dir, int noNumber) { - this.dir = dir; - this.noEntryValue = noNumber; - keys = dir.create("osmid_map_keys"); - keys.create(2000); - values = dir.create("osmid_map_values"); - values.create(1000); - } - - static long binarySearch(DataAccess da, long start, long len, long key) { - long high = start + len, low = start - 1, guess; - byte[] longBytes = new byte[8]; - while (high - low > 1) { - // use >>> for average or we could get an integer overflow. - guess = (high + low) >>> 1; - long tmp = guess << 3; - da.getBytes(tmp, longBytes, 8); - long guessedKey = bitUtil.toLong(longBytes); - if (guessedKey < key) - low = guess; - else - high = guess; - } - - if (high == start + len) - return ~(start + len); - - long tmp = high << 3; - da.getBytes(tmp, longBytes, 8); - long highKey = bitUtil.toLong(longBytes); - if (highKey == key) - return high; - else - return ~high; - } - - public void remove() { - dir.remove(keys); - } - - @Override - public int put(long key, int value) { - if (key <= lastKey) { - long oldValueIndex = binarySearch(keys, 0, getSize(), key); - if (oldValueIndex < 0) { - throw new IllegalStateException("Cannot insert keys lower than " - + "the last key " + key + " < " + lastKey + ". Only updating supported"); - } - oldValueIndex *= 4; - int oldValue = values.getInt(oldValueIndex); - values.setInt(oldValueIndex, value); - return oldValue; - } - - values.ensureCapacity(size + 4); - values.setInt(size, value); - long doubleSize = size * 2; - keys.ensureCapacity(doubleSize + 8); - - // store long => double of the orig size - byte[] longBytes = bitUtil.fromLong(key); - keys.setBytes(doubleSize, longBytes, 8); - lastKey = key; - size += 4; - return -1; - } - - @Override - public int get(long key) { - long retIndex = binarySearch(keys, 0, getSize(), key); - if (retIndex < 0) - return noEntryValue; - - return values.getInt(retIndex * 4); - } - - @Override - public long getSize() { - return size / 4; - } - - public long getCapacity() { - return keys.getCapacity(); - } - - @Override - public int getMemoryUsage() { - return Math.round(getCapacity() / Helper.MB); - } - - @Override - public void optimize() { - } -} diff --git a/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java b/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java deleted file mode 100644 index d035a0de0bc..00000000000 --- a/core/src/test/java/com/graphhopper/coll/OSMIDMapTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to GraphHopper GmbH under one or more contributor - * license agreements. See the NOTICE file distributed with this work for - * additional information regarding copyright ownership. - * - * GraphHopper GmbH licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except in - * compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.graphhopper.coll; - -import com.graphhopper.storage.DataAccess; -import com.graphhopper.storage.RAMDirectory; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author Peter Karich - */ -public class OSMIDMapTest { - @Test - public void testGet() { - OSMIDMap map = new OSMIDMap(new RAMDirectory()); - map.put(9, 0); - map.put(10, -50); - map.put(11, 2); - map.put(12, 3); - map.put(20, 6); - map.put(21, 5); - map.put(31, 2); - - assertEquals(7, map.getSize()); - assertEquals(-1, map.get(8)); - assertEquals(0, map.get(9)); - assertEquals(-50, map.get(10)); - assertEquals(2, map.get(11)); - assertEquals(3, map.get(12)); - assertEquals(-1, map.get(13)); - assertEquals(-1, map.get(19)); - assertEquals(6, map.get(20)); - assertEquals(5, map.get(21)); - assertEquals(2, map.get(31)); - assertEquals(-1, map.get(32)); - - for (int i = 0; i < 50; i++) { - map.put(i + 50, i + 7); - } - assertEquals(57, map.getSize()); - } - - @Test - public void testBinSearch() { - DataAccess da = new RAMDirectory().create(""); - da.create(100); - - da.setInt(0 * 4, 1); - da.setInt(1 * 4, 0); - - da.setInt(2 * 4, 5); - da.setInt(3 * 4, 0); - - da.setInt(4 * 4, 100); - da.setInt(5 * 4, 0); - - da.setInt(6 * 4, 300); - da.setInt(7 * 4, 0); - - da.setInt(8 * 4, 333); - da.setInt(9 * 4, 0); - - assertEquals(2, OSMIDMap.binarySearch(da, 0, 5, 100)); - assertEquals(3, OSMIDMap.binarySearch(da, 0, 5, 300)); - assertEquals(~3, OSMIDMap.binarySearch(da, 0, 5, 200)); - assertEquals(0, OSMIDMap.binarySearch(da, 0, 5, 1)); - assertEquals(1, OSMIDMap.binarySearch(da, 0, 5, 5)); - } - - @Test - public void testGetLong() { - OSMIDMap map = new OSMIDMap(new RAMDirectory()); - map.put(12, 0); - map.put(Long.MAX_VALUE / 10, 1); - map.put(Long.MAX_VALUE / 9, 2); - map.put(Long.MAX_VALUE / 7, 3); - - assertEquals(1, map.get(Long.MAX_VALUE / 10)); - assertEquals(3, map.get(Long.MAX_VALUE / 7)); - assertEquals(-1, map.get(13)); - } - - @Test - public void testGet2() { - OSMIDMap map = new OSMIDMap(new RAMDirectory()); - map.put(9, 0); - map.put(10, 1); - map.put(11, 2); - map.put(12, 3); - map.put(13, 4); - map.put(14, 5); - map.put(16, 6); - map.put(18, 7); - map.put(19, 8); - - assertEquals(9, map.getSize()); - assertEquals(-1, map.get(8)); - assertEquals(0, map.get(9)); - assertEquals(1, map.get(10)); - assertEquals(2, map.get(11)); - assertEquals(3, map.get(12)); - assertEquals(4, map.get(13)); - assertEquals(5, map.get(14)); - assertEquals(6, map.get(16)); - assertEquals(-1, map.get(17)); - assertEquals(7, map.get(18)); - assertEquals(8, map.get(19)); - } - - @Test - public void testUpdateOfLowerKeys() { - OSMIDMap map = new OSMIDMap(new RAMDirectory()); - map.put(9, 0); - map.put(10, 1); - map.put(11, 2); - map.put(9, 3); - - assertEquals(2, map.get(11)); - assertEquals(3, map.get(9)); - } -} From 98cdf7e9986161ba10035d78eec00dc0c0a14b45 Mon Sep 17 00:00:00 2001 From: Andi Date: Sun, 17 Oct 2021 10:46:04 +0200 Subject: [PATCH 31/53] Remove AbstractFlagEncoder#encoderBit, replace AbstractFlagEncoder#handleNodeTags with isBarrier (#2434) --- CHANGELOG.md | 1 + .../routing/util/AbstractFlagEncoder.java | 43 ++++-------------- .../routing/util/BikeCommonFlagEncoder.java | 4 +- .../routing/util/CarFlagEncoder.java | 4 +- .../routing/util/EncodingManager.java | 13 +++--- .../routing/util/FootFlagEncoder.java | 4 +- .../routing/util/MotorcycleFlagEncoder.java | 4 +- .../util/AbstractBikeFlagEncoderTester.java | 16 +++---- .../routing/util/BikeFlagEncoderTest.java | 6 +-- .../routing/util/CarFlagEncoderTest.java | 24 +++++----- .../routing/util/EncodingManagerTest.java | 28 +----------- .../routing/util/FootFlagEncoderTest.java | 44 +++++++++---------- .../util/MountainBikeFlagEncoderTest.java | 6 +-- .../util/WheelchairFlagEncoderTest.java | 34 +++++++------- 14 files changed, 89 insertions(+), 142 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf3c7ae09e1..20d14b3769c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 5.0 [not yet released] +- AbstractFlagEncoder#handleNodeTags was replaced by AbstractFlagEncoder#isBarrier (#2434) - consider heading when snapping coordinates to the road network, this is especially important for navigation (#2411) - OSMReader no longer sets the artificial 'estimated_center' tag and processNode also receives EMPTY_NODEs (971d686) diff --git a/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java b/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java index 72a1903c96b..b4a912c19d9 100644 --- a/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java +++ b/core/src/main/java/com/graphhopper/routing/util/AbstractFlagEncoder.java @@ -52,7 +52,6 @@ public abstract class AbstractFlagEncoder implements FlagEncoder { protected final int speedBits; protected final double speedFactor; private final int maxTurnCosts; - private long encoderBit; protected BooleanEncodedValue accessEnc; protected BooleanEncodedValue roundaboutEnc; protected DecimalEncodedValue avgSpeedEnc; @@ -130,11 +129,10 @@ public ConditionalTagInspector getConditionalTagInspector() { /** * Defines bits used for edge flags used for access, speed etc. */ - public void createEncodedValues(List registerNewEncodedValue, String prefix, int index) { + public void createEncodedValues(List registerNewEncodedValue, String prefix) { // define the first 2 bits in flags for access registerNewEncodedValue.add(accessEnc = new SimpleBooleanEncodedValue(EncodingManager.getKey(prefix, "access"), true)); roundaboutEnc = getBooleanEncodedValue(Roundabout.KEY); - encoderBit = 1L << index; } /** @@ -156,36 +154,30 @@ public int getMaxTurnCosts() { public abstract EncodingManager.Access getAccess(ReaderWay way); /** - * Parse tags on nodes. Node tags can add to speed (like traffic_signals) where the value is - * strict negative or blocks access (like a barrier), then the value is strictly positive. This - * method is called in the second parsing step. - * - * @return encoded values or 0 if not blocking or no value stored + * @return true if the given OSM node blocks access for this vehicle, false otherwise */ - public long handleNodeTags(ReaderNode node) { + public boolean isBarrier(ReaderNode node) { boolean blockByDefault = node.hasTag("barrier", blockByDefaultBarriers); if (blockByDefault || node.hasTag("barrier", passByDefaultBarriers)) { boolean locked = node.hasTag("locked", "yes"); for (String res : restrictions) { if (!locked && node.hasTag(res, intendedValues)) - return 0; + return false; if (node.hasTag(res, restrictedValues)) - return encoderBit; + return true; } - if (blockByDefault) - return encoderBit; - return 0; + return blockByDefault; } if ((node.hasTag("highway", "ford") || node.hasTag("ford", "yes")) && (blockFords && !node.hasTag(restrictions, intendedValues) || node.hasTag(restrictions, restrictedValues))) { - return encoderBit; + return true; } - return 0; + return false; } @Override @@ -216,25 +208,6 @@ protected boolean isValidSpeed(double speed) { return !Double.isNaN(speed); } - @Override - public int hashCode() { - int hash = 7; - hash = 61 * hash + this.accessEnc.hashCode(); - hash = 61 * hash + this.toString().hashCode(); - return hash; - } - - @Override - public boolean equals(Object obj) { - if (obj == null) - return false; - - if (getClass() != obj.getClass()) - return false; - AbstractFlagEncoder afe = (AbstractFlagEncoder) obj; - return toString().equals(afe.toString()) && encoderBit == afe.encoderBit && accessEnc.equals(afe.accessEnc); - } - /** * Second parsing step. Invoked after splitting the edges. Currently used to offer a hook to * calculate precise speed values based on elevation data stored in the specified edge. diff --git a/core/src/main/java/com/graphhopper/routing/util/BikeCommonFlagEncoder.java b/core/src/main/java/com/graphhopper/routing/util/BikeCommonFlagEncoder.java index c1b02f30e92..404f2f81eda 100644 --- a/core/src/main/java/com/graphhopper/routing/util/BikeCommonFlagEncoder.java +++ b/core/src/main/java/com/graphhopper/routing/util/BikeCommonFlagEncoder.java @@ -190,9 +190,9 @@ public TransportationMode getTransportationMode() { } @Override - public void createEncodedValues(List registerNewEncodedValue, String prefix, int index) { + public void createEncodedValues(List registerNewEncodedValue, String prefix) { // first two bits are reserved for route handling in superclass - super.createEncodedValues(registerNewEncodedValue, prefix, index); + super.createEncodedValues(registerNewEncodedValue, prefix); registerNewEncodedValue.add(avgSpeedEnc = new UnsignedDecimalEncodedValue(getKey(prefix, "average_speed"), speedBits, speedFactor, speedTwoDirections)); registerNewEncodedValue.add(priorityEnc = new UnsignedDecimalEncodedValue(getKey(prefix, "priority"), 4, PriorityCode.getFactor(1), false)); diff --git a/core/src/main/java/com/graphhopper/routing/util/CarFlagEncoder.java b/core/src/main/java/com/graphhopper/routing/util/CarFlagEncoder.java index 53471613a50..9ae19a1fd7c 100644 --- a/core/src/main/java/com/graphhopper/routing/util/CarFlagEncoder.java +++ b/core/src/main/java/com/graphhopper/routing/util/CarFlagEncoder.java @@ -156,9 +156,9 @@ public TransportationMode getTransportationMode() { * Define the place of the speedBits in the edge flags for car. */ @Override - public void createEncodedValues(List registerNewEncodedValue, String prefix, int index) { + public void createEncodedValues(List registerNewEncodedValue, String prefix) { // first two bits are reserved for route handling in superclass - super.createEncodedValues(registerNewEncodedValue, prefix, index); + super.createEncodedValues(registerNewEncodedValue, prefix); registerNewEncodedValue.add(avgSpeedEnc = new UnsignedDecimalEncodedValue(EncodingManager.getKey(prefix, "average_speed"), speedBits, speedFactor, speedTwoDirections)); } diff --git a/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java b/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java index 2dfd8a2bbf2..be43f91c9a6 100644 --- a/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java +++ b/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java @@ -421,15 +421,11 @@ private void setPreferredLanguage(String preferredLanguage) { } private void addEncoder(AbstractFlagEncoder encoder) { - int encoderCount = edgeEncoders.size(); - encoder.setEncodedValueLookup(this); List list = new ArrayList<>(); - encoder.createEncodedValues(list, encoder.toString(), encoderCount); - for (EncodedValue ev : list) { + encoder.createEncodedValues(list, encoder.toString()); + for (EncodedValue ev : list) addEncodedValue(ev, true); - } - edgeEncoders.add(encoder); } @@ -653,8 +649,9 @@ public int hashCode() { */ public long handleNodeTags(ReaderNode node) { long flags = 0; - for (AbstractFlagEncoder encoder : edgeEncoders) { - flags |= encoder.handleNodeTags(node); + for (int i = 0, edgeEncodersSize = edgeEncoders.size(); i < edgeEncodersSize; i++) { + AbstractFlagEncoder encoder = edgeEncoders.get(i); + flags |= (encoder.isBarrier(node) ? 1L << i : 0); } return flags; diff --git a/core/src/main/java/com/graphhopper/routing/util/FootFlagEncoder.java b/core/src/main/java/com/graphhopper/routing/util/FootFlagEncoder.java index ae8edfa0654..232d46a7ea0 100644 --- a/core/src/main/java/com/graphhopper/routing/util/FootFlagEncoder.java +++ b/core/src/main/java/com/graphhopper/routing/util/FootFlagEncoder.java @@ -143,9 +143,9 @@ public TransportationMode getTransportationMode() { } @Override - public void createEncodedValues(List registerNewEncodedValue, String prefix, int index) { + public void createEncodedValues(List registerNewEncodedValue, String prefix) { // first two bits are reserved for route handling in superclass - super.createEncodedValues(registerNewEncodedValue, prefix, index); + super.createEncodedValues(registerNewEncodedValue, prefix); // larger value required - ferries are faster than pedestrians registerNewEncodedValue.add(avgSpeedEnc = new UnsignedDecimalEncodedValue(getKey(prefix, "average_speed"), speedBits, speedFactor, speedTwoDirections)); registerNewEncodedValue.add(priorityWayEncoder = new UnsignedDecimalEncodedValue(getKey(prefix, "priority"), 4, PriorityCode.getFactor(1), speedTwoDirections)); diff --git a/core/src/main/java/com/graphhopper/routing/util/MotorcycleFlagEncoder.java b/core/src/main/java/com/graphhopper/routing/util/MotorcycleFlagEncoder.java index 79f062fb437..c1f7184a576 100644 --- a/core/src/main/java/com/graphhopper/routing/util/MotorcycleFlagEncoder.java +++ b/core/src/main/java/com/graphhopper/routing/util/MotorcycleFlagEncoder.java @@ -107,9 +107,9 @@ public MotorcycleFlagEncoder(PMap properties) { * Define the place of the speedBits in the edge flags for car. */ @Override - public void createEncodedValues(List registerNewEncodedValue, String prefix, int index) { + public void createEncodedValues(List registerNewEncodedValue, String prefix) { // first two bits are reserved for route handling in superclass - super.createEncodedValues(registerNewEncodedValue, prefix, index); + super.createEncodedValues(registerNewEncodedValue, prefix); registerNewEncodedValue.add(priorityWayEncoder = new UnsignedDecimalEncodedValue(getKey(prefix, "priority"), 4, PriorityCode.getFactor(1), false)); registerNewEncodedValue.add(curvatureEncoder = new UnsignedDecimalEncodedValue(getKey(prefix, "curvature"), 4, 0.1, false)); diff --git a/core/src/test/java/com/graphhopper/routing/util/AbstractBikeFlagEncoderTester.java b/core/src/test/java/com/graphhopper/routing/util/AbstractBikeFlagEncoderTester.java index 6e945a4689f..eeff7221a1b 100644 --- a/core/src/test/java/com/graphhopper/routing/util/AbstractBikeFlagEncoderTester.java +++ b/core/src/test/java/com/graphhopper/routing/util/AbstractBikeFlagEncoderTester.java @@ -366,38 +366,38 @@ public void testBarrierAccess() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); node.setTag("bicycle", "yes"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "no"); // barrier! - assertTrue(encoder.handleNodeTags(node) > 0); + assertTrue(encoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "yes"); node.setTag("bicycle", "no"); // barrier! - assertTrue(encoder.handleNodeTags(node) > 0); + assertTrue(encoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "no"); node.setTag("foot", "yes"); // barrier! - assertTrue(encoder.handleNodeTags(node) > 0); + assertTrue(encoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "no"); node.setTag("bicycle", "yes"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); } @Test @@ -405,11 +405,11 @@ public void testBarrierAccessFord() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("ford", "yes"); // barrier! - assertTrue(encoder.handleNodeTags(node) > 0); + assertTrue(encoder.isBarrier(node)); node.setTag("bicycle", "yes"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); } @Test diff --git a/core/src/test/java/com/graphhopper/routing/util/BikeFlagEncoderTest.java b/core/src/test/java/com/graphhopper/routing/util/BikeFlagEncoderTest.java index 4f8abfafeb3..e113cfdefb5 100644 --- a/core/src/test/java/com/graphhopper/routing/util/BikeFlagEncoderTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/BikeFlagEncoderTest.java @@ -626,19 +626,19 @@ public void testBarrierAccess() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "kissing_gate"); // barrier! - assertFalse(encoder.handleNodeTags(node) == 0); + assertTrue(encoder.isBarrier(node)); // kissing_gate with bicycle tag node = new ReaderNode(1, -1, -1); node.setTag("barrier", "kissing_gate"); node.setTag("bicycle", "yes"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); // Test if cattle_grid is non blocking node = new ReaderNode(1, -1, -1); node.setTag("barrier", "cattle_grid"); - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); } @Test diff --git a/core/src/test/java/com/graphhopper/routing/util/CarFlagEncoderTest.java b/core/src/test/java/com/graphhopper/routing/util/CarFlagEncoderTest.java index 2712e0e3d5d..21a91cf6970 100644 --- a/core/src/test/java/com/graphhopper/routing/util/CarFlagEncoderTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/CarFlagEncoderTest.java @@ -151,12 +151,12 @@ public void testFordAccess() { // Node and way are initially blocking assertTrue(encoder.isBlockFords()); assertTrue(encoder.getAccess(way).canSkip()); - assertTrue(encoder.handleNodeTags(node) > 0); + assertTrue(encoder.isBarrier(node)); CarFlagEncoder tmpEncoder = new CarFlagEncoder(new PMap("block_fords=false")); EncodingManager.create(tmpEncoder); assertTrue(tmpEncoder.getAccess(way).isWay()); - assertFalse(tmpEncoder.handleNodeTags(node) > 0); + assertFalse(tmpEncoder.isBarrier(node)); } @Test @@ -539,32 +539,32 @@ public void testBarrierAccess() { node.setTag("barrier", "lift_gate"); node.setTag("access", "yes"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "lift_gate"); node.setTag("bicycle", "yes"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "lift_gate"); node.setTag("access", "yes"); node.setTag("bicycle", "yes"); // should this be a barrier for motorcars too? - // assertTrue(encoder.handleNodeTags(node) > 0); + // assertTrue(encoder.handleNodeTags(node) == true); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "lift_gate"); node.setTag("access", "no"); node.setTag("motorcar", "yes"); // no barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "bollard"); // barrier! - assertTrue(encoder.handleNodeTags(node) > 0); + assertTrue(encoder.isBarrier(node)); CarFlagEncoder tmpEncoder = new CarFlagEncoder(); EncodingManager.create(tmpEncoder); @@ -572,7 +572,7 @@ public void testBarrierAccess() { // Test if cattle_grid is not blocking node = new ReaderNode(1, -1, -1); node.setTag("barrier", "cattle_grid"); - assertTrue(tmpEncoder.handleNodeTags(node) == 0); + assertFalse(tmpEncoder.isBarrier(node)); } @Test @@ -580,11 +580,11 @@ public void testChainBarrier() { // by default allow access through the gate for bike & foot! ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "chain"); - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); node.setTag("motor_vehicle", "no"); - assertTrue(encoder.handleNodeTags(node) > 0); + assertTrue(encoder.isBarrier(node)); node.setTag("motor_vehicle", "yes"); - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); } @Test @@ -662,7 +662,7 @@ public void testIssue_1256() { EncodingManager.create(lowFactorCar); List list = new ArrayList<>(); lowFactorCar.setEncodedValueLookup(em); - lowFactorCar.createEncodedValues(list, "car", 0); + lowFactorCar.createEncodedValues(list, "car"); assertEquals(2.5, encoder.ferrySpeedCalc.getSpeed(way), .1); assertEquals(.5, lowFactorCar.ferrySpeedCalc.getSpeed(way), .1); } diff --git a/core/src/test/java/com/graphhopper/routing/util/EncodingManagerTest.java b/core/src/test/java/com/graphhopper/routing/util/EncodingManagerTest.java index e58095a56e9..6db31711ab1 100644 --- a/core/src/test/java/com/graphhopper/routing/util/EncodingManagerTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/EncodingManagerTest.java @@ -36,32 +36,8 @@ public class EncodingManagerTest { @Test - public void testCompatibility() { - EncodingManager manager = EncodingManager.create("car,bike,foot"); - BikeFlagEncoder bike = (BikeFlagEncoder) manager.getEncoder("bike"); - CarFlagEncoder car = (CarFlagEncoder) manager.getEncoder("car"); - FootFlagEncoder foot = (FootFlagEncoder) manager.getEncoder("foot"); - assertNotEquals(car, bike); - assertNotEquals(car, foot); - assertNotEquals(car.hashCode(), bike.hashCode()); - assertNotEquals(car.hashCode(), foot.hashCode()); - - FootFlagEncoder foot2 = new FootFlagEncoder(); - EncodingManager.create(foot2); - assertNotEquals(foot, foot2); - assertNotEquals(foot.hashCode(), foot2.hashCode()); - - FootFlagEncoder foot3 = new FootFlagEncoder(); - EncodingManager.create(foot3); - assertEquals(foot3, foot2); - assertEquals(foot3.hashCode(), foot2.hashCode()); - - try { - EncodingManager.create("car,car"); - fail("there should have been an exception, do not allow duplicate flag encoders"); - } catch (Exception ex) { - // ok - } + public void duplicateNamesNotAllowed() { + assertThrows(IllegalArgumentException.class, () -> EncodingManager.create("car,car")); } @Test diff --git a/core/src/test/java/com/graphhopper/routing/util/FootFlagEncoderTest.java b/core/src/test/java/com/graphhopper/routing/util/FootFlagEncoderTest.java index 4c73f6c85fe..79e173bcf20 100644 --- a/core/src/test/java/com/graphhopper/routing/util/FootFlagEncoderTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/FootFlagEncoderTest.java @@ -343,39 +343,39 @@ public void testBarrierAccess() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); // no barrier! - assertEquals(0, footEncoder.handleNodeTags(node)); + assertFalse(footEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "yes"); // no barrier! - assertEquals(0, footEncoder.handleNodeTags(node)); + assertFalse(footEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "no"); // barrier! - assertTrue(footEncoder.handleNodeTags(node) > 0); + assertTrue(footEncoder.isBarrier(node)); node.setTag("bicycle", "yes"); // no barrier!? - // assertTrue(footEncoder.handleNodeTags(node) == 0); + // assertTrue(footEncoder.handleNodeTags(node) == false); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "no"); node.setTag("foot", "yes"); // no barrier! - assertEquals(0, footEncoder.handleNodeTags(node)); + assertFalse(footEncoder.isBarrier(node)); node.setTag("locked", "yes"); // barrier! - assertTrue(footEncoder.handleNodeTags(node) > 0); + assertTrue(footEncoder.isBarrier(node)); node.clearTags(); node.setTag("barrier", "yes"); node.setTag("access", "no"); - assertTrue(footEncoder.handleNodeTags(node) > 0); + assertTrue(footEncoder.isBarrier(node)); } @Test @@ -383,9 +383,9 @@ public void testChainBarrier() { // by default allow access through the gate for bike & foot! ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "chain"); - assertEquals(0, footEncoder.handleNodeTags(node)); + assertFalse(footEncoder.isBarrier(node)); node.setTag("foot", "no"); - assertTrue(footEncoder.handleNodeTags(node) > 0); + assertTrue(footEncoder.isBarrier(node)); } @Test @@ -393,26 +393,26 @@ public void testFord() { // by default do not block access due to fords! ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("ford", "no"); - assertEquals(0, footEncoder.handleNodeTags(node)); + assertFalse(footEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("ford", "yes"); // no barrier! - assertEquals(0, footEncoder.handleNodeTags(node)); + assertFalse(footEncoder.isBarrier(node)); // barrier! node.setTag("foot", "no"); - assertTrue(footEncoder.handleNodeTags(node) > 0); + assertTrue(footEncoder.isBarrier(node)); FootFlagEncoder tmpEncoder = new FootFlagEncoder(new PMap("block_fords=true")); EncodingManager.create(tmpEncoder); node = new ReaderNode(1, -1, -1); node.setTag("ford", "no"); - assertEquals(0, tmpEncoder.handleNodeTags(node)); + assertFalse(tmpEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("ford", "yes"); - assertTrue(tmpEncoder.handleNodeTags(node) != 0); + assertTrue(tmpEncoder.isBarrier(node)); } @Test @@ -423,36 +423,36 @@ public void testBlockByDefault() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); // potential barriers are no barrier by default - assertEquals(0, tmpFootEncoder.handleNodeTags(node)); + assertFalse(tmpFootEncoder.isBarrier(node)); node.setTag("access", "no"); - assertTrue(tmpFootEncoder.handleNodeTags(node) > 0); + assertTrue(tmpFootEncoder.isBarrier(node)); // absolute barriers always block node = new ReaderNode(1, -1, -1); node.setTag("barrier", "fence"); - assertTrue(tmpFootEncoder.handleNodeTags(node) > 0); + assertTrue(tmpFootEncoder.isBarrier(node)); node.setTag("barrier", "fence"); node.setTag("access", "yes"); - assertFalse(tmpFootEncoder.handleNodeTags(node) > 0); + assertFalse(tmpFootEncoder.isBarrier(node)); // pass potential barriers per default (if no other access tag exists) tmpFootEncoder = new FootFlagEncoder(); EncodingManager.create(tmpFootEncoder); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); - assertFalse(tmpFootEncoder.handleNodeTags(node) > 0); + assertFalse(tmpFootEncoder.isBarrier(node)); node.setTag("access", "yes"); - assertEquals(0, tmpFootEncoder.handleNodeTags(node)); + assertFalse(tmpFootEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "fence"); - assertTrue(tmpFootEncoder.handleNodeTags(node) > 0); + assertTrue(tmpFootEncoder.isBarrier(node)); // don't block potential barriers: barrier:cattle_grid should not block here tmpFootEncoder = new FootFlagEncoder(); EncodingManager.create(tmpFootEncoder); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "cattle_grid"); - assertEquals(0, tmpFootEncoder.handleNodeTags(node)); + assertFalse(tmpFootEncoder.isBarrier(node)); } } diff --git a/core/src/test/java/com/graphhopper/routing/util/MountainBikeFlagEncoderTest.java b/core/src/test/java/com/graphhopper/routing/util/MountainBikeFlagEncoderTest.java index 33c12fb48c6..ffa0793a191 100644 --- a/core/src/test/java/com/graphhopper/routing/util/MountainBikeFlagEncoderTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/MountainBikeFlagEncoderTest.java @@ -195,21 +195,21 @@ public void testBarrierAccess() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "kissing_gate"); // No barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); // kissing_gate with bicycle tag = no node = new ReaderNode(1, -1, -1); node.setTag("barrier", "kissing_gate"); node.setTag("bicycle", "no"); // barrier! - assertFalse(encoder.handleNodeTags(node) == 0); + assertTrue(encoder.isBarrier(node)); // kissing_gate with bicycle tag node = new ReaderNode(1, -1, -1); node.setTag("barrier", "kissing_gate"); node.setTag("bicycle", "yes"); // No barrier! - assertTrue(encoder.handleNodeTags(node) == 0); + assertFalse(encoder.isBarrier(node)); } } diff --git a/core/src/test/java/com/graphhopper/routing/util/WheelchairFlagEncoderTest.java b/core/src/test/java/com/graphhopper/routing/util/WheelchairFlagEncoderTest.java index 1d3a5db7ddc..3cab78f40f3 100644 --- a/core/src/test/java/com/graphhopper/routing/util/WheelchairFlagEncoderTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/WheelchairFlagEncoderTest.java @@ -361,34 +361,34 @@ public void testBarrierAccess() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); // no barrier! - assertEquals(0, wheelchairEncoder.handleNodeTags(node)); + assertFalse(wheelchairEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "yes"); // no barrier! - assertEquals(0, wheelchairEncoder.handleNodeTags(node)); + assertFalse(wheelchairEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "no"); // barrier! - assertTrue(wheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(wheelchairEncoder.isBarrier(node)); node.setTag("bicycle", "yes"); // no barrier!? - // assertTrue(wheelchairEncoder.handleNodeTags(node) == 0); + // assertTrue(wheelchairEncoder.handleNodeTags(node) == false); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "no"); node.setTag("foot", "yes"); // no barrier! - assertEquals(0, wheelchairEncoder.handleNodeTags(node)); + assertFalse(wheelchairEncoder.isBarrier(node)); node.setTag("locked", "yes"); // barrier! - assertTrue(wheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(wheelchairEncoder.isBarrier(node)); } @Test @@ -399,39 +399,39 @@ public void testBlockByDefault() { ReaderNode node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); // passByDefaultBarriers are no barrier by default - assertEquals(0, tmpWheelchairEncoder.handleNodeTags(node)); + assertFalse(tmpWheelchairEncoder.isBarrier(node)); node.setTag("access", "no"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(tmpWheelchairEncoder.isBarrier(node)); // these barriers block node = new ReaderNode(1, -1, -1); node.setTag("barrier", "fence"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(tmpWheelchairEncoder.isBarrier(node)); node.setTag("barrier", "wall"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(tmpWheelchairEncoder.isBarrier(node)); node.setTag("barrier", "handrail"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(tmpWheelchairEncoder.isBarrier(node)); node.setTag("barrier", "turnstile"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(tmpWheelchairEncoder.isBarrier(node)); // Explictly allowed access is allowed node.setTag("barrier", "fence"); node.setTag("access", "yes"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) == 0); + assertFalse(tmpWheelchairEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "gate"); node.setTag("access", "yes"); - assertEquals(0, tmpWheelchairEncoder.handleNodeTags(node)); + assertFalse(tmpWheelchairEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "kerb"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) == 0); + assertFalse(tmpWheelchairEncoder.isBarrier(node)); node.setTag("wheelchair", "yes"); - assertEquals(0, tmpWheelchairEncoder.handleNodeTags(node)); + assertFalse(tmpWheelchairEncoder.isBarrier(node)); node = new ReaderNode(1, -1, -1); node.setTag("barrier", "fence"); - assertTrue(tmpWheelchairEncoder.handleNodeTags(node) > 0); + assertTrue(tmpWheelchairEncoder.isBarrier(node)); } @Test From af5475772dd7b22e7dbf658c69be96473013f7a6 Mon Sep 17 00:00:00 2001 From: Andi Date: Sun, 17 Oct 2021 13:40:56 +0200 Subject: [PATCH 32/53] Ignore barrier nodes at junctions (#2433) --- CHANGELOG.md | 3 +- .../com/graphhopper/reader/osm/OSMReader.java | 58 ++++++++++--------- .../java/com/graphhopper/GraphHopperTest.java | 22 ++++--- .../graphhopper/reader/osm/OSMReaderTest.java | 24 ++++---- 4 files changed, 59 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20d14b3769c..5b7a069dff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 5.0 [not yet released] +- barrier nodes at junctions are now ignored (#2433) - AbstractFlagEncoder#handleNodeTags was replaced by AbstractFlagEncoder#isBarrier (#2434) - consider heading when snapping coordinates to the road network, this is especially important for navigation (#2411) - OSMReader no longer sets the artificial 'estimated_center' tag and processNode also receives EMPTY_NODEs (971d686) @@ -7,7 +8,7 @@ ### 4.0 [29 Sep 2021] - faster node-based CH preparation (~20%), (#2390) -- more flexible ElevationProvider interface, support providing elevation via node tags (#2374, #23281) +- more flexible ElevationProvider interface, support providing elevation via node tags (#2374, #2381) - added country encoded value for all countries (#2353) - bike improvements (#2357, #2371, #2389) - improved handling of barriers (#2345, #2340, #2406) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 56fd9165dc8..ecf1088f3dd 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -75,10 +75,14 @@ * @author Peter Karich */ public class OSMReader implements TurnCostParser.ExternalInternalMap { - protected static final int EMPTY_NODE = -1; - protected static final int PILLAR_NODE = 1; - protected static final int TOWER_NODE = -2; private static final Logger LOGGER = LoggerFactory.getLogger(OSMReader.class); + private static final int JUNCTION_NODE = -2; + private static final int EMPTY_NODE = -1; + private static final int END_NODE = 0; + private static final int INTERMEDIATE_NODE = 1; + // connection nodes are those where (only) two OSM ways are connected at their ends, so they are still no junctions + private static final int CONNECTION_NODE = 2; + private final GraphHopperStorage ghStorage; private final OSMReaderConfig config; private final Graph graph; @@ -90,6 +94,7 @@ public class OSMReader implements TurnCostParser.ExternalInternalMap { protected long zeroCounter = 0; protected PillarInfo pillarInfo; private long locations; + private long ignoredBarrierNodes; private final EncodingManager encodingManager; // Choosing the best Map is hard. We need a memory efficient and fast solution for big data sets! // @@ -176,12 +181,17 @@ void preProcess(File osmFile) { while ((item = in.getNext()) != null) { if (item.isType(ReaderElement.WAY)) { final ReaderWay way = (ReaderWay) item; - boolean valid = filterWay(way); - if (valid) { + if (filterWay(way)) { LongIndexedContainer wayNodes = way.getNodes(); int s = wayNodes.size(); for (int index = 0; index < s; index++) { - prepareHighwayNode(wayNodes.get(index)); + final boolean isEnd = index == 0 || index == s - 1; + final long osmId = wayNodes.get(index); + int curr = getNodeMap().get(osmId); + if (curr == EMPTY_NODE) + getNodeMap().put(osmId, isEnd ? END_NODE : INTERMEDIATE_NODE); + else + getNodeMap().put(osmId, curr == END_NODE && isEnd ? CONNECTION_NODE : JUNCTION_NODE); } if (++tmpWayCounter % 10_000_000 == 0) { @@ -506,9 +516,9 @@ protected void processNode(ReaderNode node) { int nodeType = getNodeMap().get(node.getId()); if (nodeType == EMPTY_NODE) { return; - } else if (nodeType == TOWER_NODE) { + } else if (nodeType == JUNCTION_NODE || nodeType == CONNECTION_NODE) { addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); - } else if (nodeType == PILLAR_NODE) { + } else if (nodeType == INTERMEDIATE_NODE || nodeType == END_NODE) { addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); } @@ -516,7 +526,11 @@ protected void processNode(ReaderNode node) { if (node.hasTags()) { long nodeFlags = encodingManager.handleNodeTags(node); if (nodeFlags != 0) - getNodeFlagsMap().put(node.getId(), nodeFlags); + if (nodeType == JUNCTION_NODE) { + LOGGER.debug("OSM node {} at {},{} is a barrier node at a junction, the barrier will be ignored", node.getId(), Helper.round(node.getLat(), 7), Helper.round(node.getLon(), 7)); + ignoredBarrierNodes++; + } else + getNodeFlagsMap().put(node.getId(), nodeFlags); } locations++; @@ -554,19 +568,6 @@ void prepareWaysWithRelationInfo(ReaderRelation osmRelation) { } } - void prepareHighwayNode(long osmId) { - int tmpGHNodeId = getNodeMap().get(osmId); - if (tmpGHNodeId == EMPTY_NODE) { - // this is the first time we see this osmId - getNodeMap().put(osmId, PILLAR_NODE); - } else if (tmpGHNodeId > EMPTY_NODE) { - // mark node as tower node as it now occurred for at least the second time - getNodeMap().put(osmId, TOWER_NODE); - } else { - // tmpIndex is already negative (already tower node) - } - } - int addTowerNode(long osmId, double lat, double lon, double ele) { if (nodeAccess.is3D()) nodeAccess.setNode(nextTowerId, lat, lon, ele); @@ -615,10 +616,10 @@ void addOSMWay(final LongIndexedContainer osmNodeIds, final IntsRef flags, final continue; // skip osmIds with no associated pillar or tower id (e.g. !OSMReader.isBounds) - if (tmpNode == TOWER_NODE) + if (tmpNode == JUNCTION_NODE || tmpNode == CONNECTION_NODE) continue; - if (tmpNode == PILLAR_NODE) { + if (tmpNode == INTERMEDIATE_NODE || tmpNode == END_NODE) { // In some cases no node information is saved for the specified osmId. // i.e. a way references a which does not exist in the current file. // => if the node before was a pillar node then convert into to tower node (as it is also end-standing). @@ -822,7 +823,7 @@ long addBarrierNode(long nodeId) { final long osmId = newNode.getId(); if (getNodeMap().get(osmId) != -1) throw new IllegalStateException("Artificial osm node id already exists: " + osmId); - getNodeMap().put(osmId, PILLAR_NODE); + getNodeMap().put(osmId, INTERMEDIATE_NODE); addPillarNode(osmId, newNode.getLat(), newNode.getLon(), eleProvider.getEle(newNode)); return osmId; } @@ -967,6 +968,7 @@ public OSMReader setFile(File osmFile) { private void printInfo(String str) { LOGGER.info("finished " + str + " processing." + " nodes: " + graph.getNodes() + + ", ignored barrier nodes at junctions: " + nf(ignoredBarrierNodes) + ", osmIdMap.size:" + getNodeMap().getSize() + ", osmIdMap:" + getNodeMap().getMemoryUsage() + "MB" + ", nodeFlagsMap.size:" + getNodeFlagsMap().size() + ", relFlagsMap.size:" + getRelFlagsMapSize() + ", zeroCounter:" + zeroCounter @@ -987,15 +989,15 @@ public String toString() { private boolean isTowerNode(int id) { // tower nodes are indexed -3, -4, -5, ... - return id < TOWER_NODE; + return id < JUNCTION_NODE; } private boolean isPillarNode(int id) { // pillar nodes are indexed 3, 4, 5, .. - return id > -TOWER_NODE; + return id > CONNECTION_NODE; } private boolean isNodeId(int id) { - return id > -TOWER_NODE || id < TOWER_NODE; + return id > CONNECTION_NODE || id < JUNCTION_NODE; } } diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index a5ec147854e..d789f05732f 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -2354,25 +2354,29 @@ public void testBarriers() { { // these are bollards that are located right on a junction. this should never happen according to OSM mapping // rules, but it still does. the problem with such barriers is that we can only block one direction and it - // is unclear which one is right + // is unclear which one is right. therefore we simply ignore such barriers. - // here the barrier node actually disconnects a dead-end road that should rather be connected! + // here the barrier node actually disconnected a dead-end road that should rather be connected before we + // started ignoring barriers at junctions. GHResponse carRsp = hopper.route(new GHRequest(51.327121, 12.572396, 51.327173, 12.574038).setProfile("car")); - assertTrue(carRsp.hasErrors() && carRsp.getErrors().toString().contains("Connection between locations not found"), carRsp.getErrors().toString()); + assertEquals(124, carRsp.getBest().getDistance(), 1); GHResponse bikeRsp = hopper.route(new GHRequest(51.327121, 12.572396, 51.327173, 12.574038).setProfile("bike")); assertEquals(124, bikeRsp.getBest().getDistance(), 1); - // Here the barrier node prevents us from travelling straight along Pufendorfstraße. Entering Pufendorfstraße - // from 'An der Streuobstwiese' is allowed though, but this seems wrong. We probably add a wrong barrier - // edge here (the mapping was fixed in newer OSM versions, so the barrier is no longer at the junction) + // Here the barrier could prevent us from travelling straight along Pufendorfstraße. But it could also + // prevent us from turning from Pufendorfstraße onto 'An der Streuobstwiese' (or vice versa). What should + // be allowed depends on whether the barrier is before or behind the junction. And since we can't tell + // we just ignore this barrier. Note that the mapping was fixed in newer OSM versions, so the barrier is no + // longer at the junction carRsp = hopper.route(new GHRequest(51.344134, 12.317986, 51.344231, 12.317482).setProfile("car")); - assertEquals(393, carRsp.getBest().getDistance(), 1); + assertEquals(36, carRsp.getBest().getDistance(), 1); bikeRsp = hopper.route(new GHRequest(51.344134, 12.317986, 51.344231, 12.317482).setProfile("bike")); assertEquals(36, bikeRsp.getBest().getDistance(), 1); - // Here we have to go all the way around, but not sure if this is the intended effect of this bollard node + // Here we'd have to go all the way around, but the bollard node could also mean that continuing on Adenauerallee + // is fine, and we just cannot enter the little path. Since we cannot tell we just ignore this barrier. carRsp = hopper.route(new GHRequest(51.355455, 12.40202, 51.355318, 12.401741).setProfile("car")); - assertEquals(1217, carRsp.getBest().getDistance(), 1); + assertEquals(24, carRsp.getBest().getDistance(), 1); bikeRsp = hopper.route(new GHRequest(51.355455, 12.40202, 51.355318, 12.401741).setProfile("bike")); assertEquals(24, bikeRsp.getBest().getDistance(), 1); } diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index bde2fbd53aa..ddc567fb762 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -39,7 +39,6 @@ import com.graphhopper.storage.index.Snap; import com.graphhopper.util.*; import com.graphhopper.util.details.PathDetail; -import com.graphhopper.util.shapes.GHPoint; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -352,7 +351,9 @@ public void testBarriers() { importOrLoad(); Graph graph = hopper.getGraphHopperStorage(); - assertEquals(8, graph.getNodes()); + // we ignore the barrier at node 50 + assertEquals(7, graph.getNodes()); + assertEquals(7, graph.getEdges()); int n10 = AbstractGraphStorageTester.getIdOf(graph, 51); int n20 = AbstractGraphStorageTester.getIdOf(graph, 52); @@ -458,21 +459,24 @@ public void testBarriersOnTowerNodes() { setMinNetworkSize(0). importOrLoad(); Graph graph = hopper.getGraphHopperStorage(); - assertEquals(8, graph.getNodes()); + // we ignore the barrier at node 50 + // 10-20-30 produces three edges: 10-20, 20-2x, 2x-30, the second one is a barrier edge + assertEquals(7, graph.getNodes()); + assertEquals(7, graph.getEdges()); int n60 = AbstractGraphStorageTester.getIdOf(graph, 56); - int newId = 5; - assertEquals(GHUtility.asSet(newId), GHUtility.getNeighbors(carOutExplorer.setBaseNode(n60))); + int n50 = AbstractGraphStorageTester.getIdOf(graph, 55); + int n30 = AbstractGraphStorageTester.getIdOf(graph, 53); + int n80 = AbstractGraphStorageTester.getIdOf(graph, 58); + assertEquals(GHUtility.asSet(n50), GHUtility.getNeighbors(carOutExplorer.setBaseNode(n60))); EdgeIterator iter = carOutExplorer.setBaseNode(n60); assertTrue(iter.next()); - assertEquals(newId, iter.getAdjNode()); + assertEquals(n50, iter.getAdjNode()); assertFalse(iter.next()); - iter = carOutExplorer.setBaseNode(newId); - assertTrue(iter.next()); - assertEquals(n60, iter.getAdjNode()); - assertFalse(iter.next()); + assertTrue(GHUtility.getNeighbors(carOutExplorer.setBaseNode(n30)).contains(n50)); + assertEquals(GHUtility.asSet(n30, n80, n60), GHUtility.getNeighbors(carOutExplorer.setBaseNode(n50))); } @Test From b7e286e3df9505bc1d121d9dfd2bafbc552c24c0 Mon Sep 17 00:00:00 2001 From: easbar Date: Sun, 17 Oct 2021 17:28:16 +0200 Subject: [PATCH 33/53] OSMReader: remove isOnePassable optimization, add edges for all barrier nodes --- .../com/graphhopper/reader/osm/OSMReader.java | 69 ++++++++----------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index ecf1088f3dd..71e97ed088b 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -353,36 +353,35 @@ protected void processWay(ReaderWay way) { long nodeFlags = getNodeFlagsMap().get(nodeId); // barrier was spotted and the way is passable for that mode of travel if (nodeFlags > 0) { - if (isOnePassable(encodingManager.getAccessEncFromNodeFlags(nodeFlags), edgeFlags)) { - // remove barrier to avoid duplicates - getNodeFlagsMap().put(nodeId, 0); - - // create shadow node copy for zero length edge - long newNodeId = addBarrierNode(nodeId); - if (i > 0) { - // start at beginning of array if there was no previous barrier - if (lastBarrier < 0) - lastBarrier = 0; - - // add way up to barrier shadow node - int length = i - lastBarrier + 1; - LongArrayList partNodeIds = new LongArrayList(); - partNodeIds.add(osmNodeIds.buffer, lastBarrier, length); - partNodeIds.set(length - 1, newNodeId); - addOSMWay(partNodeIds, edgeFlags, way); - - // create zero length edge for barrier - addBarrierEdge(newNodeId, nodeId, edgeFlags, nodeFlags, way); - } else { - // run edge from real first node to shadow node - addBarrierEdge(nodeId, newNodeId, edgeFlags, nodeFlags, way); - - // exchange first node for created barrier node - osmNodeIds.set(0, newNodeId); - } - // remember barrier for processing the way behind it - lastBarrier = i; + // create shadow node copy for zero length edge + long newNodeId = addBarrierNode(nodeId); + if (i > 0) { + // start at beginning of array if there was no previous barrier + if (lastBarrier < 0) + lastBarrier = 0; + + // add way up to barrier shadow node + int length = i - lastBarrier + 1; + LongArrayList partNodeIds = new LongArrayList(); + partNodeIds.add(osmNodeIds.buffer, lastBarrier, length); + partNodeIds.set(length - 1, newNodeId); + addOSMWay(partNodeIds, edgeFlags, way); + + // create zero length edge for barrier + addBarrierEdge(newNodeId, nodeId, edgeFlags, nodeFlags, way); + } else { + // run edge from real first node to shadow node + addBarrierEdge(nodeId, newNodeId, edgeFlags, nodeFlags, way); + + // exchange first node for created barrier node + osmNodeIds.set(0, newNodeId); } + // remember barrier for processing the way behind it + lastBarrier = i; + + // ignore this barrier node from now. for example a barrier can be connecting two ways (appear in both + // ways) and we only want to add a barrier edge once (but we want to add one). + getNodeFlagsMap().put(nodeId, 0); } } @@ -542,18 +541,6 @@ private void addPillarNode(long osmId, double lat, double lon, double ele) { nextPillarId++; } - /** - * The nodeFlags store the encoders to check for accessibility in edgeFlags. E.g. if nodeFlags==3, then the - * accessibility of the first two encoders will be check in edgeFlags - */ - private static boolean isOnePassable(List checkEncoders, IntsRef edgeFlags) { - for (BooleanEncodedValue accessEnc : checkEncoders) { - if (accessEnc.getBool(false, edgeFlags) || accessEnc.getBool(true, edgeFlags)) - return true; - } - return false; - } - void prepareWaysWithRelationInfo(ReaderRelation osmRelation) { for (ReaderRelation.Member member : osmRelation.getMembers()) { if (member.getType() != ReaderRelation.Member.WAY) From af58f6620853b4aedb4eec9faa92122d980aa34d Mon Sep 17 00:00:00 2001 From: easbar Date: Mon, 18 Oct 2021 08:36:53 +0200 Subject: [PATCH 34/53] Minor cleanup in GHLongIntBTree --- .../com/graphhopper/coll/GHLongIntBTree.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/graphhopper/coll/GHLongIntBTree.java b/core/src/main/java/com/graphhopper/coll/GHLongIntBTree.java index b6ded2c3d28..e556083e464 100644 --- a/core/src/main/java/com/graphhopper/coll/GHLongIntBTree.java +++ b/core/src/main/java/com/graphhopper/coll/GHLongIntBTree.java @@ -31,13 +31,13 @@ * @author Peter Karich */ public class GHLongIntBTree implements LongIntMap { + private final static Logger logger = LoggerFactory.getLogger(GHLongIntBTree.class); private final int noNumberValue = -1; - private Logger logger = LoggerFactory.getLogger(getClass()); + private final int maxLeafEntries; + private final int initLeafSize; + private final int splitIndex; + private final float factor; private long size; - private int maxLeafEntries; - private int initLeafSize; - private int splitIndex; - private float factor; private int height; private BTreeEntry root; @@ -64,8 +64,7 @@ public GHLongIntBTree(int maxLeafEntries) { clear(); } - // LATER: see OSMIDMap for a version where we use DataAccess - static int binarySearch(long keys[], int start, int len, long key) { + static int binarySearch(long[] keys, int start, int len, long key) { int high = start + len, low = start - 1, guess; while (high - low > 1) { // use >>> for average or we could get an integer overflow. @@ -184,9 +183,9 @@ public ReturnValue(int oldValue) { class BTreeEntry { int entrySize; - long keys[]; - int values[]; - BTreeEntry children[]; + long[] keys; + int[] values; + BTreeEntry[] children; boolean isLeaf; public BTreeEntry(int tmpSize, boolean leaf) { From 7a1f30b699da7a562107c8addeabb565d050e9da Mon Sep 17 00:00:00 2001 From: easbar Date: Mon, 18 Oct 2021 10:35:09 +0200 Subject: [PATCH 35/53] minor --- .../routing/ch/CHPreparationHandler.java | 16 ++++------------ .../java/com/graphhopper/tools/Measurement.java | 4 ++-- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java b/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java index d7e8b0be344..2190db55b47 100644 --- a/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java +++ b/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java @@ -41,7 +41,7 @@ * @author easbar */ public class CHPreparationHandler { - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); + private static final Logger LOGGER = LoggerFactory.getLogger(CHPreparationHandler.class); private final List preparations = new ArrayList<>(); // we first add the profiles and later read them to create the config objects (because they require // the actual Weightings) @@ -128,10 +128,6 @@ public CHPreparationHandler setCHProfiles(CHProfile... chProfiles) { return this; } - /** - * Enables the use of contraction hierarchies to reduce query times. - * "fastest|u_turn_costs=30 or your own weight-calculation type. - */ public CHPreparationHandler setCHProfiles(Collection chProfiles) { this.chProfiles.clear(); this.chProfiles.addAll(chProfiles); @@ -146,25 +142,21 @@ public List getPreparations() { return preparations; } - public PrepareContractionHierarchies getPreparation(String profile) { + public PrepareContractionHierarchies getPreparation(String chGraphName) { if (preparations.isEmpty()) throw new IllegalStateException("No CH preparations added yet"); List profileNames = new ArrayList<>(preparations.size()); for (PrepareContractionHierarchies preparation : preparations) { profileNames.add(preparation.getCHConfig().getName()); - if (preparation.getCHConfig().getName().equalsIgnoreCase(profile)) { + if (preparation.getCHConfig().getName().equalsIgnoreCase(chGraphName)) { return preparation; } } - throw new IllegalArgumentException("Cannot find CH preparation for the requested profile: '" + profile + "'" + + throw new IllegalArgumentException("Cannot find CH preparation for the requested profile: '" + chGraphName + "'" + "\nYou can try disabling CH using " + CH.DISABLE + "=true" + "\navailable CH profiles: " + profileNames); } - public PrepareContractionHierarchies getPreparation(CHConfig chConfig) { - return getPreparation(chConfig.getName()); - } - public int getPreparationThreads() { return preparationThreads; } diff --git a/tools/src/main/java/com/graphhopper/tools/Measurement.java b/tools/src/main/java/com/graphhopper/tools/Measurement.java index 5c02adb5152..8ffa8b5bb15 100644 --- a/tools/src/main/java/com/graphhopper/tools/Measurement.java +++ b/tools/src/main/java/com/graphhopper/tools/Measurement.java @@ -131,13 +131,13 @@ protected void prepareCH(boolean closeEarly) { CHConfig chConfig = getCHPreparationHandler().getNodeBasedCHConfigs().get(0); int edgesAndShortcuts = getGraphHopperStorage().getRoutingCHGraph(chConfig.getName()).getEdges(); put(Parameters.CH.PREPARE + "node.shortcuts", edgesAndShortcuts - edges); - put(Parameters.CH.PREPARE + "node.time", getCHPreparationHandler().getPreparation(chConfig).getTotalPrepareTime()); + put(Parameters.CH.PREPARE + "node.time", getCHPreparationHandler().getPreparation(chConfig.getName()).getTotalPrepareTime()); } if (!getCHPreparationHandler().getEdgeBasedCHConfigs().isEmpty()) { CHConfig chConfig = getCHPreparationHandler().getEdgeBasedCHConfigs().get(0); int edgesAndShortcuts = getGraphHopperStorage().getRoutingCHGraph(chConfig.getName()).getEdges(); put(Parameters.CH.PREPARE + "edge.shortcuts", edgesAndShortcuts - edges); - put(Parameters.CH.PREPARE + "edge.time", getCHPreparationHandler().getPreparation(chConfig).getTotalPrepareTime()); + put(Parameters.CH.PREPARE + "edge.time", getCHPreparationHandler().getPreparation(chConfig.getName()).getTotalPrepareTime()); } } From 8f516e94204d12c2404009e44e02b79423813b38 Mon Sep 17 00:00:00 2001 From: easbar Date: Mon, 18 Oct 2021 10:37:06 +0200 Subject: [PATCH 36/53] Increase Xmx for tests from 180m to 190m, probably necessary bc we add more barrier edges since b7e286e --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b8d82eb02c7..0c5e56ab473 100644 --- a/pom.xml +++ b/pom.xml @@ -222,7 +222,7 @@ maven-surefire-plugin 2.22.2 - -Xmx180m -Xms180m -Duser.language=en + -Xmx190m -Xms190m -Duser.language=en From c8afa7cca78f4b1fb41d74161e8d4116b72b9d32 Mon Sep 17 00:00:00 2001 From: otbutz Date: Mon, 18 Oct 2021 12:42:41 +0200 Subject: [PATCH 37/53] Improve LMPreparationHandler logging (#2426) Co-authored-by: Thomas Butz Co-authored-by: Andi --- .../graphhopper/routing/lm/LMPreparationHandler.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java b/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java index fca66f1e8af..234dfbfe381 100644 --- a/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java +++ b/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; +import java.net.URL; import java.util.*; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; @@ -50,7 +51,7 @@ * @author Peter Karich */ public class LMPreparationHandler { - private final Logger LOGGER = LoggerFactory.getLogger(LMPreparationHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(LMPreparationHandler.class); private int landmarkCount = 16; private final List preparations = new ArrayList<>(); @@ -290,11 +291,16 @@ public void createPreparations(GraphHopperStorage ghStorage, LocationIndex locat private JsonFeatureCollection loadLandmarkSplittingFeatureCollection(String splitAreaLocation) { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JtsModule()); + URL builtinSplittingFile = LandmarkStorage.class.getResource("map.geo.json"); try (Reader reader = splitAreaLocation.isEmpty() ? - new InputStreamReader(LandmarkStorage.class.getResource("map.geo.json").openStream(), UTF_CS) : + new InputStreamReader(builtinSplittingFile.openStream(), UTF_CS) : new InputStreamReader(new FileInputStream(splitAreaLocation), UTF_CS)) { JsonFeatureCollection result = objectMapper.readValue(reader, JsonFeatureCollection.class); - LOGGER.info("Loaded landmark splitting collection from " + splitAreaLocation); + if (splitAreaLocation.isEmpty()) { + LOGGER.info("Loaded built-in landmark splitting collection from {}", builtinSplittingFile); + } else { + LOGGER.info("Loaded landmark splitting collection from {}", splitAreaLocation); + } return result; } catch (IOException e) { LOGGER.error("Problem while reading border map GeoJSON. Skipping this.", e); From 48d2c93916c2364baa6d4c3cdc1299a4a7d0056d Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 19 Oct 2021 21:36:10 +0200 Subject: [PATCH 38/53] Move graph sorting and bridge interpolation to process method. It should only run for import, not load --- .../java/com/graphhopper/GraphHopper.java | 48 ++++++------------- .../graphhopper/reader/osm/OSMReaderTest.java | 9 +--- 2 files changed, 17 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index c1d50eb9db8..099e8c076b4 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -75,7 +75,7 @@ * @author Peter Karich */ public class GraphHopper { - private final Logger logger = LoggerFactory.getLogger(getClass()); + private static final Logger logger = LoggerFactory.getLogger(GraphHopper.class); private final Map profilesByName = new LinkedHashMap<>(); private final String fileLockName = "gh.lock"; // utils @@ -623,6 +623,7 @@ private void process(String graphHopperLocation, boolean closeEarly) { ensureWriteAccess(); importOSM(); cleanUp(); + postImport(); postProcessing(closeEarly); flush(); } finally { @@ -631,6 +632,18 @@ private void process(String graphHopperLocation, boolean closeEarly) { } } + protected void postImport() { + if (sortGraph) { + GraphHopperStorage newGraph = GHUtility.newStorage(ghStorage); + GHUtility.sortDFS(ghStorage, newGraph); + logger.info("graph sorted (" + getMemInfo() + ")"); + ghStorage = newGraph; + } + + if (hasElevation()) + interpolateBridgesTunnelsAndFerries(); + } + protected void importOSM() { if (osmFile == null) throw new IllegalStateException("Couldn't load from existing folder: " + ghLocation @@ -877,35 +890,11 @@ private void initLMPreparationHandler() { } /** - * Does the preparation and creates the location index - */ - public final void postProcessing() { - postProcessing(false); - } - - /** - * Does the preparation and creates the location index + * Runs both after the import and when loading an existing Graph * * @param closeEarly release resources as early as possible */ protected void postProcessing(boolean closeEarly) { - // Later: move this into the GraphStorage.optimize method - // Or: Doing it after preparation to optimize shortcuts too. But not possible yet #12 - - if (sortGraph) { - if (ghStorage.isCHPossible() && isCHPrepared()) - throw new IllegalArgumentException("Sorting a prepared CH is not possible yet. See #12"); - - GraphHopperStorage newGraph = GHUtility.newStorage(ghStorage); - GHUtility.sortDFS(ghStorage, newGraph); - logger.info("graph sorted (" + getMemInfo() + ")"); - ghStorage = newGraph; - } - - if (!hasInterpolated() && hasElevation()) { - interpolateBridgesTunnelsAndFerries(); - } - initLocationIndex(); importPublicTransit(); @@ -930,12 +919,6 @@ protected void postProcessing(boolean closeEarly) { protected void importPublicTransit() { } - private static final String INTERPOLATION_KEY = "prepare.elevation_interpolation.done"; - - private boolean hasInterpolated() { - return "true".equals(ghStorage.getProperties().get(INTERPOLATION_KEY)); - } - void interpolateBridgesTunnelsAndFerries() { if (ghStorage.getEncodingManager().hasEncodedValue(RoadEnvironment.KEY)) { EnumEncodedValue roadEnvEnc = ghStorage.getEncodingManager().getEnumEncodedValue(RoadEnvironment.KEY, RoadEnvironment.class); @@ -949,7 +932,6 @@ void interpolateBridgesTunnelsAndFerries() { // See #2098 for mor information sw = new StopWatch().start(); new EdgeElevationInterpolator(ghStorage, roadEnvEnc, RoadEnvironment.FERRY).execute(); - ghStorage.getProperties().put(INTERPOLATION_KEY, true); logger.info("Bridge interpolation " + (int) bridge + "s, " + "tunnel interpolation " + (int) tunnel + "s, ferry interpolation " + (int) sw.stop().getSeconds() + "s"); } } diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index ddc567fb762..7bd8adedc69 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -82,10 +82,6 @@ public void tearDown() { Helper.removeDir(new File(dir)); } - GraphHopperStorage newGraph(String directory, EncodingManager encodingManager, boolean is3D, boolean turnRestrictionsImport) { - return new GraphHopperStorage(new RAMDirectory(directory, false), encodingManager, is3D, turnRestrictionsImport); - } - @Test public void testMain() { GraphHopper hopper = new GraphHopperFacade(file1).importOrLoad(); @@ -627,7 +623,7 @@ public void testRoadAttributes() { public void testEstimatedDistance() { final CarFlagEncoder encoder = new CarFlagEncoder(); EncodingManager manager = EncodingManager.create(encoder); - GraphHopperStorage ghStorage = newGraph(dir, manager, false, false); + GraphHopperStorage ghStorage = new GraphHopperStorage(new RAMDirectory(dir, false), manager, false, false); final Map latMap = new HashMap<>(); final Map lonMap = new HashMap<>(); latMap.put(1, 1.1d); @@ -1003,8 +999,7 @@ public GraphHopperFacade(String osmFile, boolean turnCosts, String prefLang) { @Override protected void importOSM() { - GraphHopperStorage tmpGraph = newGraph(dir, getEncodingManager(), hasElevation(), - getEncodingManager().needsTurnCostsSupport()); + GraphHopperStorage tmpGraph = new GraphHopperStorage(new RAMDirectory(dir, false), getEncodingManager(), hasElevation(), getEncodingManager().needsTurnCostsSupport()); setGraphHopperStorage(tmpGraph); super.importOSM(); carAccessEnc = carEncoder.getAccessEnc(); From 334e052e77f4ebc1e03aa2948a452de732ecb18b Mon Sep 17 00:00:00 2001 From: easbar Date: Wed, 20 Oct 2021 11:59:06 +0200 Subject: [PATCH 39/53] Clean up LM/CH preparation config handling * there is no need to first add the configs and then the preparations (at least not anymore) --- .../java/com/graphhopper/GraphHopper.java | 50 ++++------ .../routing/ch/CHPreparationHandler.java | 64 +------------ .../routing/lm/LMPreparationHandler.java | 54 ++--------- .../routing/lm/LandmarkSuggestion.java | 8 +- .../storage/GraphHopperStorage.java | 3 +- .../reader/osm/GraphHopperOSMTest.java | 14 +-- .../routing/ch/CHPreparationHandlerTest.java | 93 +------------------ .../routing/lm/LMPreparationHandlerTest.java | 11 ++- .../com/graphhopper/tools/Measurement.java | 24 +++-- .../java/com/graphhopper/ui/MiniGraphUI.java | 6 +- 10 files changed, 59 insertions(+), 268 deletions(-) diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index 099e8c076b4..16729d797f5 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -67,7 +67,6 @@ import static com.graphhopper.util.GHUtility.readCountries; import static com.graphhopper.util.Helper.*; import static com.graphhopper.util.Parameters.Algorithms.RoundTrip; -import static java.util.Collections.emptyList; /** * Easy to use access point to configure import and (offline) routing. @@ -746,19 +745,12 @@ public boolean load(String graphHopperFolder) { ghStorage = new GraphHopperStorage(dir, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize); checkProfilesConsistency(); - if (lmPreparationHandler.isEnabled()) - initLMPreparationHandler(); - - List chConfigs; + // todo: move this after base graph loading/import, just like for LM. there is no real reason we have to setup CH before if (chPreparationHandler.isEnabled()) { - initCHPreparationHandler(); - chConfigs = chPreparationHandler.getCHConfigs(); - } else { - chConfigs = emptyList(); + List chConfigs = createCHConfigs(chPreparationHandler.getCHProfiles()); + ghStorage.addCHGraphs(chConfigs); } - ghStorage.addCHGraphs(chConfigs); - if (!new File(graphHopperFolder).exists()) return false; @@ -852,30 +844,26 @@ public final CHPreparationHandler getCHPreparationHandler() { return chPreparationHandler; } - private void initCHPreparationHandler() { - if (chPreparationHandler.hasCHConfigs()) { - return; - } - - for (CHProfile chProfile : chPreparationHandler.getCHProfiles()) { + private List createCHConfigs(List chProfiles) { + List chConfigs = new ArrayList<>(); + for (CHProfile chProfile : chProfiles) { Profile profile = profilesByName.get(chProfile.getProfile()); if (profile.isTurnCosts()) { - chPreparationHandler.addCHConfig(CHConfig.edgeBased(profile.getName(), createWeighting(profile, new PMap()))); + chConfigs.add(CHConfig.edgeBased(profile.getName(), createWeighting(profile, new PMap()))); } else { - chPreparationHandler.addCHConfig(CHConfig.nodeBased(profile.getName(), createWeighting(profile, new PMap()))); + chConfigs.add(CHConfig.nodeBased(profile.getName(), createWeighting(profile, new PMap()))); } } + return chConfigs; } public final LMPreparationHandler getLMPreparationHandler() { return lmPreparationHandler; } - private void initLMPreparationHandler() { - if (lmPreparationHandler.hasLMProfiles()) - return; - - for (LMProfile lmProfile : lmPreparationHandler.getLMProfiles()) { + private List createLMConfigs(List lmProfiles) { + List lmConfigs = new ArrayList<>(); + for (LMProfile lmProfile : lmProfiles) { if (lmProfile.usesOtherPreparation()) continue; Profile profile = profilesByName.get(lmProfile.getProfile()); @@ -885,8 +873,9 @@ private void initLMPreparationHandler() { // Running the preparation without turn costs is also useful to allow e.g. changing the u_turn_costs per // request (we have to use the minimum weight settings (= no turn costs) for the preparation) Weighting weighting = createWeighting(profile, new PMap(), true); - lmPreparationHandler.addLMConfig(new LMConfig(profile.getName(), weighting)); + lmConfigs.add(new LMConfig(profile.getName(), weighting)); } + return lmConfigs; } /** @@ -896,12 +885,10 @@ private void initLMPreparationHandler() { */ protected void postProcessing(boolean closeEarly) { initLocationIndex(); - importPublicTransit(); if (lmPreparationHandler.isEnabled()) - lmPreparationHandler.createPreparations(ghStorage, locationIndex); - loadOrPrepareLM(closeEarly); + loadOrPrepareLM(closeEarly); if (chPreparationHandler.isEnabled()) chPreparationHandler.createPreparations(ghStorage); @@ -1053,10 +1040,6 @@ protected void prepareCH(boolean closeEarly) { * For landmarks it is required to always call this method: either it creates the landmark data or it loads it. */ protected void loadOrPrepareLM(boolean closeEarly) { - if (!lmPreparationHandler.isEnabled() || lmPreparationHandler.getPreparations().isEmpty()) { - return; - } - for (LMProfile profile : lmPreparationHandler.getLMProfiles()) { if (!getProfileVersion(profile.getProfile()).isEmpty() && !getProfileVersion(profile.getProfile()).equals("" + profilesByName.get(profile.getProfile()).getVersion())) @@ -1064,7 +1047,8 @@ protected void loadOrPrepareLM(boolean closeEarly) { } ensureWriteAccess(); ghStorage.freeze(); - if (lmPreparationHandler.loadOrDoWork(ghStorage.getProperties(), closeEarly)) { + List lmConfigs = createLMConfigs(lmPreparationHandler.getLMProfiles()); + if (lmPreparationHandler.loadOrDoWork(lmConfigs, ghStorage, locationIndex, closeEarly)) { ghStorage.getProperties().put(Landmark.PREPARE + "done", true); for (LMProfile profile : lmPreparationHandler.getLMProfiles()) { // potentially overwrite existing keys from CH diff --git a/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java b/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java index 2190db55b47..074f44182e4 100644 --- a/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java +++ b/core/src/main/java/com/graphhopper/routing/ch/CHPreparationHandler.java @@ -46,7 +46,6 @@ public class CHPreparationHandler { // we first add the profiles and later read them to create the config objects (because they require // the actual Weightings) private final List chProfiles = new ArrayList<>(); - private final List chConfigs = new ArrayList<>(); private int preparationThreads; private ExecutorService threadPool; private PMap pMap = new PMap(); @@ -70,57 +69,7 @@ public void init(GraphHopperConfig ghConfig) { } public final boolean isEnabled() { - return !chProfiles.isEmpty() || !chConfigs.isEmpty() || !preparations.isEmpty(); - } - - /** - * Decouple CH profiles from PrepareContractionHierarchies as we need CH profiles for the - * graphstorage and the graphstorage for the preparation. - */ - public CHPreparationHandler addCHConfig(CHConfig chConfig) { - chConfigs.add(chConfig); - return this; - } - - public CHPreparationHandler addPreparation(PrepareContractionHierarchies pch) { - // we want to make sure that CH preparations are added in the same order as their corresponding profiles - if (preparations.size() >= chConfigs.size()) { - throw new IllegalStateException("You need to add the corresponding CH configs before adding preparations."); - } - CHConfig expectedConfig = chConfigs.get(preparations.size()); - if (!pch.getCHConfig().equals(expectedConfig)) { - throw new IllegalArgumentException("CH config of preparation: " + pch + " needs to be identical to previously added CH config: " + expectedConfig); - } - preparations.add(pch); - return this; - } - - public final boolean hasCHConfigs() { - return !chConfigs.isEmpty(); - } - - public List getCHConfigs() { - return chConfigs; - } - - public List getNodeBasedCHConfigs() { - List result = new ArrayList<>(); - for (CHConfig chConfig : chConfigs) { - if (!chConfig.getTraversalMode().isEdgeBased()) { - result.add(chConfig); - } - } - return result; - } - - public List getEdgeBasedCHConfigs() { - List result = new ArrayList<>(); - for (CHConfig chConfig : chConfigs) { - if (chConfig.getTraversalMode().isEdgeBased()) { - result.add(chConfig); - } - } - return result; + return !chProfiles.isEmpty() || !preparations.isEmpty(); } public CHPreparationHandler setCHProfiles(CHProfile... chProfiles) { @@ -202,14 +151,11 @@ public void prepare(final StorableProperties properties, final boolean closeEarl } public void createPreparations(GraphHopperStorage ghStorage) { - if (!isEnabled() || !preparations.isEmpty()) - return; - if (!hasCHConfigs()) - throw new IllegalStateException("No CH profiles found"); - + if (!preparations.isEmpty()) + throw new IllegalStateException("CH preparations were created already"); LOGGER.info("Creating CH preparations, {}", getMemInfo()); - for (CHConfig chConfig : chConfigs) { - addPreparation(createCHPreparation(ghStorage, chConfig)); + for (CHConfig chConfig : ghStorage.getCHConfigs()) { + preparations.add(createCHPreparation(ghStorage, chConfig)); } } diff --git a/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java b/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java index 234dfbfe381..8852a924d8c 100644 --- a/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java +++ b/core/src/main/java/com/graphhopper/routing/lm/LMPreparationHandler.java @@ -55,10 +55,7 @@ public class LMPreparationHandler { private int landmarkCount = 16; private final List preparations = new ArrayList<>(); - // we first add the profiles and later read them to create the config objects (because they require - // the actual Weightings) private final List lmProfiles = new ArrayList<>(); - private final List lmConfigs = new ArrayList<>(); private final Map maximumWeights = new HashMap<>(); private int minNodes = -1; private final List lmSuggestionsLocations = new ArrayList<>(5); @@ -107,7 +104,7 @@ public int getLandmarks() { } public final boolean isEnabled() { - return !lmProfiles.isEmpty() || !lmConfigs.isEmpty() || !preparations.isEmpty(); + return !lmProfiles.isEmpty() || !preparations.isEmpty(); } public int getPreparationThreads() { @@ -146,40 +143,10 @@ public List getLMProfiles() { return lmProfiles; } - /** - * Decouple weightings from PrepareLandmarks as we need weightings for the graphstorage and the - * graphstorage for the preparation. - */ - public LMPreparationHandler addLMConfig(LMConfig lmConfig) { - lmConfigs.add(lmConfig); - return this; - } - - public LMPreparationHandler addPreparation(PrepareLandmarks plm) { - preparations.add(plm); - int lastIndex = preparations.size() - 1; - if (lastIndex >= lmConfigs.size()) - throw new IllegalStateException("Cannot access profile for PrepareLandmarks with " + plm.getLMConfig() - + ". Call add(LMConfig) before"); - - if (preparations.get(lastIndex).getLMConfig() != lmConfigs.get(lastIndex)) - throw new IllegalArgumentException("LMConfig of PrepareLandmarks " + preparations.get(lastIndex).getLMConfig() - + " needs to be identical to previously added " + lmConfigs.get(lastIndex)); - return this; - } - - public boolean hasLMProfiles() { - return !lmConfigs.isEmpty(); - } - public int size() { return preparations.size(); } - public List getLMConfigs() { - return lmConfigs; - } - public List getPreparations() { return preparations; } @@ -206,13 +173,15 @@ public PrepareLandmarks getPreparation(String profile) { * @return true if the preparation data for at least one profile was calculated. * @see CHPreparationHandler#prepare(StorableProperties, boolean) for a very similar method */ - public boolean loadOrDoWork(final StorableProperties properties, final boolean closeEarly) { + public boolean loadOrDoWork(List lmConfigs, GraphHopperStorage ghStorage, LocationIndex locationIndex, final boolean closeEarly) { + createPreparations(lmConfigs, ghStorage, locationIndex); for (PrepareLandmarks prep : preparations) { // using the area index we separate certain areas from each other but we do not change the base graph for this // so that other algorithms still can route between these areas if (areaIndex != null) prep.setAreaIndex(areaIndex); } + StorableProperties properties = ghStorage.getProperties(); ExecutorCompletionService completionService = new ExecutorCompletionService<>(threadPool); int counter = 0; final AtomicBoolean prepared = new AtomicBoolean(false); @@ -253,12 +222,9 @@ public boolean loadOrDoWork(final StorableProperties properties, final boolean c /** * This method creates the landmark storages ready for landmark creation. */ - public void createPreparations(GraphHopperStorage ghStorage, LocationIndex locationIndex) { - if (!isEnabled() || !preparations.isEmpty()) - return; - if (lmConfigs.isEmpty()) - throw new IllegalStateException("No landmark weightings found"); - + void createPreparations(List lmConfigs, GraphHopperStorage ghStorage, LocationIndex locationIndex) { + if (!preparations.isEmpty()) + throw new IllegalStateException("LM preparations were created already"); LOGGER.info("Creating LM preparations, {}", getMemInfo()); List lmSuggestions = new ArrayList<>(lmSuggestionsLocations.size()); if (!lmSuggestionsLocations.isEmpty()) { @@ -277,14 +243,14 @@ public void createPreparations(GraphHopperStorage ghStorage, LocationIndex locat throw new IllegalStateException("maximumWeight cannot be null. Default should be just negative. " + "Couldn't find " + lmConfig.getName() + " in " + maximumWeights); - PrepareLandmarks tmpPrepareLM = new PrepareLandmarks(ghStorage.getDirectory(), ghStorage, + PrepareLandmarks prepareLandmarks = new PrepareLandmarks(ghStorage.getDirectory(), ghStorage, lmConfig, landmarkCount). setLandmarkSuggestions(lmSuggestions). setMaximumWeight(maximumWeight). setLogDetails(logDetails); if (minNodes > 1) - tmpPrepareLM.setMinimumNodes(minNodes); - addPreparation(tmpPrepareLM); + prepareLandmarks.setMinimumNodes(minNodes); + preparations.add(prepareLandmarks); } } diff --git a/core/src/main/java/com/graphhopper/routing/lm/LandmarkSuggestion.java b/core/src/main/java/com/graphhopper/routing/lm/LandmarkSuggestion.java index 9fe0d1f2ea2..1f48738e82e 100644 --- a/core/src/main/java/com/graphhopper/routing/lm/LandmarkSuggestion.java +++ b/core/src/main/java/com/graphhopper/routing/lm/LandmarkSuggestion.java @@ -16,8 +16,8 @@ * suboptimal automatic landmark finding process. */ public class LandmarkSuggestion { - private List nodeIds; - private BBox box; + private final List nodeIds; + private final BBox box; public LandmarkSuggestion(List nodeIds, BBox box) { this.nodeIds = nodeIds; @@ -36,10 +36,10 @@ public BBox getBox() { * The expected format is lon,lat per line where lines starting with characters will be ignored. You can create * such a file manually via geojson.io -> Save as CSV. Optionally add a second line with * 

#BBOX:minLat,minLon,maxLat,maxLon
- * + *

* to specify an explicit bounding box. TODO: support GeoJSON instead. */ - public static final LandmarkSuggestion readLandmarks(String file, LocationIndex locationIndex) throws IOException { + public static LandmarkSuggestion readLandmarks(String file, LocationIndex locationIndex) throws IOException { // landmarks should be suited for all vehicles EdgeFilter edgeFilter = EdgeFilter.ALL_EDGES; List lines = Helper.readFile(file); diff --git a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java index bd03a44d47e..fa03b28d4bb 100644 --- a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java +++ b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java @@ -103,9 +103,8 @@ public GraphHopperStorage addCHGraph(CHConfig chConfig) { * @see #addCHGraph(CHConfig) */ public GraphHopperStorage addCHGraphs(List chConfigs) { - for (CHConfig chConfig : chConfigs) { + for (CHConfig chConfig : chConfigs) addCHGraph(chConfig); - } return this; } diff --git a/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java b/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java index 5aba44a274f..3d78089fe49 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java @@ -848,22 +848,10 @@ public String toString() { CHConfig simpleTruckConfig = CHConfig.nodeBased("simple_truck", fwSimpleTruck); CHConfig truckConfig = CHConfig.nodeBased("truck", fwTruck); GraphHopperStorage storage = new GraphBuilder(em).setCHConfigs(Arrays.asList(simpleTruckConfig, truckConfig)).build(); - chHandler.addCHConfig(simpleTruckConfig); - chHandler.addCHConfig(truckConfig); - chHandler.addPreparation(PrepareContractionHierarchies.fromGraphHopperStorage(storage, simpleTruckConfig)); - chHandler.addPreparation(PrepareContractionHierarchies.fromGraphHopperStorage(storage, truckConfig)); + chHandler.createPreparations(storage); assertEquals("fastest|truck", chHandler.getPreparation("truck").getCHConfig().getWeighting().toString()); assertEquals("fastest|simple_truck", chHandler.getPreparation("simple_truck").getCHConfig().getWeighting().toString()); - - // make sure weighting cannot be mixed - chHandler.addCHConfig(truckConfig); - chHandler.addCHConfig(simpleTruckConfig); - try { - chHandler.addPreparation(PrepareContractionHierarchies.fromGraphHopperStorage(storage, simpleTruckConfig)); - fail(); - } catch (Exception ex) { - } } @Test diff --git a/core/src/test/java/com/graphhopper/routing/ch/CHPreparationHandlerTest.java b/core/src/test/java/com/graphhopper/routing/ch/CHPreparationHandlerTest.java index e12ee8e5b6a..6416f581a7a 100644 --- a/core/src/test/java/com/graphhopper/routing/ch/CHPreparationHandlerTest.java +++ b/core/src/test/java/com/graphhopper/routing/ch/CHPreparationHandlerTest.java @@ -33,104 +33,13 @@ * @author Peter Karich */ public class CHPreparationHandlerTest { - private CHPreparationHandler instance; - private CHConfig configNode1; - private CHConfig configNode2; - private CHConfig configNode3; - private CHConfig configEdge1; - private CHConfig configEdge2; - private CHConfig configEdge3; - private GraphHopperStorage ghStorage; - - @BeforeEach - public void setup() { - instance = new CHPreparationHandler(); - EncodingManager encodingManager = EncodingManager.create("car"); - ghStorage = new GraphBuilder(encodingManager).withTurnCosts(true) - .setCHConfigStrings( - "p1|car|fastest|node", - "p2|car|shortest|node", - "p3|car|short_fastest|node", - "p4|car|fastest|edge|30", - "p5|car|shortest|edge|30", - "p6|car|short_fastest|edge|30" - ) - .create(); - List chConfigs = ghStorage.getCHConfigs(); - configNode1 = chConfigs.get(0); - configNode2 = chConfigs.get(1); - configNode3 = chConfigs.get(2); - configEdge1 = chConfigs.get(3); - configEdge2 = chConfigs.get(4); - configEdge3 = chConfigs.get(5); - } @Test public void testEnabled() { + CHPreparationHandler instance = new CHPreparationHandler(); assertFalse(instance.isEnabled()); instance.setCHProfiles(new CHProfile("myconfig")); assertTrue(instance.isEnabled()); } - @Test - public void testAddingPreparationBeforeProfile_throws() { - assertThrows(IllegalStateException.class, () -> { - PrepareContractionHierarchies preparation = createPreparation(configNode1); - instance.addPreparation(preparation); - }); - } - - @Test - public void testAddingPreparationWithWrongProfile_throws() { - assertThrows(IllegalArgumentException.class, () -> { - instance.addCHConfig(configNode1); - PrepareContractionHierarchies preparation = createPreparation(configNode2); - instance.addPreparation(preparation); - }); - } - - @Test - public void testAddingPreparationsInWrongOrder_throws() { - assertThrows(IllegalArgumentException.class, () -> { - instance.addCHConfig(configNode1); - instance.addCHConfig(configNode2); - instance.addPreparation(createPreparation(configNode2)); - instance.addPreparation(createPreparation(configNode1)); - }); - } - - @Test - public void testAddingPreparationsWithEdgeAndNodeBasedIntermixed_works() { - instance.addCHConfig(configNode1); - instance.addCHConfig(configEdge1); - instance.addCHConfig(configNode2); - instance.addPreparation(createPreparation(configNode1)); - instance.addPreparation(createPreparation(configEdge1)); - instance.addPreparation(createPreparation(configNode2)); - } - - @Test - public void testAddingEdgeAndNodeBased_works() { - instance.addCHConfig(configNode1); - instance.addCHConfig(configNode2); - instance.addCHConfig(configEdge1); - instance.addCHConfig(configEdge2); - instance.addCHConfig(configNode3); - instance.addPreparation(createPreparation(configNode1)); - instance.addPreparation(createPreparation(configNode2)); - instance.addPreparation(createPreparation(configEdge1)); - instance.addPreparation(createPreparation(configEdge2)); - instance.addPreparation(createPreparation(configNode3)); - - CHConfig[] expectedConfigs = new CHConfig[]{configNode1, configNode2, configEdge1, configEdge2, configNode3}; - List preparations = instance.getPreparations(); - for (int i = 0; i < preparations.size(); ++i) { - assertSame(expectedConfigs[i], preparations.get(i).getCHConfig()); - } - } - - private PrepareContractionHierarchies createPreparation(CHConfig chConfig) { - return PrepareContractionHierarchies.fromGraphHopperStorage(ghStorage, chConfig); - } - } diff --git a/core/src/test/java/com/graphhopper/routing/lm/LMPreparationHandlerTest.java b/core/src/test/java/com/graphhopper/routing/lm/LMPreparationHandlerTest.java index 1460742fe32..ab6b28e6e79 100644 --- a/core/src/test/java/com/graphhopper/routing/lm/LMPreparationHandlerTest.java +++ b/core/src/test/java/com/graphhopper/routing/lm/LMPreparationHandlerTest.java @@ -12,7 +12,9 @@ import com.graphhopper.storage.RAMDirectory; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -36,10 +38,11 @@ public void maximumLMWeight() { ); FlagEncoder car = new CarFlagEncoder(); EncodingManager em = EncodingManager.create(car); - handler - .addLMConfig(new LMConfig("conf1", new FastestWeighting(car))) - .addLMConfig(new LMConfig("conf2", new ShortestWeighting(car))); - handler.createPreparations(new GraphHopperStorage(new RAMDirectory(), em, false), null); + List lmConfigs = Arrays.asList( + new LMConfig("conf1", new FastestWeighting(car)), + new LMConfig("conf2", new ShortestWeighting(car)) + ); + handler.createPreparations(lmConfigs, new GraphHopperStorage(new RAMDirectory(), em, false), null); assertEquals(1, handler.getPreparations().get(0).getLandmarkStorage().getFactor(), .1); assertEquals(0.3, handler.getPreparations().get(1).getLandmarkStorage().getFactor(), .1); } diff --git a/tools/src/main/java/com/graphhopper/tools/Measurement.java b/tools/src/main/java/com/graphhopper/tools/Measurement.java index 8ffa8b5bb15..ec310f0c47d 100644 --- a/tools/src/main/java/com/graphhopper/tools/Measurement.java +++ b/tools/src/main/java/com/graphhopper/tools/Measurement.java @@ -127,17 +127,15 @@ protected void prepareCH(boolean closeEarly) { // note that we measure the total time of all (possibly edge&node) CH preparations put(Parameters.CH.PREPARE + "time", sw.stop().getMillis()); int edges = getGraphHopperStorage().getEdges(); - if (!getCHPreparationHandler().getNodeBasedCHConfigs().isEmpty()) { - CHConfig chConfig = getCHPreparationHandler().getNodeBasedCHConfigs().get(0); - int edgesAndShortcuts = getGraphHopperStorage().getRoutingCHGraph(chConfig.getName()).getEdges(); + if (getGraphHopperStorage().getRoutingCHGraph("profile_no_tc") != null) { + int edgesAndShortcuts = getGraphHopperStorage().getRoutingCHGraph("profile_no_tc").getEdges(); put(Parameters.CH.PREPARE + "node.shortcuts", edgesAndShortcuts - edges); - put(Parameters.CH.PREPARE + "node.time", getCHPreparationHandler().getPreparation(chConfig.getName()).getTotalPrepareTime()); + put(Parameters.CH.PREPARE + "node.time", getCHPreparationHandler().getPreparation("profile_no_tc").getTotalPrepareTime()); } - if (!getCHPreparationHandler().getEdgeBasedCHConfigs().isEmpty()) { - CHConfig chConfig = getCHPreparationHandler().getEdgeBasedCHConfigs().get(0); - int edgesAndShortcuts = getGraphHopperStorage().getRoutingCHGraph(chConfig.getName()).getEdges(); + if (getGraphHopperStorage().getRoutingCHGraph("profile_tc") != null) { + int edgesAndShortcuts = getGraphHopperStorage().getRoutingCHGraph("profile_tc").getEdges(); put(Parameters.CH.PREPARE + "edge.shortcuts", edgesAndShortcuts - edges); - put(Parameters.CH.PREPARE + "edge.time", getCHPreparationHandler().getPreparation(chConfig.getName()).getTotalPrepareTime()); + put(Parameters.CH.PREPARE + "edge.time", getCHPreparationHandler().getPreparation("profile_tc").getTotalPrepareTime()); } } @@ -226,10 +224,9 @@ protected void importOSM() { boolean isCH = true; boolean isLM = false; gcAndWait(); - if (!hopper.getCHPreparationHandler().getNodeBasedCHConfigs().isEmpty()) { - CHConfig chConfig = hopper.getCHPreparationHandler().getNodeBasedCHConfigs().get(0); - RoutingCHGraph lg = g.getRoutingCHGraph(chConfig.getName()); - measureGraphTraversalCH(lg, count * 100); + RoutingCHGraph nodeBasedCH = g.getRoutingCHGraph("profile_no_tc"); + if (nodeBasedCH != null) { + measureGraphTraversalCH(nodeBasedCH, count * 100); gcAndWait(); measureRouting(hopper, new QuerySettings("routingCH", count, isCH, isLM). withInstructions().sod()); @@ -249,7 +246,8 @@ protected void importOSM() { measureRouting(hopper, new QuerySettings("routingCH_via_100_full", count / 100, isCH, isLM). withPoints(100).sod().withInstructions().simplify().pathDetails()); } - if (!hopper.getCHPreparationHandler().getEdgeBasedCHConfigs().isEmpty()) { + RoutingCHGraph edgeBasedCH = g.getRoutingCHGraph("profile_tc"); + if (edgeBasedCH != null) { measureRouting(hopper, new QuerySettings("routingCH_edge", count, isCH, isLM). edgeBased().withInstructions()); measureRouting(hopper, new QuerySettings("routingCH_edge_alt", count / 10, isCH, isLM). diff --git a/tools/src/main/java/com/graphhopper/ui/MiniGraphUI.java b/tools/src/main/java/com/graphhopper/ui/MiniGraphUI.java index ef99a8ea82a..bca6bedcd56 100644 --- a/tools/src/main/java/com/graphhopper/ui/MiniGraphUI.java +++ b/tools/src/main/java/com/graphhopper/ui/MiniGraphUI.java @@ -336,10 +336,8 @@ public void paintComponent(final Graphics2D g2) { private RoutingAlgorithm createAlgo(GraphHopper hopper) { Profile profile = hopper.getProfiles().iterator().next(); if (useCH) { - CHConfig chConfig = hopper.getCHPreparationHandler().getNodeBasedCHConfigs().get(0); - Weighting weighting = chConfig.getWeighting(); - RoutingCHGraph chGraph = hopper.getGraphHopperStorage().getRoutingCHGraph(chConfig.getName()); - logger.info("CH algo, weighting: " + weighting); + RoutingCHGraph chGraph = hopper.getGraphHopperStorage().getRoutingCHGraph(profile.getName()); + logger.info("CH algo, profile: " + profile.getName()); QueryGraph qGraph = QueryGraph.create(hopper.getGraphHopperStorage(), fromRes, toRes); QueryRoutingCHGraph queryRoutingCHGraph = new QueryRoutingCHGraph(chGraph, qGraph); return new CHDebugAlgo(queryRoutingCHGraph, mg); From 98c2f3c427ced5c65d5cfbf0b9bba8a62cc14455 Mon Sep 17 00:00:00 2001 From: Andi Date: Thu, 21 Oct 2021 12:41:24 +0200 Subject: [PATCH 40/53] Remove ghLocation parameter from GraphHopper#load (#2437) * use hopper.setGraphHopperLocation(ghLocation);hopper.load() instead of hopper.load(ghLocation) --- CHANGELOG.md | 1 + .../java/com/graphhopper/GraphHopper.java | 37 ++-- .../graphhopper/GraphHopperProfileTest.java | 90 ++------- .../java/com/graphhopper/GraphHopperTest.java | 5 +- .../reader/osm/GraphHopperOSMTest.java | 187 ++++++++++-------- .../storage/GraphHopperStorageTest.java | 9 +- 6 files changed, 143 insertions(+), 186 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b7a069dff7..32841909f14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 5.0 [not yet released] +- use GraphHopper#setGraphHopperLocation before calling load() instead of GraphHopper#load(graphHopperLocation) (#2437) - barrier nodes at junctions are now ignored (#2433) - AbstractFlagEncoder#handleNodeTags was replaced by AbstractFlagEncoder#isBarrier (#2434) - consider heading when snapping coordinates to the road network, this is especially important for navigation (#2411) diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index 16729d797f5..02af0e6b075 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -581,9 +581,9 @@ private void printInfo() { * disc which is usually a lot faster. */ public GraphHopper importOrLoad() { - if (!load(ghLocation)) { + if (!load()) { printInfo(); - process(ghLocation, false); + process(false); } else { printInfo(); } @@ -594,9 +594,9 @@ public GraphHopper importOrLoad() { * Imports and processes data, storing it to disk when complete. */ public void importAndClose() { - if (!load(ghLocation)) { + if (!load()) { printInfo(); - process(ghLocation, true); + process(true); } else { printInfo(); logger.info("Graph already imported into " + ghLocation); @@ -607,17 +607,17 @@ public void importAndClose() { /** * Creates the graph from OSM data. */ - private void process(String graphHopperLocation, boolean closeEarly) { - setGraphHopperLocation(graphHopperLocation); + private void process(boolean closeEarly) { + setGraphHopperLocation(ghLocation); GHLock lock = null; try { if (ghStorage == null) throw new IllegalStateException("GraphHopperStorage must be initialized before starting the import"); if (ghStorage.getDirectory().getDefaultType().isStoring()) { - lockFactory.setLockDir(new File(graphHopperLocation)); + lockFactory.setLockDir(new File(ghLocation)); lock = lockFactory.create(fileLockName, true); if (!lock.tryLock()) - throw new RuntimeException("To avoid multiple writers we need to obtain a write lock but it failed. In " + graphHopperLocation, lock.getObtainFailedReason()); + throw new RuntimeException("To avoid multiple writers we need to obtain a write lock but it failed. In " + ghLocation, lock.getObtainFailedReason()); } ensureWriteAccess(); importOSM(); @@ -702,35 +702,32 @@ protected File _getOSMFile() { } /** - * Opens existing graph folder. - * - * @param graphHopperFolder is the folder containing graphhopper files. Can be a compressed file - * too ala folder-content.ghz. + * Load from existing graph folder. */ - public boolean load(String graphHopperFolder) { - if (isEmpty(graphHopperFolder)) + public boolean load() { + if (isEmpty(ghLocation)) throw new IllegalStateException("GraphHopperLocation is not specified. Call setGraphHopperLocation or init before"); if (fullyLoaded) throw new IllegalStateException("graph is already successfully loaded"); - File tmpFileOrFolder = new File(graphHopperFolder); + File tmpFileOrFolder = new File(ghLocation); if (!tmpFileOrFolder.isDirectory() && tmpFileOrFolder.exists()) { throw new IllegalArgumentException("GraphHopperLocation cannot be an existing file. Has to be either non-existing or a folder."); } else { - File compressed = new File(graphHopperFolder + ".ghz"); + File compressed = new File(ghLocation + ".ghz"); if (compressed.exists() && !compressed.isDirectory()) { try { - new Unzipper().unzip(compressed.getAbsolutePath(), graphHopperFolder, removeZipped); + new Unzipper().unzip(compressed.getAbsolutePath(), ghLocation, removeZipped); } catch (IOException ex) { throw new RuntimeException("Couldn't extract file " + compressed.getAbsolutePath() - + " to " + graphHopperFolder, ex); + + " to " + ghLocation, ex); } } } - setGraphHopperLocation(graphHopperFolder); + setGraphHopperLocation(ghLocation); if (!allowWrites && dataAccessType.isMMap()) dataAccessType = DAType.MMAP_RO; @@ -751,7 +748,7 @@ public boolean load(String graphHopperFolder) { ghStorage.addCHGraphs(chConfigs); } - if (!new File(graphHopperFolder).exists()) + if (!new File(ghLocation).exists()) return false; GHLock lock = null; diff --git a/core/src/test/java/com/graphhopper/GraphHopperProfileTest.java b/core/src/test/java/com/graphhopper/GraphHopperProfileTest.java index d5569597e07..b2c9c6edf55 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperProfileTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperProfileTest.java @@ -52,16 +52,11 @@ public void deserialize() throws IOException { @Test public void duplicateProfileName_error() { final GraphHopper hopper = createHopper(); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.setProfiles( - new Profile("my_profile").setVehicle("car").setWeighting("fastest"), - new Profile("your_profile").setVehicle("car").setWeighting("short_fastest"), - new Profile("my_profile").setVehicle("car").setWeighting("shortest") - ); - } - }, "Profile names must be unique. Duplicate name: 'my_profile'"); + assertIllegalArgument(() -> hopper.setProfiles( + new Profile("my_profile").setVehicle("car").setWeighting("fastest"), + new Profile("your_profile").setVehicle("car").setWeighting("short_fastest"), + new Profile("my_profile").setVehicle("car").setWeighting("shortest") + ), "Profile names must be unique. Duplicate name: 'my_profile'"); } @Test @@ -70,25 +65,14 @@ public void vehicleDoesNotExist_error() { hopper.getEncodingManagerBuilder().add(new CarFlagEncoder()); hopper.setGraphHopperLocation(GH_LOCATION).setStoreOnFlush(false). setProfiles(new Profile("profile").setVehicle("your_car")); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "entry in encoder list not supported: your_car"); + assertIllegalArgument(hopper::load, "entry in encoder list not supported: your_car"); } @Test public void vehicleDoesNotExist_error2() { final GraphHopper hopper = new GraphHopper().setGraphHopperLocation(GH_LOCATION).setStoreOnFlush(false). setProfiles(new Profile("profile").setVehicle("your_car")); - - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "entry in encoder list not supported: your_car"); + assertIllegalArgument(hopper::load, "entry in encoder list not supported: your_car"); } @Test @@ -97,7 +81,7 @@ public void oneVehicleTwoProfilesWithAndWithoutTC_noError() { hopper.setProfiles( new Profile("profile1").setVehicle("car").setTurnCosts(false), new Profile("profile2").setVehicle("car").setTurnCosts(true)); - hopper.load(GH_LOCATION); + hopper.load(); } @Test @@ -106,19 +90,14 @@ public void oneVehicleTwoProfilesWithAndWithoutTC2_noError() { hopper.setProfiles( new Profile("profile2").setVehicle("car").setTurnCosts(true), new Profile("profile1").setVehicle("car").setTurnCosts(false)); - hopper.load(GH_LOCATION); + hopper.load(); } @Test public void profileWithUnknownWeighting_error() { final GraphHopper hopper = createHopper(); hopper.setProfiles(new Profile("profile").setVehicle("car").setWeighting("your_weighting")); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, + assertIllegalArgument(hopper::load, "Could not create weighting for profile: 'profile'", "Weighting 'your_weighting' not supported" ); @@ -129,12 +108,7 @@ public void chProfileDoesNotExist_error() { final GraphHopper hopper = createHopper(); hopper.setProfiles(new Profile("profile1").setVehicle("car")); hopper.getCHPreparationHandler().setCHProfiles(new CHProfile("other_profile")); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "CH profile references unknown profile 'other_profile'"); + assertIllegalArgument(hopper::load, "CH profile references unknown profile 'other_profile'"); } @Test @@ -145,12 +119,7 @@ public void duplicateCHProfile_error() { new CHProfile("profile"), new CHProfile("profile") ); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "Duplicate CH reference to profile 'profile'"); + assertIllegalArgument(hopper::load, "Duplicate CH reference to profile 'profile'"); } @Test @@ -158,12 +127,7 @@ public void lmProfileDoesNotExist_error() { final GraphHopper hopper = createHopper(); hopper.setProfiles(new Profile("profile1").setVehicle("car")); hopper.getLMPreparationHandler().setLMProfiles(new LMProfile("other_profile")); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "LM profile references unknown profile 'other_profile'"); + assertIllegalArgument(hopper::load, "LM profile references unknown profile 'other_profile'"); } @Test @@ -174,12 +138,7 @@ public void duplicateLMProfile_error() { new LMProfile("profile"), new LMProfile("profile") ); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "Multiple LM profiles are using the same profile 'profile'"); + assertIllegalArgument(hopper::load, "Multiple LM profiles are using the same profile 'profile'"); } @Test @@ -189,12 +148,7 @@ public void unknownLMPreparationProfile_error() { hopper.getLMPreparationHandler().setLMProfiles( new LMProfile("profile").setPreparationProfile("xyz") ); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "LM profile references unknown preparation profile 'xyz'"); + assertIllegalArgument(hopper::load, "LM profile references unknown preparation profile 'xyz'"); } @Test @@ -210,12 +164,7 @@ public void lmPreparationProfileChain_error() { new LMProfile("profile2").setPreparationProfile("profile1"), new LMProfile("profile3").setPreparationProfile("profile2") ); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "Cannot use 'profile2' as preparation_profile for LM profile 'profile3', because it uses another profile for preparation itself."); + assertIllegalArgument(hopper::load, "Cannot use 'profile2' as preparation_profile for LM profile 'profile3', because it uses another profile for preparation itself."); } @Test @@ -229,12 +178,7 @@ public void noLMProfileForPreparationProfile_error() { hopper.getLMPreparationHandler().setLMProfiles( new LMProfile("profile1").setPreparationProfile("profile2") ); - assertIllegalArgument(new Runnable() { - @Override - public void run() { - hopper.load(GH_LOCATION); - } - }, "Unknown LM preparation profile 'profile2' in LM profile 'profile1' cannot be used as preparation_profile"); + assertIllegalArgument(hopper::load, "Unknown LM preparation profile 'profile2' in LM profile 'profile1' cannot be used as preparation_profile"); } private GraphHopper createHopper() { diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index d789f05732f..da4fb5b8d24 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -2263,8 +2263,9 @@ void interpolateBridgesTunnelsAndFerries() { } }. setProfiles(new Profile("profile")). - setElevation(true); - hopper.load(GH_LOCATION); + setElevation(true). + setGraphHopperLocation(GH_LOCATION); + hopper.load(); } assertEquals(1, counter.get()); } diff --git a/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java b/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java index 3d78089fe49..93a2ea1cd4c 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/GraphHopperOSMTest.java @@ -26,7 +26,10 @@ import com.graphhopper.routing.ch.CHPreparationHandler; import com.graphhopper.routing.ch.PrepareContractionHierarchies; import com.graphhopper.routing.lm.PrepareLandmarks; -import com.graphhopper.routing.util.*; +import com.graphhopper.routing.util.CarFlagEncoder; +import com.graphhopper.routing.util.EdgeFilter; +import com.graphhopper.routing.util.EncodingManager; +import com.graphhopper.routing.util.FlagEncoder; import com.graphhopper.routing.weighting.FastestWeighting; import com.graphhopper.routing.weighting.Weighting; import com.graphhopper.routing.weighting.custom.CustomProfile; @@ -97,7 +100,8 @@ public void testLoadOSM() { setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting)). setStoreOnFlush(true); hopper.getCHPreparationHandler().setCHProfiles(new CHProfile(profile)); - assertTrue(hopper.load(ghLoc)); + hopper.setGraphHopperLocation(ghLoc); + assertTrue(hopper.load()); rsp = hopper.route(new GHRequest(51.2492152, 9.4317166, 51.2, 9.4). setProfile(profile)); assertFalse(rsp.hasErrors()); @@ -142,8 +146,9 @@ public void testLoadOSMNoCH() { gh.close(); gh = new GraphHopper(). setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting)). - setStoreOnFlush(true); - assertTrue(gh.load(ghLoc)); + setStoreOnFlush(true). + setGraphHopperLocation(ghLoc); + assertTrue(gh.load()); rsp = gh.route(new GHRequest(51.2492152, 9.4317166, 51.2, 9.4) .setProfile(profile)); assertFalse(rsp.hasErrors()); @@ -235,8 +240,9 @@ public void testLoadingWithDifferentCHConfig_issue471_pr1488() { // now load GH without CH profile gh = new GraphHopper(). setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting)). - setStoreOnFlush(true); - gh.load(ghLoc); + setStoreOnFlush(true). + setGraphHopperLocation(ghLoc); + gh.load(); // no error Helper.removeDir(new File(ghLoc)); @@ -257,13 +263,10 @@ public void testLoadingWithDifferentCHConfig_issue471_pr1488() { setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting)). setStoreOnFlush(true); gh.getCHPreparationHandler().setCHProfiles(new CHProfile("profile")); - - try { - gh.load(ghLoc); - fail(); - } catch (Exception ex) { - assertTrue(ex.getMessage().contains("is not contained in loaded CH profiles"), ex.getMessage()); - } + gh.setGraphHopperLocation(ghLoc); + final GraphHopper tmpGh = gh; + Exception ex = assertThrows(Exception.class, tmpGh::load); + assertTrue(ex.getMessage().contains("is not contained in loaded CH profiles"), ex.getMessage()); } @Test @@ -278,13 +281,15 @@ public void testAllowMultipleReadingInstances() { GraphHopper instance2 = new GraphHopper(). setStoreOnFlush(true). - setOSMFile(testOsm); - instance2.load(ghLoc); + setOSMFile(testOsm). + setGraphHopperLocation(ghLoc); + instance2.load(); GraphHopper instance3 = new GraphHopper(). setStoreOnFlush(true). - setOSMFile(testOsm); - instance3.load(ghLoc); + setOSMFile(testOsm). + setGraphHopperLocation(ghLoc); + instance3.load(); instance1.close(); instance2.close(); @@ -326,12 +331,13 @@ public void run() { GraphHopper instance2 = new GraphHopper(). setProfiles(new Profile("car").setVehicle("car").setWeighting("fastest")). setStoreOnFlush(true). - setOSMFile(testOsm); + setOSMFile(testOsm). + setGraphHopperLocation(ghLoc); try { // let thread reach the CountDownLatch latch2.await(3, TimeUnit.SECONDS); // now importOrLoad should have create a lock which this load call does not like - instance2.load(ghLoc); + instance2.load(); fail("There should have been an error because of the lock"); } catch (RuntimeException ex) { assertNotNull(ex); @@ -467,14 +473,14 @@ public void testFootAndCar() { @Test public void testFailsForWrongConfig() { instance = new GraphHopper().init( - new GraphHopperConfig(). - putObject("datareader.file", testOsm3). - putObject("datareader.dataaccess", "RAM"). - putObject("graph.flag_encoders", "foot,car"). - setProfiles(Arrays.asList( - new Profile("foot").setVehicle("foot").setWeighting("fastest"), - new Profile("car").setVehicle("car").setWeighting("fastest") - ))). + new GraphHopperConfig(). + putObject("datareader.file", testOsm3). + putObject("datareader.dataaccess", "RAM"). + putObject("graph.flag_encoders", "foot,car"). + setProfiles(Arrays.asList( + new Profile("foot").setVehicle("foot").setWeighting("fastest"), + new Profile("car").setVehicle("car").setWeighting("fastest") + ))). setGraphHopperLocation(ghLoc); instance.importOrLoad(); assertEquals(5, instance.getGraphHopperStorage().getNodes()); @@ -483,15 +489,16 @@ public void testFailsForWrongConfig() { // different config (flagEncoder list) try { GraphHopper tmpGH = new GraphHopper().init( - new GraphHopperConfig(). - putObject("datareader.file", testOsm3). - putObject("datareader.dataaccess", "RAM"). - putObject("graph.flag_encoders", "foot"). - setProfiles(Collections.singletonList( - new Profile("foot").setVehicle("foot").setWeighting("fastest") - ))). - setOSMFile(testOsm3); - tmpGH.load(ghLoc); + new GraphHopperConfig(). + putObject("datareader.file", testOsm3). + putObject("datareader.dataaccess", "RAM"). + putObject("graph.flag_encoders", "foot"). + setProfiles(Collections.singletonList( + new Profile("foot").setVehicle("foot").setWeighting("fastest") + ))). + setOSMFile(testOsm3). + setGraphHopperLocation(ghLoc); + tmpGH.load(); fail(); } catch (Exception ex) { assertTrue(ex.getMessage().startsWith("Encoding does not match"), ex.getMessage()); @@ -500,15 +507,16 @@ public void testFailsForWrongConfig() { // different order is no longer okay, see #350 try { GraphHopper tmpGH = new GraphHopper().init(new GraphHopperConfig(). - putObject("datareader.file", testOsm3). - putObject("datareader.dataaccess", "RAM"). - putObject("graph.flag_encoders", "car,foot"). - setProfiles(Arrays.asList( - new Profile("car").setVehicle("car").setWeighting("fastest"), - new Profile("foot").setVehicle("foot").setWeighting("fastest") - ))). - setOSMFile(testOsm3); - tmpGH.load(ghLoc); + putObject("datareader.file", testOsm3). + putObject("datareader.dataaccess", "RAM"). + putObject("graph.flag_encoders", "car,foot"). + setProfiles(Arrays.asList( + new Profile("car").setVehicle("car").setWeighting("fastest"), + new Profile("foot").setVehicle("foot").setWeighting("fastest") + ))). + setOSMFile(testOsm3) + .setGraphHopperLocation(ghLoc); + tmpGH.load(); fail(); } catch (Exception ex) { assertTrue(ex.getMessage().startsWith("Encoding does not match"), ex.getMessage()); @@ -516,18 +524,19 @@ public void testFailsForWrongConfig() { // different encoded values should fail to load instance = new GraphHopper().init( - new GraphHopperConfig(). - putObject("datareader.file", testOsm3). - putObject("datareader.dataaccess", "RAM"). - putObject("graph.encoded_values", "road_class"). - putObject("graph.flag_encoders", "foot,car"). - setProfiles(Arrays.asList( - new Profile("foot").setVehicle("foot").setWeighting("fastest"), - new Profile("car").setVehicle("car").setWeighting("fastest") - ))). - setOSMFile(testOsm3); + new GraphHopperConfig(). + putObject("datareader.file", testOsm3). + putObject("datareader.dataaccess", "RAM"). + putObject("graph.encoded_values", "road_class"). + putObject("graph.flag_encoders", "foot,car"). + setProfiles(Arrays.asList( + new Profile("foot").setVehicle("foot").setWeighting("fastest"), + new Profile("car").setVehicle("car").setWeighting("fastest") + ))). + setOSMFile(testOsm3). + setGraphHopperLocation(ghLoc); try { - instance.load(ghLoc); + instance.load(); fail(); } catch (Exception ex) { assertTrue(ex.getMessage().startsWith("Encoded values do not match"), ex.getMessage()); @@ -537,14 +546,14 @@ public void testFailsForWrongConfig() { @Test public void testFailsForWrongEVConfig() { instance = new GraphHopper().init( - new GraphHopperConfig(). - putObject("datareader.file", testOsm3). - putObject("datareader.dataaccess", "RAM"). - putObject("graph.flag_encoders", "foot,car"). - setProfiles(Arrays.asList( - new Profile("foot").setVehicle("foot").setWeighting("fastest"), - new Profile("car").setVehicle("car").setWeighting("fastest") - ))). + new GraphHopperConfig(). + putObject("datareader.file", testOsm3). + putObject("datareader.dataaccess", "RAM"). + putObject("graph.flag_encoders", "foot,car"). + setProfiles(Arrays.asList( + new Profile("foot").setVehicle("foot").setWeighting("fastest"), + new Profile("car").setVehicle("car").setWeighting("fastest") + ))). setGraphHopperLocation(ghLoc); instance.importOrLoad(); // older versions <= 0.12 did not store this property, ensure that we fail to load it @@ -555,22 +564,19 @@ public void testFailsForWrongEVConfig() { // different encoded values should fail to load instance = new GraphHopper().init( - new GraphHopperConfig(). - putObject("datareader.file", testOsm3). - putObject("datareader.dataaccess", "RAM"). - putObject("graph.encoded_values", "road_environment,road_class"). - putObject("graph.flag_encoders", "foot,car"). - setProfiles(Arrays.asList( - new Profile("foot").setVehicle("foot").setWeighting("fastest"), - new Profile("car").setVehicle("car").setWeighting("fastest") - ))). + new GraphHopperConfig(). + putObject("datareader.file", testOsm3). + putObject("datareader.dataaccess", "RAM"). + putObject("graph.location", ghLoc). + putObject("graph.encoded_values", "road_environment,road_class"). + putObject("graph.flag_encoders", "foot,car"). + setProfiles(Arrays.asList( + new Profile("foot").setVehicle("foot").setWeighting("fastest"), + new Profile("car").setVehicle("car").setWeighting("fastest") + ))). setOSMFile(testOsm3); - try { - instance.load(ghLoc); - fail(); - } catch (Exception ex) { - assertTrue(ex.getMessage().startsWith("Encoded values do not match"), ex.getMessage()); - } + Exception ex = assertThrows(Exception.class, () -> instance.load()); + assertTrue(ex.getMessage().startsWith("Encoded values do not match"), ex.getMessage()); } @Test @@ -580,11 +586,12 @@ public void testNoNPE_ifLoadNotSuccessful() { String weighting = "fastest"; instance = new GraphHopper(). setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting)). - setStoreOnFlush(true); + setStoreOnFlush(true). + setGraphHopperLocation(ghLoc); try { // loading from empty directory new File(ghLoc).mkdirs(); - assertFalse(instance.load(ghLoc)); + assertFalse(instance.load()); instance.route(new GHRequest(10, 40, 12, 32).setProfile(profile)); fail(); } catch (IllegalStateException ex) { @@ -596,7 +603,8 @@ public void testNoNPE_ifLoadNotSuccessful() { public void testDoesNotCreateEmptyFolderIfLoadingFromNonExistingPath() { instance = new GraphHopper(); instance.setProfiles(new Profile("car").setVehicle("car").setWeighting("fastest")); - assertFalse(instance.load(ghLoc)); + instance.setGraphHopperLocation(ghLoc); + assertFalse(instance.load()); assertFalse(new File(ghLoc).exists()); } @@ -890,16 +898,18 @@ public void testLoadingLMAndCHProfiles() { .setProfiles(new Profile("car").setVehicle("car").setWeighting("fastest")); hopper.getLMPreparationHandler().setLMProfiles(new LMProfile("car")); hopper.getCHPreparationHandler().setCHProfiles(new CHProfile("car")); - assertTrue(hopper.load(ghLoc)); + hopper.setGraphHopperLocation(ghLoc); + assertTrue(hopper.load()); hopper.close(); // problem: changed weighting in profile although LM preparation was enabled hopper = new GraphHopper() .setProfiles(new Profile("car").setVehicle("car").setWeighting("shortest")); hopper.getLMPreparationHandler().setLMProfiles(new LMProfile("car")); + hopper.setGraphHopperLocation(ghLoc); // do not load CH try { - assertFalse(hopper.load(ghLoc)); + assertFalse(hopper.load()); fail("load should fail"); } catch (Exception ex) { assertEquals("LM preparation of car already exists in storage and doesn't match configuration", ex.getMessage()); @@ -911,9 +921,10 @@ public void testLoadingLMAndCHProfiles() { hopper = new GraphHopper() .setProfiles(new Profile("car").setVehicle("car").setWeighting("shortest")); hopper.getCHPreparationHandler().setCHProfiles(new CHProfile("car")); + hopper.setGraphHopperLocation(ghLoc); // do not load LM try { - assertFalse(hopper.load(ghLoc)); + assertFalse(hopper.load()); fail("load should fail"); } catch (Exception ex) { assertEquals("CH preparation of car already exists in storage and doesn't match configuration", ex.getMessage()); @@ -937,7 +948,8 @@ public void testLoadingCustomProfiles() { hopper = new GraphHopper() .setProfiles(new CustomProfile("car").setCustomModel(customModel)); hopper.getLMPreparationHandler().setLMProfiles(new LMProfile("car")); - assertTrue(hopper.load(ghLoc)); + hopper.setGraphHopperLocation(ghLoc); + assertTrue(hopper.load()); hopper.close(); // do not load changed CustomModel @@ -945,8 +957,9 @@ public void testLoadingCustomProfiles() { hopper = new GraphHopper() .setProfiles(new CustomProfile("car").setCustomModel(customModel)); hopper.getLMPreparationHandler().setLMProfiles(new LMProfile("car")); + hopper.setGraphHopperLocation(ghLoc); try { - assertFalse(hopper.load(ghLoc)); + assertFalse(hopper.load()); fail("load should fail"); } catch (Exception ex) { assertEquals("LM preparation of car already exists in storage and doesn't match configuration", ex.getMessage()); diff --git a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java index a9c10df705c..ba91fc9ed3f 100644 --- a/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java +++ b/core/src/test/java/com/graphhopper/storage/GraphHopperStorageTest.java @@ -250,7 +250,8 @@ public void testLoadGraph_implicitEncodedValues_issue1862() { if (ch) { hopper.getCHPreparationHandler().setCHProfiles(new CHProfile("p_car")); } - assertTrue(hopper.load(defaultGraphLoc)); + hopper.setGraphHopperLocation(defaultGraphLoc); + assertTrue(hopper.load()); graph = hopper.getGraphHopperStorage(); assertEquals(nodes, graph.getNodes()); assertEquals(edges, graph.getAllEdges().length()); @@ -260,10 +261,10 @@ public void testLoadGraph_implicitEncodedValues_issue1862() { // load via explicitly configured FlagEncoders then we can define only one profile hopper.getEncodingManagerBuilder().add(createCarFlagEncoder()).add(new BikeFlagEncoder()); hopper.setProfiles(Collections.singletonList(new Profile("p_car").setVehicle("car").setWeighting("fastest"))); - if (ch) { + if (ch) hopper.getCHPreparationHandler().setCHProfiles(new CHProfile("p_car")); - } - assertTrue(hopper.load(defaultGraphLoc)); + hopper.setGraphHopperLocation(defaultGraphLoc); + assertTrue(hopper.load()); graph = hopper.getGraphHopperStorage(); assertEquals(nodes, graph.getNodes()); assertEquals(edges, graph.getAllEdges().length()); From 14bd13a78bd494173109016b58c05759d9cc680d Mon Sep 17 00:00:00 2001 From: easbar Date: Thu, 21 Oct 2021 13:21:48 +0200 Subject: [PATCH 41/53] Minor follow-up to #2437 --- .../java/com/graphhopper/GraphHopper.java | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index 02af0e6b075..2f53e458593 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -124,7 +124,7 @@ public EncodingManager.Builder getEncodingManagerBuilder() { public EncodingManager getEncodingManager() { if (encodingManager == null) - throw new IllegalStateException("EncodingManager not yet build"); + throw new IllegalStateException("EncodingManager not yet built"); return encodingManager; } @@ -394,6 +394,7 @@ public CountryRuleFactory getCountryRuleFactory() { * is read from `config.yml`. */ public GraphHopper init(GraphHopperConfig ghConfig) { + ensureNotLoaded(); // disabling_allowed config options were removed for GH 3.0 if (ghConfig.has("routing.ch.disabling_allowed")) throw new IllegalArgumentException("The 'routing.ch.disabling_allowed' configuration option is no longer supported"); @@ -413,12 +414,11 @@ public GraphHopper init(GraphHopperConfig ghConfig) { graphHopperFolder = pruneFileEnd(osmFile) + "-gh"; } + ghLocation = graphHopperFolder; countryRuleFactory = ghConfig.getBool("country_rules.enabled", false) ? new CountryRuleFactory() : null; customAreasDirectory = ghConfig.getString("custom_areas.directory", customAreasDirectory); - // graph - setGraphHopperLocation(graphHopperFolder); defaultSegmentSize = ghConfig.getInt("graph.dataaccess.segment_size", defaultSegmentSize); String graphDATypeStr = ghConfig.getString("graph.dataaccess", "RAM_STORE"); @@ -437,7 +437,6 @@ public GraphHopper init(GraphHopperConfig ghConfig) { if (encodingManager != null) throw new IllegalStateException("Cannot call init twice. EncodingManager was already initialized."); - emBuilder.setEnableInstructions(ghConfig.getBool("datareader.instructions", true)); emBuilder.setPreferredLanguage(ghConfig.getString("datareader.preferred_language", "")); emBuilder.setDateRangeParser(DateRangeParser.createInstance(ghConfig.getString("datareader.date_range_parser_day", ""))); @@ -489,10 +488,12 @@ public GraphHopper init(GraphHopperConfig ghConfig) { } private EncodingManager buildEncodingManager(GraphHopperConfig ghConfig) { + if (profilesByName.isEmpty()) + throw new IllegalStateException("no profiles exist but assumed to create EncodingManager. E.g. provide them in GraphHopperConfig when calling GraphHopper.init"); + String flagEncodersStr = ghConfig.getString("graph.flag_encoders", ""); - String encodedValueStr = ghConfig.getString("graph.encoded_values", ""); - Map flagEncoderMap = new LinkedHashMap<>(), implicitFlagEncoderMap = new HashMap<>(); - for (String encoderStr : Arrays.asList(flagEncodersStr.split(","))) { + Map flagEncoderMap = new LinkedHashMap<>(); + for (String encoderStr : flagEncodersStr.split(",")) { String key = encoderStr.split("\\|")[0]; if (!key.isEmpty()) { if (flagEncoderMap.containsKey(key)) @@ -500,8 +501,7 @@ private EncodingManager buildEncodingManager(GraphHopperConfig ghConfig) { flagEncoderMap.put(key, encoderStr); } } - if (profilesByName.isEmpty()) - throw new IllegalStateException("no profiles exist but assumed to create EncodingManager. E.g. provide them in GraphHopperConfig when calling GraphHopper.init"); + Map implicitFlagEncoderMap = new HashMap<>(); for (Profile profile : profilesByName.values()) { emBuilder.add(Subnetwork.create(profile.getName())); if (!flagEncoderMap.containsKey(profile.getVehicle()) @@ -510,7 +510,9 @@ private EncodingManager buildEncodingManager(GraphHopperConfig ghConfig) { implicitFlagEncoderMap.put(profile.getVehicle(), profile.getVehicle() + (profile.isTurnCosts() ? "|turn_costs=true" : "")); } flagEncoderMap.putAll(implicitFlagEncoderMap); - flagEncoderMap.values().stream().forEach(s -> emBuilder.addIfAbsent(flagEncoderFactory, s)); + flagEncoderMap.values().forEach(s -> emBuilder.addIfAbsent(flagEncoderFactory, s)); + + String encodedValueStr = ghConfig.getString("graph.encoded_values", ""); for (String tpStr : encodedValueStr.split(",")) { if (!tpStr.isEmpty()) emBuilder.addIfAbsent(tagParserFactory, tpStr); } @@ -608,7 +610,6 @@ public void importAndClose() { * Creates the graph from OSM data. */ private void process(boolean closeEarly) { - setGraphHopperLocation(ghLocation); GHLock lock = null; try { if (ghStorage == null) @@ -712,7 +713,6 @@ public boolean load() { throw new IllegalStateException("graph is already successfully loaded"); File tmpFileOrFolder = new File(ghLocation); - if (!tmpFileOrFolder.isDirectory() && tmpFileOrFolder.exists()) { throw new IllegalArgumentException("GraphHopperLocation cannot be an existing file. Has to be either non-existing or a folder."); } else { @@ -727,8 +727,6 @@ public boolean load() { } } - setGraphHopperLocation(ghLocation); - if (!allowWrites && dataAccessType.isMMap()) dataAccessType = DAType.MMAP_RO; if (encodingManager == null) { From 84a80fae7ad49322edbf4a2ca8d67c77508fe91a Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 25 Oct 2021 19:49:53 +0200 Subject: [PATCH 42/53] OSMReader: Keep node tags until we create edges (#2430) --- .../com/graphhopper/reader/ReaderElement.java | 8 +- .../com/graphhopper/reader/ReaderNode.java | 8 ++ .../graphhopper/reader/ReaderRelation.java | 3 +- .../com/graphhopper/reader/osm/OSMReader.java | 118 ++++++++---------- .../routing/util/EncodingManager.java | 30 ++--- .../java/com/graphhopper/GraphHopperTest.java | 24 ++-- .../graphhopper/reader/osm/OSMReaderTest.java | 19 ++- .../routing/RoutingAlgorithmWithOSMTest.java | 2 +- .../http/resources/RouteResourceTest.java | 8 +- .../http/resources/SPTResourceTest.java | 6 +- 10 files changed, 108 insertions(+), 118 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/ReaderElement.java b/core/src/main/java/com/graphhopper/reader/ReaderElement.java index db6f3582bee..ab4895ed2c6 100644 --- a/core/src/main/java/com/graphhopper/reader/ReaderElement.java +++ b/core/src/main/java/com/graphhopper/reader/ReaderElement.java @@ -37,13 +37,13 @@ public abstract class ReaderElement { private final Map properties; protected ReaderElement(long id, int type) { - this(id, type, 4); + this(id, type, new HashMap<>(4)); } - protected ReaderElement(long id, int type, int propertyMapSize) { + protected ReaderElement(long id, int type, Map properties) { this.id = id; this.type = type; - properties = new HashMap<>(propertyMapSize); + this.properties = properties; } public long getId() { @@ -64,7 +64,7 @@ protected String tagsToString() { return tagTxt.toString(); } - protected Map getTags() { + public Map getTags() { return properties; } diff --git a/core/src/main/java/com/graphhopper/reader/ReaderNode.java b/core/src/main/java/com/graphhopper/reader/ReaderNode.java index 6d322335c36..9a89b25e64d 100644 --- a/core/src/main/java/com/graphhopper/reader/ReaderNode.java +++ b/core/src/main/java/com/graphhopper/reader/ReaderNode.java @@ -17,6 +17,8 @@ */ package com.graphhopper.reader; +import java.util.Map; + /** * Represents a node received from the reader. *

@@ -33,6 +35,12 @@ public ReaderNode(long id, double lat, double lon) { this.lon = lon; } + public ReaderNode(long id, double lat, double lon, Map tags) { + super(id, NODE, tags); + this.lat = lat; + this.lon = lon; + } + public double getLat() { return lat; } diff --git a/core/src/main/java/com/graphhopper/reader/ReaderRelation.java b/core/src/main/java/com/graphhopper/reader/ReaderRelation.java index e3c41549e31..4011fbb530a 100644 --- a/core/src/main/java/com/graphhopper/reader/ReaderRelation.java +++ b/core/src/main/java/com/graphhopper/reader/ReaderRelation.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; /** @@ -31,7 +32,7 @@ public class ReaderRelation extends ReaderElement { protected List members; public ReaderRelation(long id) { - super(id, RELATION, 2); + super(id, RELATION, new HashMap<>(2)); } @Override diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 71e97ed088b..01dd36d870b 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -17,15 +17,16 @@ */ package com.graphhopper.reader.osm; -import com.carrotsearch.hppc.*; -import com.graphhopper.coll.LongIntMap; +import com.carrotsearch.hppc.IntLongMap; +import com.carrotsearch.hppc.LongArrayList; +import com.carrotsearch.hppc.LongIndexedContainer; +import com.carrotsearch.hppc.LongSet; import com.graphhopper.coll.*; import com.graphhopper.reader.*; import com.graphhopper.reader.dem.EdgeSampling; import com.graphhopper.reader.dem.ElevationProvider; import com.graphhopper.reader.dem.GraphElevationSmoothing; import com.graphhopper.routing.OSMReaderConfig; -import com.graphhopper.routing.ev.BooleanEncodedValue; import com.graphhopper.routing.ev.Country; import com.graphhopper.routing.util.AreaIndex; import com.graphhopper.routing.util.CustomArea; @@ -42,13 +43,11 @@ import javax.xml.stream.XMLStreamException; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; +import java.util.*; import static com.graphhopper.util.Helper.nf; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; /** * This class parses an OSM xml or pbf file and creates a graph from it. It does so in a two phase @@ -58,9 +57,8 @@ * node occurs once it is a pillar node and if more it is a tower node, otherwise * {@link #osmNodeIdToInternalNodeMap} returns EMPTY. *

- * 1. b) Reads relations from OSM file. In case that the relation is a route relation, it stores - * specific relation attributes required for routing into {@link #osmWayIdToRelationFlagsMap} for all the ways - * of the relation. + * 1. b) Reads relations from OSM file. We keep all route relations for each way ID. We also keep the way IDs for + * all restriction relations, so later we know which edge IDs we need to keep to create turn relation entries *

* 2.a) Reads nodes from OSM file and stores lat+lon information either into the intermediate * data structure for the pillar nodes (pillarLats/pillarLons) or, if a tower node, directly into the @@ -106,8 +104,9 @@ public class OSMReader implements TurnCostParser.ExternalInternalMap { // smaller memory overhead for bigger data sets because of avoiding a "rehash" // remember how many times a node was used to identify tower nodes private LongIntMap osmNodeIdToInternalNodeMap; - private GHLongLongHashMap osmNodeIdToNodeFlagsMap; private GHLongLongHashMap osmWayIdToRelationFlagsMap; + private List> nodeTags = new ArrayList<>(); + private LongIntMap nodeTagIndicesByOsmNodeID = new GHLongIntBTree(200); // stores osm way ids used by relations to identify which edge ids needs to be mapped later private GHLongHashSet osmWayIdSet = new GHLongHashSet(); private IntLongMap edgeIdToOsmWayIdMap; @@ -135,13 +134,11 @@ public OSMReader(GraphHopperStorage ghStorage, OSMReaderConfig config) { simplifyAlgo.setElevationMaxDistance(config.getElevationMaxWayPointDistance()); osmNodeIdToInternalNodeMap = new GHLongIntBTree(200); - osmNodeIdToNodeFlagsMap = new GHLongLongHashMap(200, .5f); osmWayIdToRelationFlagsMap = new GHLongLongHashMap(200, .5f); pillarInfo = new PillarInfo(nodeAccess.is3D(), ghStorage.getDirectory()); tempRelFlags = encodingManager.createRelationFlags(); if (tempRelFlags.length != 2) throw new IllegalArgumentException("Cannot use relation flags with != 2 integers"); - tcs = graph.getTurnCostStorage(); } @@ -345,43 +342,42 @@ protected void processWay(ReaderWay way) { return; LongArrayList osmNodeIds = way.getNodes(); - // look for barriers along the way final int size = osmNodeIds.size(); int lastBarrier = -1; for (int i = 0; i < size; i++) { long nodeId = osmNodeIds.get(i); - long nodeFlags = getNodeFlagsMap().get(nodeId); - // barrier was spotted and the way is passable for that mode of travel - if (nodeFlags > 0) { - // create shadow node copy for zero length edge - long newNodeId = addBarrierNode(nodeId); + int nodeTagIndex = nodeTagIndicesByOsmNodeID.get(nodeId); + if (nodeTagIndex >= 0) { + // this node is a barrier. we will add an extra edge to block access + // create an extra node for the barrier edge + long extraNodeId = addBarrierNode(nodeId); if (i > 0) { // start at beginning of array if there was no previous barrier if (lastBarrier < 0) lastBarrier = 0; - // add way up to barrier shadow node + // add way up to the new extra node int length = i - lastBarrier + 1; LongArrayList partNodeIds = new LongArrayList(); partNodeIds.add(osmNodeIds.buffer, lastBarrier, length); - partNodeIds.set(length - 1, newNodeId); + partNodeIds.set(length - 1, extraNodeId); addOSMWay(partNodeIds, edgeFlags, way); - // create zero length edge for barrier - addBarrierEdge(newNodeId, nodeId, edgeFlags, nodeFlags, way); + // create zero length barrier edge + addBarrierEdge(extraNodeId, nodeId, edgeFlags, nodeTags.get(nodeTagIndex), way); } else { - // run edge from real first node to shadow node - addBarrierEdge(nodeId, newNodeId, edgeFlags, nodeFlags, way); - + // make sure the extra node is not at the beginning of the edge + addBarrierEdge(nodeId, extraNodeId, edgeFlags, nodeTags.get(nodeTagIndex), way); // exchange first node for created barrier node - osmNodeIds.set(0, newNodeId); + osmNodeIds.set(0, extraNodeId); } // remember barrier for processing the way behind it lastBarrier = i; // ignore this barrier node from now. for example a barrier can be connecting two ways (appear in both // ways) and we only want to add a barrier edge once (but we want to add one). - getNodeFlagsMap().put(nodeId, 0); + nodeTagIndicesByOsmNodeID.put(nodeId, -1); + nodeTags.set(nodeTagIndex, emptyMap()); } } @@ -513,23 +509,27 @@ public int getInternalNodeIdOfOsmNode(long nodeOsmId) { protected void processNode(ReaderNode node) { int nodeType = getNodeMap().get(node.getId()); - if (nodeType == EMPTY_NODE) { + if (nodeType == EMPTY_NODE) return; - } else if (nodeType == JUNCTION_NODE || nodeType == CONNECTION_NODE) { + else if (nodeType == JUNCTION_NODE || nodeType == CONNECTION_NODE) addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); - } else if (nodeType == INTERMEDIATE_NODE || nodeType == END_NODE) { + else if (nodeType == INTERMEDIATE_NODE || nodeType == END_NODE) addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); - } - // analyze node tags for barriers - if (node.hasTags()) { - long nodeFlags = encodingManager.handleNodeTags(node); - if (nodeFlags != 0) - if (nodeType == JUNCTION_NODE) { - LOGGER.debug("OSM node {} at {},{} is a barrier node at a junction, the barrier will be ignored", node.getId(), Helper.round(node.getLat(), 7), Helper.round(node.getLon(), 7)); - ignoredBarrierNodes++; - } else - getNodeFlagsMap().put(node.getId(), nodeFlags); + // we keep node tags for barrier nodes + if (node.getTags().containsKey("barrier")) { + if (nodeType == JUNCTION_NODE) { + LOGGER.debug("OSM node {} at {},{} is a barrier node at a junction, the barrier will be ignored", node.getId(), Helper.round(node.getLat(), 7), Helper.round(node.getLon(), 7)); + ignoredBarrierNodes++; + } else { + int tagIndex = nodeTagIndicesByOsmNodeID.get(node.getId()); + if (tagIndex == -1) { + nodeTagIndicesByOsmNodeID.put(node.getId(), nodeTags.size()); + nodeTags.add(node.getTags()); + } else { + throw new IllegalStateException("Duplicate node OSM ID: " + node.getId()); + } + } } locations++; @@ -541,17 +541,13 @@ private void addPillarNode(long osmId, double lat, double lon, double ele) { nextPillarId++; } - void prepareWaysWithRelationInfo(ReaderRelation osmRelation) { - for (ReaderRelation.Member member : osmRelation.getMembers()) { + void prepareWaysWithRelationInfo(ReaderRelation relation) { + for (ReaderRelation.Member member : relation.getMembers()) { if (member.getType() != ReaderRelation.Member.WAY) continue; - - long osmId = member.getRef(); - IntsRef oldRelationFlags = getRelFlagsMap(osmId); - - // Check if our new relation data is better compared to the last one - IntsRef newRelationFlags = encodingManager.handleRelationTags(osmRelation, oldRelationFlags); - putRelFlagsMap(osmId, newRelationFlags); + IntsRef oldRelationFlags = getRelFlagsMap(member.getRef()); + IntsRef newRelationFlags = encodingManager.handleRelationTags(relation, oldRelationFlags); + putRelFlagsMap(member.getRef(), newRelationFlags); } } @@ -785,7 +781,8 @@ protected void finishedReading() { encodingManager.releaseParsers(); eleProvider.release(); osmNodeIdToInternalNodeMap = null; - osmNodeIdToNodeFlagsMap = null; + nodeTags = null; + nodeTagIndicesByOsmNodeID = null; osmWayIdToRelationFlagsMap = null; osmWayIdSet = null; edgeIdToOsmWayIdMap = null; @@ -822,13 +819,9 @@ private long createArtificialOSMNodeId() { /** * Add a zero length edge with reduced routing options to the graph. */ - void addBarrierEdge(long fromId, long toId, IntsRef inEdgeFlags, long nodeFlags, ReaderWay way) { - IntsRef edgeFlags = IntsRef.deepCopyOf(inEdgeFlags); - // clear blocked directions from flags - for (BooleanEncodedValue accessEnc : encodingManager.getAccessEncFromNodeFlags(nodeFlags)) { - accessEnc.setBool(false, edgeFlags, false); - accessEnc.setBool(true, edgeFlags, false); - } + void addBarrierEdge(long fromId, long toId, IntsRef edgeFlags, Map nodeTags, ReaderWay way) { + // update edge flags to block access depending on the node tags + edgeFlags = encodingManager.handleNodeTags(nodeTags, IntsRef.deepCopyOf(edgeFlags)); // add edge barrierNodeIds.clear(); barrierNodeIds.add(fromId); @@ -872,7 +865,8 @@ List createTurnRelations(ReaderRelation relation) { return osmTurnRelations; } - OSMTurnRelation createTurnRelation(ReaderRelation relation, String restrictionType, String vehicleTypeRestricted, List vehicleTypesExcept) { + OSMTurnRelation createTurnRelation(ReaderRelation relation, String restrictionType, String + vehicleTypeRestricted, List vehicleTypesExcept) { OSMTurnRelation.Type type = OSMTurnRelation.Type.getRestrictionType(restrictionType); if (type != OSMTurnRelation.Type.UNSUPPORTED) { long fromWayID = -1; @@ -907,10 +901,6 @@ protected LongIntMap getNodeMap() { return osmNodeIdToInternalNodeMap; } - protected LongLongMap getNodeFlagsMap() { - return osmNodeIdToNodeFlagsMap; - } - int getRelFlagsMapSize() { return osmWayIdToRelationFlagsMap.size(); } @@ -957,7 +947,7 @@ private void printInfo(String str) { LOGGER.info("finished " + str + " processing." + " nodes: " + graph.getNodes() + ", ignored barrier nodes at junctions: " + nf(ignoredBarrierNodes) + ", osmIdMap.size:" + getNodeMap().getSize() + ", osmIdMap:" + getNodeMap().getMemoryUsage() + "MB" - + ", nodeFlagsMap.size:" + getNodeFlagsMap().size() + ", relFlagsMap.size:" + getRelFlagsMapSize() + + ", nodeTags.size:" + nodeTags.size() + ", relationsMap.size:" + getRelFlagsMapSize() + ", zeroCounter:" + zeroCounter + " " + Helper.getMemInfo()); } diff --git a/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java b/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java index be43f91c9a6..bebeda2f465 100644 --- a/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java +++ b/core/src/main/java/com/graphhopper/routing/util/EncodingManager.java @@ -645,16 +645,20 @@ public int hashCode() { } /** - * Analyze tags on osm node. Store node tags (barriers etc) for later usage while parsing way. + * Updates the given edge flags based on node tags */ - public long handleNodeTags(ReaderNode node) { - long flags = 0; - for (int i = 0, edgeEncodersSize = edgeEncoders.size(); i < edgeEncodersSize; i++) { - AbstractFlagEncoder encoder = edgeEncoders.get(i); - flags |= (encoder.isBarrier(node) ? 1L << i : 0); + public IntsRef handleNodeTags(Map nodeTags, IntsRef edgeFlags) { + for (AbstractFlagEncoder encoder : edgeEncoders) { + // for now we just create a dummy reader node, because our encoders do not make use of the coordinates anyway + ReaderNode readerNode = new ReaderNode(0, 0, 0, nodeTags); + // block access for all encoders that treat this node as a barrier + if (encoder.isBarrier(readerNode)) { + BooleanEncodedValue accessEnc = encoder.getAccessEnc(); + accessEnc.setBool(false, edgeFlags, false); + accessEnc.setBool(true, edgeFlags, false); + } } - - return flags; + return edgeFlags; } public void applyWayTags(ReaderWay way, EdgeIteratorState edge) { @@ -698,16 +702,6 @@ public boolean needsTurnCostsSupport() { return false; } - public List getAccessEncFromNodeFlags(long importNodeFlags) { - List list = new ArrayList<>(edgeEncoders.size()); - for (int i = 0; i < edgeEncoders.size(); i++) { - FlagEncoder encoder = edgeEncoders.get(i); - if (((1L << i) & importNodeFlags) != 0) - list.add(encoder.getAccessEnc()); - } - return list; - } - @Override public List getEncodedValues() { return Collections.unmodifiableList(new ArrayList<>(encodedValueMap.values())); diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index da4fb5b8d24..4a3864e513c 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -90,12 +90,12 @@ public void setup() { @ParameterizedTest @CsvSource({ - DIJKSTRA + ",false,505", - ASTAR + ",false,438", - DIJKSTRA_BI + ",false,224", - ASTAR_BI + ",false,180", - ASTAR_BI + ",true,41", - DIJKSTRA_BI + ",true,41" + DIJKSTRA + ",false,511", + ASTAR + ",false,444", + DIJKSTRA_BI + ",false,228", + ASTAR_BI + ",false,184", + ASTAR_BI + ",true,36", + DIJKSTRA_BI + ",true,30" }) public void testMonacoDifferentAlgorithms(String algo, boolean withCH, int expectedVisitedNodes) { final String vehicle = "car"; @@ -142,7 +142,7 @@ public void testMonacoWithInstructions() { setAlgorithm(ASTAR).setProfile(profile)); // identify the number of counts to compare with CH foot route - assertEquals(700, rsp.getHints().getLong("visited_nodes.sum", 0)); + assertEquals(706, rsp.getHints().getLong("visited_nodes.sum", 0)); ResponsePath res = rsp.getBest(); assertEquals(3437.1, res.getDistance(), .1); @@ -1566,17 +1566,17 @@ public void testCrossQuery() { hopper.importOrLoad(); // flex - testCrossQueryAssert(profile1, hopper, 528.3, 160, true); - testCrossQueryAssert(profile2, hopper, 635.8, 158, true); - testCrossQueryAssert(profile3, hopper, 815.2, 154, true); + testCrossQueryAssert(profile1, hopper, 528.3, 166, true); + testCrossQueryAssert(profile2, hopper, 635.8, 160, true); + testCrossQueryAssert(profile3, hopper, 815.2, 158, true); // LM (should be the same as flex, but with less visited nodes!) testCrossQueryAssert(profile1, hopper, 528.3, 74, false); - testCrossQueryAssert(profile2, hopper, 635.8, 82, false); + testCrossQueryAssert(profile2, hopper, 635.8, 124, false); // this is actually interesting: the number of visited nodes *increases* once again (while it strictly decreases // with rising distance factor for flex): cross-querying 'works', but performs *worse*, because the landmarks // were not customized for the weighting in use. Creating a separate LM preparation for profile3 yields 74 - testCrossQueryAssert(profile3, hopper, 815.2, 148, false); + testCrossQueryAssert(profile3, hopper, 815.2, 162, false); } private void testCrossQueryAssert(String profile, GraphHopper hopper, double expectedWeight, int expectedVisitedNodes, boolean disableLM) { diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index 7bd8adedc69..ca908bdef72 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -347,7 +347,7 @@ public void testBarriers() { importOrLoad(); Graph graph = hopper.getGraphHopperStorage(); - // we ignore the barrier at node 50 + // we ignore the barrier at node 50, but not the one at node 20 assertEquals(7, graph.getNodes()); assertEquals(7, graph.getEdges()); @@ -478,29 +478,26 @@ public void testBarriersOnTowerNodes() { @Test public void testRelation() { EncodingManager manager = EncodingManager.create("bike"); - GraphHopperStorage ghStorage = new GraphHopperStorage(new RAMDirectory(), manager, false); - OSMReader reader = new OSMReader(ghStorage, new OSMReaderConfig()); ReaderRelation osmRel = new ReaderRelation(1); osmRel.add(new ReaderRelation.Member(ReaderRelation.WAY, 1, "")); osmRel.add(new ReaderRelation.Member(ReaderRelation.WAY, 2, "")); osmRel.setTag("route", "bicycle"); osmRel.setTag("network", "lcn"); - reader.prepareWaysWithRelationInfo(osmRel); - IntsRef flags = IntsRef.deepCopyOf(reader.getRelFlagsMap(1)); + IntsRef flags = manager.createRelationFlags(); + manager.handleRelationTags(osmRel, flags); assertFalse(flags.isEmpty()); // unchanged network - reader.prepareWaysWithRelationInfo(osmRel); - IntsRef flags2 = reader.getRelFlagsMap(1); - assertEquals(flags, flags2); + IntsRef before = IntsRef.deepCopyOf(flags); + manager.handleRelationTags(osmRel, flags); + assertEquals(before, flags); // overwrite network osmRel.setTag("network", "ncn"); - reader.prepareWaysWithRelationInfo(osmRel); - IntsRef flags3 = reader.getRelFlagsMap(1); - assertNotEquals(flags, flags3); + manager.handleRelationTags(osmRel, flags); + assertNotEquals(before, flags); } @Test diff --git a/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java b/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java index 1cee2453c74..9d2e7a89158 100644 --- a/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java +++ b/core/src/test/java/com/graphhopper/routing/RoutingAlgorithmWithOSMTest.java @@ -335,7 +335,7 @@ public void testMonacoBike3D_twoSpeedsPerEdge() { // 1. queries.add(new Query(43.727687, 7.418737, 43.730864, 7.420771, 2599, 115)); queries.add(new Query(43.74958, 7.436566, 43.728499, 7.417907, 4180, 165)); - queries.add(new Query(43.739213, 7.427806, 43.728677, 7.41016, 3244, 177)); + queries.add(new Query(43.739213, 7.427806, 43.728677, 7.41016, 3244, 179)); // 4. avoid tunnel(s)! queries.add(new Query(43.739662, 7.424355, 43.733802, 7.413433, 2436, 112)); GraphHopper hopper = createHopper(MONACO, new Profile("bike2").setVehicle("bike2").setWeighting("fastest")); diff --git a/web/src/test/java/com/graphhopper/http/resources/RouteResourceTest.java b/web/src/test/java/com/graphhopper/http/resources/RouteResourceTest.java index b6c9799f261..09600480a10 100644 --- a/web/src/test/java/com/graphhopper/http/resources/RouteResourceTest.java +++ b/web/src/test/java/com/graphhopper/http/resources/RouteResourceTest.java @@ -295,9 +295,9 @@ public void testPathDetails() { List edgeIdDetails = pathDetails.get("edge_id"); assertEquals(77, edgeIdDetails.size()); - assertEquals(880L, edgeIdDetails.get(0).getValue()); + assertEquals(882L, edgeIdDetails.get(0).getValue()); assertEquals(2, edgeIdDetails.get(0).getLength()); - assertEquals(881L, edgeIdDetails.get(1).getValue()); + assertEquals(883L, edgeIdDetails.get(1).getValue()); assertEquals(8, edgeIdDetails.get(1).getLength()); long expectedTime = rsp.getBest().getTime(); @@ -352,8 +352,8 @@ public void testPathDetailsWithoutGraphHopperWeb() { JsonNode edgeIds = details.get("edge_id"); int firstLink = edgeIds.get(0).get(2).asInt(); int lastLink = edgeIds.get(edgeIds.size() - 1).get(2).asInt(); - assertEquals(880, firstLink); - assertEquals(1420, lastLink); + assertEquals(882, firstLink); + assertEquals(1425, lastLink); JsonNode maxSpeed = details.get("max_speed"); assertEquals(-1, maxSpeed.get(0).get(2).asDouble(-1), .01); diff --git a/web/src/test/java/com/graphhopper/http/resources/SPTResourceTest.java b/web/src/test/java/com/graphhopper/http/resources/SPTResourceTest.java index e92a100c71a..29509764072 100644 --- a/web/src/test/java/com/graphhopper/http/resources/SPTResourceTest.java +++ b/web/src/test/java/com/graphhopper/http/resources/SPTResourceTest.java @@ -96,9 +96,9 @@ public void requestSPTEdgeBased() { String[] lines = rspCsvString.split("\n"); assertTrue(lines.length > 500); assertEquals("prev_node_id,edge_id,node_id,time,distance", lines[0]); - assertEquals("-1,-1,1941,0,0", lines[1]); - assertEquals("1941,2270,1324,3817,74", lines[2]); - assertEquals("1941,2269,263,13495,262", lines[3]); + assertEquals("-1,-1,1948,0,0", lines[1]); + assertEquals("1948,2277,1324,3817,74", lines[2]); + assertEquals("1948,2276,263,13495,262", lines[3]); } @Test From d33067dc8495b0316486124f856499d60553b0e5 Mon Sep 17 00:00:00 2001 From: easbar Date: Tue, 26 Oct 2021 08:21:36 +0200 Subject: [PATCH 43/53] Minor cleanup --- .../graphhopper/storage/AbstractDataAccess.java | 1 - .../java/com/graphhopper/storage/Directory.java | 1 - .../com/graphhopper/storage/MMapDataAccess.java | 2 +- .../com/graphhopper/storage/RAMDataAccess.java | 14 ++++---------- .../graphhopper/storage/RAMIntDataAccess.java | 16 +++++----------- .../graphhopper/storage/index/LineIntIndex.java | 10 +++++----- .../com/graphhopper/storage/DataAccessTest.java | 9 +++++++++ 7 files changed, 24 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java index 5c03f4edeb0..88f54b2b481 100644 --- a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java @@ -20,7 +20,6 @@ import com.graphhopper.util.BitUtil; import com.graphhopper.util.Helper; -import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteOrder; diff --git a/core/src/main/java/com/graphhopper/storage/Directory.java b/core/src/main/java/com/graphhopper/storage/Directory.java index 91e5f730794..d334712a249 100644 --- a/core/src/main/java/com/graphhopper/storage/Directory.java +++ b/core/src/main/java/com/graphhopper/storage/Directory.java @@ -18,7 +18,6 @@ package com.graphhopper.storage; import java.nio.ByteOrder; -import java.util.Collection; /** * Maintains a collection of DataAccess objects stored at the same location. One GraphStorage per diff --git a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java index 6d7873862db..16d450ac96a 100644 --- a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java @@ -61,7 +61,7 @@ public final class MMapDataAccess extends AbstractDataAccess { private final boolean allowWrites; private RandomAccessFile raFile; - private List segments = new ArrayList<>(); + private final List segments = new ArrayList<>(); MMapDataAccess(String name, String location, ByteOrder order, boolean allowWrites, int segmentSize) { super(name, location, order, segmentSize); diff --git a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java index 1549c2130a9..5bdffe2f23a 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java @@ -105,14 +105,13 @@ public boolean loadExisting() { return false; try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r")) { long byteCount = readHeader(raFile) - HEADER_OFFSET; if (byteCount < 0) return false; raFile.seek(HEADER_OFFSET); - // raFile.readInt() <- too slow + // raFile.readInt() <- too slow int segmentCount = (int) (byteCount / segmentSizeInBytes); if (byteCount % segmentSizeInBytes != 0) segmentCount++; @@ -127,8 +126,6 @@ public boolean loadExisting() { segments[s] = bytes; } return true; - } finally { - raFile.close(); } } catch (IOException ex) { throw new RuntimeException("Problem while loading " + getFullName(), ex); @@ -144,18 +141,15 @@ public void flush() { return; try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw")) { long len = getCapacity(); writeHeader(raFile, len, segmentSizeInBytes); raFile.seek(HEADER_OFFSET); // raFile.writeInt() <- too slow, so copy into byte array for (int s = 0; s < segments.length; s++) { - byte area[] = segments[s]; + byte[] area = segments[s]; raFile.write(area); } - } finally { - raFile.close(); } } catch (Exception ex) { throw new RuntimeException("Couldn't store bytes to " + toString(), ex); diff --git a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java index e899d75204c..db8783ca89c 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java @@ -107,15 +107,14 @@ public boolean loadExisting() { return false; } try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r")) { long byteCount = readHeader(raFile) - HEADER_OFFSET; if (byteCount < 0) { return false; } byte[] bytes = new byte[segmentSizeInBytes]; raFile.seek(HEADER_OFFSET); - // raFile.readInt() <- too slow + // raFile.readInt() <- too slow int segmentCount = (int) (byteCount / segmentSizeInBytes); if (byteCount % segmentSizeInBytes != 0) segmentCount++; @@ -123,15 +122,13 @@ public boolean loadExisting() { segments = new int[segmentCount][]; for (int s = 0; s < segmentCount; s++) { int read = raFile.read(bytes) / 4; - int area[] = new int[read]; + int[] area = new int[read]; for (int j = 0; j < read; j++) { area[j] = bitUtil.toInt(bytes, j * 4); } segments[s] = area; } return true; - } finally { - raFile.close(); } } catch (IOException ex) { throw new RuntimeException("Problem while loading " + getFullName(), ex); @@ -147,14 +144,13 @@ public void flush() { return; } try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw")) { long len = getCapacity(); writeHeader(raFile, len, segmentSizeInBytes); raFile.seek(HEADER_OFFSET); // raFile.writeInt() <- too slow, so copy into byte array for (int s = 0; s < segments.length; s++) { - int area[] = segments[s]; + int[] area = segments[s]; int intLen = area.length; byte[] byteArea = new byte[intLen * 4]; for (int i = 0; i < intLen; i++) { @@ -162,8 +158,6 @@ public void flush() { } raFile.write(byteArea); } - } finally { - raFile.close(); } } catch (Exception ex) { throw new RuntimeException("Couldn't store integers to " + toString(), ex); diff --git a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java index 6e0e3a8657f..548631df800 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java +++ b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java @@ -180,9 +180,9 @@ public void onEdge(int edgeId) { } private void query(int intPointer, BBox queryBBox, - double minLat, double minLon, - double deltaLatPerDepth, double deltaLonPerDepth, - LocationIndex.Visitor function, int depth) { + double minLat, double minLon, + double deltaLatPerDepth, double deltaLonPerDepth, + LocationIndex.Visitor function, int depth) { long pointer = (long) intPointer * 4; if (depth == entries.length) { int nextIntPointer = dataAccess.getInt(pointer); @@ -225,11 +225,11 @@ private void query(int intPointer, BBox queryBBox, /** * This method collects edge ids from the neighborhood of a point and puts them into foundEntries. - * + *

* If it is called with iteration = 0, it just looks in the tile the query point is in. * If it is called with iteration = 0,1,2,.., it will look in additional tiles further and further * from the start tile. (In a square that grows by one pixel in all four directions per iteration). - * + *

* See discussion at issue #221. *

*/ diff --git a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java index dd842de36c3..c36e0be27a3 100644 --- a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java @@ -178,6 +178,15 @@ public void testSegments() { @Test public void testSegmentSize() { DataAccess da = createDataAccess(name, 20); + da.create(10); + // a minimum segment size is applied + assertEquals(128, da.getSegmentSize()); + da.flush(); + da.close(); + + da = createDataAccess(name, 256); + da.loadExisting(); + // we chose a different segment size, but it is ignored assertEquals(128, da.getSegmentSize()); da.close(); } From 5a5fe4da5acd348115870f3fc9ae413ce0189d1b Mon Sep 17 00:00:00 2001 From: Andi Date: Tue, 26 Oct 2021 10:46:02 +0200 Subject: [PATCH 44/53] Remove byteOrder option for Directory, always use little endianess for all DataAccess (#2442) --- .../dem/AbstractSRTMElevationProvider.java | 5 ++--- .../graphhopper/storage/AbstractDataAccess.java | 8 +++----- .../java/com/graphhopper/storage/BaseGraph.java | 2 +- .../java/com/graphhopper/storage/Directory.java | 7 ------- .../com/graphhopper/storage/GHDirectory.java | 17 +++++------------ .../graphhopper/storage/GraphHopperStorage.java | 5 ----- .../com/graphhopper/storage/MMapDataAccess.java | 5 ++--- .../com/graphhopper/storage/RAMDataAccess.java | 5 ++--- .../graphhopper/storage/RAMIntDataAccess.java | 5 ++--- .../graphhopper/geohash/SpatialKeyAlgoTest.java | 2 +- .../ch/PrepareContractionHierarchiesTest.java | 4 ++-- .../com/graphhopper/storage/CHStorageTest.java | 3 +-- .../com/graphhopper/storage/DataAccessTest.java | 8 +++----- .../graphhopper/storage/MMapDataAccessTest.java | 6 +++--- .../graphhopper/storage/RAMDataAccessTest.java | 2 +- .../storage/RAMIntDataAccessTest.java | 2 +- 16 files changed, 29 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 68f46b0552d..9e066730d35 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -34,8 +34,6 @@ * @author Robin Boldt */ public abstract class AbstractSRTMElevationProvider extends TileBasedElevationProvider { - - private static final BitUtil BIT_UTIL = BitUtil.BIG; private final int DEFAULT_WIDTH; private final int MIN_LAT; private final int MAX_LAT; @@ -146,7 +144,8 @@ private void updateHeightsFromFile(double lat, double lon, DataAccess heights) t byte[] bytes = getByteArrayFromFile(lat, lon); heights.create(bytes.length); for (int bytePos = 0; bytePos < bytes.length; bytePos += 2) { - short val = BIT_UTIL.toShort(bytes, bytePos); + // we need big endianess to read the SRTM files + short val = BitUtil.BIG.toShort(bytes, bytePos); if (val < -1000 || val > 12000) val = Short.MIN_VALUE; diff --git a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java index 88f54b2b481..2b4f378ce91 100644 --- a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java @@ -33,8 +33,8 @@ public abstract class AbstractDataAccess implements DataAccess { protected static final int HEADER_OFFSET = 20 * 4 + 20; protected static final byte[] EMPTY = new byte[1024]; private static final int SEGMENT_SIZE_DEFAULT = 1 << 20; - protected final ByteOrder byteOrder; - protected final BitUtil bitUtil; + protected final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; + protected final BitUtil bitUtil = BitUtil.LITTLE; private final String location; protected int[] header = new int[(HEADER_OFFSET - 20) / 4]; protected String name; @@ -43,9 +43,7 @@ public abstract class AbstractDataAccess implements DataAccess { protected int indexDivisor; protected boolean closed = false; - public AbstractDataAccess(String name, String location, ByteOrder order, int segmentSize) { - byteOrder = order; - bitUtil = BitUtil.get(order); + public AbstractDataAccess(String name, String location, int segmentSize) { this.name = name; if (!Helper.isEmpty(location) && !location.endsWith("/")) throw new IllegalArgumentException("Create DataAccess object via its corresponding Directory!"); diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 0153688f874..01214dc844e 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -57,7 +57,7 @@ class BaseGraph implements Graph { public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { this.dir = dir; - this.bitUtil = BitUtil.get(dir.getByteOrder()); + this.bitUtil = BitUtil.LITTLE; this.wayGeometry = dir.create("geometry", segmentSize); this.stringIndex = new StringIndex(dir, 1000, segmentSize); this.store = new BaseGraphNodesAndEdges(dir, intsForFlags, withElevation, withTurnCosts, segmentSize); diff --git a/core/src/main/java/com/graphhopper/storage/Directory.java b/core/src/main/java/com/graphhopper/storage/Directory.java index d334712a249..61c8bf7d6ee 100644 --- a/core/src/main/java/com/graphhopper/storage/Directory.java +++ b/core/src/main/java/com/graphhopper/storage/Directory.java @@ -17,8 +17,6 @@ */ package com.graphhopper.storage; -import java.nio.ByteOrder; - /** * Maintains a collection of DataAccess objects stored at the same location. One GraphStorage per * Directory as we need one to maintain one DataAccess object for nodes, edges and location2id @@ -33,11 +31,6 @@ public interface Directory { */ String getLocation(); - /** - * @return the order in which the data is stored - */ - ByteOrder getByteOrder(); - /** * Creates a new DataAccess object with the given name in the location of this Directory. Each name can only * be used once. diff --git a/core/src/main/java/com/graphhopper/storage/GHDirectory.java b/core/src/main/java/com/graphhopper/storage/GHDirectory.java index a266d9c8336..a9990502d1d 100644 --- a/core/src/main/java/com/graphhopper/storage/GHDirectory.java +++ b/core/src/main/java/com/graphhopper/storage/GHDirectory.java @@ -18,7 +18,6 @@ package com.graphhopper.storage; import java.io.File; -import java.nio.ByteOrder; import java.util.HashMap; import java.util.Map; @@ -32,7 +31,6 @@ public class GHDirectory implements Directory { protected final String location; private final DAType defaultType; - private final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; protected Map map = new HashMap<>(); public GHDirectory(String _location, DAType defaultType) { @@ -49,11 +47,6 @@ public GHDirectory(String _location, DAType defaultType) { throw new RuntimeException("file '" + dir + "' exists but is not a directory"); } - @Override - public ByteOrder getByteOrder() { - return byteOrder; - } - @Override public DataAccess create(String name) { return create(name, defaultType); @@ -83,15 +76,15 @@ public DataAccess create(String name, DAType type, int segmentSize) { if (type.isInMemory()) { if (type.isInteg()) { if (type.isStoring()) - da = new RAMIntDataAccess(name, location, true, byteOrder, segmentSize); + da = new RAMIntDataAccess(name, location, true, segmentSize); else - da = new RAMIntDataAccess(name, location, false, byteOrder, segmentSize); + da = new RAMIntDataAccess(name, location, false, segmentSize); } else if (type.isStoring()) - da = new RAMDataAccess(name, location, true, byteOrder, segmentSize); + da = new RAMDataAccess(name, location, true, segmentSize); else - da = new RAMDataAccess(name, location, false, byteOrder, segmentSize); + da = new RAMDataAccess(name, location, false, segmentSize); } else if (type.isMMap()) { - da = new MMapDataAccess(name, location, byteOrder, type.isAllowWrites(), segmentSize); + da = new MMapDataAccess(name, location, type.isAllowWrites(), segmentSize); } else { throw new IllegalArgumentException("DAType not supported " + type); } diff --git a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java index fa03b28d4bb..e944cf512c6 100644 --- a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java +++ b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java @@ -217,7 +217,6 @@ public GraphHopperStorage create(long byteCount) { properties.put("graph.encoded_values", encodingManager.toEncodedValuesAsString()); properties.put("graph.flag_encoders", encodingManager.toFlagEncodersAsString()); - properties.put("graph.byte_order", dir.getByteOrder()); properties.put("graph.dimension", baseGraph.nodeAccess.getDimension()); properties.putCurrentVersions(); @@ -264,10 +263,6 @@ public boolean loadExisting() { + "\nChange configuration to match the graph or delete " + dir.getLocation()); } - String byteOrder = properties.get("graph.byte_order"); - if (!byteOrder.equalsIgnoreCase("" + dir.getByteOrder())) - throw new IllegalStateException("Configured graph.byte_order (" + dir.getByteOrder() + ") is not equal to loaded " + byteOrder + ""); - String dim = properties.get("graph.dimension"); baseGraph.loadExisting(dim); diff --git a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java index 16d450ac96a..2dace19736d 100644 --- a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java @@ -29,7 +29,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.security.AccessController; @@ -63,8 +62,8 @@ public final class MMapDataAccess extends AbstractDataAccess { private RandomAccessFile raFile; private final List segments = new ArrayList<>(); - MMapDataAccess(String name, String location, ByteOrder order, boolean allowWrites, int segmentSize) { - super(name, location, order, segmentSize); + MMapDataAccess(String name, String location, boolean allowWrites, int segmentSize) { + super(name, location, segmentSize); this.allowWrites = allowWrites; } diff --git a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java index 5bdffe2f23a..49918b85ce4 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.ByteOrder; import java.util.Arrays; /** @@ -34,8 +33,8 @@ public class RAMDataAccess extends AbstractDataAccess { private byte[][] segments = new byte[0][]; private boolean store; - RAMDataAccess(String name, String location, boolean store, ByteOrder order, int segmentSize) { - super(name, location, order, segmentSize); + RAMDataAccess(String name, String location, boolean store, int segmentSize) { + super(name, location, segmentSize); this.store = store; } diff --git a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java index db8783ca89c..c5cc0ccb39d 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.ByteOrder; import java.util.Arrays; /** @@ -36,8 +35,8 @@ class RAMIntDataAccess extends AbstractDataAccess { private boolean store; private int segmentSizeIntsPower; - RAMIntDataAccess(String name, String location, boolean store, ByteOrder order, int segmentSize) { - super(name, location, order, segmentSize); + RAMIntDataAccess(String name, String location, boolean store, int segmentSize) { + super(name, location, segmentSize); this.store = store; } diff --git a/core/src/test/java/com/graphhopper/geohash/SpatialKeyAlgoTest.java b/core/src/test/java/com/graphhopper/geohash/SpatialKeyAlgoTest.java index 61426432187..58b9ddb600d 100644 --- a/core/src/test/java/com/graphhopper/geohash/SpatialKeyAlgoTest.java +++ b/core/src/test/java/com/graphhopper/geohash/SpatialKeyAlgoTest.java @@ -31,7 +31,7 @@ public class SpatialKeyAlgoTest { public void testEncode() { SpatialKeyAlgo algo = new SpatialKeyAlgo(32, new BBox(-180, 180, -90, 90)); long val = algo.encodeLatLon(-24.235345f, 47.234234f); - assertEquals("01100110101000111100000110010100", BitUtil.BIG.toLastBitString(val, 32)); + assertEquals("01100110101000111100000110010100", BitUtil.LITTLE.toLastBitString(val, 32)); } @Test diff --git a/core/src/test/java/com/graphhopper/routing/ch/PrepareContractionHierarchiesTest.java b/core/src/test/java/com/graphhopper/routing/ch/PrepareContractionHierarchiesTest.java index b41966165e2..4dccec482cf 100644 --- a/core/src/test/java/com/graphhopper/routing/ch/PrepareContractionHierarchiesTest.java +++ b/core/src/test/java/com/graphhopper/routing/ch/PrepareContractionHierarchiesTest.java @@ -465,8 +465,8 @@ public void testBits() { int endNode = Integer.MAX_VALUE / 37 * 17; long edgeId = (long) fromNode << 32 | endNode; - assertEquals((BitUtil.BIG.toBitString(edgeId)), - BitUtil.BIG.toLastBitString(fromNode, 32) + BitUtil.BIG.toLastBitString(endNode, 32)); + assertEquals((BitUtil.LITTLE.toBitString(edgeId)), + BitUtil.LITTLE.toLastBitString(fromNode, 32) + BitUtil.LITTLE.toLastBitString(endNode, 32)); } @Test diff --git a/core/src/test/java/com/graphhopper/storage/CHStorageTest.java b/core/src/test/java/com/graphhopper/storage/CHStorageTest.java index 6569cf364fd..b2ee50dbfd4 100644 --- a/core/src/test/java/com/graphhopper/storage/CHStorageTest.java +++ b/core/src/test/java/com/graphhopper/storage/CHStorageTest.java @@ -4,7 +4,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import java.nio.ByteOrder; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -71,7 +70,7 @@ public void testBigWeight() { @Test public void testLargeNodeA() { int nodeA = Integer.MAX_VALUE; - RAMIntDataAccess access = new RAMIntDataAccess("", "", false, ByteOrder.LITTLE_ENDIAN, -1); + RAMIntDataAccess access = new RAMIntDataAccess("", "", false, -1); access.create(1000); access.setInt(0, nodeA << 1 | 1 & PrepareEncoder.getScFwdDir()); assertTrue(access.getInt(0) < 0); diff --git a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java index c36e0be27a3..caee43a03b0 100644 --- a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.api.Test; import java.io.File; -import java.nio.ByteOrder; import static org.junit.jupiter.api.Assertions.*; @@ -33,7 +32,6 @@ */ public abstract class DataAccessTest { private final File folder = new File("./target/tmp/da"); - protected ByteOrder defaultOrder = ByteOrder.LITTLE_ENDIAN; protected String directory; protected String name = "dataacess"; @@ -196,15 +194,15 @@ public void testSet_GetBytes() { DataAccess da = createDataAccess(name); da.create(300); assertEquals(128, da.getSegmentSize()); - byte[] bytes = BitUtil.BIG.fromInt(Integer.MAX_VALUE / 3); + byte[] bytes = BitUtil.LITTLE.fromInt(Integer.MAX_VALUE / 3); da.setBytes(8, bytes, bytes.length); bytes = new byte[4]; da.getBytes(8, bytes, bytes.length); - assertEquals(Integer.MAX_VALUE / 3, BitUtil.BIG.toInt(bytes)); + assertEquals(Integer.MAX_VALUE / 3, BitUtil.LITTLE.toInt(bytes)); da.setBytes(127, bytes, bytes.length); da.getBytes(127, bytes, bytes.length); - assertEquals(Integer.MAX_VALUE / 3, BitUtil.BIG.toInt(bytes)); + assertEquals(Integer.MAX_VALUE / 3, BitUtil.LITTLE.toInt(bytes)); da.close(); diff --git a/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java index a6db8a011b4..1082e3d9a98 100644 --- a/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java @@ -27,12 +27,12 @@ public class MMapDataAccessTest extends DataAccessTest { @Override public DataAccess createDataAccess(String name, int segmentSize) { - return new MMapDataAccess(name, directory, defaultOrder, true, segmentSize); + return new MMapDataAccess(name, directory, true, segmentSize); } @Test public void textMixRAM2MMAP() { - DataAccess da = new RAMDataAccess(name, directory, true, defaultOrder, -1); + DataAccess da = new RAMDataAccess(name, directory, true, -1); assertFalse(da.loadExisting()); da.create(100); da.setInt(7 * 4, 123); @@ -54,7 +54,7 @@ public void textMixMMAP2RAM() { // TODO "memory mapped flush" is expensive and not required. only writing the header is required. da.flush(); da.close(); - da = new RAMDataAccess(name, directory, true, defaultOrder, -1); + da = new RAMDataAccess(name, directory, true, -1); assertTrue(da.loadExisting()); assertEquals(123, da.getInt(7 * 4)); da.close(); diff --git a/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java index cb5ca3c867e..4406f57053a 100644 --- a/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java @@ -23,6 +23,6 @@ public class RAMDataAccessTest extends DataAccessTest { @Override public DataAccess createDataAccess(String name, int segmentSize) { - return new RAMDataAccess(name, directory, true, defaultOrder, segmentSize); + return new RAMDataAccess(name, directory, true, segmentSize); } } diff --git a/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java index 4610d9369c9..bfdc069836e 100644 --- a/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java @@ -23,7 +23,7 @@ public class RAMIntDataAccessTest extends DataAccessTest { @Override public DataAccess createDataAccess(String name, int segmentSize) { - return new RAMIntDataAccess(name, directory, true, defaultOrder, segmentSize); + return new RAMIntDataAccess(name, directory, true, segmentSize); } @Override From ab0cd6fbac790a3757fdbcc77b6e3c0037b34a86 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 26 Oct 2021 16:40:30 +0200 Subject: [PATCH 45/53] custom model docs: minor change to make example working in custom model editor --- docs/core/custom-models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/core/custom-models.md b/docs/core/custom-models.md index 3283d1adf63..a18940d307f 100644 --- a/docs/core/custom-models.md +++ b/docs/core/custom-models.md @@ -120,7 +120,7 @@ Here is a complete request example for a POST /route query in berlin that includ "custom_model": { "speed": [ { - "if": true, + "if": "true", "limit_to": 100 } ], From 61d977ef4e8a15016e82e91a3bd6161927a5b9ed Mon Sep 17 00:00:00 2001 From: easbar Date: Wed, 27 Oct 2021 07:07:07 +0200 Subject: [PATCH 46/53] Stop using GHPoint in DirectionResolver * GHPoint's hashCode/equals contract is broken and we used it in a hash container * no need to use GHPoint anyway --- .../routing/DirectionResolver.java | 83 +++++++++++++------ 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/com/graphhopper/routing/DirectionResolver.java b/core/src/main/java/com/graphhopper/routing/DirectionResolver.java index 5310c348b5c..e9dc4873a6f 100644 --- a/core/src/main/java/com/graphhopper/routing/DirectionResolver.java +++ b/core/src/main/java/com/graphhopper/routing/DirectionResolver.java @@ -79,7 +79,7 @@ public DirectionResolverResult resolveDirections(int node, GHPoint location) { if (adjacentEdges.numLoops > 0) { return DirectionResolverResult.unrestricted(); } - GHPoint snappedPoint = new GHPoint(nodeAccess.getLat(node), nodeAccess.getLon(node)); + Point snappedPoint = new Point(nodeAccess.getLat(node), nodeAccess.getLon(node)); if (adjacentEdges.nextPoints.contains(snappedPoint)) { // this might happen if a pillar node of an adjacent edge has the same coordinates as the snapped point, // but this should be prevented by the map import already @@ -87,7 +87,7 @@ public DirectionResolverResult resolveDirections(int node, GHPoint location) { } // we can classify the different cases by the number of different next points! if (adjacentEdges.nextPoints.size() == 1) { - GHPoint neighbor = adjacentEdges.nextPoints.iterator().next(); + Point neighbor = adjacentEdges.nextPoints.iterator().next(); List inEdges = adjacentEdges.getInEdges(neighbor); List outEdges = adjacentEdges.getOutEdges(neighbor); assert inEdges.size() > 0 && outEdges.size() > 0 : "if there is only one next point there has to be an in edge and an out edge connected with it"; @@ -100,9 +100,9 @@ public DirectionResolverResult resolveDirections(int node, GHPoint location) { // side are treated equally and for both cases we use the only possible edge ids. return DirectionResolverResult.restricted(inEdges.get(0).edgeId, outEdges.get(0).edgeId, inEdges.get(0).edgeId, outEdges.get(0).edgeId); } else if (adjacentEdges.nextPoints.size() == 2) { - Iterator iter = adjacentEdges.nextPoints.iterator(); - GHPoint p1 = iter.next(); - GHPoint p2 = iter.next(); + Iterator iter = adjacentEdges.nextPoints.iterator(); + Point p1 = iter.next(); + Point p2 = iter.next(); List in1 = adjacentEdges.getInEdges(p1); List in2 = adjacentEdges.getInEdges(p2); List out1 = adjacentEdges.getOutEdges(p1); @@ -116,12 +116,13 @@ public DirectionResolverResult resolveDirections(int node, GHPoint location) { if (in1.size() + out1.size() == 0 || in2.size() + out2.size() == 0) { throw new IllegalStateException("there has to be at least one in or one out edge for each of the two next points"); } + Point locationPoint = new Point(location.lat, location.lon); if (in1.isEmpty() || out2.isEmpty()) { - return resolveDirections(snappedPoint, location, in2.get(0), out1.get(0)); + return resolveDirections(snappedPoint, locationPoint, in2.get(0), out1.get(0)); } else if (in2.isEmpty() || out1.isEmpty()) { - return resolveDirections(snappedPoint, location, in1.get(0), out2.get(0)); + return resolveDirections(snappedPoint, locationPoint, in1.get(0), out2.get(0)); } else { - return resolveDirections(snappedPoint, location, in1.get(0), out2.get(0), in2.get(0).edgeId, out1.get(0).edgeId); + return resolveDirections(snappedPoint, locationPoint, in1.get(0), out2.get(0), in2.get(0).edgeId, out1.get(0).edgeId); } } else { // we snapped to a junction, in this case we do not apply restrictions @@ -131,7 +132,7 @@ public DirectionResolverResult resolveDirections(int node, GHPoint location) { } } - private DirectionResolverResult resolveDirections(GHPoint snappedPoint, GHPoint queryPoint, Edge inEdge, Edge outEdge) { + private DirectionResolverResult resolveDirections(Point snappedPoint, Point queryPoint, Edge inEdge, Edge outEdge) { boolean rightLane = isOnRightLane(queryPoint, snappedPoint, inEdge.nextPoint, outEdge.nextPoint); if (rightLane) { return DirectionResolverResult.onlyRight(inEdge.edgeId, outEdge.edgeId); @@ -140,9 +141,9 @@ private DirectionResolverResult resolveDirections(GHPoint snappedPoint, GHPoint } } - private DirectionResolverResult resolveDirections(GHPoint snappedPoint, GHPoint queryPoint, Edge inEdge, Edge outEdge, int altInEdge, int altOutEdge) { - GHPoint inPoint = inEdge.nextPoint; - GHPoint outPoint = outEdge.nextPoint; + private DirectionResolverResult resolveDirections(Point snappedPoint, Point queryPoint, Edge inEdge, Edge outEdge, int altInEdge, int altOutEdge) { + Point inPoint = inEdge.nextPoint; + Point outPoint = outEdge.nextPoint; boolean rightLane = isOnRightLane(queryPoint, snappedPoint, inPoint, outPoint); if (rightLane) { return DirectionResolverResult.restricted(inEdge.edgeId, outEdge.edgeId, altInEdge, altOutEdge); @@ -151,7 +152,7 @@ private DirectionResolverResult resolveDirections(GHPoint snappedPoint, GHPoint } } - private boolean isOnRightLane(GHPoint queryPoint, GHPoint snappedPoint, GHPoint inPoint, GHPoint outPoint) { + private boolean isOnRightLane(Point queryPoint, Point snappedPoint, Point inPoint, Point outPoint) { double qX = diffLon(snappedPoint, queryPoint); double qY = diffLat(snappedPoint, queryPoint); double iX = diffLon(snappedPoint, inPoint); @@ -161,11 +162,11 @@ private boolean isOnRightLane(GHPoint queryPoint, GHPoint snappedPoint, GHPoint return !AngleCalc.ANGLE_CALC.isClockwise(iX, iY, oX, oY, qX, qY); } - private double diffLon(GHPoint p, GHPoint q) { + private double diffLon(Point p, Point q) { return q.lon - p.lon; } - private double diffLat(GHPoint p, GHPoint q) { + private double diffLat(Point p, Point q) { return q.lat - p.lat; } @@ -196,7 +197,7 @@ private AdjacentEdges calcAdjEdges(int node) { nextPointLat = geometry.getLat(2); nextPointLon = geometry.getLon(2); } - GHPoint nextPoint = new GHPoint(nextPointLat, nextPointLon); + Point nextPoint = new Point(nextPointLat, nextPointLon); Edge edge = new Edge(iter.getEdge(), iter.getAdjNode(), nextPoint); adjacentEdges.addEdge(edge, isIn, isOut); } @@ -204,9 +205,9 @@ private AdjacentEdges calcAdjEdges(int node) { } private static class AdjacentEdges { - private final Map> inEdgesByNextPoint = new HashMap<>(2); - private final Map> outEdgesByNextPoint = new HashMap<>(2); - final Set nextPoints = new HashSet<>(2); + private final Map> inEdgesByNextPoint = new HashMap<>(2); + private final Map> outEdgesByNextPoint = new HashMap<>(2); + final Set nextPoints = new HashSet<>(2); int numLoops; int numNonLoops; @@ -220,14 +221,14 @@ void addEdge(Edge edge, boolean isIn, boolean isOut) { addNextPoint(edge); } - List getInEdges(GHPoint p) { + List getInEdges(Point p) { List result = inEdgesByNextPoint.get(p); - return result == null ? Collections.emptyList() : result; + return result == null ? Collections.emptyList() : result; } - List getOutEdges(GHPoint p) { + List getOutEdges(Point p) { List result = outEdgesByNextPoint.get(p); - return result == null ? Collections.emptyList() : result; + return result == null ? Collections.emptyList() : result; } boolean hasInEdges() { @@ -250,7 +251,7 @@ private void addNextPoint(Edge edge) { nextPoints.add(edge.nextPoint); } - private void addEdge(Map> edgesByNextPoint, Edge edge) { + private static void addEdge(Map> edgesByNextPoint, Edge edge) { List edges = edgesByNextPoint.get(edge.nextPoint); if (edges == null) { edges = new ArrayList<>(2); @@ -262,6 +263,36 @@ private void addEdge(Map> edgesByNextPoint, Edge edge) { } } + private static class Point { + final double lat; + final double lon; + + Point(double lat, double lon) { + this.lat = lat; + this.lon = lon; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Point other = (Point) o; + return NumHelper.equalsEps(lat, other.lat) && NumHelper.equalsEps(lon, other.lon); + } + + @Override + public int hashCode() { + // it does not matter, because we only use maps with very few elements. not using GHPoint because of it's + // broken hashCode implementation (#2445) and there is no good reason need to depend on it either + return 0; + } + + @Override + public String toString() { + return lat + ", " + lon; + } + } + private static class Edge { final int edgeId; final int adjNode; @@ -269,9 +300,9 @@ private static class Edge { * the next point of this edge, not necessarily the point corresponding to adjNode, but often this is the * next pillar (!) node. */ - final GHPoint nextPoint; + final Point nextPoint; - Edge(int edgeId, int adjNode, GHPoint nextPoint) { + Edge(int edgeId, int adjNode, Point nextPoint) { this.edgeId = edgeId; this.adjNode = adjNode; this.nextPoint = nextPoint; From d4804ecddda04b1e259d8b996536d544cef9a3f1 Mon Sep 17 00:00:00 2001 From: Andi Date: Wed, 27 Oct 2021 21:00:12 +0200 Subject: [PATCH 47/53] Split ways at junctions first (#2444) --- .../com/graphhopper/reader/osm/OSMReader.java | 319 ++++++++---------- .../graphhopper/reader/osm/OSMReaderTest.java | 27 +- 2 files changed, 152 insertions(+), 194 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 01dd36d870b..de15f8d2935 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.LongIndexedContainer; import com.carrotsearch.hppc.LongSet; +import com.carrotsearch.hppc.cursors.LongCursor; import com.graphhopper.coll.*; import com.graphhopper.reader.*; import com.graphhopper.reader.dem.EdgeSampling; @@ -85,7 +86,6 @@ public class OSMReader implements TurnCostParser.ExternalInternalMap { private final OSMReaderConfig config; private final Graph graph; private final NodeAccess nodeAccess; - private final LongIndexedContainer barrierNodeIds = new LongArrayList(); private final DistanceCalc distCalc = DistanceCalcEarth.DIST_EARTH; private final DouglasPeucker simplifyAlgo = new DouglasPeucker(); private CountryRuleFactory countryRuleFactory = null; @@ -341,57 +341,125 @@ protected void processWay(ReaderWay way) { if (edgeFlags.isEmpty()) return; - LongArrayList osmNodeIds = way.getNodes(); - final int size = osmNodeIds.size(); - int lastBarrier = -1; - for (int i = 0; i < size; i++) { - long nodeId = osmNodeIds.get(i); - int nodeTagIndex = nodeTagIndicesByOsmNodeID.get(nodeId); + splitWayAtJunctionsAndEmptySections(way, edgeFlags); + } + + private void splitWayAtJunctionsAndEmptySections(ReaderWay way, IntsRef edgeFlags) { + List fullSegment = new ArrayList<>(); + for (LongCursor node : way.getNodes()) + fullSegment.add(new SegmentNode(node.value, getNodeMap().get(node.value))); + + List segment = new ArrayList<>(); + for (SegmentNode node : fullSegment) { + if (!isNodeId(node.id)) { + // this node exists in ways, but not in nodes. we ignore it, but we split the way when we encounter + // such a missing node. for example an OSM way might lead out of an area where nodes are available and + // back into it. we do not want to connect the exit/entry points using a straight line. this usually + // should only happen for OSM extracts + if (segment.size() > 1) { + splitLoopSegments(segment, way, edgeFlags); + segment = new ArrayList<>(); + } + } else if (isTowerNode(node.id)) { + if (!segment.isEmpty()) { + segment.add(node); + splitLoopSegments(segment, way, edgeFlags); + segment = new ArrayList<>(); + } + segment.add(node); + } else { + segment.add(node); + } + } + // the last segment might end at the end of the way + if (segment.size() > 1) + splitLoopSegments(segment, way, edgeFlags); + } + + private void splitLoopSegments(List segment, ReaderWay way, IntsRef edgeFlags) { + if (segment.size() < 2) + throw new IllegalStateException("Segment size must be >= 2, but was: " + segment.size()); + + boolean isLoop = segment.get(0).osmNodeId == segment.get(segment.size() - 1).osmNodeId; + if (segment.size() == 2 && isLoop) { + LOGGER.warn("Loop in OSM way: {}, will be ignored, duplicate node: {}", way.getId(), segment.get(0).osmNodeId); + } else if (isLoop) { + // split into two segments + splitSegmentAtSplitNodes(segment.subList(0, segment.size() - 1), way, edgeFlags); + splitSegmentAtSplitNodes(segment.subList(segment.size() - 2, segment.size()), way, edgeFlags); + } else { + splitSegmentAtSplitNodes(segment, way, edgeFlags); + } + } + + private void splitSegmentAtSplitNodes(List parentSegment, ReaderWay way, IntsRef edgeFlags) { + List segment = new ArrayList<>(); + for (int i = 0; i < parentSegment.size(); i++) { + SegmentNode node = parentSegment.get(i); + int nodeTagIndex = nodeTagIndicesByOsmNodeID.get(node.osmNodeId); if (nodeTagIndex >= 0) { - // this node is a barrier. we will add an extra edge to block access - // create an extra node for the barrier edge - long extraNodeId = addBarrierNode(nodeId); - if (i > 0) { - // start at beginning of array if there was no previous barrier - if (lastBarrier < 0) - lastBarrier = 0; - - // add way up to the new extra node - int length = i - lastBarrier + 1; - LongArrayList partNodeIds = new LongArrayList(); - partNodeIds.add(osmNodeIds.buffer, lastBarrier, length); - partNodeIds.set(length - 1, extraNodeId); - addOSMWay(partNodeIds, edgeFlags, way); - - // create zero length barrier edge - addBarrierEdge(extraNodeId, nodeId, edgeFlags, nodeTags.get(nodeTagIndex), way); - } else { - // make sure the extra node is not at the beginning of the edge - addBarrierEdge(nodeId, extraNodeId, edgeFlags, nodeTags.get(nodeTagIndex), way); - // exchange first node for created barrier node - osmNodeIds.set(0, extraNodeId); + // this node is a barrier. we will add an extra edge to be able to block access + SegmentNode barrierFrom = node; + SegmentNode barrierTo = addBarrierNode(barrierFrom); + if (i == parentSegment.size() - 1) { + // make sure the barrier node is always on the inside of the segment + SegmentNode tmp = barrierFrom; + barrierFrom = barrierTo; + barrierTo = tmp; + } + if (!segment.isEmpty()) { + segment.add(barrierFrom); + handleSegment(segment, way, edgeFlags, emptyMap()); + segment = new ArrayList<>(); } - // remember barrier for processing the way behind it - lastBarrier = i; + segment.add(barrierFrom); + segment.add(barrierTo); + Map nodeTags = this.nodeTags.get(nodeTagIndex); + handleSegment(segment, way, edgeFlags, nodeTags); + segment = new ArrayList<>(); + segment.add(barrierTo); // ignore this barrier node from now. for example a barrier can be connecting two ways (appear in both // ways) and we only want to add a barrier edge once (but we want to add one). - nodeTagIndicesByOsmNodeID.put(nodeId, -1); - nodeTags.set(nodeTagIndex, emptyMap()); + nodeTagIndicesByOsmNodeID.put(node.osmNodeId, -1); + this.nodeTags.set(nodeTagIndex, emptyMap()); + } else { + segment.add(node); } } + if (segment.size() > 1) + handleSegment(segment, way, edgeFlags, emptyMap()); + } + + void handleSegment(List segment, ReaderWay way, IntsRef edgeFlags, Map nodeTags) { + final PointList pointList = new PointList(segment.size(), nodeAccess.is3D()); + int from = -1; + int to = -1; + for (int i = 0; i < segment.size(); i++) { + SegmentNode node = segment.get(i); + int id = node.id; + if (!isNodeId(id)) + throw new IllegalStateException("Invalid id for node: " + node.osmNodeId + " when handling segment " + segment + " for way: " + way.getId()); + if (isPillarNode(id)) { + // PILLAR node, but convert to towerNode if end-standing + boolean convertToTowerNode = i == 0 || i == segment.size() - 1; + id = node.id = handlePillarNode(id, node.osmNodeId, pointList, convertToTowerNode); + } - // just add remainder of way to graph if barrier was not the last node - if (lastBarrier >= 0) { - if (lastBarrier < size - 1) { - LongArrayList partNodeIds = new LongArrayList(); - partNodeIds.add(osmNodeIds.buffer, lastBarrier, size - lastBarrier); - addOSMWay(partNodeIds, edgeFlags, way); + if (isTowerNode(id)) { + id = -id - 3; + if (i == 0) + from = id; + else if (i == segment.size() - 1) + to = id; + else + throw new IllegalStateException("Tower nodes should only appear at the end of segments, way: " + way.getId()); + pointList.add(nodeAccess, id); } - } else { - // no barriers - simply add the whole way - addOSMWay(way.getNodes(), edgeFlags, way); } + if (from < 0 || to < 0) + throw new IllegalStateException("The first and last nodes of a segment must be tower nodes, way: " + way.getId()); + addEdge(from, to, pointList, edgeFlags, way, nodeTags); } private void setArtificialWayTags(ReaderWay way) { @@ -405,7 +473,7 @@ private void setArtificialWayTags(ReaderWay way) { GHPoint estimatedCenter = null; if (!Double.isNaN(firstLat) && !Double.isNaN(firstLon) && !Double.isNaN(lastLat) && !Double.isNaN(lastLon)) { double estimatedDist = distCalc.calcDist(firstLat, firstLon, lastLat, lastLon); - // Add artificial tag for the estimated distance and center + // Add artificial tag for the estimated distance way.setTag("estimated_distance", estimatedDist); estimatedCenter = new GHPoint((firstLat + lastLat) / 2, (firstLon + lastLon) / 2); } @@ -423,7 +491,7 @@ private void setArtificialWayTags(ReaderWay way) { List customAreas = estimatedCenter == null || areaIndex == null ? emptyList() : areaIndex.query(estimatedCenter.lat, estimatedCenter.lon); - // special handling for countries: since they are built-in with GraphHopper they are always fed to the encodingmanager + // special handling for countries: since they are built-in with GraphHopper they are always fed to the EncodingManager Country country = Country.MISSING; for (CustomArea customArea : customAreas) { Object countryCode = customArea.getProperties().get("ISO3166-1:alpha3"); @@ -515,6 +583,8 @@ else if (nodeType == JUNCTION_NODE || nodeType == CONNECTION_NODE) addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); else if (nodeType == INTERMEDIATE_NODE || nodeType == END_NODE) addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); + else + throw new IllegalStateException("Unknown node type: " + nodeType + ", duplicate OSM node ID: " + node.getId()); // we keep node tags for barrier nodes if (node.getTags().containsKey("barrier")) { @@ -535,10 +605,12 @@ else if (nodeType == INTERMEDIATE_NODE || nodeType == END_NODE) locations++; } - private void addPillarNode(long osmId, double lat, double lon, double ele) { + private int addPillarNode(long osmId, double lat, double lon, double ele) { pillarInfo.setNode(nextPillarId, lat, lon, ele); - getNodeMap().put(osmId, nextPillarId + 3); + int id = nextPillarId + 3; + getNodeMap().put(osmId, id); nextPillarId++; + return id; } void prepareWaysWithRelationInfo(ReaderRelation relation) { @@ -563,113 +635,7 @@ int addTowerNode(long osmId, double lat, double lon, double ele) { return id; } - /** - * This method creates from an OSM way (via the osm ids) one or more edges in the graph. - */ - void addOSMWay(final LongIndexedContainer osmNodeIds, final IntsRef flags, final ReaderWay way) { - final PointList pointList = new PointList(osmNodeIds.size(), nodeAccess.is3D()); - int firstNode = -1; - int lastInBoundsPillarNode = -1; - try { - // #2221: ways might include nodes at the beginning or end that do not exist -> skip them - int firstExisting = -1; - int lastExisting = -1; - for (int i = 0; i < osmNodeIds.size(); ++i) { - final int tmpNode = getNodeMap().get(osmNodeIds.get(i)); - if (isNodeId(tmpNode)) { - firstExisting = i; - break; - } - } - for (int i = osmNodeIds.size() - 1; i >= 0; --i) { - final int tmpNode = getNodeMap().get(osmNodeIds.get(i)); - if (isNodeId(tmpNode)) { - lastExisting = i; - break; - } - } - if (firstExisting < 0) { - assert lastExisting < 0; - return; - } - for (int i = firstExisting; i <= lastExisting; i++) { - final long osmNodeId = osmNodeIds.get(i); - int tmpNode = getNodeMap().get(osmNodeId); - if (tmpNode == EMPTY_NODE) - continue; - - // skip osmIds with no associated pillar or tower id (e.g. !OSMReader.isBounds) - if (tmpNode == JUNCTION_NODE || tmpNode == CONNECTION_NODE) - continue; - - if (tmpNode == INTERMEDIATE_NODE || tmpNode == END_NODE) { - // In some cases no node information is saved for the specified osmId. - // i.e. a way references a which does not exist in the current file. - // => if the node before was a pillar node then convert into to tower node (as it is also end-standing). - if (!pointList.isEmpty() && isPillarNode(lastInBoundsPillarNode)) { - // transform the pillar node to a tower node - tmpNode = lastInBoundsPillarNode; - tmpNode = handlePillarNode(tmpNode, osmNodeId, null, true); - tmpNode = -tmpNode - 3; - if (pointList.size() > 1 && firstNode >= 0) { - // TOWER node - addEdge(firstNode, tmpNode, pointList, flags, way); - pointList.clear(); - pointList.add(nodeAccess, tmpNode); - } - firstNode = tmpNode; - lastInBoundsPillarNode = -1; - } - continue; - } - - if (!isNodeId(tmpNode)) - throw new AssertionError("Mapped index not in correct bounds " + tmpNode + ", " + osmNodeId); - - if (isPillarNode(tmpNode)) { - // PILLAR node, but convert to towerNode if end-standing - boolean convertToTowerNode = i == firstExisting || i == lastExisting; - if (!convertToTowerNode) - lastInBoundsPillarNode = tmpNode; - tmpNode = handlePillarNode(tmpNode, osmNodeId, pointList, convertToTowerNode); - } - - if (isTowerNode(tmpNode)) { - tmpNode = -tmpNode - 3; - - if (firstNode >= 0 && firstNode == tmpNode) { - // loop detected. See #1525 and #1533. Insert last OSM ID as tower node. Do this for all loops so that users can manipulate loops later arbitrarily. - long lastOsmNodeId = osmNodeIds.get(i - 1); - int lastGHNodeId = getNodeMap().get(lastOsmNodeId); - if (isTowerNode(lastGHNodeId)) { - LOGGER.warn("Pillar node " + lastOsmNodeId + " is already a tower node and used in loop, see #1533. " + - "Fix mapping for way " + way.getId() + ", nodes:" + osmNodeIds); - break; - } - - int newEndNode = -handlePillarNode(lastGHNodeId, lastOsmNodeId, pointList, true) - 3; - addEdge(firstNode, newEndNode, pointList, flags, way); - pointList.clear(); - pointList.add(nodeAccess, newEndNode); - firstNode = newEndNode; - } - - pointList.add(nodeAccess, tmpNode); - if (firstNode >= 0) { - addEdge(firstNode, tmpNode, pointList, flags, way); - pointList.clear(); - pointList.add(nodeAccess, tmpNode); - } - firstNode = tmpNode; - } - } - } catch (RuntimeException ex) { - LOGGER.error("Couldn't properly add edge with osm ids:" + osmNodeIds, ex); - throw ex; - } - } - - void addEdge(int fromIndex, int toIndex, PointList pointList, IntsRef flags, ReaderWay way) { + void addEdge(int fromIndex, int toIndex, PointList pointList, IntsRef flags, ReaderWay way, Map nodeTags) { // sanity checks if (fromIndex < 0 || toIndex < 0) throw new AssertionError("to or from index is invalid for this edge " + fromIndex + "->" + toIndex + ", points:" + pointList); @@ -709,6 +675,10 @@ void addEdge(int fromIndex, int toIndex, PointList pointList, IntsRef flags, Rea towerNodeDistance = maxDistance; } + // update edge flags to potentially block access in case there are node tags + if (!nodeTags.isEmpty()) + flags = encodingManager.handleNodeTags(nodeTags, IntsRef.deepCopyOf(flags)); + EdgeIteratorState iter = graph.edge(fromIndex, toIndex).setDistance(towerNodeDistance).setFlags(flags); // If the entire way is just the first and last point, do not waste space storing an empty way geometry @@ -791,15 +761,15 @@ protected void finishedReading() { /** * Create a copy of the barrier node */ - long addBarrierNode(long nodeId) { + SegmentNode addBarrierNode(SegmentNode node) { ReaderNode newNode; - int graphIndex = getNodeMap().get(nodeId); - if (isTowerNode(graphIndex)) { - graphIndex = -graphIndex - 3; - newNode = new ReaderNode(createArtificialOSMNodeId(), nodeAccess.getLat(graphIndex), nodeAccess.getLon(graphIndex)); - } else if (isPillarNode(graphIndex)) { - graphIndex = graphIndex - 3; - newNode = new ReaderNode(createArtificialOSMNodeId(), pillarInfo.getLat(graphIndex), pillarInfo.getLon(graphIndex)); + int id = node.id; + if (isTowerNode(id)) { + id = -id - 3; + newNode = new ReaderNode(createArtificialOSMNodeId(), nodeAccess.getLat(id), nodeAccess.getLon(id)); + } else if (isPillarNode(id)) { + id = id - 3; + newNode = new ReaderNode(createArtificialOSMNodeId(), pillarInfo.getLat(id), pillarInfo.getLon(id)); } else { throw new IllegalStateException("Cannot add barrier nodes for osm node ids that do not appear in ways or nodes"); } @@ -808,27 +778,14 @@ long addBarrierNode(long nodeId) { if (getNodeMap().get(osmId) != -1) throw new IllegalStateException("Artificial osm node id already exists: " + osmId); getNodeMap().put(osmId, INTERMEDIATE_NODE); - addPillarNode(osmId, newNode.getLat(), newNode.getLon(), eleProvider.getEle(newNode)); - return osmId; + int newId = addPillarNode(osmId, newNode.getLat(), newNode.getLon(), eleProvider.getEle(newNode)); + return new SegmentNode(osmId, newId); } private long createArtificialOSMNodeId() { return newUniqueOsmId++; } - /** - * Add a zero length edge with reduced routing options to the graph. - */ - void addBarrierEdge(long fromId, long toId, IntsRef edgeFlags, Map nodeTags, ReaderWay way) { - // update edge flags to block access depending on the node tags - edgeFlags = encodingManager.handleNodeTags(nodeTags, IntsRef.deepCopyOf(edgeFlags)); - // add edge - barrierNodeIds.clear(); - barrierNodeIds.add(fromId); - barrierNodeIds.add(toId); - addOSMWay(barrierNodeIds, edgeFlags, way); - } - /** * Creates turn relations out of an unspecified OSM relation */ @@ -977,4 +934,14 @@ private boolean isPillarNode(int id) { private boolean isNodeId(int id) { return id > CONNECTION_NODE || id < JUNCTION_NODE; } + + static class SegmentNode { + long osmNodeId; + int id; + + public SegmentNode(long osmNodeId, int id) { + this.osmNodeId = osmNodeId; + this.id = id; + } + } } diff --git a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java index ca908bdef72..7ec4b2a4271 100644 --- a/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java +++ b/core/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java @@ -17,7 +17,7 @@ */ package com.graphhopper.reader.osm; -import com.carrotsearch.hppc.LongIndexedContainer; +import com.carrotsearch.hppc.LongArrayList; import com.graphhopper.GHRequest; import com.graphhopper.GHResponse; import com.graphhopper.GraphHopper; @@ -386,9 +386,9 @@ public void testBarrierBetweenWays() { importOrLoad(); Graph graph = hopper.getGraphHopperStorage(); - // there are seven ways, but there should also be five barrier edges - // note that because of the extra edge at the loop way we do not split the loop - assertEquals(12, graph.getEdges()); + // there are seven ways, but there should also be six barrier edges + // we first split the loop way into two parts, and then we split the barrier node => 3 edges total + assertEquals(7 + 6, graph.getEdges()); int loops = 0; AllEdgesIterator iter = graph.getAllEdges(); while (iter.next()) { @@ -397,7 +397,7 @@ public void testBarrierBetweenWays() { if (graph.getNodeAccess().getLat(iter.getBaseNode()) == graph.getNodeAccess().getLat(iter.getAdjNode())) loops++; } - assertEquals(5 + 1, loops); + assertEquals(5, loops); } @Test @@ -434,19 +434,10 @@ public void avoidsLoopEdgesIdenticalLatLon_1533() { @Test public void avoidsLoopEdgesIdenticalNodeIds_1533() { - // We can handle the following case with the proper result: + // BDCBB checkLoop(new GraphHopperFacade("test-avoid-loops3.xml").importOrLoad()); - // We cannot handle the following case, i.e. no loop is created. so we only check that there are no loops - GraphHopper hopper = new GraphHopperFacade("test-avoid-loops4.xml").importOrLoad(); - GraphHopperStorage graph = hopper.getGraphHopperStorage(); - AllEdgesIterator iter = graph.getAllEdges(); - assertEquals(2, iter.length()); - while (iter.next()) { - assertTrue(iter.getAdjNode() != iter.getBaseNode(), "found a loop"); - } - int nodeB = AbstractGraphStorageTester.getIdOf(graph, 12); - assertTrue(nodeB > -1, "could not find OSM node B"); - assertEquals(2, GHUtility.count(graph.createEdgeExplorer().setBaseNode(nodeB))); + // BBCDB + checkLoop(new GraphHopperFacade("test-avoid-loops4.xml").importOrLoad()); } @Test @@ -642,7 +633,7 @@ public void testEstimatedDistance() { } @Override - void addOSMWay(LongIndexedContainer osmNodeIds, IntsRef wayFlags, ReaderWay way) { + void handleSegment(List segment, ReaderWay way, IntsRef edgeFlags, Map nodeTags) { } }; From 27b9ad3edf8c2e8c07c946f6293b1c265b3aa6b4 Mon Sep 17 00:00:00 2001 From: Andi Date: Wed, 27 Oct 2021 21:02:35 +0200 Subject: [PATCH 48/53] Snap to tower nodes when they are close (#2446) --- .../storage/index/LocationIndexTree.java | 85 ++++++++--------- .../java/com/graphhopper/GraphHopperTest.java | 92 ++++++++----------- .../storage/index/LocationIndexTreeTest.java | 73 +++++++++++++-- 3 files changed, 148 insertions(+), 102 deletions(-) diff --git a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java index 7a19325d715..0a0ca9c5092 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java +++ b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java @@ -162,7 +162,7 @@ public LocationIndex prepareIndex(EdgeFilter edgeFilter) { // I want to be able to create a location index for the empty graph without error, but for that // I need valid bounds so that the initialization logic works. if (!bounds.isValid()) - bounds = new BBox(-10.0,10.0,-10.0,10.0); + bounds = new BBox(-10.0, 10.0, -10.0, 10.0); InMemConstructionIndex inMemConstructionIndex = prepareInMemConstructionIndex(bounds, edgeFilter); @@ -319,58 +319,59 @@ public interface EdgeCheck { public void traverseEdge(double queryLat, double queryLon, EdgeIteratorState currEdge, EdgeCheck edgeCheck) { int baseNode = currEdge.getBaseNode(); - double currLat = nodeAccess.getLat(baseNode); - double currLon = nodeAccess.getLon(baseNode); - double currNormedDist = DIST_PLANE.calcNormalizedDist(queryLat, queryLon, currLat, currLon); - - int tmpClosestNode = baseNode; - edgeCheck.check(tmpClosestNode, currNormedDist, 0, Snap.Position.TOWER); - if (currNormedDist <= equalNormedDelta) - return; + double baseLat = nodeAccess.getLat(baseNode); + double baseLon = nodeAccess.getLon(baseNode); + double baseDist = DIST_PLANE.calcNormalizedDist(queryLat, queryLon, baseLat, baseLon); int adjNode = currEdge.getAdjNode(); double adjLat = nodeAccess.getLat(adjNode); double adjLon = nodeAccess.getLon(adjNode); - double adjDist = DIST_PLANE.calcNormalizedDist(adjLat, adjLon, queryLat, queryLon); - // if there are wayPoints this is only an approximation - if (adjDist < currNormedDist) - tmpClosestNode = adjNode; - - double tmpLat = currLat; - double tmpLon = currLon; - double tmpNormedDist; + double adjDist = DIST_PLANE.calcNormalizedDist(queryLat, queryLon, adjLat, adjLon); + PointList pointList = currEdge.fetchWayGeometry(FetchMode.PILLAR_AND_ADJ); - int len = pointList.size(); - for (int pointIndex = 0; pointIndex < len; pointIndex++) { - double wayLat = pointList.getLat(pointIndex); - double wayLon = pointList.getLon(pointIndex); - Snap.Position pos = Snap.Position.EDGE; - if (DIST_PLANE.isCrossBoundary(tmpLon, wayLon)) { - tmpLat = wayLat; - tmpLon = wayLon; + final int len = pointList.size(); + + int closestTowerNode; + double closestDist; + if (baseDist < adjDist) { + closestTowerNode = baseNode; + closestDist = baseDist; + edgeCheck.check(baseNode, baseDist, 0, Snap.Position.TOWER); + } else { + closestTowerNode = adjNode; + closestDist = adjDist; + edgeCheck.check(adjNode, adjDist, len, Snap.Position.TOWER); + } + if (closestDist <= equalNormedDelta) + // if a tower node is close to the query point we stop + return; + + double lastLat = baseLat; + double lastLon = baseLon; + for (int i = 0; i < len; i++) { + double lat = pointList.getLat(i); + double lon = pointList.getLon(i); + if (DIST_PLANE.isCrossBoundary(lastLon, lon)) { + lastLat = lat; + lastLon = lon; continue; } - if (DIST_PLANE.validEdgeDistance(queryLat, queryLon, tmpLat, tmpLon, wayLat, wayLon)) { - tmpNormedDist = DIST_PLANE.calcNormalizedEdgeDistance(queryLat, queryLon, - tmpLat, tmpLon, wayLat, wayLon); - edgeCheck.check(tmpClosestNode, tmpNormedDist, pointIndex, pos); + // +1 because we skipped the base node + final int indexInFullPointList = i + 1; + if (DIST_PLANE.validEdgeDistance(queryLat, queryLon, lastLat, lastLon, lat, lon)) { + closestDist = DIST_PLANE.calcNormalizedEdgeDistance(queryLat, queryLon, lastLat, lastLon, lat, lon); + edgeCheck.check(closestTowerNode, closestDist, indexInFullPointList - 1, Snap.Position.EDGE); + } else if (i < len - 1) { + closestDist = DIST_PLANE.calcNormalizedDist(queryLat, queryLon, lat, lon); + edgeCheck.check(closestTowerNode, closestDist, indexInFullPointList, Snap.Position.PILLAR); } else { - if (pointIndex + 1 == len) { - tmpNormedDist = adjDist; - pos = Snap.Position.TOWER; - } else { - tmpNormedDist = DIST_PLANE.calcNormalizedDist(queryLat, queryLon, wayLat, wayLon); - pos = Snap.Position.PILLAR; - } - edgeCheck.check(tmpClosestNode, tmpNormedDist, pointIndex + 1, pos); + // we snapped onto the last tower node, but we already handled this before so do nothing } - - if (tmpNormedDist <= equalNormedDelta) + if (closestDist <= equalNormedDelta) return; - - tmpLat = wayLat; - tmpLon = wayLon; + lastLat = lat; + lastLon = lon; } } diff --git a/core/src/test/java/com/graphhopper/GraphHopperTest.java b/core/src/test/java/com/graphhopper/GraphHopperTest.java index 4a3864e513c..2097ec92439 100644 --- a/core/src/test/java/com/graphhopper/GraphHopperTest.java +++ b/core/src/test/java/com/graphhopper/GraphHopperTest.java @@ -1024,47 +1024,9 @@ public void testSRTMWithInstructions() { assertEquals(52, res.getPoints().get(10).getEle(), 1e-2); } - @Test - public void testSRTMWithoutTunnelInterpolation() { - final String profile = "profile"; - final String vehicle = "foot"; - final String weighting = "shortest"; - - GraphHopper hopper = new GraphHopper(); - hopper.getEncodingManagerBuilder().add(new OSMRoadEnvironmentParser() { - @Override - public IntsRef handleWayTags(IntsRef edgeFlags, ReaderWay readerWay, boolean ferry, IntsRef relationFlags) { - // do not change RoadEnvironment to avoid triggering tunnel interpolation - return edgeFlags; - } - }).addIfAbsent(new DefaultFlagEncoderFactory(), vehicle); - hopper.setOSMFile(MONACO) - .setStoreOnFlush(true) - .setGraphHopperLocation(GH_LOCATION) - .setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting)); - - hopper.setElevationProvider(new SRTMProvider(DIR)); - hopper.importOrLoad(); - - GHResponse rsp = hopper.route(new GHRequest(43.74056471749763, 7.4299266210693755, - 43.73790260334179, 7.427984089259056).setAlgorithm(ASTAR) - .setProfile(profile)); - ResponsePath res = rsp.getBest(); - assertEquals(356.5, res.getDistance(), .1); - PointList pointList = res.getPoints(); - assertEquals(6, pointList.size()); - assertTrue(pointList.is3D()); - - assertEquals(20.0, pointList.getEle(0), .1); - assertEquals(23.0, pointList.getEle(1), .1); - assertEquals(23.0, pointList.getEle(2), .1); - assertEquals(41.0, pointList.getEle(3), .1); - assertEquals(19.0, pointList.getEle(4), .1); - assertEquals(26.5, pointList.getEle(5), .1); - } - - @Test - public void testSRTMWithTunnelInterpolation() { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testSRTMWithTunnelInterpolation(boolean withTunnelInterpolation) { final String profile = "profile"; final String vehicle = "foot"; final String weighting = "shortest"; @@ -1072,29 +1034,53 @@ public void testSRTMWithTunnelInterpolation() { GraphHopper hopper = new GraphHopper(). setGraphHopperLocation(GH_LOCATION). setOSMFile(MONACO). - setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting), - new Profile("car").setVehicle("foot").setWeighting(weighting)). + setProfiles(new Profile(profile).setVehicle(vehicle).setWeighting(weighting)). setStoreOnFlush(true); + if (!withTunnelInterpolation) { + hopper.getEncodingManagerBuilder().add(new OSMRoadEnvironmentParser() { + @Override + public IntsRef handleWayTags(IntsRef edgeFlags, ReaderWay readerWay, boolean ferry, IntsRef relationFlags) { + // do not change RoadEnvironment to avoid triggering tunnel interpolation + return edgeFlags; + } + }).addIfAbsent(new DefaultFlagEncoderFactory(), vehicle); + } + hopper.setElevationProvider(new SRTMProvider(DIR)); hopper.importOrLoad(); - GHResponse rsp = hopper.route(new GHRequest(43.74056471749763, 7.4299266210693755, - 43.73790260334179, 7.427984089259056).setAlgorithm(ASTAR) + GHPoint from = new GHPoint(43.7405647, 7.4299266); + GHPoint to = new GHPoint(43.7378990, 7.4279780); + + // make sure we hit tower nodes, because all we really want is test the elevation interpolation + assertEquals(Snap.Position.TOWER, hopper.getLocationIndex().findClosest(from.lat, from.lon, EdgeFilter.ALL_EDGES).getSnappedPosition()); + assertEquals(Snap.Position.TOWER, hopper.getLocationIndex().findClosest(to.lat, to.lon, EdgeFilter.ALL_EDGES).getSnappedPosition()); + + GHResponse rsp = hopper.route(new GHRequest(from, to) .setProfile(profile)); ResponsePath res = rsp.getBest(); - // Without interpolation: 356.5 - assertEquals(351, res.getDistance(), .1); PointList pointList = res.getPoints(); assertEquals(6, pointList.size()); assertTrue(pointList.is3D()); - assertEquals(18, pointList.getEle(0), .1); - assertEquals(19.04, pointList.getEle(1), .1); - assertEquals(21.67, pointList.getEle(2), .1); - assertEquals(25.03, pointList.getEle(3), .1); - assertEquals(28.65, pointList.getEle(4), .1); - assertEquals(31.32, pointList.getEle(5), .1); + if (withTunnelInterpolation) { + assertEquals(351.8, res.getDistance(), .1); + assertEquals(17, pointList.getEle(0), .1); + assertEquals(19.04, pointList.getEle(1), .1); + assertEquals(21.67, pointList.getEle(2), .1); + assertEquals(25.03, pointList.getEle(3), .1); + assertEquals(28.65, pointList.getEle(4), .1); + assertEquals(34.00, pointList.getEle(5), .1); + } else { + assertEquals(358.3, res.getDistance(), .1); + assertEquals(17.0, pointList.getEle(0), .1); + assertEquals(23.0, pointList.getEle(1), .1); + assertEquals(23.0, pointList.getEle(2), .1); + assertEquals(41.0, pointList.getEle(3), .1); + assertEquals(19.0, pointList.getEle(4), .1); + assertEquals(34.0, pointList.getEle(5), .1); + } } @Test diff --git a/core/src/test/java/com/graphhopper/storage/index/LocationIndexTreeTest.java b/core/src/test/java/com/graphhopper/storage/index/LocationIndexTreeTest.java index c042d57ebdf..38625cc2693 100644 --- a/core/src/test/java/com/graphhopper/storage/index/LocationIndexTreeTest.java +++ b/core/src/test/java/com/graphhopper/storage/index/LocationIndexTreeTest.java @@ -25,12 +25,11 @@ import com.graphhopper.util.shapes.BBox; import com.graphhopper.util.shapes.GHPoint; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.Closeable; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Random; +import java.util.*; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -42,14 +41,14 @@ public class LocationIndexTreeTest { protected final EncodingManager encodingManager = EncodingManager.create("car"); public static void initSimpleGraph(Graph g, EncodingManager em) { - // 6 | 4 + // 6 | 4 // 5 | // | 6 // 4 | 5 // 3 | // 2 | 1 // 1 | 3 - // 0 | 2 + // 0 | 2 // -1 | 0 // ---|------------------- // |-2 -1 0 1 2 3 4 @@ -62,7 +61,8 @@ public static void initSimpleGraph(Graph g, EncodingManager em) { na.setNode(4, 6, 1); na.setNode(5, 4, 4); na.setNode(6, 4.5, -0.5); - List list = Arrays.asList(g.edge(0, 1), + List list = Arrays.asList( + g.edge(0, 1), g.edge(0, 2), g.edge(2, 3), g.edge(3, 4), @@ -613,4 +613,63 @@ public void testDifferentVehicles() { assertEquals(2, idx.findClosest(1, -1, AccessFilter.allEdges(footEncoder.getAccessEnc())).getClosestNode()); Helper.close((Closeable) g); } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void closeToTowerNode(boolean snapAtBase) { + // 0 - 1 + GraphHopperStorage graph = new GraphBuilder(encodingManager).create(); + NodeAccess na = graph.getNodeAccess(); + na.setNode(0, 51.985500, 19.254000); + na.setNode(1, 51.986000, 19.255000); + DistancePlaneProjection distCalc = new DistancePlaneProjection(); + // we query the location index close to node 0. since the query point is so close to the tower node we expect + // a TOWER snap. this should not depend on whether node 0 is the base or adj node of our edge. + final int snapNode = 0; + final int base = snapAtBase ? 0 : 1; + final int adj = snapAtBase ? 1 : 0; + graph.edge(base, adj).setDistance(distCalc.calcDist(na.getLat(0), na.getLon(0), na.getLat(1), na.getLon(1))); + LocationIndexTree index = new LocationIndexTree(graph, graph.getDirectory()); + index.prepareIndex(); + + GHPoint queryPoint = new GHPoint(51.9855003, 19.2540003); + double distFromTower = distCalc.calcDist(queryPoint.lat, queryPoint.lon, na.getLat(snapNode), na.getLon(snapNode)); + assertTrue(distFromTower < 0.1); + Snap snap = index.findClosest(queryPoint.lat, queryPoint.lon, EdgeFilter.ALL_EDGES); + assertEquals(Snap.Position.TOWER, snap.getSnappedPosition()); + } + + @Test + public void queryBehindBeforeOrBehindLastTowerNode() { + // 0 -x- 1 + GraphHopperStorage graph = new GraphBuilder(encodingManager).create(); + NodeAccess na = graph.getNodeAccess(); + na.setNode(0, 51.985000, 19.254000); + na.setNode(1, 51.986000, 19.255000); + DistancePlaneProjection distCalc = new DistancePlaneProjection(); + EdgeIteratorState edge = graph.edge(0, 1).setDistance(distCalc.calcDist(na.getLat(0), na.getLon(0), na.getLat(1), na.getLon(1))); + edge.setWayGeometry(Helper.createPointList(51.985500, 19.254500)); + LocationIndexTree index = new LocationIndexTree(graph, graph.getDirectory()); + index.prepareIndex(); + { + // snap before last tower node + List output = new ArrayList<>(); + index.traverseEdge(51.985700, 19.254700, edge, (node, normedDist, wayIndex, pos) -> + output.add(node + ", " + Math.round(distCalc.calcDenormalizedDist(normedDist)) + ", " + wayIndex + ", " + pos)); + assertEquals(Arrays.asList( + "1, 39, 2, TOWER", + "1, 26, 1, PILLAR", + "1, 0, 1, EDGE"), output); + } + + { + // snap behind last tower node + List output = new ArrayList<>(); + index.traverseEdge(51.986100, 19.255100, edge, (node, normedDist, wayIndex, pos) -> + output.add(node + ", " + Math.round(distCalc.calcDenormalizedDist(normedDist)) + ", " + wayIndex + ", " + pos)); + assertEquals(Arrays.asList( + "1, 13, 2, TOWER", + "1, 78, 1, PILLAR"), output); + } + } } From e79f74fda44d8ac2dec72370ac7a1ae89f9cfde8 Mon Sep 17 00:00:00 2001 From: Andi Date: Thu, 28 Oct 2021 08:30:49 +0200 Subject: [PATCH 49/53] Fix curbside handling at barrier nodes (#2447) --- .../routing/DirectionResolver.java | 49 +++++++++++++------ .../DirectionResolverOnQueryGraphTest.java | 45 +++++++++++++++-- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/graphhopper/routing/DirectionResolver.java b/core/src/main/java/com/graphhopper/routing/DirectionResolver.java index e9dc4873a6f..26dec3941da 100644 --- a/core/src/main/java/com/graphhopper/routing/DirectionResolver.java +++ b/core/src/main/java/com/graphhopper/routing/DirectionResolver.java @@ -67,7 +67,7 @@ public DirectionResolver(Graph graph, BiPredicate is */ public DirectionResolverResult resolveDirections(int node, GHPoint location) { AdjacentEdges adjacentEdges = calcAdjEdges(node); - if (adjacentEdges.numNonLoops == 0) { + if (adjacentEdges.numStandardEdges == 0) { return DirectionResolverResult.impossible(); } if (!adjacentEdges.hasInEdges() || !adjacentEdges.hasOutEdges()) { @@ -79,11 +79,16 @@ public DirectionResolverResult resolveDirections(int node, GHPoint location) { if (adjacentEdges.numLoops > 0) { return DirectionResolverResult.unrestricted(); } + if (adjacentEdges.numZeroDistanceEdges > 0) { + // if we snap to a tower node that is adjacent to a barrier edge we apply no restrictions. this is the + // easiest thing to do, but maybe we need a more sophisticated handling of this case in the future. + return DirectionResolverResult.unrestricted(); + } Point snappedPoint = new Point(nodeAccess.getLat(node), nodeAccess.getLon(node)); if (adjacentEdges.nextPoints.contains(snappedPoint)) { // this might happen if a pillar node of an adjacent edge has the same coordinates as the snapped point, // but this should be prevented by the map import already - throw new IllegalArgumentException("Pillar node of adjacent edge matches snapped point, this should not happen"); + throw new IllegalStateException("Pillar node of adjacent edge matches snapped point, this should not happen"); } // we can classify the different cases by the number of different next points! if (adjacentEdges.nextPoints.size() == 1) { @@ -127,7 +132,7 @@ public DirectionResolverResult resolveDirections(int node, GHPoint location) { } else { // we snapped to a junction, in this case we do not apply restrictions // note: TOWER and PILLAR mostly occur when location is near the end of a dead end street or a sharp - // curve, like switchbacks in the mountains of andorra + // curve, like switchbacks in the mountains of Andorra return DirectionResolverResult.unrestricted(); } } @@ -176,30 +181,41 @@ private AdjacentEdges calcAdjEdges(int node) { while (iter.next()) { boolean isIn = isAccessible.test(iter, true); boolean isOut = isAccessible.test(iter, false); - if (!isIn && !isOut) { + if (!isIn && !isOut) continue; - } - if (iter.getBaseNode() == iter.getAdjNode()) { - adjacentEdges.numLoops++; - } else { - adjacentEdges.numNonLoops++; - } // we are interested in the coordinates of the next point on this edge, it could be the adj tower node // but also a pillar node final PointList geometry = iter.fetchWayGeometry(FetchMode.ALL); double nextPointLat = geometry.getLat(1); double nextPointLon = geometry.getLon(1); - // todo: special treatment in case the coordinates of the first pillar node equal those of the base tower - // node, see #1694 - if (geometry.size() > 2 && PointList.equalsEps(nextPointLat, geometry.getLat(0)) && + boolean isZeroDistanceEdge = false; + if (PointList.equalsEps(nextPointLat, geometry.getLat(0)) && PointList.equalsEps(nextPointLon, geometry.getLon(0))) { - nextPointLat = geometry.getLat(2); - nextPointLon = geometry.getLon(2); + if (geometry.size() > 2) { + // todo: special treatment in case the coordinates of the first pillar node equal those of the base tower + // node, see #1694 + nextPointLat = geometry.getLat(2); + nextPointLon = geometry.getLon(2); + } else if (geometry.size() == 2) { + // an edge where base and adj node share the same coordinates. this is the case for barrier edges that + // we create artificially + isZeroDistanceEdge = true; + } else { + throw new IllegalStateException("Geometry has less than two points"); + } } Point nextPoint = new Point(nextPointLat, nextPointLon); Edge edge = new Edge(iter.getEdge(), iter.getAdjNode(), nextPoint); adjacentEdges.addEdge(edge, isIn, isOut); + + if (iter.getBaseNode() == iter.getAdjNode()) + adjacentEdges.numLoops++; + else if (isZeroDistanceEdge) + adjacentEdges.numZeroDistanceEdges++; + else + adjacentEdges.numStandardEdges++; + } return adjacentEdges; } @@ -209,7 +225,8 @@ private static class AdjacentEdges { private final Map> outEdgesByNextPoint = new HashMap<>(2); final Set nextPoints = new HashSet<>(2); int numLoops; - int numNonLoops; + int numStandardEdges; + int numZeroDistanceEdges; void addEdge(Edge edge, boolean isIn, boolean isOut) { if (isIn) { diff --git a/core/src/test/java/com/graphhopper/routing/DirectionResolverOnQueryGraphTest.java b/core/src/test/java/com/graphhopper/routing/DirectionResolverOnQueryGraphTest.java index 053bd1ad081..0c626c0ce73 100644 --- a/core/src/test/java/com/graphhopper/routing/DirectionResolverOnQueryGraphTest.java +++ b/core/src/test/java/com/graphhopper/routing/DirectionResolverOnQueryGraphTest.java @@ -27,10 +27,7 @@ import com.graphhopper.storage.RAMDirectory; import com.graphhopper.storage.index.LocationIndexTree; import com.graphhopper.storage.index.Snap; -import com.graphhopper.util.EdgeExplorer; -import com.graphhopper.util.EdgeIterator; -import com.graphhopper.util.EdgeIteratorState; -import com.graphhopper.util.GHUtility; +import com.graphhopper.util.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,7 +37,8 @@ import static com.graphhopper.routing.DirectionResolverResult.unrestricted; import static com.graphhopper.util.EdgeIterator.NO_EDGE; import static com.graphhopper.util.Helper.createPointList; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * This test simulates incoming lat/lon coordinates that get snapped to graph edges (using {@link QueryGraph}) and the @@ -255,6 +253,43 @@ public void duplicateCoordinatesAtBaseOrAdjNode() { checkResult(0.9, 0.9, restricted(edge(0, 2), edge(2, 1), edge(1, 2), edge(2, 0))); } + @Test + public void closeToTowerNode_issue2443() { + // 0x-1 + addNode(0, 51.986000, 19.255000); + addNode(1, 51.985500, 19.254000); + DistancePlaneProjection distCalc = new DistancePlaneProjection(); + addEdge(0, 1, true).setDistance(distCalc.calcDist(na.getLat(0), na.getLon(0), na.getLat(1), na.getLon(1))); + init(); + + double lat = 51.9855003; + double lon = 19.2540003; + Snap snap = snapCoordinate(lat, lon); + queryGraph = QueryGraph.create(graph, snap); + DirectionResolver resolver = new DirectionResolver(queryGraph, this::isAccessible); + DirectionResolverResult result = resolver.resolveDirections(snap.getClosestNode(), snap.getQueryPoint()); + assertEquals(0, result.getInEdgeRight()); + assertEquals(0, result.getOutEdgeRight()); + assertEquals(0, result.getInEdgeLeft()); + assertEquals(0, result.getOutEdgeRight()); + } + + @Test + public void unblockedBarrierEdge_issue2443() { + // 0---1-2 + addNode(0, 51.9860, 19.2550); + addNode(1, 51.9861, 19.2551); + addNode(2, 51.9861, 19.2551); + DistancePlaneProjection distCalc = new DistancePlaneProjection(); + addEdge(0, 1, true).setDistance(distCalc.calcDist(na.getLat(0), na.getLon(0), na.getLat(1), na.getLon(1))); + // a barrier edge connects two different nodes (it is not a loop), but they have the same coordinates (distance is 0) + // barrier edges **can** be accessible, for example they could be blocked only for certain vehicles + addEdge(1, 2, true).setDistance(0); + init(); + // currently we just use unrestricted when we snap to a barrier edge node, see #2447 + assertUnrestricted(51.9861, 19.2551); + } + private void addNode(int nodeId, double lat, double lon) { na.setNode(nodeId, lat, lon); } From 65fdf55e9efb0ffb3669982cbff8ffa27f7cd9f7 Mon Sep 17 00:00:00 2001 From: otbutz Date: Fri, 29 Oct 2021 11:32:58 +0200 Subject: [PATCH 50/53] Allow CountryRules to provide fallback values for toll (#2164) * Improve readability * Allow country rules to provide fallback toll values * Assume HGV toll for german motorways, trunks and primary roads * Assume general toll for austrian motorways and trunks * Adapt unittests * Update docs * Update changelog Co-authored-by: Thomas Butz Co-authored-by: Peter --- CHANGELOG.md | 1 + .../java/com/graphhopper/routing/ev/Toll.java | 17 ++--------- .../util/countryrules/AustriaCountryRule.java | 15 ++++++++++ .../util/countryrules/CountryRule.java | 7 +++-- .../util/countryrules/GermanyCountryRule.java | 15 ++++++++++ .../routing/util/parsers/OSMTollParser.java | 30 ++++++++++++++----- .../util/parsers/OSMTollParserTest.java | 2 +- .../weighting/custom/CustomWeightingTest.java | 2 +- docs/core/custom-models.md | 2 +- 9 files changed, 64 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32841909f14..a9c4ed2f48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - AbstractFlagEncoder#handleNodeTags was replaced by AbstractFlagEncoder#isBarrier (#2434) - consider heading when snapping coordinates to the road network, this is especially important for navigation (#2411) - OSMReader no longer sets the artificial 'estimated_center' tag and processNode also receives EMPTY_NODEs (971d686) +- added Toll.MISSING; custom models must be adapted to check for explicit toll values e.g `toll != NO` -> `toll == HGV || toll == ALL` (#2164) ### 4.0 [29 Sep 2021] diff --git a/core/src/main/java/com/graphhopper/routing/ev/Toll.java b/core/src/main/java/com/graphhopper/routing/ev/Toll.java index ef8bdc5db0f..a01bc548063 100644 --- a/core/src/main/java/com/graphhopper/routing/ev/Toll.java +++ b/core/src/main/java/com/graphhopper/routing/ev/Toll.java @@ -17,13 +17,12 @@ */ package com.graphhopper.routing.ev; -import com.graphhopper.util.Helper; - /** - * This enum defines the toll value like NO (default), ALL (all vehicles) and HGV (toll for heavy goods vehicles) + * This enum defines the toll value like MISSING (default), NO (no toll), HGV + * (toll for heavy goods vehicles) and ALL (all vehicles) */ public enum Toll { - NO("no"), ALL("all"), HGV("hgv"); + MISSING("missing"), NO("no"), HGV("hgv"), ALL("all"); public static final String KEY = "toll"; @@ -33,16 +32,6 @@ public enum Toll { this.name = name; } - public static Toll find(String name) { - if (name == null) - return NO; - try { - return Toll.valueOf(Helper.toUpperCase(name)); - } catch (IllegalArgumentException ex) { - return NO; - } - } - @Override public String toString() { return name; diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java index e214ac0f8ad..64b0a73e87f 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/AustriaCountryRule.java @@ -21,6 +21,7 @@ import com.graphhopper.reader.ReaderWay; import com.graphhopper.routing.ev.RoadAccess; import com.graphhopper.routing.ev.RoadClass; +import com.graphhopper.routing.ev.Toll; import com.graphhopper.routing.util.TransportationMode; public class AustriaCountryRule implements CountryRule { @@ -71,4 +72,18 @@ public RoadAccess getAccess(ReaderWay readerWay, TransportationMode transportati return RoadAccess.YES; } } + + @Override + public Toll getToll(ReaderWay readerWay, TransportationMode transportationMode, Toll currentToll) { + if (!transportationMode.isMotorVehicle() || currentToll != Toll.MISSING) { + return currentToll; + } + + RoadClass roadClass = RoadClass.find(readerWay.getTag("highway", "")); + if (roadClass == RoadClass.MOTORWAY || roadClass == RoadClass.TRUNK) { + return Toll.ALL; + } + + return currentToll; + } } diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRule.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRule.java index 447e2f51d2e..cf67f15761a 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRule.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/CountryRule.java @@ -19,9 +19,8 @@ package com.graphhopper.routing.util.countryrules; import com.graphhopper.reader.ReaderWay; -import com.graphhopper.routing.ev.Country; import com.graphhopper.routing.ev.RoadAccess; -import com.graphhopper.routing.ev.RoadClass; +import com.graphhopper.routing.ev.Toll; import com.graphhopper.routing.util.TransportationMode; /** @@ -35,4 +34,8 @@ default double getMaxSpeed(ReaderWay readerWay, TransportationMode transportatio default RoadAccess getAccess(ReaderWay readerWay, TransportationMode transportationMode, RoadAccess currentRoadAccess) { return currentRoadAccess; } + + default Toll getToll(ReaderWay readerWay, TransportationMode transportationMode, Toll currentToll) { + return currentToll; + } } diff --git a/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java b/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java index d78c800da9d..49a9d01aa39 100644 --- a/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java +++ b/core/src/main/java/com/graphhopper/routing/util/countryrules/GermanyCountryRule.java @@ -22,6 +22,7 @@ import com.graphhopper.routing.ev.MaxSpeed; import com.graphhopper.routing.ev.RoadAccess; import com.graphhopper.routing.ev.RoadClass; +import com.graphhopper.routing.ev.Toll; import com.graphhopper.routing.util.TransportationMode; /** @@ -79,4 +80,18 @@ public RoadAccess getAccess(ReaderWay readerWay, TransportationMode transportati return RoadAccess.YES; } } + + @Override + public Toll getToll(ReaderWay readerWay, TransportationMode transportationMode, Toll currentToll) { + if (!transportationMode.isMotorVehicle() || currentToll != Toll.MISSING) { + return currentToll; + } + + RoadClass roadClass = RoadClass.find(readerWay.getTag("highway", "")); + if (roadClass == RoadClass.MOTORWAY || roadClass == RoadClass.TRUNK || roadClass == RoadClass.PRIMARY) { + return Toll.HGV; + } + + return currentToll; + } } diff --git a/core/src/main/java/com/graphhopper/routing/util/parsers/OSMTollParser.java b/core/src/main/java/com/graphhopper/routing/util/parsers/OSMTollParser.java index c9c566b94b6..ec26822358e 100644 --- a/core/src/main/java/com/graphhopper/routing/util/parsers/OSMTollParser.java +++ b/core/src/main/java/com/graphhopper/routing/util/parsers/OSMTollParser.java @@ -22,12 +22,17 @@ import com.graphhopper.routing.ev.EncodedValueLookup; import com.graphhopper.routing.ev.EnumEncodedValue; import com.graphhopper.routing.ev.Toll; +import com.graphhopper.routing.util.TransportationMode; +import com.graphhopper.routing.util.countryrules.CountryRule; import com.graphhopper.storage.IntsRef; +import java.util.Arrays; +import java.util.Collections; import java.util.List; public class OSMTollParser implements TagParser { + private static final List HGV_TAGS = Collections.unmodifiableList(Arrays.asList("toll:hgv", "toll:N2", "toll:N3")); private final EnumEncodedValue tollEnc; public OSMTollParser() { @@ -45,14 +50,23 @@ public void createEncodedValues(EncodedValueLookup lookup, List li @Override public IntsRef handleWayTags(IntsRef edgeFlags, ReaderWay readerWay, boolean ferry, IntsRef relationFlags) { - if (readerWay.hasTag("toll", "yes")) - tollEnc.setEnum(false, edgeFlags, Toll.ALL); - else if (readerWay.hasTag("toll:hgv", "yes")) - tollEnc.setEnum(false, edgeFlags, Toll.HGV); - else if (readerWay.hasTag("toll:N2", "yes")) - tollEnc.setEnum(false, edgeFlags, Toll.HGV); - else if (readerWay.hasTag("toll:N3", "yes")) - tollEnc.setEnum(false, edgeFlags, Toll.HGV); + Toll toll; + if (readerWay.hasTag("toll", "yes")) { + toll = Toll.ALL; + } else if (readerWay.hasTag(HGV_TAGS, Collections.singletonList("yes"))) { + toll = Toll.HGV; + } else if (readerWay.hasTag("toll", "no")) { + toll = Toll.NO; + } else { + toll = Toll.MISSING; + } + + CountryRule countryRule = readerWay.getTag("country_rule", null); + if (countryRule != null) + toll = countryRule.getToll(readerWay, TransportationMode.CAR, toll); + + tollEnc.setEnum(false, edgeFlags, toll); + return edgeFlags; } } diff --git a/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTollParserTest.java b/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTollParserTest.java index bd645c022bf..4510d49f500 100644 --- a/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTollParserTest.java +++ b/core/src/test/java/com/graphhopper/routing/util/parsers/OSMTollParserTest.java @@ -29,7 +29,7 @@ public void testSimpleTags() { IntsRef intsRef = em.createEdgeFlags(); readerWay.setTag("highway", "primary"); parser.handleWayTags(intsRef, readerWay, false, relFlags); - assertEquals(Toll.NO, tollEnc.getEnum(false, intsRef)); + assertEquals(Toll.MISSING, tollEnc.getEnum(false, intsRef)); intsRef = em.createEdgeFlags(); readerWay.setTag("highway", "primary"); diff --git a/core/src/test/java/com/graphhopper/routing/weighting/custom/CustomWeightingTest.java b/core/src/test/java/com/graphhopper/routing/weighting/custom/CustomWeightingTest.java index 95f9978b9eb..40ab0b5332c 100644 --- a/core/src/test/java/com/graphhopper/routing/weighting/custom/CustomWeightingTest.java +++ b/core/src/test/java/com/graphhopper/routing/weighting/custom/CustomWeightingTest.java @@ -170,7 +170,7 @@ public void testIssueSameKey() { set(avSpeedEnc, 80).set(accessEnc, true, true); CustomModel vehicleModel = new CustomModel(); - vehicleModel.addToSpeed(If("toll != NO", MULTIPLY, 0.8)); + vehicleModel.addToSpeed(If("toll == HGV || toll == ALL", MULTIPLY, 0.8)); vehicleModel.addToSpeed(If("hazmat != NO", MULTIPLY, 0.8)); assertEquals(1.26, createWeighting(vehicleModel).calcEdgeWeight(withToll, false), 0.01); assertEquals(1.26, createWeighting(vehicleModel).calcEdgeWeight(noToll, false), 0.01); diff --git a/docs/core/custom-models.md b/docs/core/custom-models.md index a18940d307f..94a56c4d3ea 100644 --- a/docs/core/custom-models.md +++ b/docs/core/custom-models.md @@ -70,7 +70,7 @@ encoded values are the following (some of their possible values are given in bra - road_access: (DESTINATION, DELIVERY, PRIVATE, NO, ...) - surface: (PAVED, DIRT, SAND, GRAVEL, ...) - smoothness: (EXCELLENT, GOOD, INTERMEDIATE, ...) -- toll: (NO, ALL, HGV) +- toll: (MISSING, NO, HGV, ALL) To learn about all available encoded values you can query the `/info` endpoint. From ada840b40ea2db491ec2b480810ea7c023986ebf Mon Sep 17 00:00:00 2001 From: Andi Date: Fri, 29 Oct 2021 18:13:01 +0200 Subject: [PATCH 51/53] Store DataAccess versions in headers instead of properties file (#2436) --- .../com/graphhopper/search/StringIndex.java | 16 ++++-- .../com/graphhopper/storage/BaseGraph.java | 12 +++- .../storage/BaseGraphNodesAndEdges.java | 52 +++++++++-------- .../com/graphhopper/storage/CHStorage.java | 32 +++++++---- .../storage/GraphHopperStorage.java | 18 +++++- .../storage/StorableProperties.java | 56 ++----------------- .../storage/index/LineIntIndex.java | 10 ++-- .../java/com/graphhopper/util/Constants.java | 13 +++-- .../java/com/graphhopper/util/GHUtility.java | 9 +++ .../storage/StorablePropertiesTest.java | 17 ------ .../com/graphhopper/util/GHUtilityTest.java | 7 +-- 11 files changed, 115 insertions(+), 127 deletions(-) diff --git a/core/src/main/java/com/graphhopper/search/StringIndex.java b/core/src/main/java/com/graphhopper/search/StringIndex.java index 98707e04adc..bda603240eb 100644 --- a/core/src/main/java/com/graphhopper/search/StringIndex.java +++ b/core/src/main/java/com/graphhopper/search/StringIndex.java @@ -20,6 +20,8 @@ import com.graphhopper.storage.DataAccess; import com.graphhopper.storage.Directory; import com.graphhopper.util.BitUtil; +import com.graphhopper.util.Constants; +import com.graphhopper.util.GHUtility; import com.graphhopper.util.Helper; import java.util.*; @@ -77,7 +79,11 @@ public boolean loadExisting() { if (vals.loadExisting()) { if (!keys.loadExisting()) throw new IllegalStateException("Loaded values but cannot load keys"); - bytePointer = BitUtil.LITTLE.combineIntsToLong(vals.getHeader(0), vals.getHeader(4)); + int stringIndexKeysVersion = keys.getHeader(0); + int stringIndexValsVersion = vals.getHeader(0); + GHUtility.checkDAVersion(keys.getName(), Constants.VERSION_STRING_IDX, stringIndexKeysVersion); + GHUtility.checkDAVersion(vals.getName(), Constants.VERSION_STRING_IDX, stringIndexValsVersion); + bytePointer = BitUtil.LITTLE.combineIntsToLong(vals.getHeader(4), vals.getHeader(8)); // load keys into memory int count = keys.getShort(0); @@ -277,7 +283,7 @@ public String get(final long entryPointer, String key) { tmpPointer += 1 + valueLength; } - // value for specified key does not existing for the specified pointer + // value for specified key does not exist for the specified pointer return null; } @@ -294,6 +300,7 @@ private byte[] getBytesForString(String info, String name) { } public void flush() { + keys.setHeader(0, Constants.VERSION_STRING_IDX); keys.ensureCapacity(2); keys.setShort(0, (short) keysInMem.size()); long keyBytePointer = 2; @@ -308,8 +315,9 @@ public void flush() { } keys.flush(); - vals.setHeader(0, BitUtil.LITTLE.getIntLow(bytePointer)); - vals.setHeader(4, BitUtil.LITTLE.getIntHigh(bytePointer)); + vals.setHeader(0, Constants.VERSION_STRING_IDX); + vals.setHeader(4, BitUtil.LITTLE.getIntLow(bytePointer)); + vals.setHeader(8, BitUtil.LITTLE.getIntHigh(bytePointer)); vals.flush(); } diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 01214dc844e..8c051cee110 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -92,12 +92,18 @@ void checkNotInitialized() { } private void loadWayGeometryHeader() { - maxGeoRef = bitUtil.combineIntsToLong(wayGeometry.getHeader(0), wayGeometry.getHeader(4)); + int geometryVersion = wayGeometry.getHeader(0); + GHUtility.checkDAVersion(wayGeometry.getName(), Constants.VERSION_GEOMETRY, geometryVersion); + maxGeoRef = bitUtil.combineIntsToLong( + wayGeometry.getHeader(4), + wayGeometry.getHeader(8) + ); } private void setWayGeometryHeader() { - wayGeometry.setHeader(0, bitUtil.getIntLow(maxGeoRef)); - wayGeometry.setHeader(4, bitUtil.getIntHigh(maxGeoRef)); + wayGeometry.setHeader(0, Constants.VERSION_GEOMETRY); + wayGeometry.setHeader(4, bitUtil.getIntLow(maxGeoRef)); + wayGeometry.setHeader(8, bitUtil.getIntHigh(maxGeoRef)); } private void setInitialized() { diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java index 2593452eaa1..21b2f41b854 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java @@ -18,7 +18,9 @@ package com.graphhopper.storage; +import com.graphhopper.util.Constants; import com.graphhopper.util.EdgeIterator; +import com.graphhopper.util.GHUtility; import com.graphhopper.util.Helper; import com.graphhopper.util.shapes.BBox; @@ -98,38 +100,44 @@ public boolean loadExisting() { return false; // now load some properties from stored data - nodeEntryBytes = nodes.getHeader(0 * 4); - nodeCount = nodes.getHeader(1 * 4); - bounds.minLon = Helper.intToDegree(nodes.getHeader(2 * 4)); - bounds.maxLon = Helper.intToDegree(nodes.getHeader(3 * 4)); - bounds.minLat = Helper.intToDegree(nodes.getHeader(4 * 4)); - bounds.maxLat = Helper.intToDegree(nodes.getHeader(5 * 4)); + final int nodesVersion = nodes.getHeader(0 * 4); + GHUtility.checkDAVersion("nodes", Constants.VERSION_NODE, nodesVersion); + nodeEntryBytes = nodes.getHeader(1 * 4); + nodeCount = nodes.getHeader(2 * 4); + bounds.minLon = Helper.intToDegree(nodes.getHeader(3 * 4)); + bounds.maxLon = Helper.intToDegree(nodes.getHeader(4 * 4)); + bounds.minLat = Helper.intToDegree(nodes.getHeader(5 * 4)); + bounds.maxLat = Helper.intToDegree(nodes.getHeader(6 * 4)); if (withElevation) { - bounds.minEle = Helper.intToEle(nodes.getHeader(6 * 4)); - bounds.maxEle = Helper.intToEle(nodes.getHeader(7 * 4)); + bounds.minEle = Helper.intToEle(nodes.getHeader(7 * 4)); + bounds.maxEle = Helper.intToEle(nodes.getHeader(8 * 4)); } - frozen = nodes.getHeader(8 * 4) == 1; + frozen = nodes.getHeader(9 * 4) == 1; - edgeEntryBytes = edges.getHeader(0 * 4); - edgeCount = edges.getHeader(1 * 4); + final int edgesVersion = edges.getHeader(0 * 4); + GHUtility.checkDAVersion("edges", Constants.VERSION_EDGE, edgesVersion); + edgeEntryBytes = edges.getHeader(1 * 4); + edgeCount = edges.getHeader(2 * 4); return true; } public void flush() { - nodes.setHeader(0 * 4, nodeEntryBytes); - nodes.setHeader(1 * 4, nodeCount); - nodes.setHeader(2 * 4, Helper.degreeToInt(bounds.minLon)); - nodes.setHeader(3 * 4, Helper.degreeToInt(bounds.maxLon)); - nodes.setHeader(4 * 4, Helper.degreeToInt(bounds.minLat)); - nodes.setHeader(5 * 4, Helper.degreeToInt(bounds.maxLat)); + nodes.setHeader(0 * 4, Constants.VERSION_NODE); + nodes.setHeader(1 * 4, nodeEntryBytes); + nodes.setHeader(2 * 4, nodeCount); + nodes.setHeader(3 * 4, Helper.degreeToInt(bounds.minLon)); + nodes.setHeader(4 * 4, Helper.degreeToInt(bounds.maxLon)); + nodes.setHeader(5 * 4, Helper.degreeToInt(bounds.minLat)); + nodes.setHeader(6 * 4, Helper.degreeToInt(bounds.maxLat)); if (withElevation) { - nodes.setHeader(6 * 4, Helper.eleToInt(bounds.minEle)); - nodes.setHeader(7 * 4, Helper.eleToInt(bounds.maxEle)); + nodes.setHeader(7 * 4, Helper.eleToInt(bounds.minEle)); + nodes.setHeader(8 * 4, Helper.eleToInt(bounds.maxEle)); } - nodes.setHeader(8 * 4, frozen ? 1 : 0); + nodes.setHeader(9 * 4, frozen ? 1 : 0); - edges.setHeader(0, edgeEntryBytes); - edges.setHeader(1 * 4, edgeCount); + edges.setHeader(0 * 4, Constants.VERSION_EDGE); + edges.setHeader(1 * 4, edgeEntryBytes); + edges.setHeader(2 * 4, edgeCount); edges.flush(); nodes.flush(); diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java index c948a70ad06..92669c7d31c 100644 --- a/core/src/main/java/com/graphhopper/storage/CHStorage.java +++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java @@ -20,6 +20,8 @@ import com.graphhopper.routing.ch.NodeOrderingProvider; import com.graphhopper.routing.ch.PrepareEncoder; +import com.graphhopper.util.Constants; +import com.graphhopper.util.GHUtility; import com.graphhopper.util.Helper; import java.util.Locale; @@ -123,15 +125,17 @@ public void init(int nodes, int expectedShortcuts) { public void flush() { // nodes - nodesCH.setHeader(0, nodeCount); - nodesCH.setHeader(4, nodeCHEntryBytes); + nodesCH.setHeader(0, Constants.VERSION_NODE_CH); + nodesCH.setHeader(4, nodeCount); + nodesCH.setHeader(8, nodeCHEntryBytes); nodesCH.flush(); // shortcuts - shortcuts.setHeader(0, shortcutCount); - shortcuts.setHeader(4, shortcutEntryBytes); - shortcuts.setHeader(8, numShortcutsExceedingWeight); - shortcuts.setHeader(12, edgeBased ? 1 : 0); + shortcuts.setHeader(0, Constants.VERSION_SHORTCUT); + shortcuts.setHeader(4, shortcutCount); + shortcuts.setHeader(8, shortcutEntryBytes); + shortcuts.setHeader(12, numShortcutsExceedingWeight); + shortcuts.setHeader(16, edgeBased ? 1 : 0); shortcuts.flush(); } @@ -140,14 +144,18 @@ public boolean loadExisting() { return false; // nodes - nodeCount = nodesCH.getHeader(0); - nodeCHEntryBytes = nodesCH.getHeader(4); + int nodesCHVersion = nodesCH.getHeader(0); + GHUtility.checkDAVersion(nodesCH.getName(), Constants.VERSION_NODE_CH, nodesCHVersion); + nodeCount = nodesCH.getHeader(4); + nodeCHEntryBytes = nodesCH.getHeader(8); // shortcuts - shortcutCount = shortcuts.getHeader(0); - shortcutEntryBytes = shortcuts.getHeader(4); - numShortcutsExceedingWeight = shortcuts.getHeader(8); - edgeBased = shortcuts.getHeader(12) == 1; + int shortcutsVersion = shortcuts.getHeader(0); + GHUtility.checkDAVersion(shortcuts.getName(), Constants.VERSION_SHORTCUT, shortcutsVersion); + shortcutCount = shortcuts.getHeader(4); + shortcutEntryBytes = shortcuts.getHeader(8); + numShortcutsExceedingWeight = shortcuts.getHeader(12); + edgeBased = shortcuts.getHeader(16) == 1; return true; } diff --git a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java index e944cf512c6..3d93aa7fb5f 100644 --- a/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java +++ b/core/src/main/java/com/graphhopper/storage/GraphHopperStorage.java @@ -21,6 +21,7 @@ import com.graphhopper.routing.util.EdgeFilter; import com.graphhopper.routing.util.EncodingManager; import com.graphhopper.routing.weighting.Weighting; +import com.graphhopper.util.Constants; import com.graphhopper.util.EdgeExplorer; import com.graphhopper.util.EdgeIteratorState; import com.graphhopper.util.Helper; @@ -218,7 +219,6 @@ public GraphHopperStorage create(long byteCount) { properties.put("graph.flag_encoders", encodingManager.toFlagEncodersAsString()); properties.put("graph.dimension", baseGraph.nodeAccess.getDimension()); - properties.putCurrentVersions(); baseGraph.create(initSize); @@ -244,7 +244,9 @@ public StorableProperties getProperties() { public boolean loadExisting() { baseGraph.checkNotInitialized(); if (properties.loadExisting()) { - properties.checkVersions(false); + if (properties.containsVersion()) + throw new IllegalStateException("The GraphHopper file format is not compatible with the data you are " + + "trying to load. You either need to use an older version of GraphHopper or run a new import"); // check encoding for compatibility String flagEncodersStr = properties.get("graph.flag_encoders"); @@ -354,7 +356,17 @@ public String toString() { + "|" + getDirectory().getDefaultType() + "|" + baseGraph.nodeAccess.getDimension() + "D" + "|" + (baseGraph.supportsTurnCosts() ? baseGraph.turnCostStorage : "no_turn_cost") - + "|" + getProperties().versionsToString(); + + "|" + getVersionsString(); + } + + private String getVersionsString() { + return "nodes:" + Constants.VERSION_NODE + + ",edges:" + Constants.VERSION_EDGE + + ",geometry:" + Constants.VERSION_GEOMETRY + + ",location_index:" + Constants.VERSION_LOCATION_IDX + + ",string_index:" + Constants.VERSION_STRING_IDX + + ",nodesCH:" + Constants.VERSION_NODE_CH + + ",shortcuts:" + Constants.VERSION_SHORTCUT; } // now delegate all Graph methods to BaseGraph to avoid ugly programming flow ala diff --git a/core/src/main/java/com/graphhopper/storage/StorableProperties.java b/core/src/main/java/com/graphhopper/storage/StorableProperties.java index 056a21873c2..2a7ec7056e7 100644 --- a/core/src/main/java/com/graphhopper/storage/StorableProperties.java +++ b/core/src/main/java/com/graphhopper/storage/StorableProperties.java @@ -17,7 +17,6 @@ */ package com.graphhopper.storage; -import com.graphhopper.util.Constants; import com.graphhopper.util.Helper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,55 +127,12 @@ public synchronized long getCapacity() { return da.getCapacity(); } - public synchronized void putCurrentVersions() { - put("nodes.version", Constants.VERSION_NODE); - put("edges.version", Constants.VERSION_EDGE); - put("geometry.version", Constants.VERSION_GEOMETRY); - put("location_index.version", Constants.VERSION_LOCATION_IDX); - put("string_index.version", Constants.VERSION_STRING_IDX); - put("shortcuts.version", Constants.VERSION_SHORTCUT); - } - - public synchronized String versionsToString() { - return get("nodes.version") + "," - + get("edges.version") + "," - + get("geometry.version") + "," - + get("location_index.version") + "," - + get("string_index.version"); - } - - public synchronized boolean checkVersions(boolean silent) { - if (!check("nodes", Constants.VERSION_NODE, silent)) - return false; - - if (!check("edges", Constants.VERSION_EDGE, silent)) - return false; - - if (!check("geometry", Constants.VERSION_GEOMETRY, silent)) - return false; - - if (!check("location_index", Constants.VERSION_LOCATION_IDX, silent)) - return false; - - if (!check("string_index", Constants.VERSION_STRING_IDX, silent)) - return false; - - if (!check("shortcuts", Constants.VERSION_SHORTCUT, silent)) - return false; - return true; - } - - boolean check(String key, int vers, boolean silent) { - String str = get(key + ".version"); - if (!str.equals(vers + "")) { - if (silent) - return false; - - throw new IllegalStateException("Version of " + key + " unsupported: " + str + ", expected:" + vers + ". " - + "Make sure you are using the same GraphHopper version for reading the files that was used for creating them. " - + "See https://discuss.graphhopper.com/t/722"); - } - return true; + public synchronized boolean containsVersion() { + return map.containsKey("nodes.version") || + map.containsKey("edges.version") || + map.containsKey("geometry.version") || + map.containsKey("location_index.version") || + map.containsKey("string_index.version"); } @Override diff --git a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java index 548631df800..ce687f468e8 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java +++ b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java @@ -24,14 +24,14 @@ import com.graphhopper.storage.DAType; import com.graphhopper.storage.DataAccess; import com.graphhopper.storage.Directory; +import com.graphhopper.util.Constants; +import com.graphhopper.util.GHUtility; import com.graphhopper.util.Helper; import com.graphhopper.util.shapes.BBox; import java.util.function.IntConsumer; public class LineIntIndex { - - private final int MAGIC_INT = Integer.MAX_VALUE / 22318; // do not start with 0 as a positive value means leaf and a negative means "entry with subentries" static final int START_POINTER = 1; @@ -59,9 +59,7 @@ public boolean loadExisting() { if (!dataAccess.loadExisting()) return false; - if (dataAccess.getHeader(0) != MAGIC_INT) - throw new IllegalStateException("incorrect location index version, expected:" + MAGIC_INT); - + GHUtility.checkDAVersion("location_index", Constants.VERSION_LOCATION_IDX, dataAccess.getHeader(0)); checksum = dataAccess.getHeader(1 * 4); minResolutionInMeter = dataAccess.getHeader(2 * 4); indexStructureInfo = IndexStructureInfo.create(bounds, minResolutionInMeter); @@ -278,7 +276,7 @@ public void setMinResolutionInMeter(int minResolutionInMeter) { } public void flush() { - dataAccess.setHeader(0, MAGIC_INT); + dataAccess.setHeader(0, Constants.VERSION_LOCATION_IDX); dataAccess.setHeader(1 * 4, checksum); dataAccess.setHeader(2 * 4, minResolutionInMeter); diff --git a/core/src/main/java/com/graphhopper/util/Constants.java b/core/src/main/java/com/graphhopper/util/Constants.java index 74ca9bd43e7..2f32414bb91 100644 --- a/core/src/main/java/com/graphhopper/util/Constants.java +++ b/core/src/main/java/com/graphhopper/util/Constants.java @@ -66,12 +66,13 @@ public class Constants { private static final int JVM_MAJOR_VERSION; private static final int JVM_MINOR_VERSION; - public static final int VERSION_NODE = 7; - public static final int VERSION_EDGE = 20; - public static final int VERSION_SHORTCUT = 7; - public static final int VERSION_GEOMETRY = 5; - public static final int VERSION_LOCATION_IDX = 4; - public static final int VERSION_STRING_IDX = 5; + public static final int VERSION_NODE = 8; + public static final int VERSION_EDGE = 21; + public static final int VERSION_SHORTCUT = 8; + public static final int VERSION_NODE_CH = 0; + public static final int VERSION_GEOMETRY = 6; + public static final int VERSION_LOCATION_IDX = 5; + public static final int VERSION_STRING_IDX = 6; /** * The version without the snapshot string */ diff --git a/core/src/main/java/com/graphhopper/util/GHUtility.java b/core/src/main/java/com/graphhopper/util/GHUtility.java index f16ac8a9ec3..3f8c03a7016 100644 --- a/core/src/main/java/com/graphhopper/util/GHUtility.java +++ b/core/src/main/java/com/graphhopper/util/GHUtility.java @@ -427,6 +427,15 @@ public static int getAdjNode(Graph g, int edge, int adjNode) { return adjNode; } + public static void checkDAVersion(String name, int expectedVersion, int version) { + if (version != expectedVersion) { + throw new IllegalStateException("Unexpected version for '" + name + "'. Got: " + version + ", " + + "expected: " + expectedVersion + ". " + + "Make sure you are using the same GraphHopper version for reading the files that was used for creating them. " + + "See https://discuss.graphhopper.com/t/722"); + } + } + public static EdgeIteratorState createMockedEdgeIteratorState(final double distance, final IntsRef flags) { return createMockedEdgeIteratorState(distance, flags, 0, 1, 2, 3, 4); } diff --git a/core/src/test/java/com/graphhopper/storage/StorablePropertiesTest.java b/core/src/test/java/com/graphhopper/storage/StorablePropertiesTest.java index 7a98e900659..8df51886b1f 100644 --- a/core/src/test/java/com/graphhopper/storage/StorablePropertiesTest.java +++ b/core/src/test/java/com/graphhopper/storage/StorablePropertiesTest.java @@ -47,23 +47,6 @@ public void testLoad() { instance.close(); } - @Test - public void testVersionCheck() { - StorableProperties instance = new StorableProperties(createDir("", false)); - instance.putCurrentVersions(); - assertTrue(instance.checkVersions(true)); - - instance.put("nodes.version", 0); - assertFalse(instance.checkVersions(true)); - - try { - instance.checkVersions(false); - assertTrue(false); - } catch (Exception ex) { - } - instance.close(); - } - @Test public void testStore() { String dir = "./target/test"; diff --git a/core/src/test/java/com/graphhopper/util/GHUtilityTest.java b/core/src/test/java/com/graphhopper/util/GHUtilityTest.java index 32ba5e1a2bc..4cdd04b2780 100644 --- a/core/src/test/java/com/graphhopper/util/GHUtilityTest.java +++ b/core/src/test/java/com/graphhopper/util/GHUtilityTest.java @@ -18,13 +18,13 @@ package com.graphhopper.util; import com.graphhopper.coll.GHIntLongHashMap; -import com.graphhopper.routing.ev.BooleanEncodedValue; import com.graphhopper.routing.util.AllEdgesIterator; import com.graphhopper.routing.util.CarFlagEncoder; import com.graphhopper.routing.util.EncodingManager; import com.graphhopper.routing.util.FlagEncoder; -import com.graphhopper.routing.weighting.FastestWeighting; -import com.graphhopper.storage.*; +import com.graphhopper.storage.Graph; +import com.graphhopper.storage.GraphBuilder; +import com.graphhopper.storage.NodeAccess; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -35,7 +35,6 @@ public class GHUtilityTest { private final FlagEncoder carEncoder = new CarFlagEncoder(); private final EncodingManager encodingManager = EncodingManager.create(carEncoder); - private final BooleanEncodedValue accessEnc = carEncoder.getAccessEnc(); Graph createGraph() { return new GraphBuilder(encodingManager).create(); From f9d2903503df5eb4f709fc251b834cf6bb05b519 Mon Sep 17 00:00:00 2001 From: aoles Date: Thu, 9 Apr 2026 11:51:55 +0200 Subject: [PATCH 52/53] fix: move storing of node tags --- .../com/graphhopper/reader/osm/OSMReader.java | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 7d7ffac4e10..07c85ff4044 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -640,6 +640,9 @@ protected void processNode(ReaderNode node) { addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); } else if (nodeType == INTERMEDIATE_NODE || nodeType == END_NODE) { addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); + // ORS-GH MOD START - Store tags from the node so that they can be accessed later + storeNodeTags(node); + // ORS-GH MOD END } // analyze node tags for barriers @@ -669,22 +672,6 @@ protected ReaderNode onProcessNode(ReaderNode node) { } // ORS-GH MOD END - boolean addNode(ReaderNode node) { - int nodeType = getNodeMap().get(node.getId()); - if (nodeType == EMPTY_NODE) - return false; - - if (nodeType == TOWER_NODE) { - addTowerNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); - } else if (nodeType == PILLAR_NODE) { - addPillarNode(node.getId(), node.getLat(), node.getLon(), eleProvider.getEle(node)); - // ORS-GH MOD START - Store tags from the node so that they can be accessed later - storeNodeTags(node); - // ORS-GH MOD END - } - return true; - } - // ORS-GH MOD START - Store tags from the node so that they can be accessed later private void storeNodeTags(ReaderNode node) { Iterator> it = node.getTags().entrySet().iterator(); From bf573f3a2876aafca16f5f823d5129ea0899af1d Mon Sep 17 00:00:00 2001 From: aoles Date: Thu, 9 Apr 2026 15:06:53 +0200 Subject: [PATCH 53/53] fix: expose methods to ORS --- .../java/com/graphhopper/reader/osm/OSMReader.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java index 07c85ff4044..84b5ab60335 100644 --- a/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java +++ b/core/src/main/java/com/graphhopper/reader/osm/OSMReader.java @@ -1178,16 +1178,26 @@ public String toString() { return getClass().getSimpleName(); } - private boolean isTowerNode(int id) { + // ORS-GH MOD START - expose method to ORS + protected boolean isTowerNode(int id) { + // ORS-GH MOD END // tower nodes are indexed -3, -4, -5, ... return id < JUNCTION_NODE; } - private boolean isPillarNode(int id) { + // ORS-GH MOD START - expose method to ORS + protected boolean isPillarNode(int id) { + // ORS-GH MOD END // pillar nodes are indexed 3, 4, 5, .. return id > CONNECTION_NODE; } + // ORS-GH MOD START - add new method for ORS + protected boolean isEmptyNode(int id) { + return id == EMPTY_NODE; + } + // ORS-GH MOD END + private boolean isNodeId(int id) { return id > CONNECTION_NODE || id < JUNCTION_NODE; }