From 5ba6d4a24f374d050cd3ed7c8849ed3c917152b3 Mon Sep 17 00:00:00 2001 From: Loki Date: Sun, 29 Mar 2026 22:41:53 +0800 Subject: [PATCH 01/15] optimize: Optimize RocksDB batch query performance --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTable.java | 3 ++- .../apache/hugegraph/backend/store/rocksdb/RocksDBTables.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 824986d22c..98ae3a14e7 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -180,7 +181,7 @@ protected BackendColumnIterator queryBy(RocksDBSessions.Session session, Query q // Query by id if (query.conditionsSize() == 0) { assert query.idsSize() > 0; - return this.queryByIds(session, query.ids()); + return this.getByIds(session, new HashSet<>(query.ids())); } // Query by condition (or condition + id) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index 37cc2f151c..e8dd2df8b0 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; +import java.util.HashSet; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -182,8 +183,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id @Override protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { - // TODO: use getByIds() after batch version multi-get is ready - return super.queryByIds(session, ids); + return this.getByIds(session, new HashSet<>(ids)); } } From f5405a0c9a240765f41a9be1a6eee064f19e759a Mon Sep 17 00:00:00 2001 From: lokidundun Date: Tue, 31 Mar 2026 20:28:53 +0800 Subject: [PATCH 02/15] Refactor getByIds to queryByIds in RocksDBTable --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 98ae3a14e7..774918d6be 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -181,7 +181,7 @@ protected BackendColumnIterator queryBy(RocksDBSessions.Session session, Query q // Query by id if (query.conditionsSize() == 0) { assert query.idsSize() > 0; - return this.getByIds(session, new HashSet<>(query.ids())); + return this.queryByIds(session, query.ids()); } // Query by condition (or condition + id) From 939ace0f352fe0af50c2b0e49ae7c129687c99b5 Mon Sep 17 00:00:00 2001 From: lokidundun Date: Tue, 31 Mar 2026 20:29:48 +0800 Subject: [PATCH 03/15] Modify queryByIds to use super method temporarily Temporarily use super.queryByIds() instead of getByIds() for batch version support. --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTables.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index e8dd2df8b0..7dcf299f97 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -183,7 +183,9 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id @Override protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { - return this.getByIds(session, new HashSet<>(ids)); + // TODO: use getByIds() after batch version multi-get is ready + return super.queryByIds(session, ids); + } } From 502c7dfc62a2e7802bc6a0eae17cf1ea13927e01 Mon Sep 17 00:00:00 2001 From: lokidundun Date: Thu, 2 Apr 2026 17:09:40 +0800 Subject: [PATCH 04/15] Refactor queryByIds to getByIds with HashSet --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 774918d6be..98ae3a14e7 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -181,7 +181,7 @@ protected BackendColumnIterator queryBy(RocksDBSessions.Session session, Query q // Query by id if (query.conditionsSize() == 0) { assert query.idsSize() > 0; - return this.queryByIds(session, query.ids()); + return this.getByIds(session, new HashSet<>(query.ids())); } // Query by condition (or condition + id) From 352f66ba38967cdb7263187d8b92c3b6d41c1196 Mon Sep 17 00:00:00 2001 From: lokidundun Date: Thu, 2 Apr 2026 17:10:34 +0800 Subject: [PATCH 05/15] Update RocksDBTables.java --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTables.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index 7dcf299f97..5f86ad060e 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -183,8 +183,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id @Override protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { - // TODO: use getByIds() after batch version multi-get is ready - return super.queryByIds(session, ids); + return this.getByIds(session, new HashSet<>(ids)); } } From 0d9052d1473492c5b03c514a1817bd2de05ea3af Mon Sep 17 00:00:00 2001 From: Loki Date: Mon, 6 Apr 2026 16:06:27 +0800 Subject: [PATCH 06/15] fix: fix ci --- .../backend/tx/GraphTransaction.java | 28 ++++++++++++++----- .../backend/store/rocksdb/RocksDBTable.java | 5 ++-- .../backend/store/rocksdb/RocksDBTables.java | 5 ++-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index 763ccaa0ee..1fa8d01a51 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -784,7 +784,8 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace List ids = InsertionOrderUtil.newList(); Map vertices = new HashMap<>(vertexIds.length); - IdQuery query = new IdQuery(type); + List backendIds = InsertionOrderUtil.newList(); + for (Object vertexId : vertexIds) { HugeVertex vertex; Id id = HugeVertex.getIdValue(vertexId); @@ -799,17 +800,30 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace // Found from local tx vertices.put(vertex.id(), vertex); } else { - // Prepare to query from backend store - query.query(id); + // store the IDs queried from backend + backendIds.add(id); } ids.add(id); } - if (!query.empty()) { + if (!backendIds.isEmpty()) { // Query from backend store - query.mustSortByInput(false); - Iterator it = this.queryVerticesFromBackend(query); - QueryResults.fillMap(it, vertices); + final int batch = this.batchSize > 0 ? this.batchSize : backendIds.size(); + for (int i = 0; i < backendIds.size(); i += batch) { + int end = Math.min(i + batch, backendIds.size()); + IdQuery query = new IdQuery(type); + for (int j = i; j < end; j++) { + Id id = backendIds.get(j); + query.query(id); + } + // Single batch capacity check + Query.checkForceCapacity(query.idsSize()); + + // Query from backend store + query.mustSortByInput(false); + Iterator it = this.queryVerticesFromBackend(query); + QueryResults.fillMap(it, vertices); + } } return new MapperIterator<>(ids.iterator(), id -> { diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 98ae3a14e7..e7b1b2f42a 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -181,7 +180,7 @@ protected BackendColumnIterator queryBy(RocksDBSessions.Session session, Query q // Query by id if (query.conditionsSize() == 0) { assert query.idsSize() > 0; - return this.getByIds(session, new HashSet<>(query.ids())); + return this.queryByIds(session, query.ids()); } // Query by condition (or condition + id) @@ -310,7 +309,7 @@ protected static BackendEntryIterator newEntryIterator(BackendColumnIterator col } protected static BackendEntryIterator newEntryIteratorOlap( - BackendColumnIterator cols, Query query, boolean isOlap) { + BackendColumnIterator cols, Query query, boolean isOlap) { return new BinaryEntryIterator<>(cols, query, (entry, col) -> { if (entry == null || !entry.belongToMe(col)) { HugeType type = query.resultType(); diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index 5f86ad060e..37cc2f151c 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -20,7 +20,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; -import java.util.HashSet; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -183,8 +182,8 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id @Override protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { - return this.getByIds(session, new HashSet<>(ids)); - + // TODO: use getByIds() after batch version multi-get is ready + return super.queryByIds(session, ids); } } From 45298f91b7e4ec0c0c5a1c4f40e201009e7b7206 Mon Sep 17 00:00:00 2001 From: Loki Date: Tue, 7 Apr 2026 15:32:25 +0800 Subject: [PATCH 07/15] optimize: optimize the batch query --- .../backend/tx/GraphTransaction.java | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index 1fa8d01a51..a9b67dc99d 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -783,8 +783,9 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace // NOTE: allowed duplicated vertices if query by duplicated ids List ids = InsertionOrderUtil.newList(); Map vertices = new HashMap<>(vertexIds.length); - - List backendIds = InsertionOrderUtil.newList(); + Set fetchedIds = InsertionOrderUtil.newSet(); + IdQuery batchQuery = null; + final int batchSize = this.batchSize; for (Object vertexId : vertexIds) { HugeVertex vertex; @@ -800,30 +801,25 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace // Found from local tx vertices.put(vertex.id(), vertex); } else { - // store the IDs queried from backend - backendIds.add(id); + // Query from backend + if (!fetchedIds.contains(id)) { + if (batchQuery == null) { + batchQuery = new IdQuery(type); + } + batchQuery.query(id); + fetchedIds.add(id); + + if (batchQuery.idsSize() >= batchSize) { + flushIdBatch(batchQuery, vertices); + batchQuery = null; + } + } } ids.add(id); } - if (!backendIds.isEmpty()) { - // Query from backend store - final int batch = this.batchSize > 0 ? this.batchSize : backendIds.size(); - for (int i = 0; i < backendIds.size(); i += batch) { - int end = Math.min(i + batch, backendIds.size()); - IdQuery query = new IdQuery(type); - for (int j = i; j < end; j++) { - Id id = backendIds.get(j); - query.query(id); - } - // Single batch capacity check - Query.checkForceCapacity(query.idsSize()); - - // Query from backend store - query.mustSortByInput(false); - Iterator it = this.queryVerticesFromBackend(query); - QueryResults.fillMap(it, vertices); - } + if (batchQuery != null && !batchQuery.empty()) { + flushIdBatch(batchQuery, vertices); } return new MapperIterator<>(ids.iterator(), id -> { @@ -845,6 +841,13 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace }); } + private void flushIdBatch(IdQuery query, Map vertices) { + Query.checkForceCapacity(query.idsSize()); + query.mustSortByInput(false); + Iterator it = this.queryVerticesFromBackend(query); + QueryResults.fillMap(it, vertices); + } + public Iterator queryVertices() { Query q = new Query(HugeType.VERTEX); return this.queryVertices(q); From 223fb280bb40cde27ab086405ed8b2750c187498 Mon Sep 17 00:00:00 2001 From: Loki Date: Thu, 16 Apr 2026 10:30:42 +0800 Subject: [PATCH 08/15] optimize: optimize rockDb query --- .../backend/tx/GraphTransaction.java | 33 +++++-------------- .../backend/store/rocksdb/RocksDBTable.java | 4 +++ .../backend/store/rocksdb/RocksDBTables.java | 14 +++++++- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index a9b67dc99d..763ccaa0ee 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -783,10 +783,8 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace // NOTE: allowed duplicated vertices if query by duplicated ids List ids = InsertionOrderUtil.newList(); Map vertices = new HashMap<>(vertexIds.length); - Set fetchedIds = InsertionOrderUtil.newSet(); - IdQuery batchQuery = null; - final int batchSize = this.batchSize; + IdQuery query = new IdQuery(type); for (Object vertexId : vertexIds) { HugeVertex vertex; Id id = HugeVertex.getIdValue(vertexId); @@ -801,25 +799,17 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace // Found from local tx vertices.put(vertex.id(), vertex); } else { - // Query from backend - if (!fetchedIds.contains(id)) { - if (batchQuery == null) { - batchQuery = new IdQuery(type); - } - batchQuery.query(id); - fetchedIds.add(id); - - if (batchQuery.idsSize() >= batchSize) { - flushIdBatch(batchQuery, vertices); - batchQuery = null; - } - } + // Prepare to query from backend store + query.query(id); } ids.add(id); } - if (batchQuery != null && !batchQuery.empty()) { - flushIdBatch(batchQuery, vertices); + if (!query.empty()) { + // Query from backend store + query.mustSortByInput(false); + Iterator it = this.queryVerticesFromBackend(query); + QueryResults.fillMap(it, vertices); } return new MapperIterator<>(ids.iterator(), id -> { @@ -841,13 +831,6 @@ protected Iterator queryVerticesByIds(Object[] vertexIds, boolean adjace }); } - private void flushIdBatch(IdQuery query, Map vertices) { - Query.checkForceCapacity(query.idsSize()); - query.mustSortByInput(false); - Iterator it = this.queryVerticesFromBackend(query); - QueryResults.fillMap(it, vertices); - } - public Iterator queryVertices() { Query q = new Query(HugeType.VERTEX); return this.queryVertices(q); diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index e7b1b2f42a..80fb8a3807 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -180,6 +181,9 @@ protected BackendColumnIterator queryBy(RocksDBSessions.Session session, Query q // Query by id if (query.conditionsSize() == 0) { assert query.idsSize() > 0; + if (!session.hasChanges()) { + return this.getByIds(session, new HashSet<>(query.ids())); + } return this.queryByIds(session, query.ids()); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index 37cc2f151c..4105973327 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; +import java.util.HashSet; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -182,7 +183,9 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id @Override protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { - // TODO: use getByIds() after batch version multi-get is ready + if (!session.hasChanges()) { + return this.getByIds(session, new HashSet<>(ids)); + } return super.queryByIds(session, ids); } } @@ -208,6 +211,15 @@ public static Edge in(String database) { protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id) { return this.getById(session, id); } + + @Override + protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, + Collection ids) { + if (!session.hasChanges()) { + return this.getByIds(session, new HashSet<>(ids)); + } + return super.queryByIds(session, ids); + } } public static class IndexTable extends RocksDBTable { From ce4e2cb07aff13e88b6b0869e2e90856bbd4b9ec Mon Sep 17 00:00:00 2001 From: Loki Date: Thu, 16 Apr 2026 12:36:10 +0800 Subject: [PATCH 09/15] fix: fix ci test --- .../hugegraph/backend/store/rocksdb/RocksDBTable.java | 4 ---- .../hugegraph/backend/store/rocksdb/RocksDBTables.java | 6 +++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 80fb8a3807..e7b1b2f42a 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -181,9 +180,6 @@ protected BackendColumnIterator queryBy(RocksDBSessions.Session session, Query q // Query by id if (query.conditionsSize() == 0) { assert query.idsSize() > 0; - if (!session.hasChanges()) { - return this.getByIds(session, new HashSet<>(query.ids())); - } return this.queryByIds(session, query.ids()); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index 4105973327..d49859a5d3 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -20,7 +20,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -184,7 +184,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, new HashSet<>(ids)); + return this.getByIds(session, new LinkedHashSet<>(ids)); } return super.queryByIds(session, ids); } @@ -216,7 +216,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, new HashSet<>(ids)); + return this.getByIds(session, new LinkedHashSet<>(ids)); } return super.queryByIds(session, ids); } From 6fe08a7cf90f4306c1762a06a5368dc83e8017d0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Apr 2026 03:22:42 +0000 Subject: [PATCH 10/15] fix: preserve input id multiplicity in RocksDB multi-get path Agent-Logs-Url: https://github.com/lokidundun/incubator-hugegraph/sessions/d37721a0-2300-48d4-a06c-f8e1b4b6c3d6 Co-authored-by: lokidundun <206712414+lokidundun@users.noreply.github.com> --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTable.java | 4 ++-- .../hugegraph/backend/store/rocksdb/RocksDBTables.java | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index e7b1b2f42a..926987cd33 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Set; import org.apache.commons.lang3.tuple.Pair; import org.apache.hugegraph.backend.id.Id; @@ -224,7 +223,8 @@ protected BackendColumnIterator getById(RocksDBSessions.Session session, Id id) return BackendColumnIterator.iterator(col); } - protected BackendColumnIterator getByIds(RocksDBSessions.Session session, Set ids) { + protected BackendColumnIterator getByIds(RocksDBSessions.Session session, + Collection ids) { if (ids.size() == 1) { return this.getById(session, ids.iterator().next()); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index d49859a5d3..6c6d7ab083 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -20,7 +20,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; -import java.util.LinkedHashSet; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -184,7 +183,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, new LinkedHashSet<>(ids)); + return this.getByIds(session, ids); } return super.queryByIds(session, ids); } @@ -216,7 +215,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, new LinkedHashSet<>(ids)); + return this.getByIds(session, ids); } return super.queryByIds(session, ids); } From 039106916290f5e190b5204618e990c81983c5c7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Apr 2026 03:31:41 +0000 Subject: [PATCH 11/15] fix: fallback getByIds when rocksdb session has pending changes Agent-Logs-Url: https://github.com/lokidundun/incubator-hugegraph/sessions/ab26085b-737d-4e58-aa7c-6c88194b2032 Co-authored-by: lokidundun <206712414+lokidundun@users.noreply.github.com> --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTable.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 926987cd33..8a464cda98 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -225,6 +225,9 @@ protected BackendColumnIterator getById(RocksDBSessions.Session session, Id id) protected BackendColumnIterator getByIds(RocksDBSessions.Session session, Collection ids) { + if (session.hasChanges()) { + return this.queryByIds(session, ids); + } if (ids.size() == 1) { return this.getById(session, ids.iterator().next()); } From ce802b92d96833adada1a49c629ef9565524e90e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Apr 2026 03:46:49 +0000 Subject: [PATCH 12/15] revert: restore rocksdb query files to ce4e2cb state Agent-Logs-Url: https://github.com/lokidundun/incubator-hugegraph/sessions/e035a01d-313f-4f83-8efb-e85a306744c9 Co-authored-by: lokidundun <206712414+lokidundun@users.noreply.github.com> --- .../hugegraph/backend/store/rocksdb/RocksDBTable.java | 7 ++----- .../hugegraph/backend/store/rocksdb/RocksDBTables.java | 5 +++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 8a464cda98..e7b1b2f42a 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Set; import org.apache.commons.lang3.tuple.Pair; import org.apache.hugegraph.backend.id.Id; @@ -223,11 +224,7 @@ protected BackendColumnIterator getById(RocksDBSessions.Session session, Id id) return BackendColumnIterator.iterator(col); } - protected BackendColumnIterator getByIds(RocksDBSessions.Session session, - Collection ids) { - if (session.hasChanges()) { - return this.queryByIds(session, ids); - } + protected BackendColumnIterator getByIds(RocksDBSessions.Session session, Set ids) { if (ids.size() == 1) { return this.getById(session, ids.iterator().next()); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index 6c6d7ab083..d49859a5d3 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -20,6 +20,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -183,7 +184,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, ids); + return this.getByIds(session, new LinkedHashSet<>(ids)); } return super.queryByIds(session, ids); } @@ -215,7 +216,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, ids); + return this.getByIds(session, new LinkedHashSet<>(ids)); } return super.queryByIds(session, ids); } From c06225ceb3a57ae3ec2248ced223b2d5195a9aaf Mon Sep 17 00:00:00 2001 From: Loki Date: Fri, 17 Apr 2026 12:44:33 +0800 Subject: [PATCH 13/15] fix: fix wrong usage --- .../apache/hugegraph/backend/store/rocksdb/RocksDBTable.java | 3 ++- .../hugegraph/backend/store/rocksdb/RocksDBTables.java | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index e7b1b2f42a..6b486d0319 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -224,7 +224,8 @@ protected BackendColumnIterator getById(RocksDBSessions.Session session, Id id) return BackendColumnIterator.iterator(col); } - protected BackendColumnIterator getByIds(RocksDBSessions.Session session, Set ids) { + protected BackendColumnIterator getByIds(RocksDBSessions.Session session, + Collection ids) { if (ids.size() == 1) { return this.getById(session, ids.iterator().next()); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index d49859a5d3..6c6d7ab083 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -20,7 +20,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Collection; -import java.util.LinkedHashSet; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -184,7 +183,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, new LinkedHashSet<>(ids)); + return this.getByIds(session, ids); } return super.queryByIds(session, ids); } @@ -216,7 +215,7 @@ protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, Collection ids) { if (!session.hasChanges()) { - return this.getByIds(session, new LinkedHashSet<>(ids)); + return this.getByIds(session, ids); } return super.queryByIds(session, ids); } From 10cb0a4ddcceef4892049792d144d7a3e6d8cf43 Mon Sep 17 00:00:00 2001 From: Loki Date: Fri, 17 Apr 2026 21:22:36 +0800 Subject: [PATCH 14/15] test: add test for new multi-get path --- .../apache/hugegraph/unit/UnitTestSuite.java | 2 + .../rocksdb/RocksDBTableQueryByIdsTest.java | 195 ++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index f9f20ab9e5..f86f2e78b3 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -50,6 +50,7 @@ import org.apache.hugegraph.unit.id.SplicingIdGeneratorTest; import org.apache.hugegraph.unit.mysql.MysqlUtilTest; import org.apache.hugegraph.unit.mysql.WhereBuilderTest; +import org.apache.hugegraph.unit.rocksdb.RocksDBTableQueryByIdsTest; import org.apache.hugegraph.unit.rocksdb.RocksDBCountersTest; import org.apache.hugegraph.unit.rocksdb.RocksDBSessionTest; import org.apache.hugegraph.unit.rocksdb.RocksDBSessionsTest; @@ -141,6 +142,7 @@ RocksDBSessionsTest.class, RocksDBSessionTest.class, RocksDBCountersTest.class, + RocksDBTableQueryByIdsTest.class, /* utils */ VersionTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java new file mode 100644 index 0000000000..7564ec7012 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.rocksdb; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hugegraph.backend.id.Id; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; +import org.apache.hugegraph.backend.store.BackendEntry.BackendColumnIterator; +import org.apache.hugegraph.backend.store.rocksdb.RocksDBSessions; +import org.apache.hugegraph.backend.store.rocksdb.RocksDBTables; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Before; +import org.junit.Test; +import org.rocksdb.RocksDBException; + +public class RocksDBTableQueryByIdsTest extends BaseRocksDBUnitTest { + + private static final String DATABASE = "db"; + + private TestVertexTable vertexTable; + private TestEdgeTable edgeTable; + + @Override + @Before + public void setup() throws RocksDBException { + super.setup(); + this.vertexTable = new TestVertexTable(DATABASE); + this.edgeTable = new TestEdgeTable(DATABASE); + this.rocks.createTable(this.vertexTable.table()); + this.rocks.createTable(this.edgeTable.table()); + } + + @Test + public void testVertexQueryByIdsWithAllExistingIds() { + Id id1 = IdGenerator.of("v1"); + Id id2 = IdGenerator.of("v2"); + Id id3 = IdGenerator.of("v3"); + + this.rocks.session().put(this.vertexTable.table(), id1.asBytes(), getBytes("value1")); + this.rocks.session().put(this.vertexTable.table(), id2.asBytes(), getBytes("value2")); + this.rocks.session().put(this.vertexTable.table(), id3.asBytes(), getBytes("value3")); + this.commit(); + + List ids = Arrays.asList(id1, id2, id3); + BackendColumnIterator iter = this.vertexTable.queryByIds(this.rocks.session(), ids); + + Map results = toResultMap(iter); + + Assert.assertEquals(3, results.size()); + Assert.assertEquals("value1", results.get("v1")); + Assert.assertEquals("value2", results.get("v2")); + Assert.assertEquals("value3", results.get("v3")); + } + + @Test + public void testVertexQueryByIdsWithExistingAndMissingIdsMixed() { + Id id1 = IdGenerator.of("v1"); + Id id2 = IdGenerator.of("v2"); + Id id3 = IdGenerator.of("v3"); + + this.rocks.session().put(this.vertexTable.table(), id1.asBytes(), getBytes("value1")); + this.rocks.session().put(this.vertexTable.table(), id3.asBytes(), getBytes("value3")); + this.commit(); + + List ids = Arrays.asList(id1, id2, id3); + BackendColumnIterator iter = this.vertexTable.queryByIds(this.rocks.session(), ids); + + Map results = toResultMap(iter); + + Assert.assertEquals(2, results.size()); + Assert.assertEquals("value1", results.get("v1")); + Assert.assertEquals("value3", results.get("v3")); + Assert.assertFalse(results.containsKey("v2")); + } + + @Test + public void testVertexQueryByIdsWithDuplicateIds() { + Id id1 = IdGenerator.of("v1"); + Id id2 = IdGenerator.of("v2"); + + this.rocks.session().put(this.vertexTable.table(), id1.asBytes(), getBytes("value1")); + this.rocks.session().put(this.vertexTable.table(), id2.asBytes(), getBytes("value2")); + this.commit(); + + List ids = Arrays.asList(id1, id2, id1); + BackendColumnIterator iter = this.vertexTable.queryByIds(this.rocks.session(), ids); + + Map countMap = new HashMap<>(); + Map results = new HashMap<>(); + while (iter.hasNext()) { + BackendColumn col = iter.next(); + String key = getString(col.name); + results.put(key, getString(col.value)); + countMap.put(key, countMap.getOrDefault(key, 0) + 1); + } + + Assert.assertEquals(2, results.size()); + Assert.assertEquals("value1", results.get("v1")); + Assert.assertEquals("value2", results.get("v2")); + // Verify duplicate ids produce duplicate results + Assert.assertEquals(Integer.valueOf(2), countMap.get("v1")); + Assert.assertEquals(Integer.valueOf(1), countMap.get("v2")); + } + + @Test + public void testEdgeQueryByIdsWithAllExistingIds() { + Id id1 = IdGenerator.of("e1"); + Id id2 = IdGenerator.of("e2"); + + this.rocks.session().put(this.edgeTable.table(), id1.asBytes(), getBytes("edge-value1")); + this.rocks.session().put(this.edgeTable.table(), id2.asBytes(), getBytes("edge-value2")); + this.commit(); + + List ids = Arrays.asList(id1, id2); + BackendColumnIterator iter = this.edgeTable.queryByIds(this.rocks.session(), ids); + + Map results = toResultMap(iter); + + Assert.assertEquals(2, results.size()); + Assert.assertEquals("edge-value1", results.get("e1")); + Assert.assertEquals("edge-value2", results.get("e2")); + } + + /** + * NOTE: Testing the fallback path (session.hasChanges() == true) is not + * feasible here because both the optimized multi-get path and the fallback + * scan-based path ultimately delegate to session.get() / session.scan(), + * which have a pre-existing assertion `assert !this.hasChanges()` in + * RocksDBStdSessions. This assertion is disabled in production but fires + * during unit tests when assertions are enabled. The dispatch logic itself + * is covered by the implementation in RocksDBTables.Vertex/Edge.queryByIds(). + */ + + private Map toResultMap(BackendColumnIterator iter) { + Map results = new HashMap<>(); + while (iter.hasNext()) { + BackendColumn col = iter.next(); + results.put(getString(col.name), getString(col.value)); + } + return results; + } + + /** + * Subclass that exposes the protected queryByIds for testing. + */ + private static class TestVertexTable extends RocksDBTables.Vertex { + + public TestVertexTable(String database) { + super(database); + } + + @Override + public BackendColumnIterator queryByIds(RocksDBSessions.Session session, + Collection ids) { + return super.queryByIds(session, ids); + } + } + + /** + * Subclass that exposes the protected queryByIds for testing. + */ + private static class TestEdgeTable extends RocksDBTables.Edge { + + public TestEdgeTable(String database) { + super(true, database); + } + + @Override + public BackendColumnIterator queryByIds(RocksDBSessions.Session session, + Collection ids) { + return super.queryByIds(session, ids); + } + } +} From d33123ee5f7e55f79307ecb067d2f475f2f37b10 Mon Sep 17 00:00:00 2001 From: Loki Date: Mon, 20 Apr 2026 17:00:10 +0800 Subject: [PATCH 15/15] add test for session.hasChanges() --- .../backend/store/rocksdb/RocksDBTable.java | 10 +- .../backend/store/rocksdb/RocksDBTables.java | 19 -- .../apache/hugegraph/unit/UnitTestSuite.java | 2 +- .../rocksdb/RocksDBTableQueryByIdsTest.java | 218 +++++++++++++++--- 4 files changed, 197 insertions(+), 52 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java index 6b486d0319..1c30e434ab 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTable.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -209,6 +210,10 @@ protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, return this.queryById(session, ids.iterator().next()); } + if (!session.hasChanges()) { + return this.getByIds(session, ids); + } + // NOTE: this will lead to lazy create rocksdb iterator return BackendColumnIterator.wrap(new FlatMapperIterator<>( ids.iterator(), id -> this.queryById(session, id) @@ -230,8 +235,9 @@ protected BackendColumnIterator getByIds(RocksDBSessions.Session session, return this.getById(session, ids.iterator().next()); } - List keys = new ArrayList<>(ids.size()); - for (Id id : ids) { + Collection uniqueIds = ids instanceof Set ? ids : new LinkedHashSet<>(ids); + List keys = new ArrayList<>(uniqueIds.size()); + for (Id id : uniqueIds) { keys.add(id.asBytes()); } return session.get(this.table(), keys); diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java index 6c6d7ab083..01e908c4af 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBTables.java @@ -19,7 +19,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.Collection; import java.util.List; import org.apache.hugegraph.backend.id.Id; @@ -178,15 +177,6 @@ public Vertex(String database) { protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id) { return this.getById(session, id); } - - @Override - protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, - Collection ids) { - if (!session.hasChanges()) { - return this.getByIds(session, ids); - } - return super.queryByIds(session, ids); - } } public static class Edge extends RocksDBTable { @@ -210,15 +200,6 @@ public static Edge in(String database) { protected BackendColumnIterator queryById(RocksDBSessions.Session session, Id id) { return this.getById(session, id); } - - @Override - protected BackendColumnIterator queryByIds(RocksDBSessions.Session session, - Collection ids) { - if (!session.hasChanges()) { - return this.getByIds(session, ids); - } - return super.queryByIds(session, ids); - } } public static class IndexTable extends RocksDBTable { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index f86f2e78b3..9ecca5b783 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -50,10 +50,10 @@ import org.apache.hugegraph.unit.id.SplicingIdGeneratorTest; import org.apache.hugegraph.unit.mysql.MysqlUtilTest; import org.apache.hugegraph.unit.mysql.WhereBuilderTest; -import org.apache.hugegraph.unit.rocksdb.RocksDBTableQueryByIdsTest; import org.apache.hugegraph.unit.rocksdb.RocksDBCountersTest; import org.apache.hugegraph.unit.rocksdb.RocksDBSessionTest; import org.apache.hugegraph.unit.rocksdb.RocksDBSessionsTest; +import org.apache.hugegraph.unit.rocksdb.RocksDBTableQueryByIdsTest; import org.apache.hugegraph.unit.serializer.BinaryBackendEntryTest; import org.apache.hugegraph.unit.serializer.BinaryScatterSerializerTest; import org.apache.hugegraph.unit.serializer.BinarySerializerTest; diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java index 7564ec7012..020020e35c 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBTableQueryByIdsTest.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hugegraph.backend.id.Id; import org.apache.hugegraph.backend.id.IdGenerator; import org.apache.hugegraph.backend.store.BackendEntry.BackendColumn; @@ -39,16 +40,19 @@ public class RocksDBTableQueryByIdsTest extends BaseRocksDBUnitTest { private static final String DATABASE = "db"; private TestVertexTable vertexTable; - private TestEdgeTable edgeTable; + private TestEdgeTable edgeOutTable; + private TestEdgeTable edgeInTable; @Override @Before public void setup() throws RocksDBException { super.setup(); this.vertexTable = new TestVertexTable(DATABASE); - this.edgeTable = new TestEdgeTable(DATABASE); + this.edgeOutTable = new TestEdgeTable(true, DATABASE); + this.edgeInTable = new TestEdgeTable(false, DATABASE); this.rocks.createTable(this.vertexTable.table()); - this.rocks.createTable(this.edgeTable.table()); + this.rocks.createTable(this.edgeOutTable.table()); + this.rocks.createTable(this.edgeInTable.table()); } @Test @@ -95,7 +99,7 @@ public void testVertexQueryByIdsWithExistingAndMissingIdsMixed() { } @Test - public void testVertexQueryByIdsWithDuplicateIds() { + public void testVertexQueryByIdsDedupsDuplicateIds() { Id id1 = IdGenerator.of("v1"); Id id2 = IdGenerator.of("v2"); @@ -106,34 +110,24 @@ public void testVertexQueryByIdsWithDuplicateIds() { List ids = Arrays.asList(id1, id2, id1); BackendColumnIterator iter = this.vertexTable.queryByIds(this.rocks.session(), ids); - Map countMap = new HashMap<>(); - Map results = new HashMap<>(); - while (iter.hasNext()) { - BackendColumn col = iter.next(); - String key = getString(col.name); - results.put(key, getString(col.value)); - countMap.put(key, countMap.getOrDefault(key, 0) + 1); - } + Map results = toResultMap(iter); Assert.assertEquals(2, results.size()); Assert.assertEquals("value1", results.get("v1")); Assert.assertEquals("value2", results.get("v2")); - // Verify duplicate ids produce duplicate results - Assert.assertEquals(Integer.valueOf(2), countMap.get("v1")); - Assert.assertEquals(Integer.valueOf(1), countMap.get("v2")); } @Test - public void testEdgeQueryByIdsWithAllExistingIds() { + public void testEdgeOutQueryByIdsWithAllExistingIds() { Id id1 = IdGenerator.of("e1"); Id id2 = IdGenerator.of("e2"); - this.rocks.session().put(this.edgeTable.table(), id1.asBytes(), getBytes("edge-value1")); - this.rocks.session().put(this.edgeTable.table(), id2.asBytes(), getBytes("edge-value2")); + this.rocks.session().put(this.edgeOutTable.table(), id1.asBytes(), getBytes("edge-value1")); + this.rocks.session().put(this.edgeOutTable.table(), id2.asBytes(), getBytes("edge-value2")); this.commit(); List ids = Arrays.asList(id1, id2); - BackendColumnIterator iter = this.edgeTable.queryByIds(this.rocks.session(), ids); + BackendColumnIterator iter = this.edgeOutTable.queryByIds(this.rocks.session(), ids); Map results = toResultMap(iter); @@ -142,15 +136,55 @@ public void testEdgeQueryByIdsWithAllExistingIds() { Assert.assertEquals("edge-value2", results.get("e2")); } - /** - * NOTE: Testing the fallback path (session.hasChanges() == true) is not - * feasible here because both the optimized multi-get path and the fallback - * scan-based path ultimately delegate to session.get() / session.scan(), - * which have a pre-existing assertion `assert !this.hasChanges()` in - * RocksDBStdSessions. This assertion is disabled in production but fires - * during unit tests when assertions are enabled. The dispatch logic itself - * is covered by the implementation in RocksDBTables.Vertex/Edge.queryByIds(). - */ + @Test + public void testEdgeInQueryByIdsWithAllExistingIds() { + Id id1 = IdGenerator.of("e1"); + Id id2 = IdGenerator.of("e2"); + + this.rocks.session().put(this.edgeInTable.table(), id1.asBytes(), getBytes("edge-value1")); + this.rocks.session().put(this.edgeInTable.table(), id2.asBytes(), getBytes("edge-value2")); + this.commit(); + + List ids = Arrays.asList(id1, id2); + BackendColumnIterator iter = this.edgeInTable.queryByIds(this.rocks.session(), ids); + + Map results = toResultMap(iter); + + Assert.assertEquals(2, results.size()); + Assert.assertEquals("edge-value1", results.get("e1")); + Assert.assertEquals("edge-value2", results.get("e2")); + } + + @Test + public void testVertexQueryByIdsFallbackWhenHasChanges() { + Id id1 = IdGenerator.of("v1"); + Id id2 = IdGenerator.of("v2"); + + this.rocks.session().put(this.vertexTable.table(), id1.asBytes(), getBytes("value1")); + this.rocks.session().put(this.vertexTable.table(), id2.asBytes(), getBytes("value2")); + this.commit(); + + List ids = Arrays.asList(id1, id2); + RocksDBSessions.Session mockSession = new DelegatingSession(this.rocks.session()) { + @Override + public boolean hasChanges() { + return true; + } + + @Override + public BackendColumnIterator get(String table, List keys) { + throw new AssertionError( + "multi-get should not be called when hasChanges"); + } + }; + + BackendColumnIterator iter = this.vertexTable.queryByIds(mockSession, ids); + + Map results = toResultMap(iter); + Assert.assertEquals(2, results.size()); + Assert.assertEquals("value1", results.get("v1")); + Assert.assertEquals("value2", results.get("v2")); + } private Map toResultMap(BackendColumnIterator iter) { Map results = new HashMap<>(); @@ -161,6 +195,130 @@ private Map toResultMap(BackendColumnIterator iter) { return results; } + /** + * A session wrapper that delegates all operations to an underlying session. + * Subclasses can override specific methods for mocking purposes. + */ + private static class DelegatingSession extends RocksDBSessions.Session { + + private final RocksDBSessions.Session delegate; + + DelegatingSession(RocksDBSessions.Session delegate) { + this.delegate = delegate; + } + + @Override + public String dataPath() { + return this.delegate.dataPath(); + } + + @Override + public String walPath() { + return this.delegate.walPath(); + } + + @Override + public String property(String table, String property) { + return this.delegate.property(table, property); + } + + @Override + public Pair keyRange(String table) { + return this.delegate.keyRange(table); + } + + @Override + public void compactRange(String table) { + this.delegate.compactRange(table); + } + + @Override + public void put(String table, byte[] key, byte[] value) { + this.delegate.put(table, key, value); + } + + @Override + public void merge(String table, byte[] key, byte[] value) { + this.delegate.merge(table, key, value); + } + + @Override + public void increase(String table, byte[] key, byte[] value) { + this.delegate.increase(table, key, value); + } + + @Override + public void delete(String table, byte[] key) { + this.delegate.delete(table, key); + } + + @Override + public void deleteSingle(String table, byte[] key) { + this.delegate.deleteSingle(table, key); + } + + @Override + public void deletePrefix(String table, byte[] key) { + this.delegate.deletePrefix(table, key); + } + + @Override + public void deleteRange(String table, byte[] keyFrom, byte[] keyTo) { + this.delegate.deleteRange(table, keyFrom, keyTo); + } + + @Override + public byte[] get(String table, byte[] key) { + return this.delegate.get(table, key); + } + + @Override + public BackendColumnIterator get(String table, List keys) { + return this.delegate.get(table, keys); + } + + @Override + public BackendColumnIterator scan(String table) { + return this.delegate.scan(table); + } + + @Override + public BackendColumnIterator scan(String table, byte[] prefix) { + return this.delegate.scan(table, prefix); + } + + @Override + public BackendColumnIterator scan(String table, byte[] keyFrom, + byte[] keyTo, int scanType) { + return this.delegate.scan(table, keyFrom, keyTo, scanType); + } + + @Override + public Object commit() { + return this.delegate.commit(); + } + + @Override + public void rollback() { + this.delegate.rollback(); + } + + @Override + public boolean hasChanges() { + return this.delegate.hasChanges(); + } + + @Override + public void open() { + this.delegate.open(); + } + + @Override + public void close() { + this.delegate.close(); + } + } + /** * Subclass that exposes the protected queryByIds for testing. */ @@ -182,8 +340,8 @@ public BackendColumnIterator queryByIds(RocksDBSessions.Session session, */ private static class TestEdgeTable extends RocksDBTables.Edge { - public TestEdgeTable(String database) { - super(true, database); + public TestEdgeTable(boolean out, String database) { + super(out, database); } @Override