[C++] Bound retained memory during hot-device metadata lookups

.github/workflows/unit-test-cpp.yml

@@ -80,6 +80,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v7
 
+      - name: Set up Python for clang-format
+        if: runner.os == 'Linux'
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
+
       - name: Set up JDK 17
         uses: actions/setup-java@v6.0.1
         with:
@@ -121,10 +127,10 @@ jobs:
             elif command -v yum >/dev/null 2>&1; then
               sudo yum install -y libuuid-devel
             fi
-            sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100
-            sudo update-alternatives --set clang-format /usr/bin/clang-format-17
-            sudo apt-get update
-            sudo apt-get install -y uuid-dev
+            python -m venv "$RUNNER_TEMP/clang-format"
+            "$RUNNER_TEMP/clang-format/bin/python" -m pip install --only-binary=:all: clang-format==17.0.6
+            "$RUNNER_TEMP/clang-format/bin/clang-format" --version
+            echo "$RUNNER_TEMP/clang-format/bin" >> "$GITHUB_PATH"
           elif [[ "$RUNNER_OS" == "Windows" ]]; then
             choco install llvm --version 17.0.6 --force
           else

.github/workflows/unit-test-python.yml

@@ -45,6 +45,13 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-latest, windows-latest ]
         python-version: ["3.9", "3.11", "3.14"]
+        exclude:
+          - os: ubuntu-latest
+            python-version: "3.9"
+        include:
+          # Python 3.9 has no setup-python build for Ubuntu 26.04.
+          - os: ubuntu-22.04
+            python-version: "3.9"
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -92,8 +99,10 @@ jobs:
         shell: bash
         run: |
           if [[ "$RUNNER_OS" == "Linux" ]]; then
-            sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100
-            sudo update-alternatives --set clang-format /usr/bin/clang-format-17
+            python -m venv "$RUNNER_TEMP/clang-format"
+            "$RUNNER_TEMP/clang-format/bin/python" -m pip install --only-binary=:all: clang-format==17.0.6
+            "$RUNNER_TEMP/clang-format/bin/clang-format" --version
+            echo "$RUNNER_TEMP/clang-format/bin" >> "$GITHUB_PATH"
             sudo apt-get update
             sudo apt-get install -y uuid-dev
           elif [[ "$RUNNER_OS" == "Windows" ]]; then

cpp/src/common/tsfile_common.cc

@@ -386,6 +386,14 @@ int MetaIndexNode::binary_search_children(
     std::shared_ptr<IComparable> key, bool exact_search,
     std::shared_ptr<IMetaIndexEntry>& ret_index_entry,
     int64_t& ret_end_offset) {
+    return binary_search_children(key, exact_search, ret_index_entry,
+                                  ret_end_offset, *pa_);
+}
+
+int MetaIndexNode::binary_search_children(
+    std::shared_ptr<IComparable> key, bool exact_search,
+    std::shared_ptr<IMetaIndexEntry>& ret_index_entry, int64_t& ret_end_offset,
+    common::PageArena& result_pa) {
 #if DEBUG_SE
     std::cout << "MetaIndexNode::binary_search_children start, name=" << key
               << ", exact_search=" << exact_search
@@ -436,7 +444,7 @@ int MetaIndexNode::binary_search_children(
             return E_NOT_EXIST;
         }
     }
-    ret_index_entry = children_[l]->clone(pa_);
+    ret_index_entry = children_[l]->clone(&result_pa);
     if (l == (int)children_.size() - 1) {
         ret_end_offset = this->end_offset_;
     } else {

cpp/src/common/tsfile_common.h

@@ -1008,6 +1008,13 @@ struct MetaIndexNode {
         std::shared_ptr<IMetaIndexEntry>& ret_index_entry,
         int64_t& ret_end_offset);
 
+    // Lookup copies can outlive the source node but must not accumulate in a
+    // cached node's arena. The caller keeps result_pa alive with the result.
+    int binary_search_children(
+        std::shared_ptr<IComparable> key, bool exact_search,
+        std::shared_ptr<IMetaIndexEntry>& ret_index_entry,
+        int64_t& ret_end_offset, common::PageArena& result_pa);
+
     int serialize_to(common::ByteStream& out) {
         int ret = common::E_OK;
 #if DEBUG_SE

cpp/src/file/tsfile_io_reader.cc

@@ -733,7 +733,7 @@ int TsFileIOReader::load_timeseries_index_for_ssi(
     int64_t measurement_ie_end_offset = 0;
     if (RET_FAIL(load_measurement_index_entry(measurement_name, top_node,
                                               measurement_index_entry,
-                                              measurement_ie_end_offset))) {
+                                              measurement_ie_end_offset, pa))) {
         return ret;
     } else if (RET_FAIL(do_load_timeseries_index(
                    measurement_name, measurement_index_entry->get_offset(),
@@ -800,19 +800,19 @@ int TsFileIOReader::load_measurement_index_entry(
     const std::string& measurement_name_str,
     std::shared_ptr<MetaIndexNode> top_node,
     std::shared_ptr<IMetaIndexEntry>& ret_measurement_index_entry,
-    int64_t& ret_end_offset) {
+    int64_t& ret_end_offset, common::PageArena& result_pa) {
     int ret = E_OK;
     // search from top_node in top-down way
     auto measurement_name =
         std::make_shared<StringComparable>(measurement_name_str);
     if (top_node->node_type_ == LEAF_MEASUREMENT) {
         ret = top_node->binary_search_children(
             measurement_name, /*exact*/ false, ret_measurement_index_entry,
-            ret_end_offset);
+            ret_end_offset, result_pa);
     } else {
         ret = search_from_internal_node(measurement_name, false, top_node,
                                         ret_measurement_index_entry,
-                                        ret_end_offset);
+                                        ret_end_offset, &result_pa);
     }
     if (ret == E_NOT_EXIST) {
         ret = E_MEASUREMENT_NOT_EXIST;
@@ -945,7 +945,7 @@ int TsFileIOReader::get_timeseries_indexes(
         timeseries_indexs[idx] = nullptr;
         ret = load_measurement_index_entry(measurement_name, top_node,
                                            measurement_index_entry,
-                                           measurement_ie_end_offset);
+                                           measurement_ie_end_offset, pa);
         if (ret == E_MEASUREMENT_NOT_EXIST || ret == E_NOT_EXIST) {
             ret = E_OK;
             idx++;
@@ -997,16 +997,18 @@ int TsFileIOReader::search_from_leaf_node(
 int TsFileIOReader::search_from_internal_node(
     std::shared_ptr<IComparable> target_name, bool is_device,
     std::shared_ptr<MetaIndexNode> index_node,
-    std::shared_ptr<IMetaIndexEntry>& ret_index_entry,
-    int64_t& ret_end_offset) {
+    std::shared_ptr<IMetaIndexEntry>& ret_index_entry, int64_t& ret_end_offset,
+    common::PageArena* result_pa) {
     int ret = E_OK;
+    PageArena& output_pa = result_pa == nullptr ? *index_node->pa_ : *result_pa;
     std::shared_ptr<IMetaIndexEntry> index_entry;
     int64_t end_offset = 0;
 
     ASSERT(index_node->node_type_ == INTERNAL_MEASUREMENT ||
            index_node->node_type_ == INTERNAL_DEVICE);
     if (RET_FAIL(index_node->binary_search_children(
-            target_name, /*exact=*/false, index_entry, end_offset))) {
+            target_name, /*exact=*/false, index_entry, end_offset,
+            output_pa))) {
         return ret;
     }
 
@@ -1045,17 +1047,20 @@ int TsFileIOReader::search_from_internal_node(
         }
         if (cur_level_index_node->node_type_ == LEAF_DEVICE) {
             ret = cur_level_index_node->binary_search_children(
-                target_name, /*exact=*/true, ret_index_entry, ret_end_offset);
+                target_name, /*exact=*/true, ret_index_entry, ret_end_offset,
+                output_pa);
             cur_level_index_node->destroy();
             return ret;  //// FIXME
         } else if (cur_level_index_node->node_type_ == LEAF_MEASUREMENT) {
             ret = cur_level_index_node->binary_search_children(
-                target_name, /*exact=*/false, ret_index_entry, ret_end_offset);
+                target_name, /*exact=*/false, ret_index_entry, ret_end_offset,
+                output_pa);
             cur_level_index_node->destroy();
             return ret;  //// FIXME
         } else {
             ret = cur_level_index_node->binary_search_children(
-                target_name, /*exact=*/false, index_entry, end_offset);
+                target_name, /*exact=*/false, index_entry, end_offset,
+                output_pa);
             cur_level_index_node->destroy();
         }
     }

cpp/src/file/tsfile_io_reader.h

@@ -165,7 +165,7 @@ class TsFileIOReader {
         const std::string& measurement_name,
         std::shared_ptr<MetaIndexNode> top_node,
         std::shared_ptr<IMetaIndexEntry>& ret_measurement_index_entry,
-        int64_t& ret_end_offset);
+        int64_t& ret_end_offset, common::PageArena& result_pa);
 
     int load_all_measurement_index_entry(
         int64_t start_offset, int64_t end_offset, common::PageArena& pa,
@@ -203,7 +203,7 @@ class TsFileIOReader {
         std::shared_ptr<IComparable> target_name, bool is_device,
         std::shared_ptr<MetaIndexNode> index_node,
         std::shared_ptr<IMetaIndexEntry>& ret_index_entry,
-        int64_t& ret_end_offset);
+        int64_t& ret_end_offset, common::PageArena* result_pa = nullptr);
 
     bool filter_stasify(ITimeseriesIndex* ts_index, Filter* time_filter);
 

cpp/test/common/tsfile_common_test.cc

@@ -456,6 +456,24 @@ TEST_F(MetaIndexNodeSearchTest, NonExactSearchNotFound) {
     ASSERT_EQ(result, common::E_NOT_EXIST);
 }
 
+TEST_F(MetaIndexNodeSearchTest, ResultCopyOutlivesSourceArena) {
+    common::PageArena result_arena;
+    std::shared_ptr<IMetaIndexEntry> result;
+    int64_t end_offset = 0;
+    ASSERT_EQ(node_.binary_search_children(
+                  std::make_shared<StringComparable>("banana"), true, result,
+                  end_offset, result_arena),
+              common::E_OK);
+    node_.children_.clear();
+    entry1_.reset();
+    entry2_.reset();
+    entry3_.reset();
+    arena_.reset();
+    EXPECT_EQ(result->get_name().to_std_string(), "banana");
+    EXPECT_EQ(result->get_offset(), 20);
+    EXPECT_EQ(end_offset, 30);
+}
+
 class TsFileMetaTest : public ::testing::Test {
    protected:
     common::PageArena pa_;

cpp/test/reader/tsfile_reader_test.cc

@@ -557,6 +557,112 @@ TEST_F(TsFileReaderTest, DeviceNodeCacheIsBoundedAndReloadsEvictedDevices) {
 #endif
 }
 
+class HotDeviceMetadataTest : public TsFileReaderTest,
+                              public ::testing::WithParamInterface<bool> {
+   protected:
+    void SetUp() override {
+        TsFileReaderTest::SetUp();
+        saved_degree_ = g_config_value_.max_degree_of_index_node_;
+        ASSERT_EQ(set_max_degree_of_index_node(2), E_OK);
+    }
+    void TearDown() override {
+        set_max_degree_of_index_node(saved_degree_);
+        TsFileReaderTest::TearDown();
+    }
+    uint32_t saved_degree_ = 0;
+};
+
+TEST_P(HotDeviceMetadataTest, RepeatedLookupsDoNotGrowRetainedArenas) {
+    const std::string device = "root.followup.d0";
+    const int measurements = GetParam() ? 12 : 1;
+    for (int i = 0; i < measurements; ++i) {
+        const std::string measurement = "value" + std::to_string(i);
+        ASSERT_EQ(tsfile_writer_->register_timeseries(
+                      device, MeasurementSchema(measurement, INT32, PLAIN,
+                                                UNCOMPRESSED)),
+                  E_OK);
+        TsRecord record(1, device);
+        record.add_point(measurement, int32_t(42 + i));
+        ASSERT_EQ(tsfile_writer_->write_record(record), E_OK);
+    }
+    ASSERT_EQ(tsfile_writer_->close(), E_OK);
+
+    TsFileIOReader reader;
+    ASSERT_EQ(reader.init(file_name_), E_OK);
+    auto id = std::make_shared<StringArrayDeviceID>(device);
+    PageArena query_pa;
+    query_pa.init(512, MOD_TSFILE_READER);
+    auto load = [&]() {
+        TsFileSeriesScanIterator* ssi = nullptr;
+        int ret = reader.alloc_ssi(id, "value0", ssi, query_pa);
+        if (ssi != nullptr) reader.revert_ssi(ssi);
+        query_pa.reset();
+        return ret;
+    };
+    ASSERT_EQ(load(), E_OK);
+#ifdef ENABLE_MEM_STAT
+    const int64_t before = reader.TEST_reader_memory_bytes();
+#endif
+    for (int i = 0; i < 10000; ++i) ASSERT_EQ(load(), E_OK);
+    EXPECT_EQ(reader.TEST_device_node_cache_size(), 1u);
+#ifdef ENABLE_MEM_STAT
+    EXPECT_EQ(reader.TEST_reader_memory_bytes(), before);
+#endif
+}
+
+INSTANTIATE_TEST_SUITE_P(IndexDepth, HotDeviceMetadataTest,
+                         ::testing::Values(false, true));
+
+TEST_F(TsFileReaderTest, ActiveIteratorSurvivesDeviceCacheEviction) {
+    const size_t capacity = TsFileIOReader::TEST_device_node_cache_capacity();
+    for (size_t i = 0; i <= capacity; ++i) {
+        const std::string device = "root.eviction.d" + std::to_string(i);
+        ASSERT_EQ(
+            tsfile_writer_->register_timeseries(
+                device, MeasurementSchema("value", INT32, PLAIN, UNCOMPRESSED)),
+            E_OK);
+        TsRecord record(1, device);
+        record.add_point("value", int32_t(42));
+        ASSERT_EQ(tsfile_writer_->write_record(record), E_OK);
+    }
+    ASSERT_EQ(tsfile_writer_->close(), E_OK);
+    TsFileIOReader reader;
+    ASSERT_EQ(reader.init(file_name_), E_OK);
+    PageArena active_pa;
+    active_pa.init(512, MOD_TSFILE_READER);
+    TsFileSeriesScanIterator* active = nullptr;
+    ASSERT_EQ(reader.alloc_ssi(
+                  std::make_shared<StringArrayDeviceID>("root.eviction.d0"),
+                  "value", active, active_pa),
+              E_OK);
+    ASSERT_NE(active, nullptr);
+    PageArena transient_pa;
+    transient_pa.init(512, MOD_TSFILE_READER);
+    for (size_t i = 1; i <= capacity; ++i) {
+        TsFileSeriesScanIterator* transient = nullptr;
+        ASSERT_EQ(reader.alloc_ssi(std::make_shared<StringArrayDeviceID>(
+                                       "root.eviction.d" + std::to_string(i)),
+                                   "value", transient, transient_pa),
+                  E_OK);
+        reader.revert_ssi(transient);
+        transient_pa.reset();
+    }
+    EXPECT_EQ(reader.TEST_device_node_cache_size(), capacity);
+    TsBlock* block = nullptr;
+    ASSERT_EQ(active->get_next(block, true), E_OK);
+    ASSERT_NE(block, nullptr);
+    ASSERT_EQ(block->get_row_count(), 1u);
+    {
+        ColIterator time(0, block);
+        ColIterator value(1, block);
+        uint32_t len = 0;
+        EXPECT_EQ(*reinterpret_cast<int64_t*>(time.read(&len)), 1);
+        EXPECT_EQ(*reinterpret_cast<int32_t*>(value.read(&len)), 42);
+    }
+    active->revert_tsblock();
+    reader.revert_ssi(active);
+}
+
 TEST_F(TsFileReaderTest, ResultSetMetadata) {
     std::string device_path = "device1";
     std::string measurement_name = "temperature";