From 21ac949b45ab73c526814bbba182892415c608d5 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 4 Aug 2026 15:24:44 +0800 Subject: [PATCH 01/14] Add TsFile v4 file format documentation --- src/.vuepress/client.ts | 2 + .../components/TsFileStructureNode.vue | 107 +++ .../components/TsFileStructureTree.vue | 475 +++++++++++++ src/.vuepress/components/tsFileStructure.ts | 643 ++++++++++++++++++ src/.vuepress/sidebar/v2.x/en.ts | 28 + src/.vuepress/sidebar/v2.x/zh.ts | 28 + .../FileFormat/Binary-Representation.md | 113 +++ .../latest/FileFormat/Bloom-Filter.md | 98 +++ .../latest/FileFormat/Configurations.md | 140 ++++ .../FileFormat/DataPages/Checksumming.md | 56 ++ .../latest/FileFormat/DataPages/Chunks.md | 181 +++++ .../FileFormat/DataPages/Compression.md | 80 +++ .../latest/FileFormat/DataPages/Encodings.md | 111 +++ .../latest/FileFormat/DataPages/Encryption.md | 76 +++ .../FileFormat/DataPages/Error-Recovery.md | 87 +++ .../latest/FileFormat/DataPages/README.md | 179 +++++ .../latest/FileFormat/Extensibility.md | 68 ++ .../latest/FileFormat/Format-Versions.md | 70 ++ src/UserGuide/latest/FileFormat/Metadata.md | 284 ++++++++ src/UserGuide/latest/FileFormat/Nulls.md | 71 ++ src/UserGuide/latest/FileFormat/README.md | 152 +++++ src/UserGuide/latest/FileFormat/Types.md | 88 +++ .../FileFormat/Binary-Representation.md | 106 +++ .../latest/FileFormat/Bloom-Filter.md | 94 +++ .../latest/FileFormat/Configurations.md | 129 ++++ .../FileFormat/DataPages/Checksumming.md | 49 ++ .../latest/FileFormat/DataPages/Chunks.md | 172 +++++ .../FileFormat/DataPages/Compression.md | 74 ++ .../latest/FileFormat/DataPages/Encodings.md | 107 +++ .../latest/FileFormat/DataPages/Encryption.md | 72 ++ .../FileFormat/DataPages/Error-Recovery.md | 83 +++ .../latest/FileFormat/DataPages/README.md | 168 +++++ .../latest/FileFormat/Extensibility.md | 61 ++ .../latest/FileFormat/Format-Versions.md | 63 ++ .../UserGuide/latest/FileFormat/Metadata.md | 275 ++++++++ src/zh/UserGuide/latest/FileFormat/Nulls.md | 68 ++ src/zh/UserGuide/latest/FileFormat/README.md | 140 ++++ src/zh/UserGuide/latest/FileFormat/Types.md | 81 +++ 38 files changed, 4879 insertions(+) create mode 100644 src/.vuepress/components/TsFileStructureNode.vue create mode 100644 src/.vuepress/components/TsFileStructureTree.vue create mode 100644 src/.vuepress/components/tsFileStructure.ts create mode 100644 src/UserGuide/latest/FileFormat/Binary-Representation.md create mode 100644 src/UserGuide/latest/FileFormat/Bloom-Filter.md create mode 100644 src/UserGuide/latest/FileFormat/Configurations.md create mode 100644 src/UserGuide/latest/FileFormat/DataPages/Checksumming.md create mode 100644 src/UserGuide/latest/FileFormat/DataPages/Chunks.md create mode 100644 src/UserGuide/latest/FileFormat/DataPages/Compression.md create mode 100644 src/UserGuide/latest/FileFormat/DataPages/Encodings.md create mode 100644 src/UserGuide/latest/FileFormat/DataPages/Encryption.md create mode 100644 src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md create mode 100644 src/UserGuide/latest/FileFormat/DataPages/README.md create mode 100644 src/UserGuide/latest/FileFormat/Extensibility.md create mode 100644 src/UserGuide/latest/FileFormat/Format-Versions.md create mode 100644 src/UserGuide/latest/FileFormat/Metadata.md create mode 100644 src/UserGuide/latest/FileFormat/Nulls.md create mode 100644 src/UserGuide/latest/FileFormat/README.md create mode 100644 src/UserGuide/latest/FileFormat/Types.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Binary-Representation.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Configurations.md create mode 100644 src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md create mode 100644 src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md create mode 100644 src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md create mode 100644 src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md create mode 100644 src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md create mode 100644 src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md create mode 100644 src/zh/UserGuide/latest/FileFormat/DataPages/README.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Extensibility.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Format-Versions.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Metadata.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Nulls.md create mode 100644 src/zh/UserGuide/latest/FileFormat/README.md create mode 100644 src/zh/UserGuide/latest/FileFormat/Types.md diff --git a/src/.vuepress/client.ts b/src/.vuepress/client.ts index 8f2fd9f874..dabe83d296 100644 --- a/src/.vuepress/client.ts +++ b/src/.vuepress/client.ts @@ -20,11 +20,13 @@ import { defineDocSearchConfig } from '@vuepress/plugin-docsearch/client'; import { computed } from 'vue'; import { defineClientConfig, usePageData, useRouter, withBase } from 'vuepress/client'; import DocSearch from './components/DocSearch.vue'; +import TsFileStructureTree from './components/TsFileStructureTree.vue'; import { getDocVersion } from './utils/index.js'; export default defineClientConfig({ enhance: ({ app }) => { app.component('DocSearch', DocSearch); + app.component('TsFileStructureTree', TsFileStructureTree); }, setup() { const pageData = usePageData(); diff --git a/src/.vuepress/components/TsFileStructureNode.vue b/src/.vuepress/components/TsFileStructureNode.vue new file mode 100644 index 0000000000..c9079afdf4 --- /dev/null +++ b/src/.vuepress/components/TsFileStructureNode.vue @@ -0,0 +1,107 @@ + + + + + diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue new file mode 100644 index 0000000000..b221f0e166 --- /dev/null +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -0,0 +1,475 @@ + + + + + + + diff --git a/src/.vuepress/components/tsFileStructure.ts b/src/.vuepress/components/tsFileStructure.ts new file mode 100644 index 0000000000..c179434960 --- /dev/null +++ b/src/.vuepress/components/tsFileStructure.ts @@ -0,0 +1,643 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type TsFileStructureLocale = 'en' | 'zh'; + +export type TsFileStructureKind = + | 'file' + | 'region' + | 'repeated' + | 'record' + | 'field' + | 'choice'; + +export interface TsFileStructureText { + title: string; + description: string; + offset?: string; + size?: string; + layout?: string; + note?: string; +} + +export interface TsFileStructureNode { + id: string; + kind: TsFileStructureKind; + text: Record; + link?: string; + children?: TsFileStructureNode[]; +} + +export const tsFileStructure: TsFileStructureNode = { + id: 'tsfile', + kind: 'file', + text: { + en: { + title: 'TsFile v4', + description: + 'The complete physical file. All offsets are absolute byte offsets measured from the first byte. Readers normally locate the footer from the tail, traverse metadata, and then read only selected Chunks and Pages.', + offset: '0 … F - 1', + size: 'F bytes', + layout: 'File Header | Data Section | Separator | Metadata | Footer', + }, + zh: { + title: 'TsFile v4', + description: '完整的物理文件。所有偏移均为从文件第一个字节开始计算的绝对字节偏移。读取器通常先从文件尾定位 Footer、遍历元数据，再按需读取选中的 Chunk 和 Page。', + offset: '0 … F - 1', + size: 'F 字节', + layout: '文件头 | 数据区 | 分隔符 | 元数据 | 文件尾', + }, + }, + children: [ + { + id: 'file-header', + kind: 'region', + link: 'Binary-Representation.html', + text: { + en: { + title: 'File Header', + description: 'A fixed seven-byte prefix identifying TsFile and format version 4. The version byte selects the complete grammar before any data marker is interpreted.', + offset: '0 … 6', + size: '7 bytes', + layout: 'magic[6] | version[1]', + }, + zh: { + title: '文件头', + description: '固定的 7 字节前缀，用于标识 TsFile 以及第 4 版文件格式。解释任何数据区标记之前，必须先用版本字节选择完整语法。', + offset: '0 … 6', + size: '7 字节', + layout: 'magic[6] | version[1]', + }, + }, + children: [ + { + id: 'head-magic', + kind: 'field', + text: { + en: { + title: 'Head Magic', + description: 'The UTF-8 bytes of the literal string "TsFile". They identify the container but are not a checksum or a content-integrity proof.', + offset: '0 … 5', + size: '6 bytes', + layout: '54 73 46 69 6c 65', + }, + zh: { + title: '头部魔数', + description: '字符串 “TsFile” 的 UTF-8 字节。它用于识别容器，但不是校验和，也不能证明内容完整。', + offset: '0 … 5', + size: '6 字节', + layout: '54 73 46 69 6c 65', + }, + }, + }, + { + id: 'version', + kind: 'field', + link: 'Format-Versions.html', + text: { + en: { + title: 'Format Version', + description: 'The one-byte on-disk format identifier. Value 0x04 selects this layout; it is independent of the project 2.x release label.', + offset: '6', + size: '1 byte', + layout: '04', + }, + zh: { + title: '格式版本', + description: '单字节磁盘格式标识。数值 0x04 选择本页面所述布局，它与项目的 2.x 发布版本标签相互独立。', + offset: '6', + size: '1 字节', + layout: '04', + }, + }, + }, + ], + }, + { + id: 'data-section', + kind: 'region', + link: 'DataPages/Chunks.html', + text: { + en: { + title: 'Data Section', + description: + 'Marker-delimited Chunk Groups and optional Operation Index Ranges. Structures are written sequentially, and every marker is interpreted only at a known top-level boundary.', + offset: '7 … meta_offset - 1', + size: 'variable', + layout: 'Chunk Group* | Operation Index Range*', + }, + zh: { + title: '数据区', + description: '由标记分隔的 Chunk Group 和可选 Operation Index Range。各结构顺序写入，只有位于已知顶层边界的字节才可解释为 marker。', + offset: '7 … meta_offset - 1', + size: '可变', + layout: 'Chunk Group* | Operation Index Range*', + }, + }, + children: [ + { + id: 'chunk-group', + kind: 'repeated', + link: 'DataPages/Chunks.html#chunk-group-header', + text: { + en: { + title: 'Chunk Group', + description: + 'A repeated device-scoped container. All following Chunks belong to its device until another top-level marker is encountered. The group has no total-length field of its own.', + offset: 'chunk_group_offset', + size: 'variable · repeated', + layout: 'Chunk Group Header | Chunk*', + }, + zh: { + title: 'Chunk Group', + description: '按设备组织的重复容器。后续 Chunk 均属于该设备，直到遇到下一个顶层标记。Chunk Group 自身没有总长度字段。', + offset: 'chunk_group_offset', + size: '可变 · 重复', + layout: 'Chunk Group Header | Chunk*', + }, + }, + children: [ + { + id: 'chunk-group-header', + kind: 'record', + link: 'DataPages/Chunks.html#chunk-group-header', + text: { + en: { + title: 'Chunk Group Header', + description: 'Starts one device group and serializes the segmented DeviceID. The segment count is an element count, and every segment is a separate VarString.', + offset: 'chunk_group_offset', + size: 'variable', + layout: 'marker 0x00 | DeviceID', + }, + zh: { + title: 'Chunk Group Header', + description: '开始一个设备分组，并序列化分片 DeviceID。片段数表示元素数量，每个片段都是独立 VarString。', + offset: 'chunk_group_offset', + size: '可变', + layout: 'marker 0x00 | DeviceID', + }, + }, + }, + { + id: 'chunk-layout', + kind: 'choice', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Chunk Layout', + description: + 'A device contains non-aligned Chunks, aligned Chunk sets, or both. Each Chunk stores one physical column.', + offset: 'after Chunk Group Header', + size: 'variable · repeated', + layout: 'Non-aligned Chunk | Aligned Chunk Set', + }, + zh: { + title: 'Chunk 布局', + description: '一个设备可包含非对齐 Chunk、对齐 Chunk 集，或同时包含两者。每个 Chunk 保存一个物理列。', + offset: 'Chunk Group Header 之后', + size: '可变 · 重复', + layout: '非对齐 Chunk | 对齐 Chunk 集', + }, + }, + children: [ + { + id: 'non-aligned-chunk', + kind: 'repeated', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Non-aligned Chunk', + description: + 'Stores one measurement and its independent timestamp stream. Every Page contains both streams, and the decoded timestamp and value counts must match.', + offset: 'chunk_offset', + size: 'header + data_size', + layout: 'Chunk Header | Non-aligned Page*', + }, + zh: { + title: '非对齐 Chunk', + description: '保存一个测量列及其独立时间戳流。每个 Page 同时包含时间流和值流，解码后的时间戳数和值数必须相等。', + offset: 'chunk_offset', + size: 'header + data_size', + layout: 'Chunk Header | 非对齐 Page*', + }, + }, + children: [ + { + id: 'non-aligned-chunk-header', + kind: 'record', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Chunk Header', + description: + 'Describes the value column and gives the exact total byte length of all Page Headers and stored Page data. This boundary lets readers skip the whole Chunk without decoding its codec.', + offset: 'chunk_offset', + size: 'variable', + layout: + 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type', + note: 'For a non-aligned Chunk, encoding_type describes the value stream.', + }, + zh: { + title: 'Chunk Header', + description: '描述值列，并给出全部 Page Header 与磁盘 Page 数据的准确总字节数。读取器可以利用该边界跳过整个 Chunk，而无需调用其编解码器。', + offset: 'chunk_offset', + size: '可变', + layout: + 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type', + note: '对于非对齐 Chunk，encoding_type 描述的是值流。', + }, + }, + }, + { + id: 'non-aligned-page', + kind: 'repeated', + link: 'DataPages/index.html#non-aligned-page', + text: { + en: { + title: 'Non-aligned Page', + description: + 'The Page Header is followed by one stored Page payload. After decryption and decompression, a length-delimited time stream comes first and the value stream occupies the remaining bytes.', + offset: 'inside Chunk data_size', + size: 'variable · repeated', + layout: + 'Page Header | time_stream_size | encoded_time_stream | encoded_value_stream', + note: 'Timestamps and values pair by position; they are not interleaved point by point.', + }, + zh: { + title: '非对齐 Page', + description: 'Page Header 后是一个磁盘 Page 载荷。解密并解压后，先存带长度的完整时间流，值流占据剩余全部字节。', + offset: 'Chunk data_size 内部', + size: '可变 · 重复', + layout: + 'Page Header | time_stream_size | encoded_time_stream | encoded_value_stream', + note: '时间戳和值按位置配对，并不是逐点交错存储。', + }, + }, + }, + ], + }, + { + id: 'aligned-chunk-set', + kind: 'repeated', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Aligned Chunk Set', + description: + 'Stores one shared time column followed by aligned measurement value columns. Chunks are contiguous rather than interleaved Page by Page, while equal Page ordinals define correspondence.', + offset: 'first aligned Chunk offset', + size: 'variable · repeated', + layout: 'Time Chunk | Value Chunk*', + }, + zh: { + title: '对齐 Chunk 集', + description: '先保存一个共享时间列，再保存各个对齐测量值列。不同 Chunk 连续存放而不按 Page 交错；相同 Page 序号建立对应关系。', + offset: '第一个对齐 Chunk 的偏移', + size: '可变 · 重复', + layout: 'Time Chunk | Value Chunk*', + }, + }, + children: [ + { + id: 'time-chunk', + kind: 'record', + link: 'DataPages/index.html#aligned-time-page', + text: { + en: { + title: 'Time Chunk', + description: 'Contains all Time Pages for the aligned set. Its Chunk Header records the time encoding, and each decoded Page establishes the logical positions shared by matching Value Pages.', + offset: 'time_chunk_offset', + size: 'header + data_size', + layout: 'Time Chunk Header | Time Page*', + }, + zh: { + title: 'Time Chunk', + description: '包含该对齐集合的全部 Time Page；其 Chunk Header 记录时间编码。每个解码后的 Page 建立对应 Value Page 共用的逻辑位置。', + offset: 'time_chunk_offset', + size: 'header + data_size', + layout: 'Time Chunk Header | Time Page*', + }, + }, + }, + { + id: 'value-chunk', + kind: 'repeated', + link: 'DataPages/index.html#aligned-value-page', + text: { + en: { + title: 'Value Chunk', + description: + 'Contains one aligned measurement column. Value Page number i corresponds to Time Page number i; its bitmap maps present values and nulls onto those shared positions.', + offset: 'value_chunk_offset', + size: 'header + data_size · repeated', + layout: 'Value Chunk Header | Value Page*', + }, + zh: { + title: 'Value Chunk', + description: '包含一个对齐测量值列。编号为 i 的 Value Page 对应编号为 i 的 Time Page，并通过位图把实际值和 null 映射到共享位置。', + offset: 'value_chunk_offset', + size: 'header + data_size · 重复', + layout: 'Value Chunk Header | Value Page*', + }, + }, + }, + ], + }, + ], + }, + ], + }, + { + id: 'operation-index-range', + kind: 'record', + link: 'DataPages/Chunks.html#operation-index-range', + text: { + en: { + title: 'Operation Index Range', + description: 'An optional storage-engine record containing two operation sequence numbers for checkpoint, snapshot, backup, or recovery coordination. They are not timestamps or file offsets.', + offset: 'between Chunk Groups', + size: '17 bytes', + layout: 'marker 0x04 | min_operation_index[8] | max_operation_index[8]', + }, + zh: { + title: 'Operation Index Range', + description: '可选的存储引擎顶层记录，包含两个操作序列号，用于协调检查点、快照、备份或恢复。它们不是时间戳，也不是文件偏移量。', + offset: 'Chunk Group 之间', + size: '17 字节', + layout: 'marker 0x04 | min_operation_index[8] | max_operation_index[8]', + }, + }, + }, + ], + }, + { + id: 'separator', + kind: 'field', + link: 'DataPages/Chunks.html#markers', + text: { + en: { + title: 'Data/Metadata Separator', + description: 'The single top-level separator ending the data section. TsFileMetadata.meta_offset points to this byte; a matching 0x02 inside payload data has no marker meaning.', + offset: 'meta_offset', + size: '1 byte', + layout: '02', + }, + zh: { + title: '数据/元数据分隔符', + description: '结束数据区的唯一顶层分隔符，TsFileMetadata.meta_offset 指向这个字节。负载内部出现相同的 0x02 不具有 marker 含义。', + offset: 'meta_offset', + size: '1 字节', + layout: '02', + }, + }, + }, + { + id: 'metadata-section', + kind: 'region', + link: 'Metadata.html', + text: { + en: { + title: 'Indexed Metadata Section', + description: + 'Timeseries Metadata and metadata index nodes. Their physical ranges can be interleaved and are delimited by absolute offsets; this is a graph of referenced byte ranges, not one self-delimiting list.', + offset: 'meta_offset + 1 … file_metadata_pos - 1', + size: 'variable', + layout: 'TimeseriesMetadata* | MetadataIndexNode*', + }, + zh: { + title: '索引元数据区', + description: 'Timeseries Metadata 与元数据索引节点。两类记录的物理区间可以交错，并由绝对偏移划分；这里是由字节范围引用形成的图，而不是单一自定界列表。', + offset: 'meta_offset + 1 … file_metadata_pos - 1', + size: '可变', + layout: 'TimeseriesMetadata* | MetadataIndexNode*', + }, + }, + children: [ + { + id: 'timeseries-metadata', + kind: 'repeated', + link: 'Metadata.html#timeseries-metadata', + text: { + en: { + title: 'Timeseries Metadata', + description: 'One logical-series record containing aggregate Statistics and a byte-length-delimited Chunk Metadata list. Entries point back to physical Chunk Headers in the data section.', + offset: 'referenced by a measurement index entry', + size: 'variable · repeated', + layout: + 'timeseries_type | measurement_id | data_type | chunk_metadata_list_size | Statistics | ChunkMetadata*', + }, + zh: { + title: 'Timeseries Metadata', + description: '每个逻辑序列对应一条记录，包含聚合 Statistics 和按字节长度定界的 Chunk Metadata 列表。各条目反向指向数据区的物理 Chunk Header。', + offset: '由测量索引条目引用', + size: '可变 · 重复', + layout: + 'timeseries_type | measurement_id | data_type | chunk_metadata_list_size | Statistics | ChunkMetadata*', + }, + }, + }, + { + id: 'metadata-index', + kind: 'repeated', + link: 'Metadata.html#metadata-index', + text: { + en: { + title: 'Metadata Index Node', + description: 'An ordered internal or leaf node used to locate devices, measurements, and metadata ranges. Each child offset starts a range whose end is the next child offset or end_offset.', + offset: 'referenced by a parent node or TsFileMetadata', + size: 'variable · repeated', + layout: 'child_count | MetadataEntry* | end_offset | node_type', + }, + zh: { + title: 'Metadata Index Node', + description: '用于定位设备、测量列和元数据区间的有序内部节点或叶子节点。每个子项的范围从自身 offset 开始，到下一子项 offset 或 end_offset 结束。', + offset: '由父节点或 TsFileMetadata 引用', + size: '可变 · 重复', + layout: 'child_count | MetadataEntry* | end_offset | node_type', + }, + }, + }, + ], + }, + { + id: 'file-metadata', + kind: 'record', + link: 'Metadata.html#file-metadata', + text: { + en: { + title: 'TsFileMetadata', + description: + 'The file-level index roots, table schemas, data-section offset, Bloom filter, and properties. Its start is derived from the trailing fixed-width length, so no preceding marker is needed.', + offset: 'file_metadata_pos = F - 6 - 4 - L', + size: 'L bytes', + layout: + 'table indexes | table schemas | meta_offset | BloomFilter | properties', + note: 'No separator or marker immediately precedes TsFileMetadata.', + }, + zh: { + title: 'TsFileMetadata', + description: '包含文件级索引根、表 Schema、数据区偏移、布隆过滤器以及文件属性。其起点由尾部定宽长度反向计算，因此前面无需 marker。', + offset: 'file_metadata_pos = F - 6 - 4 - L', + size: 'L 字节', + layout: 'table indexes | table schemas | meta_offset | BloomFilter | properties', + note: 'TsFileMetadata 前面没有独立分隔符或标记。', + }, + }, + children: [ + { + id: 'table-index-roots', + kind: 'repeated', + text: { + en: { + title: 'Table Index Roots', + description: 'Named device-index roots for tree-model and table-model data. A root stores the entry range used to descend toward one device and then one measurement.', + offset: 'inside TsFileMetadata', + size: 'variable · repeated', + }, + zh: { + title: '表索引根', + description: '树模型与表模型数据使用的具名设备索引根。根节点保存向下定位某个设备、再定位某个测点所需的条目范围。', + offset: 'TsFileMetadata 内部', + size: '可变 · 重复', + }, + }, + }, + { + id: 'table-schemas', + kind: 'repeated', + link: 'Metadata.html#table-schema', + text: { + en: { + title: 'Table Schemas', + description: 'Named schemas for explicit table-model data. Column names, physical types, and categories preserve a common column order.', + offset: 'inside TsFileMetadata', + size: 'variable · repeated', + }, + zh: { + title: '表 Schema', + description: '显式表模型数据使用的具名 Schema。列名、物理类型和列类别共同保持同一个列顺序。', + offset: 'TsFileMetadata 内部', + size: '可变 · 重复', + }, + }, + }, + { + id: 'meta-offset', + kind: 'field', + text: { + en: { + title: 'meta_offset', + description: 'Absolute offset of the top-level 0x02 separator at the end of the data section. The indexed metadata section begins at the following byte.', + offset: 'inside TsFileMetadata', + size: '8 bytes', + layout: 'int64, big-endian', + }, + zh: { + title: 'meta_offset', + description: '数据区末尾顶层 0x02 分隔符的绝对偏移；索引元数据区从它的下一个字节开始。', + offset: 'TsFileMetadata 内部', + size: '8 字节', + layout: 'int64，大端序', + }, + }, + }, + { + id: 'bloom-filter', + kind: 'record', + link: 'Bloom-Filter.html', + text: { + en: { + title: 'Bloom Filter', + description: 'Optional membership filter used to reject missing full paths without index traversal. A negative answer is definitive; a positive answer still requires metadata lookup.', + offset: 'inside TsFileMetadata', + size: 'variable', + layout: 'filter bytes | bit count | hash-function count', + }, + zh: { + title: '布隆过滤器', + description: '可选的成员过滤器，无需遍历索引即可排除不存在的完整路径。否定结果是确定的，肯定结果仍需查询元数据确认。', + offset: 'TsFileMetadata 内部', + size: '可变', + layout: 'filter bytes | bit count | hash-function count', + }, + }, + }, + { + id: 'file-properties', + kind: 'repeated', + text: { + en: { + title: 'File Properties', + description: 'A length-delimited map of string keys and values, including encryption properties when present. Unknown properties remain structurally consumable.', + offset: 'end of TsFileMetadata', + size: 'variable · repeated', + layout: 'property_count | (key | value)*', + }, + zh: { + title: '文件属性', + description: '按长度定界的字符串键值映射；启用加密时也包含加密属性。未知属性在结构上仍可读取和跳过。', + offset: 'TsFileMetadata 尾部', + size: '可变 · 重复', + layout: 'property_count | (key | value)*', + }, + }, + }, + ], + }, + { + id: 'metadata-size', + kind: 'field', + text: { + en: { + title: 'TsFileMetadata Size', + description: 'The big-endian serialized byte length L of TsFileMetadata. With file size F, metadata occupies exactly [F - 10 - L, F - 10).', + offset: 'F - 10 … F - 7', + size: '4 bytes', + layout: 'int32 L, big-endian', + }, + zh: { + title: 'TsFileMetadata 长度', + description: 'TsFileMetadata 的大端序列化字节长度 L。文件大小为 F 时，元数据恰好位于 [F - 10 - L, F - 10)。', + offset: 'F - 10 … F - 7', + size: '4 字节', + layout: 'int32 L，大端序', + }, + }, + }, + { + id: 'tail-magic', + kind: 'field', + text: { + en: { + title: 'Tail Magic', + description: 'The same six-byte magic string as the file header. It supports completion checks but is not sufficient without validating the footer length and structure.', + offset: 'F - 6 … F - 1', + size: '6 bytes', + layout: '54 73 46 69 6c 65', + }, + zh: { + title: '尾部魔数', + description: '与文件头相同的 6 字节魔数字符串。它可用于完成性检查，但还必须同时校验 Footer 长度与结构。', + offset: 'F - 6 … F - 1', + size: '6 字节', + layout: '54 73 46 69 6c 65', + }, + }, + }, + ], +}; diff --git a/src/.vuepress/sidebar/v2.x/en.ts b/src/.vuepress/sidebar/v2.x/en.ts index c7ae96653d..2514bb7122 100644 --- a/src/.vuepress/sidebar/v2.x/en.ts +++ b/src/.vuepress/sidebar/v2.x/en.ts @@ -33,6 +33,34 @@ export const enSidebar = { collapsible: true, link: 'QuickStart/Data-Model', }, + { + text: 'File Format', + collapsible: true, + link: '/UserGuide/latest/FileFormat/', + children: [ + { text: 'Binary Representation', link: '/UserGuide/latest/FileFormat/Binary-Representation' }, + { text: 'Configurations', link: '/UserGuide/latest/FileFormat/Configurations' }, + { text: 'Extensibility', link: '/UserGuide/latest/FileFormat/Extensibility' }, + { text: 'Metadata', link: '/UserGuide/latest/FileFormat/Metadata' }, + { text: 'Types', link: '/UserGuide/latest/FileFormat/Types' }, + { text: 'Bloom Filter', link: '/UserGuide/latest/FileFormat/Bloom-Filter' }, + { + text: 'Data Pages', + collapsible: true, + link: '/UserGuide/latest/FileFormat/DataPages/', + children: [ + { text: 'Chunks', link: '/UserGuide/latest/FileFormat/DataPages/Chunks' }, + { text: 'Compression', link: '/UserGuide/latest/FileFormat/DataPages/Compression' }, + { text: 'Encodings', link: '/UserGuide/latest/FileFormat/DataPages/Encodings' }, + { text: 'Encryption', link: '/UserGuide/latest/FileFormat/DataPages/Encryption' }, + { text: 'Checksumming', link: '/UserGuide/latest/FileFormat/DataPages/Checksumming' }, + { text: 'Error Recovery', link: '/UserGuide/latest/FileFormat/DataPages/Error-Recovery' }, + ], + }, + { text: 'Nulls', link: '/UserGuide/latest/FileFormat/Nulls' }, + { text: 'Format Versions', link: '/UserGuide/latest/FileFormat/Format-Versions' }, + ], + }, { text: 'Quick Start', collapsible: true, diff --git a/src/.vuepress/sidebar/v2.x/zh.ts b/src/.vuepress/sidebar/v2.x/zh.ts index f1a40e158c..a0fe9fe165 100644 --- a/src/.vuepress/sidebar/v2.x/zh.ts +++ b/src/.vuepress/sidebar/v2.x/zh.ts @@ -33,6 +33,34 @@ export const zhSidebar = { collapsible: true, link: 'QuickStart/Data-Model', }, + { + text: '文件格式', + collapsible: true, + link: '/zh/UserGuide/latest/FileFormat/', + children: [ + { text: '二进制表示', link: '/zh/UserGuide/latest/FileFormat/Binary-Representation' }, + { text: '配置', link: '/zh/UserGuide/latest/FileFormat/Configurations' }, + { text: '扩展机制', link: '/zh/UserGuide/latest/FileFormat/Extensibility' }, + { text: '元数据', link: '/zh/UserGuide/latest/FileFormat/Metadata' }, + { text: '类型', link: '/zh/UserGuide/latest/FileFormat/Types' }, + { text: '布隆过滤器', link: '/zh/UserGuide/latest/FileFormat/Bloom-Filter' }, + { + text: '数据页', + collapsible: true, + link: '/zh/UserGuide/latest/FileFormat/DataPages/', + children: [ + { text: '数据块', link: '/zh/UserGuide/latest/FileFormat/DataPages/Chunks' }, + { text: '压缩', link: '/zh/UserGuide/latest/FileFormat/DataPages/Compression' }, + { text: '编码', link: '/zh/UserGuide/latest/FileFormat/DataPages/Encodings' }, + { text: '加密', link: '/zh/UserGuide/latest/FileFormat/DataPages/Encryption' }, + { text: '校验和', link: '/zh/UserGuide/latest/FileFormat/DataPages/Checksumming' }, + { text: '错误恢复', link: '/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery' }, + ], + }, + { text: '空值', link: '/zh/UserGuide/latest/FileFormat/Nulls' }, + { text: '格式版本', link: '/zh/UserGuide/latest/FileFormat/Format-Versions' }, + ], + }, { text: '快速上手', collapsible: true, diff --git a/src/UserGuide/latest/FileFormat/Binary-Representation.md b/src/UserGuide/latest/FileFormat/Binary-Representation.md new file mode 100644 index 0000000000..a75c937d92 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Binary-Representation.md @@ -0,0 +1,113 @@ + + +# Binary Representation + +All offsets are measured from the first byte of the file, and all lengths are in bytes. Unless +otherwise specified, fixed-width integers and IEEE 754 floating-point values use big-endian byte +order. Strings use UTF-8. + +## Fixed-width Values + +| Notation | Bytes | Meaning | +| --- | ---: | --- | +| `byte` | 1 | An 8-bit raw value | +| `bool` | 1 | `0x00` for false and `0x01` for true | +| `int32` | 4 | Big-endian signed 32-bit integer | +| `int64` | 8 | Big-endian signed 64-bit integer | +| `float32` | 4 | Big-endian IEEE 754 binary32 | +| `float64` | 8 | Big-endian IEEE 754 binary64 | + +## Unsigned Variable-length Integer + +`UVarInt` is an unsigned LEB128-style variable-length 32-bit integer. The low seven bits of each +byte carry data. A set high bit indicates that another byte follows. The least-significant group +is written first. + +```text +value = Σ ((byte[i] & 0x7f) << (7 * i)) +``` + +The encoding occupies one to five bytes and uses the shortest form. + +For example, decimal `300` is split into low-order seven-bit groups `0x2c` and `0x02`, producing +`ac 02`. A reader must stop after at most five bytes for a 32-bit field and must detect a shift or +value overflow. Writers emit the canonical shortest representation; overlong representations are +not generated. + +## Signed Variable-length Integer + +`VarInt` first maps a signed 32-bit integer with ZigZag encoding, then writes the result as a +`UVarInt`: + +```text +zigzag(n) = (n << 1) XOR (n >> 31) +``` + +| Original value | Unsigned wire value | Byte | +| ---: | ---: | --- | +| 0 | 0 | `00` | +| -1 | 1 | `01` | +| 1 | 2 | `02` | +| -2 | 3 | `03` | + +## Strings and Byte Arrays + +| Notation | Format | +| --- | --- | +| `VarString` | `VarInt byte_length` + `byte[byte_length]` | +| `String32` | `int32 byte_length` + `byte[byte_length]` | +| `Binary32` | `int32 byte_length` + `byte[byte_length]` | + +A length of `-1` represents null, and a length of `0` represents an empty string or byte array. +`VarString` and `String32` contain UTF-8; `Binary32` contains raw bytes. + +The length counts encoded bytes, not Unicode code points or characters. For example, a UTF-8 +string containing one non-ASCII character can have a byte length greater than one. Null and empty +are distinct wire values and must remain distinct in structures whose fields are nullable. + +## Boundaries and Defensive Decoding + +Every length-delimited field is contained by a higher-level boundary: a string by its enclosing +record, a Page by its Chunk, and footer objects by metadata offsets or the trailing metadata +length. Before allocating or slicing a buffer for length `N`, a reader validates: + +```text +N >= 0 # except the documented null sentinel +current_offset + N >= current_offset +current_offset + N <= enclosing_end +``` + +The second comparison detects integer wraparound. A reader should additionally enforce a local +resource limit; a structurally valid length does not require an implementation to allocate an +unbounded amount of memory. + +Fixed-width values require their complete byte width. A truncated `int64`, unterminated VarInt, +negative non-null length, invalid UTF-8 where a string is required, or bytes left over after a +length-delimited record are format errors. + +## Offset Conventions + +Offsets stored in metadata are signed 64-bit values but represent non-negative absolute byte +positions. An offset points to the first byte of the named structure: for example, +`offset_of_chunk_header` points to the Chunk marker, and `meta_offset` points to the `0x02` +separator. Lengths are not implicit offsets; arithmetic that combines an offset and length must +be checked for overflow and against the enclosing file or record boundary. diff --git a/src/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/UserGuide/latest/FileFormat/Bloom-Filter.md new file mode 100644 index 0000000000..558f93059c --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -0,0 +1,98 @@ + + +# Bloom Filter + +TsFile v4 stores a file-level Bloom filter in `TsFileMetadata` to test whether a timeseries path +may exist. The filter can report that a path is definitely absent or possibly present. It cannot +confirm that a path exists. + +## File Format + +```text +BloomFilter := + UVarInt byte_length + byte bits[byte_length] + UVarInt bit_count? # present when byte_length > 0 + UVarInt hash_function_count? # present when byte_length > 0 +``` + +When `byte_length = 0`, the file has no Bloom filter and the final two fields are absent. + +Bit `i` is stored at bit `i mod 8` in `bits[i / 8]`; bits within each byte are numbered from the +least-significant bit. Trailing all-zero bytes are omitted from the file. `bit_count` records the +logical number of bits. + +## Filter Input + +The filter input is the UTF-8 representation of the complete timeseries path: + +```text +UTF8(join(DeviceID segments, ".") + "." + measurement_id) +``` + +A null device segment is represented by the ASCII text `null` in the path. + +## Hashing + +Hashing uses the MurmurHash3 x64 128-bit variant. The final two 64-bit results are added and the +low 32 bits are used. If that result is the minimum 32-bit integer, the value is replaced with +zero; otherwise its absolute value is taken and reduced modulo `bit_count`. + +The hash seeds, in order, are: + +```text +5, 7, 11, 19, 31, 37, 43, 59 +``` + +`hash_function_count` selects the number of seeds used, with a maximum of 8. + +## Lookup Procedure + +For an exact path lookup, a reader: + +1. constructs the path bytes using the same `DeviceID` segment and measurement rules; +2. evaluates the first `hash_function_count` seeds; +3. tests the corresponding positions in the logical `bit_count`-bit vector; +4. returns **definitely absent** if any tested bit is clear, otherwise **possibly present**. + +A positive result must still be checked in the metadata index. A negative result is reliable only +when the query uses exactly the same path canonicalization and UTF-8 bytes as the writer. + +## Space and Probability + +Let `n` be the number of inserted paths, `m` the logical `bit_count`, and `k` the +`hash_function_count`. The usual Bloom-filter approximation is: + +```text +false_positive_probability ≈ (1 - exp(-k * n / m))^k +optimal_k ≈ (m / n) * ln(2) +``` + +These equations explain the writer's space/accuracy tradeoff; they are not additional file fields. +The serialized `bit_count` and `hash_function_count` are authoritative for reading. + +## Validation + +When `byte_length > 0`, `bit_count` must be positive, `hash_function_count` must be in the range +`1..8`, and `byte_length` must not exceed `ceil(bit_count / 8)`. Omitted trailing zero bytes are +logically reconstructed as zero. The reader rejects a bit-vector length that crosses the +`TsFileMetadata` boundary. diff --git a/src/UserGuide/latest/FileFormat/Configurations.md b/src/UserGuide/latest/FileFormat/Configurations.md new file mode 100644 index 0000000000..f329c1b6c6 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Configurations.md @@ -0,0 +1,140 @@ + + +# Configurations + +Most TsFile configurations are writer policies. They control buffering, Page boundaries, +Chunk Group boundaries, and the shape of metadata indexes, but they are not additional fields in +the file. A reader determines the actual layout from markers, length fields, Chunk Headers, and +serialized metadata. + +The names below identify format-level concepts. Individual APIs may expose different setting names, +but the effect on bytes and the reader requirement are the same. + +| Conceptual setting | Observable file effect | Setting serialized directly? | Reader needs original setting? | +| --- | --- | --- | --- | +| Chunk Group size | Where a device group is flushed | No | No | +| Page size target | Page boundaries and Statistics granularity | No | No | +| Maximum points per Page | Page boundaries | No | No | +| Metadata index fanout | Index-tree depth and node widths | No | No | +| Bloom false-positive rate | Generated bit count and hash-function count | No; generated values are serialized | No | +| Value encoding | Encoded Page stream | Yes, as `encoding_type` in each Chunk | No | +| Compression | Stored Page payload | Yes, as `compression_type` in each Chunk | No | +| Time precision and zone | Interpretation of `int64` timestamps | No | Yes, from the surrounding protocol | +| Non-aligned time encoding | Encoded time stream in each non-aligned Page | No | Yes | +| Encryption | Stored Page payload and footer properties | Properties are serialized; key context may be external | Yes, when encrypted | + +“Not serialized directly” means that the configured threshold or rate is absent as a named field. +Its result can still be visible in the physical layout. + +## Chunk Group Size + +The Chunk Group size is the amount of data that a writer buffers before it flushes the Chunks of +the current device. Larger Chunk Groups reduce marker and metadata overhead and favor sequential +I/O, but require more writer memory. Smaller Chunk Groups reduce peak memory and make data +available to the output sooner. + +The configured size is a flush threshold, not an exact on-disk size. The format does not prescribe +a default. Chunk Group boundaries are identified by the `0x00` Chunk Group Header marker and the +next top-level marker. + +## Data Page Size + +The Data Page size is a target for the in-memory data accumulated by a series writer. Smaller +Pages provide finer-grained Statistics and skipping. Larger Pages reduce Page Header overhead and +can improve compression, at the cost of more buffering and coarser skipping. + +The threshold is not an exact serialized Page size. A writer can finish a Page earlier because of +the maximum point count, an explicit flush, or aligned-page boundary requirements. The actual +Page boundary is determined by the Page Header lengths. The format does not prescribe a default +Page size. + +## Maximum Points per Page + +A writer may limit the number of logical positions in one Page independently of the Page size +target. This prevents a Page containing many compact values from growing to an excessive number +of points. The reference writer configuration uses `10,000` points. + +For aligned series, the Time Page and its corresponding Value Pages must cover the same logical +position range. A writer therefore applies a common boundary to the aligned columns. + +The configured limit is not stored as a separate field. Readers use Page boundaries and, for an +aligned Value Page, `position_count`. + +## Metadata Index Fanout + +The metadata index fanout limits the number of children in one internal metadata index node. +A larger fanout produces a shallower tree with larger nodes; a smaller fanout produces a deeper +tree with smaller nodes. The value must be at least `2`; the reference writer configuration uses +`256`. + +The configured limit is not serialized. Each metadata index node stores its node type, children, +and end offset, so a reader traverses the resulting tree without knowing the writer setting. + +## Bloom Filter False-positive Rate + +The Bloom filter false-positive rate controls the space/accuracy tradeoff for path lookup. A lower +rate uses more bits and hash functions. Standard writers constrain the value to the range `0.01` +through `0.10` and use `0.05` by default. + +The rate itself is not serialized. The resulting filter bytes, bit count, and hash-function count +are stored in `TsFileMetadata`, so a reader does not need the writer setting. See +[Bloom Filter](Bloom-Filter.md). + +## Value Encoding and Compression + +A writer may select a default value encoding by physical type and a default compression method. +These defaults are schema and writer policies rather than shared reader configuration: + +- `encoding_type` in the Chunk Header identifies the value encoding, or the time encoding for an + aligned Time Chunk. +- `compression_type` in the Chunk Header identifies how every Page in the Chunk is compressed. + +Consequently, a reader uses the Chunk Header and does not need the writer's defaults. Supported +identifiers and their payload formats are defined in [Encodings](DataPages/Encodings.md) and +[Compression](DataPages/Compression.md). + +## Time Precision and Time Zone + +A physical timestamp is a signed 64-bit integer. TsFile v4 does not record a time unit or time +zone in the file. The protocol that uses the file defines whether the precision is milliseconds, +microseconds, nanoseconds, or another unit, as well as any time-zone semantics. + +## Non-aligned Time Encoding + +The `encoding_type` in a non-aligned Chunk Header describes the value stream, not the time stream +in the same Page. Readers and writers determine the non-aligned time-stream encoding from shared +configuration. The default is `TS_2DIFF`. + +This restriction does not apply to an aligned Time Chunk. Its Chunk Header records the time +encoding in `encoding_type`. + +## Encryption + +The `encryptLevel`, `encryptType`, and `encryptKey` properties in `TsFileMetadata` describe whether +Page data is encrypted and which algorithm is used. The key material required to open an encrypted +file remains external to the file. See [Encryption](DataPages/Encryption.md). + +## Implementation Settings + +Settings for memory allocation, memory-check cadence, batch size, parallel execution, thread +count, storage backend, and close-time synchronization do not define TsFile bytes. They are +implementation and deployment concerns and are outside this file-format specification. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md b/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md new file mode 100644 index 0000000000..915e2b3af1 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md @@ -0,0 +1,56 @@ + + +# Checksumming + +TsFile v4 does not store a general Page checksum, Chunk checksum, or file checksum. + +The leading and trailing magic strings, length fields, markers, offsets, and structure boundaries +can expose some truncation or structural corruption. They do not provide content verification or +a cryptographic integrity guarantee. + +## What Structural Validation Detects + +A strict reader can detect many malformed layouts by requiring: + +- both magic strings and the version byte to be present; +- the footer length and `meta_offset` to resolve to valid boundaries; +- every `data_size`, `compressed_size`, string length, and metadata-list size to remain inside its + containing structure; +- decompression to produce exactly `uncompressed_size` bytes; +- decoded element counts, aligned bitmaps, Statistics, and index ranges to be internally + consistent. + +These checks are necessary even when an external checksum exists. They can detect truncation and +many accidental changes, but a byte substitution may preserve every length and boundary and remain +undetected. + +## Integrity at Rest or in Transit + +Systems that need corruption detection should protect the complete file with an external checksum +or object-store integrity field. Systems that need protection against intentional modification +should use an authenticated digest or signature whose key or trusted value is stored outside the +TsFile. The external value must cover the exact complete-file byte sequence, including the footer +length and trailing magic string. + +Encryption of Page payloads is a separate feature. Unless the selected encryption algorithm +explicitly authenticates its ciphertext, encryption alone does not add a TsFile checksum or verify +the unencrypted headers and metadata. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md new file mode 100644 index 0000000000..fe2be94f43 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -0,0 +1,181 @@ + + +# Chunks + +The TsFile data section starts at offset 7 and consists of marker-delimited Chunk Groups, Chunks, +and Operation Index Ranges. + +The top-level grammar is: + +```text +DataSection := (ChunkGroup | OperationIndexRange)* Separator +ChunkGroup := ChunkGroupHeader Chunk* +Separator := byte(0x02) +``` + +`Chunk*` is not length-prefixed as a group. A reader recognizes the end of a Chunk Group only +after consuming each Chunk by its `data_size` and then reading the next top-level marker. + +```text +File +└── Chunk Group a batch of data for one device + ├── Non-aligned Chunk one measurement column with its own time stream + │ └── Page(s) + └── Aligned Chunk Set measurement columns sharing a time column + ├── Time Chunk + │ └── Time Page(s) + └── Value Chunk(s) + └── Value Page(s) +``` + +## Markers + +| Value | Name | Following structure | +| ---: | --- | --- | +| `0x00` | `CHUNK_GROUP_HEADER` | `DeviceID` | +| `0x01` | `CHUNK_HEADER` | Multi-page non-aligned Chunk | +| `0x02` | `SEPARATOR` | End of data section and start of metadata section | +| `0x03` | `VERSION` | Unused since v3; identifier reserved | +| `0x04` | `OPERATION_INDEX_RANGE` | Two `int64` values | +| `0x05` | `ONLY_ONE_PAGE_CHUNK_HEADER` | Single-page non-aligned Chunk | +| `0x41` | `VALUE_CHUNK_HEADER` | Multi-page aligned Value Chunk | +| `0x45` | `ONLY_ONE_PAGE_VALUE_CHUNK_HEADER` | Single-page aligned Value Chunk | +| `0x81` | `TIME_CHUNK_HEADER` | Multi-page aligned Time Chunk | +| `0x85` | `ONLY_ONE_PAGE_TIME_CHUNK_HEADER` | Single-page aligned Time Chunk | + +The high two bits of a Chunk marker identify the column role. The low six bits identify the Page +count category: + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0x01: more than one Page +low 6 bits = 0x05: one Page or no Page +``` + +An aligned Value Chunk with `data_size = 0` can contain no Page. + +A marker is interpreted only at a known structure boundary. Marker values can occur inside an +encoded or compressed Page payload and have no structural meaning there. + +## Chunk Group Header + +```text +ChunkGroupHeader := + byte marker = 0x00 + DeviceID device_id + +DeviceID := + UVarInt segment_count + VarString segment[segment_count] +``` + +A device identifier is an ordered tuple of nullable string segments. Trailing null segments are +removed during serialization, and a device identifier cannot consist entirely of null segments. +On read, `segment_count = 0` represents a device containing one empty-string segment. + +Chunks following a Chunk Group Header belong to that device until the next Chunk Group Header, +Operation Index Range, or Separator. + +`segment_count` counts serialized tuple elements, not UTF-8 bytes. Each segment is independently +length-delimited. Device identity is the ordered tuple itself; joining segments with a display +separator is not part of the binary format and can be ambiguous when a segment contains that +separator. + +## Chunk Header + +The marker is the first field of the Chunk Header: + +```text +ChunkHeader := + byte chunk_type + VarString measurement_id + UVarInt data_size + byte data_type + byte compression_type + byte encoding_type +``` + +`data_size` is the total size of all Page Headers and stored Page data in the Chunk. It does not +include the Chunk Header. A reader consumes exactly `data_size` bytes after the Chunk Header. + +If `H` is the offset immediately after the Chunk Header, then the next structure begins at: + +```text +next_structure_offset = H + data_size +``` + +The sum of every serialized Page Header and its `compressed_size` bytes must equal `data_size`. +The value is therefore both a skip length and a containment boundary for validating Pages. + +For an aligned Time Chunk, `measurement_id` is an empty string and `data_type` is `VECTOR (0x06)`. +For an aligned Value Chunk or non-aligned Chunk, `data_type` is the measurement value type. + +Aligned data is stored Chunk by Chunk, not as interleaved Time Pages and Value Pages. The Time +Chunk stores the complete sequence of Time Pages, and each Value Chunk stores its own sequence of +Value Pages. Pages correspond by ordinal position. + +An aligned set has the following invariants: + +- The Time Chunk appears before the Value Chunks that use it. +- A Value Chunk identifies an individual measurement; the Time Chunk uses an empty + `measurement_id` and `VECTOR (0x06)` as its physical type marker. +- Value Page `i` covers the logical positions of Time Page `i`. +- A Value Chunk may be absent when the measurement has no data in the set. An existing aligned + Value Chunk can also have `data_size = 0`. + +For a non-aligned Chunk, every logical point carries a timestamp and a value. Consequently, the +decoded time stream and decoded value stream of each Page must contain the same number of +elements. + +## Chunk Scan State Machine + +A streaming or recovery reader can process the data section with this state machine: + +1. At offset 7, read one top-level marker. +2. For `0x00`, read the DeviceID and enter that Chunk Group. +3. For a Chunk marker, read its Header and advance by exactly `data_size` bytes after optionally + parsing its Pages. +4. For `0x04`, consume the following 16 bytes as an operation-index range. +5. For `0x02`, stop. The following byte belongs to the indexed metadata region. +6. Reject an unrecognized top-level marker or a structure whose declared boundary exceeds the + available bytes. + +## Operation Index Range + +```text +OperationIndexRange := + byte marker = 0x04 + int64 min_operation_index + int64 max_operation_index +``` + +This structure records the minimum and maximum operation indexes involved in the most recent +write batch. Operation indexes are storage-engine sequence numbers; they are not timestamps, +measurement indexes, or file offsets. They can support checkpoint, snapshot, backup, and recovery +logic by describing which logical operations have reached the file. + +The record is optional for a general-purpose writer. A writer without operation-index semantics +may omit it or write a placeholder range. A reader that does not use this information still has +to recognize marker `0x04` and skip the complete 16-byte payload so that the next marker is read +at the correct boundary. Multiple records can occur because multiple write batches can be +flushed into one file. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/UserGuide/latest/FileFormat/DataPages/Compression.md new file mode 100644 index 0000000000..190bf995a7 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -0,0 +1,80 @@ + + +# Compression + +The single-byte `compression_type` in a Chunk Header selects the compression format for every +Page in that Chunk. + +| Value | Name | Page data format | +| ---: | --- | --- | +| 0 | `UNCOMPRESSED` | Raw bytes | +| 1 | `SNAPPY` | Snappy raw block | +| 2 | `GZIP` | One gzip member | +| 3 | `LZO` | LZO block | +| 7 | `LZ4` | LZ4 raw block | +| 8 | `ZSTD` | Zstandard frame | +| 9 | `LZMA2` | LZMA2 stream in an XZ container | + +Each Page is compressed independently; compression state is not shared across Pages. The Page +Header's `uncompressed_size` and `compressed_size` record the payload lengths before and after +compression, respectively. + +The numeric values are stable wire identifiers rather than the ordinal positions of a compressor +library. Values `4` through `6` are not assigned by this version. `LZO` is retained as a historical +identifier; a reader that does not provide the codec cannot decode that Chunk. + +## Page-local Framing + +If a Page begins at offset `P` and its serialized Page Header occupies `H` bytes, the stored payload +is exactly the half-open interval: + +```text +[P + H, P + H + compressed_size) +``` + +For `UNCOMPRESSED`, `compressed_size` equals `uncompressed_size` and the bytes are already the +encoded Page payload. For every other method, decompression must produce exactly +`uncompressed_size` bytes. A shorter result, a longer result, or a codec request for bytes outside +the interval makes the Page invalid. + +Because each Page is an independent compression unit, a reader can use Page Statistics to reject a +Page and advance by `compressed_size` without initializing the codec. Random access does not need +the preceding Page's dictionary or history. + +## Read Pipeline and Validation + +The Page Header is read before the payload, so the reader knows both bounds before allocating or +decompressing: + +```text +read Page Header + -> read exactly compressed_size bytes + -> decrypt when file properties require it + -> decompress according to compression_type + -> require exactly uncompressed_size bytes + -> decode the Page streams +``` + +Implementations should reject sizes that overflow offset arithmetic or exceed configured resource +limits before allocating memory. An unsupported `compression_type` prevents decoding the affected +Chunk, but the Chunk remains skippable through its `data_size`; it must not be interpreted as +`UNCOMPRESSED`. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Encodings.md b/src/UserGuide/latest/FileFormat/DataPages/Encodings.md new file mode 100644 index 0000000000..3eaf1190cf --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Encodings.md @@ -0,0 +1,111 @@ + + +# Encodings + +The single-byte `encoding_type` in a Chunk Header selects the value-stream encoding. In an +aligned Time Chunk, it selects the time-stream encoding. In a non-aligned Chunk, it selects only +the value-stream encoding. + +| Value | Name | +| ---: | --- | +| 0 | `PLAIN` | +| 1 | `DICTIONARY` | +| 2 | `RLE` | +| 3 | `DIFF` | +| 4 | `TS_2DIFF` | +| 5 | `BITMAP` | +| 6 | `GORILLA_V1` | +| 7 | `REGULAR` | +| 8 | `GORILLA` | +| 9 | `ZIGZAG` | +| 10 | `FREQ` (deprecated) | +| 11 | `CHIMP` | +| 12 | `SPRINTZ` | +| 13 | `RLBE` | +| 14 | `CAMEL` | + +The following data-type and encoding combinations are defined: + +| Data type | Available encodings | +| --- | --- | +| `BOOLEAN` | `PLAIN`, `RLE` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `FLOAT` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `DOUBLE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE`, `CAMEL` | +| `TEXT`, `STRING` | `PLAIN`, `DICTIONARY` | +| `BLOB`, `OBJECT` | `PLAIN` | + +## Encoding Families + +The identifiers describe different ways to transform a logical sequence into bytes. They are wire +format identifiers, not implementation-specific enumeration ordinals. + +| Family | Encodings | Data property typically exploited | +| --- | --- | --- | +| Literal | `PLAIN` | No prediction; values are serialized independently | +| Dictionary | `DICTIONARY` | Repeated binary values are replaced by dictionary references | +| Run length / bit packing | `RLE`, `RLBE` | Repeated values or small bit widths | +| Delta and prediction | `DIFF`, `TS_2DIFF`, `REGULAR`, `SPRINTZ` | Small or regular changes between adjacent values | +| Integer remapping | `ZIGZAG` | Small signed integers represented as small unsigned integers | +| XOR-based floating-point | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | Stable leading and trailing bits between adjacent floating-point values | +| Legacy or specialized | `BITMAP`, `FREQ` | Reserved historical layouts; `FREQ` is deprecated | + +The best encoding depends on the data distribution. The file does not record why a writer selected +an encoding, and a reader must select its decoder from both `data_type` and `encoding_type`. + +## Plain Encoding + +A `PLAIN` stream does not store an element count. The Page context, time stream, or null bitmap +determines the number of elements. Values are written consecutively: + +| Data type | Encoding of one value | +| --- | --- | +| `BOOLEAN` | `0x00` or `0x01` | +| `INT32`, `DATE` | `VarInt` | +| `INT64`, `TIMESTAMP` | Big-endian `int64` | +| `FLOAT` | Big-endian `float32` | +| `DOUBLE` | Big-endian `float64` | +| `TEXT`, `STRING`, `BLOB`, `OBJECT` | `VarInt byte_length` + raw bytes | + +The length of a `PLAIN` binary value is non-negative. A non-aligned value stream does not contain +nulls; the Value Page bitmap represents nulls in an aligned value stream. + +The other encoding identifiers select their respective value-stream bit layouts. A reader cannot +decode an unknown encoding as `PLAIN`. + +## Decoder Contract + +Encoding state is scoped to one uncompressed Page stream. After Page decompression, a reader: + +1. creates a decoder for the Chunk's `data_type` and `encoding_type`; +2. decodes exactly the number of logical elements required by the Page context; +3. verifies that decoding neither reads past the Page boundary nor leaves an invalid partial block; +4. discards the decoder state before processing the next Page. + +For a non-aligned Page, the time stream has its own encoding declared by `time_encoder` in the file +metadata; `encoding_type` applies to the value stream only. For an aligned Time Page, +`encoding_type` applies to the Page's time stream. For an aligned Value Page, null positions are +removed before value encoding, so the value decoder must produce exactly the number of set bits in +the bitmap. + +An unsupported encoding prevents decoding the affected Chunk, but its byte range can still be +skipped using `data_size`. Falling back to another decoder is not valid. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md new file mode 100644 index 0000000000..218bf98736 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -0,0 +1,76 @@ + + +# Encryption + +Encryption applies only to the compressed Page `stored_data`. Page Headers, Chunk Headers, +Statistics, Metadata Indexes, and File Metadata are not encrypted. + +The write pipeline is: + +```text +encoded page payload + -> compression + -> encryption + -> stored_data +``` + +Reading decrypts before decompression and decoding. + +## File Properties + +The property map in `TsFileMetadata` uses these keys to describe encryption: + +| Property | Meaning | +| --- | --- | +| `encryptLevel` | `0` for no encryption; `1` for a directly stored data key; `2` for a data key encrypted by an external master key | +| `encryptType` | Encryption and decryption algorithm identifier | +| `encryptKey` | Text representation of the data key or key-wrapping result | + +A Page Header has no separate ciphertext-length field. The encrypted result has the same length +as the compressed input, so `compressed_size` remains the length of `stored_data` in the file. + +When `encryptLevel` is absent, it is interpreted as `0` for compatibility. For levels `1` and `2`, +both `encryptType` and a non-empty `encryptKey` are required. An unknown level or unavailable +algorithm makes the encrypted Page data unreadable; readers must not silently treat it as +unencrypted. + +## Key Interpretation + +At level `1`, `encryptKey` represents the data key used with `encryptType`. At level `2`, it +represents a wrapped data key: the reader obtains the external master-key decryptor, unwraps the +data key, and then decrypts Page payloads with `encryptType`. Key derivation, master-key storage, +rotation, and access control are outside the file format. + +## Security Scope + +Page encryption protects the confidentiality of encoded sample data. It does not hide the file +shape, device and measurement identifiers, types, time ranges, value Statistics, schemas, indexes, +or the encryption properties themselves. Applications should treat this metadata as visible. + +TsFile v4 does not define a general authentication tag for Pages or for the file. Whether a +ciphertext modification is detected depends on the selected algorithm; the structural validation +and external integrity guidance in [Checksumming](Checksumming.md) still apply. + +Readers should resolve the encryption properties and key material before allocating Page buffers, +then apply the exact pipeline `read -> decrypt -> decompress -> decode`. A decryption failure, +wrong output length, unknown algorithm, or missing key is an error for the affected data and must +not trigger plaintext fallback. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md b/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md new file mode 100644 index 0000000000..2876e7cdb7 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md @@ -0,0 +1,87 @@ + + +# Error Recovery + +A completed TsFile contains the leading magic string, version byte, `0x02` separator, footer +metadata, 4-byte `TsFileMetadata` length, and trailing magic string. A file without the footer +length or trailing magic string is incomplete. + +A recovery reader can scan the data section from offset 7 by markers and use the `data_size` in +each Chunk Header to skip a complete Chunk. Within a Chunk, the sum of the serialized Page lengths +equals `data_size`. + +A scanner cannot treat an arbitrary `0x02` byte in Page data as a separator. It may read the next +marker only after completely parsing or skipping the structure delimited by the preceding marker. + +## Completion Check + +Given file size `F` and the big-endian footer length `L` stored at `[F - 10, F - 6)`, a completed +file must satisfy all of the following before ordinary metadata-first reading begins: + +```text +bytes[F - 6, F) = "TsFile" +TsFileMetadata interval = [F - 10 - L, F - 10) +meta_offset = offset of the top-level 0x02 separator +meta_offset < F - 10 - L +``` + +The leading magic and version must also be valid, and the `TsFileMetadata` decoder must consume +exactly `L` bytes. A valid tail magic by itself does not prove that the file is complete. + +## Safe Forward Scan + +A recovery scanner can identify the longest structurally complete prefix of the data section: + +1. validate the leading magic and version and set the cursor to offset `7`; +2. at a known top-level boundary, read one marker; +3. for a Chunk Group Header (`0x00`), decode its complete `DeviceID` and remember the group context; +4. for a Chunk marker, decode the complete Chunk Header, validate `cursor + data_size`, then either + skip that range or validate all Page boundaries inside it; +5. for an Operation Index Range (`0x04`), consume exactly two big-endian `int64` values; +6. stop successfully at the top-level separator (`0x02`), or stop at the last known-good boundary + when a field or payload is incomplete. + +The last known-good boundary is the end of a complete delimited structure, not the last byte that +happened to be readable. A recovery tool may use the complete Chunks before that boundary to +rebuild indexes and a new footer; it must not invent or partially decode the truncated Chunk. + +The file is incomplete or corrupt when any of these conditions is detected: + +- A Chunk's `data_size` extends beyond the file or the next structure boundary. +- A Page length extends beyond its Chunk boundary. +- The footer length cannot locate a complete `TsFileMetadata` structure. +- `meta_offset` does not point to the `0x02` separator. +- An index offset is outside the file or points into the middle of a structure. + +Other failures include an unknown top-level marker, an unterminated VarInt or VarString, integer +overflow while computing an end offset, a Page decoder that produces the wrong number of elements, +and an aligned Time/Value Page mismatch. + +## Recovery Limits + +Recovery reconstructs structure, not missing information. A complete compressed or encrypted Page +still requires its declared codec and, where applicable, the external key. Without a file-level +checksum, a structurally valid byte substitution may be indistinguishable from original data. +Operation Index Range records can help a storage engine relate the salvaged prefix to its external +operation log, but their meaning does not replace Chunk and Page boundary validation. + +TsFile v4 has no general checksum. See [Checksumming](Checksumming.md) for the resulting limits. diff --git a/src/UserGuide/latest/FileFormat/DataPages/README.md b/src/UserGuide/latest/FileFormat/DataPages/README.md new file mode 100644 index 0000000000..97e5add290 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/README.md @@ -0,0 +1,179 @@ + + +# Data Pages + +One or more Pages follow a Chunk Header without padding between Pages. The Chunk Header's +`data_size` is the total byte length of all Page Headers and stored Page data in that Chunk. + +## Page Header + +A non-empty Page has the following format: + +```text +Page := + UVarInt uncompressed_size + UVarInt compressed_size + Statistics statistics? # determined by the Chunk marker + byte stored_data[compressed_size] +``` + +`uncompressed_size` is the size of the Page payload after decryption and decompression. +`compressed_size` is the size of the Page payload stored in the file. The two values are equal +when `compression_type = UNCOMPRESSED`. + +The low six bits of the Chunk marker determine whether the Page Header contains Statistics: + +- `0x01`: the Chunk has multiple Pages; every non-empty Page Header contains Statistics. +- `0x05`: the Chunk has one Page or no Page; the Page Header does not contain Statistics. + +See [Metadata](../Metadata.md#statistics) for the binary format of Statistics. + +The Header shape can therefore be selected without inspecting the Page payload: + +| Chunk marker low bits | Page count | Page Statistics | +| ---: | --- | --- | +| `0x01` | More than one | Present in every non-empty Page Header | +| `0x05` | Zero or one | Absent | + +If a Page starts at `P`, let `h` be the serialized size of its two length fields and optional +Statistics. The stored payload occupies the half-open interval `[P + h, P + h + compressed_size)` +and the next Page, if any, starts at `P + h + compressed_size`. That end position must not exceed +the enclosing Chunk boundary. + +After decryption and decompression, the payload must contain exactly `uncompressed_size` bytes. +A decoder rejects a payload whose produced size differs, even if the compression library reports +successful termination. + +## Page Encoding + +When writing a non-empty Page, the data passes through these stages: + +```text +logical values + -> value encoding + -> assemble uncompressed page payload + -> compression + -> optional encryption + -> stored_data +``` + +Reading applies the inverse order: decrypt, decompress, then decode values. + +Encoding and compression are independent. The encoding converts typed values into an +uncompressed byte stream; compression treats that stream as bytes. Page boundaries reset both +the value encoder/decoder and the compressor/decompressor, so no state is carried from one Page +to the next. + +## Non-aligned Page + +The uncompressed payload of a non-aligned Page is: + +```text +NonAlignedPagePayload := + UVarInt time_stream_size + byte encoded_time_stream[time_stream_size] + byte encoded_value_stream[remaining bytes] +``` + +A non-aligned Page stores both time and values. It does not use a separate Time Page followed by +a Value Page. The complete time stream is stored first, followed by the complete value stream; +`timestamp, value` pairs are not interleaved point by point. The two streams contain the same +number of elements and pair by position to form `(timestamp, value)` points. + +For an uncompressed payload of size `U` and a serialized `time_stream_size` field of `v` bytes, +the value stream length is: + +```text +value_stream_size = U - v - time_stream_size +``` + +The result must be non-negative. After decoding, both stream decoders must reach their boundaries +and produce the same point count. A trailing byte, an early end, or unequal counts is a malformed +Page. + +A non-aligned Chunk Header records only the value stream's `encoding_type`. See +[Configurations](../Configurations.md#non-aligned-time-encoding) for the time-stream encoding. + +## Aligned Time Page + +The uncompressed payload of an aligned Time Page consists entirely of the encoded time stream: + +```text +AlignedTimePagePayload := byte encoded_time_stream[remaining bytes] +``` + +The `encoding_type` in the Time Chunk Header identifies the time encoding. + +The number of decoded timestamps is the logical position count of this Page. That count controls +the bitmap width and null reconstruction of every corresponding aligned Value Page. + +## Aligned Value Page + +The uncompressed payload of a non-empty aligned Value Page is: + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +Within one aligned set, Value Page number `i` corresponds to Time Page number `i`. See +[Nulls](../Nulls.md) for the null bitmap and the all-null empty Page representation. + +`position_count` must equal the decoded timestamp count of the corresponding Time Page. The +bitmap has exactly `ceil(position_count / 8)` bytes; unused low-order bits in its last byte do not +describe positions. The value decoder is invoked only for set bits and must produce exactly the +number of set bits. + +## Page Selection and Skipping + +Page Statistics allow a reader to evaluate time and value predicates before reading stored Page +data. A Page can be skipped only when its Statistics prove that no point can satisfy the +predicate. Statistics are conservative summaries: overlap means “possibly matches,” not +“definitely matches.” + +For a one-Page Chunk, the Page Header omits Statistics. The equivalent Chunk or Timeseries +Statistics in the footer can be used before seeking to the Chunk. If Statistics for a data type do +not contain the requested summary, the reader must decode the Page or rely on another predicate. + +## Empty Aligned Value Page + +An aligned value column can be null for every position in a Time Page. This case is encoded as a +single unsigned-varint zero: + +```text +EmptyAlignedValuePage := UVarInt(0) # uncompressed_size +``` + +There is no `compressed_size`, Page Statistics, bitmap, or stored payload after that zero. The +position count is inherited from the corresponding Time Page. This representation is valid only +where the aligned Page sequence supplies that external position count. + +## Data Page Details + +- [Chunks](Chunks.md) +- [Compression](Compression.md) +- [Encodings](Encodings.md) +- [Encryption](Encryption.md) +- [Checksumming](Checksumming.md) +- [Error Recovery](Error-Recovery.md) diff --git a/src/UserGuide/latest/FileFormat/Extensibility.md b/src/UserGuide/latest/FileFormat/Extensibility.md new file mode 100644 index 0000000000..a3059220f4 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Extensibility.md @@ -0,0 +1,68 @@ + + +# Extensibility + +TsFile v4 provides numbered or self-describing extension points in the following locations: + +- Data types, encodings, and compression methods use single-byte identifiers. +- A length-delimited string property map is stored at the end of `TsFileMetadata`. +- The file version byte selects the complete file layout. + +A new data type, encoding, or compression method requires a new identifier. The wire meaning of +an existing identifier does not change. + +Each footer property consists of a `VarString` key and value. A reader can consume an +unrecognized property without changing the boundaries of the other footer fields. + +Data-section markers do not have a common length field. A reader cannot safely skip an unknown +marker. A new data-section structure that cannot be fully delimited by an existing marker +therefore requires either a new file version or an encoding contained within an existing +structure. + +## Safe Evolution Rules + +An extension is safely skippable only when an older reader can determine its complete byte range +without interpreting the new payload. Examples are a new footer property value and a new codec +inside a Chunk whose `data_size` is known. Adding bytes to a non-length-delimited header is not +skippable because every following field would move. + +Within v4, extension authors must: + +- allocate a previously unassigned identifier and never reuse an existing value; +- preserve byte order, length interpretation, and containment boundaries of existing structures; +- define which physical data types are valid for a new encoding; +- define output-size, element-count, and malformed-input behavior for new codecs; +- require an explicit capability error when a reader cannot decode the extension. + +Reserved or unknown identifiers are not aliases for a default. In particular, an unknown encoding +is not `PLAIN`, an unknown compression method is not `UNCOMPRESSED`, and an unknown marker is not +padding. + +## Properties and Semantics + +Length delimitation makes an unknown footer property structurally readable, but it does not make +the property's semantics optional. A property that changes how existing bytes must be interpreted +must define how readers advertise and enforce support. Security-critical properties must fail +closed when recognized semantics cannot be applied. + +Changes that alter the grammar of top-level structures, reinterpret assigned identifiers, or make +old boundaries ambiguous require a new format version. diff --git a/src/UserGuide/latest/FileFormat/Format-Versions.md b/src/UserGuide/latest/FileFormat/Format-Versions.md new file mode 100644 index 0000000000..45eb8cc0d1 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Format-Versions.md @@ -0,0 +1,70 @@ + + +# Format Versions + +The TsFile format version follows the leading magic string: + +```text +6-byte magic string "TsFile" +1-byte format version +``` + +This documentation describes version byte `0x04`. After validating the leading magic string, a +reader selects the corresponding file layout from the version byte. + +Version 4 uses segmented `DeviceID` values and stores per-table device-index roots and Table +Schemas in `TsFileMetadata`. Version 3 uses a single-string device identifier and a different +footer layout; it cannot be parsed with the version 4 `DeviceID` or `TsFileMetadata` structures. + +## Reader Dispatch + +The version byte is interpreted before any data-section marker. A reader therefore follows this +order: + +```text +validate leading magic -> read version -> select complete version grammar -> parse data/footer +``` + +An implementation that supports version 4 but not the observed byte must report an unsupported +version. It must not probe the file as v4 based on a familiar marker or footer suffix, because the +meaning and width of structures may differ between versions. + +## Compatibility Contract + +Compatibility is defined at the byte-format level: + +- A v4 reader may read a v4 file when it supports every identifier required by the selected data. +- A v4 writer emits the v4 grammar described by this documentation, independent of its programming + language or in-memory classes. +- Unknown footer properties can be consumed because each key and value is length-delimited. +- Unknown data-section markers cannot generally be skipped because markers have no common payload + length. +- Numeric identifiers already assigned to types, encodings, compression methods, markers, and + index-node kinds retain their meanings within v4. + +Support for a version does not imply support for every optional codec or encryption algorithm. +When such a feature is unavailable, the containing Chunk may remain structurally skippable, but +its sample values are not decodable. + +The project release series and the on-disk format byte are separate version spaces. A product in +the 2.x release series can read and write format byte `0x04`; the release label is not serialized +in the TsFile header. diff --git a/src/UserGuide/latest/FileFormat/Metadata.md b/src/UserGuide/latest/FileFormat/Metadata.md new file mode 100644 index 0000000000..50d49e464d --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Metadata.md @@ -0,0 +1,284 @@ + + +# Metadata + +TsFile metadata falls into two categories: + +- Chunk Headers, Page Headers, and Page Statistics stored with the data. +- Timeseries Metadata, Metadata Indexes, and File Metadata stored after the data section. + +Footer metadata starts after `SEPARATOR (0x02)`. Ranges of `TimeseriesMetadata` and measurement +index nodes may be interleaved. The `offset` and `end_offset` fields in the indexes delimit each +range. Every offset is an absolute byte offset measured from the beginning of the file. + +The metadata hierarchy supports progressively finer pruning: + +| Level | Identifies | Primary use | +| --- | --- | --- | +| File metadata | Tables and device-index roots | Enter the correct index tree | +| Device index | Devices | Locate one device's measurement index | +| Measurement index | Measurement-name ranges | Locate Timeseries Metadata | +| Timeseries Metadata | One logical series | Select Chunk offsets and series Statistics | +| Chunk Metadata | One physical Chunk | Seek to data and prune by Chunk Statistics | +| Page Header | One Page | Prune by Page Statistics and read its payload | + +## Statistics + +Page-, Chunk-, and timeseries-level Statistics use the same type-specific structure: + +```text +Statistics := + UVarInt count + int64 start_time + int64 end_time + ... type-specific fields +``` + +`count` is the number of values covered by the structure. It equals the number of points in a +non-aligned series, the number of non-null values in an aligned value column, or the number of +timestamps in an aligned time column. + +Type-specific fields immediately follow the common prefix in the order shown below: + +| Data type | Type-specific fields | +| --- | --- | +| `BOOLEAN` | `bool first`, `bool last`, `int64 true_count` | +| `INT32`, `DATE` | `int32 min`, `int32 max`, `int32 first`, `int32 last`, `int64 sum` | +| `INT64`, `TIMESTAMP` | `int64 min`, `int64 max`, `int64 first`, `int64 last`, `float64 sum` | +| `FLOAT` | `float32 min`, `float32 max`, `float32 first`, `float32 last`, `float64 sum` | +| `DOUBLE` | `float64 min`, `float64 max`, `float64 first`, `float64 last`, `float64 sum` | +| `TEXT` | `Binary32 first`, `Binary32 last` | +| `STRING` | `Binary32 first`, `Binary32 last`, `Binary32 min`, `Binary32 max` | +| `BLOB`, `OBJECT`, `VECTOR` | No type-specific fields | + +Statistics do not carry a data type identifier. The type used to parse Statistics comes from the +Chunk Header or Timeseries Metadata. + +`first` and `last` describe value order, not the minimum and maximum. For an ordered timeseries, +they correspond to the values at the first and last covered timestamps. The `start_time` and +`end_time` fields are inclusive. Numeric `sum` fields are intended for aggregation; notably, an +`INT64` sum is stored as `float64` and can lose integer precision for large magnitudes. + +Statistics are conservative pruning metadata. A reader may skip a structure when its time range +does not intersect the query range, or when a supported value summary proves that no value can +match. It must not infer a match merely from overlapping ranges. Types without minimum/maximum +fields, such as `BLOB` and `OBJECT`, cannot support range pruning through Statistics. + +Statistics at a parent level summarize all values represented by that parent. For a series with +multiple Chunks, series Statistics cover the union of the Chunk Statistics. For a multi-Page +Chunk, Chunk Statistics cover the union of Page Statistics. Counts and time bounds must be +consistent with those lower-level records. + +## Timeseries Metadata + +Each timeseries has one `TimeseriesMetadata` structure: + +```text +TimeseriesMetadata := + byte timeseries_type + VarString measurement_id + byte data_type + UVarInt chunk_metadata_list_size + Statistics series_statistics + byte chunk_metadata_list[chunk_metadata_list_size] +``` + +The high bits of `timeseries_type` identify the column role, and the low six bits identify the +Chunk Metadata layout: + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0: exactly one Chunk; ChunkMetadata omits Statistics +low 6 bits = 1: more than one Chunk; every ChunkMetadata contains Statistics +``` + +## Chunk Metadata + +When `timeseries_type & 0x3f == 0`, the timeseries has exactly one Chunk: + +```text +ChunkMetadata := int64 offset_of_chunk_header +``` + +The Chunk uses `series_statistics` as its Statistics. + +When `timeseries_type & 0x3f != 0`: + +```text +ChunkMetadata := + int64 offset_of_chunk_header + Statistics chunk_statistics +``` + +`offset_of_chunk_header` points to the Chunk marker, which is the first byte of the Chunk Header. + +`chunk_metadata_list_size` is a byte boundary, not a Chunk count. A reader first computes the end +of the list, then deserializes Chunk Metadata records until it reaches exactly that end. Ending +early, crossing the boundary, or leaving trailing bytes is a format error. + +## Metadata Index + +```text +MetadataIndexNode := + UVarInt child_count + MetadataEntry child[child_count] + int64 end_offset + byte node_type +``` + +`node_type` determines the node level, Entry encoding, and referenced data: + +| Value | Name | Entry encoding | Entry refers to | +| ---: | --- | --- | --- | +| 0 | `INTERNAL_DEVICE` | `DeviceIndexEntry` | A child device-index node | +| 1 | `LEAF_DEVICE` | `DeviceIndexEntry` | The measurement-index root of a device | +| 2 | `INTERNAL_MEASUREMENT` | `MeasurementIndexEntry` | A child measurement-index node | +| 3 | `LEAF_MEASUREMENT` | `MeasurementIndexEntry` | A range of Timeseries Metadata | + +The Entry structures are: + +```text +DeviceIndexEntry := + DeviceID first_device_id + int64 offset + +MeasurementIndexEntry := + VarString first_measurement_id + int64 offset +``` + +The `offset` of an Entry is the start of its range. Its end is the `offset` of the next Entry; the +last Entry ends at the node's `end_offset`. + +```text +entry[0].offset <= entry[1].offset <= ... <= end_offset +``` + +An Entry key is the first key in the range it covers. Device indexes compare `DeviceID` values +segment by segment, with null before non-null and a shorter tuple before a longer tuple when their +common prefixes are equal. Measurement indexes use lexicographic measurement-name order. + +### Index Lookup + +To find key `K` in an index node, select the last Entry whose first key is less than or equal to +`K`. The selected byte range begins at that Entry's `offset` and ends at the next Entry's offset, +or at the node's `end_offset` for the last Entry. An internal node range contains another index +node; a leaf range contains the object named in the table above. + +An index reader validates the node before following it: + +- `child_count` is consistent with the number of complete Entries in the node. +- Entry keys are sorted according to the node's key comparator. +- Entry offsets are monotonic and each referenced range is non-negative. +- The last Entry offset does not exceed `end_offset`. +- Child nodes have the level expected by the parent node type. + +The first-key representation allows an implementation to binary-search Entries within a node. +The file format does not require a particular in-memory search algorithm. + +## File Metadata + +`TsFileMetadata` immediately precedes the file's trailing length field: + +```text +TsFileMetadata := + UVarInt table_index_count + TableIndexRoot table_index[table_index_count] + UVarInt table_schema_count + NamedTableSchema table_schema[table_schema_count] + int64 meta_offset + BloomFilter bloom_filter + VarInt property_count + Property property[property_count] + +TableIndexRoot := + VarString table_name + MetadataIndexNode device_index_root + +NamedTableSchema := + VarString table_name + TableSchema schema + +Property := + VarString key + VarString value +``` + +`meta_offset` points to the `0x02` separator at the end of the data section. There is no marker +before `TsFileMetadata`; its starting position is determined by the 4-byte length stored at the +end of the file. + +The physical order of map entries has no file-format meaning. `table_index` contains device-index +roots for both tree-model and table-model data. `table_schema` contains schemas for the explicit +table model. + +The trailing four-byte metadata length is outside `TsFileMetadata`. For a file length `F` and +metadata length `L`, a reader obtains the exact byte interval using: + +```text +file_metadata_start = F - 6 - 4 - L +file_metadata_end = F - 6 - 4 # exclusive +``` + +The reader must consume exactly `L` bytes. This check detects a schema mismatch or malformed +length-delimited field before the trailing magic is accepted as the file terminator. + +## Table Schema + +```text +TableSchema := + UVarInt column_count + TableColumn column[column_count] + +TableColumn := + MeasurementSchema schema + int32 column_category + +MeasurementSchema := + String32 column_name + byte data_type + byte encoding_type + byte compression_type + int32 property_count + SchemaProperty property[property_count] + +SchemaProperty := + String32 key + String32 value +``` + +`MeasurementSchema` uses `String32`, rather than the `VarString` used by most other footer fields. + +Column category identifiers are: + +| Value | Category | +| ---: | --- | +| 0 | `TAG` | +| 1 | `FIELD` | +| 2 | `ATTRIBUTE` | +| 3 | `TIME` | + +Column order is significant because table records and aligned value columns refer to the schema +positionally. `TAG` columns participate in device identity, `FIELD` columns hold measured values, +`ATTRIBUTE` columns describe entities, and `TIME` identifies the time column. The category does +not replace `data_type`; both fields are required to interpret a column. diff --git a/src/UserGuide/latest/FileFormat/Nulls.md b/src/UserGuide/latest/FileFormat/Nulls.md new file mode 100644 index 0000000000..f7b9914e15 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Nulls.md @@ -0,0 +1,71 @@ + + +# Nulls + +A non-aligned timeseries stores only the `(timestamp, value)` points that exist. Its Pages do not +contain null markers. + +An aligned value column records nulls in the bitmap of each Value Page: + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +Position `i` corresponds to bit `7 - (i mod 8)` of `present_bitmap[i / 8]`: + +- A set bit means that the position has a value. +- A clear bit means that the position is null. + +The encoded value stream contains only values whose bitmap bits are set, in position order. +`position_count` equals the number of timestamps in the corresponding Time Page. + +For example, with `position_count = 10`, bitmap bytes `10110000 01000000` indicate values at +positions `0`, `2`, `3`, and `9`. The value decoder therefore produces four values and places them +at those positions; the remaining six positions are null. Unused low-order bits in the final byte +are not logical positions and are ignored. + +For a non-empty Value Page the following invariants hold: + +```text +bitmap_bytes = ceil(position_count / 8) +decoded_value_count = popcount(present_bitmap over position_count bits) +position_count = decoded_timestamp_count of the corresponding Time Page +``` + +A mismatch is structural corruption; the reader must not infer additional positions from trailing +bitmap bits or encoded values. + +If an entire Value Page is null, the Page consists only of an `uncompressed_size` equal to zero: + +```text +EmptyPage := UVarInt(0) +``` + +An empty Page has no `compressed_size`, Statistics, bitmap, or value stream. Its position count is +obtained from the corresponding Time Page. + +An absent Value Chunk and an empty Value Page are different. An absent Chunk means that the +measurement has no physical column in that Chunk Group. An empty Page preserves the Page-to-Page +alignment of an existing value column while stating that every position in that interval is null. diff --git a/src/UserGuide/latest/FileFormat/README.md b/src/UserGuide/latest/FileFormat/README.md new file mode 100644 index 0000000000..62f928339b --- /dev/null +++ b/src/UserGuide/latest/FileFormat/README.md @@ -0,0 +1,152 @@ + + +# File Format + +This document describes the language-independent binary format of TsFile version 4. The file +version byte is `0x04`. + + + +## Scope and Terminology + +This specification defines the bytes required to exchange a completed TsFile v4 between +independent implementations. It distinguishes four levels of organization: + +- A **Page** is the smallest independently compressed and decoded data unit. +- A **Chunk** contains the Pages of one physical column and fixes their data type, encoding, and + compression method. +- A **Chunk Group** associates a sequence of Chunks with one device identifier. +- A **file** contains the sequential data section followed by indexes and file-level metadata. + +“Aligned” describes measurements that share one physical time column. “Non-aligned” describes a +measurement that carries its own time stream. Alignment changes the Page payload layout; it does +not change the meaning of a timestamp or value. + +All byte intervals in this specification are inclusive unless expressed as a half-open range. +An absolute offset is measured from the first byte of the file. A serialized length excludes the +field that stores that length unless stated otherwise. + +## Physical Regions + +For a file of length `F`, let `L` be the four-byte length of `TsFileMetadata` and let +`meta_offset` be the value stored inside `TsFileMetadata`: + +| Region | Inclusive byte interval | How it is located | +| --- | --- | --- | +| Header magic | `0 … 5` | Fixed position | +| Version | `6` | Fixed position | +| Data section | `7 … meta_offset - 1` | Ends immediately before the separator | +| Separator | `meta_offset` | `0x02`, referenced by `TsFileMetadata` | +| Indexed metadata | `meta_offset + 1 … F - 11 - L` | Delimited by metadata offsets | +| `TsFileMetadata` | `F - 10 - L … F - 11` | Located from the trailing length | +| Metadata length | `F - 10 … F - 7` | Fixed position relative to the end | +| Tail magic | `F - 6 … F - 1` | Fixed position relative to the end | + +The indexed metadata interval can contain Timeseries Metadata and Metadata Index Nodes in an +implementation-dependent physical order. Their absolute offsets, rather than adjacency, define +their relationships. + +## Write Model + +The data is divided into Chunk Groups by device. A Chunk stores one measurement column and +contains one or more Pages. A non-aligned Chunk stores its time stream and value stream in the +same Page. Aligned data stores one shared time column in a Time Chunk and the measurement columns +in Value Chunks. + +File metadata follows the data, so a writer can write all Chunk Groups sequentially before +writing the indexes and footer. A reader normally starts at the end of the file, uses the +`TsFile Metadata size` to locate `TsFileMetadata`, and follows its indexes to the required Chunks. + +Writers may choose Page sizes, Chunk Group flush thresholds, metadata-index fanout, encodings, +and compression methods. Those choices affect performance and physical boundaries, but readers +derive the resulting layout entirely from the bytes in the file. + +## Metadata-first Read Algorithm + +A random-access reader can open a completed file without scanning the data section: + +1. Verify the six-byte tail magic. +2. Read `L` at `F - 10` and compute `file_metadata_pos = F - 10 - L`. +3. Deserialize `TsFileMetadata`, then verify that `meta_offset` points to `0x02`. +4. Optionally test the complete path with the Bloom filter. A negative result proves absence; a + positive result requires index lookup. +5. Traverse the table/device index and then the measurement index to locate the requested + Timeseries Metadata range. +6. Read the Chunk Metadata offsets selected by the time and value predicates. +7. Seek to each Chunk Header, use Page Statistics when present, and decode only the required + Pages. + +This path makes metadata size proportional to the selected devices, measurements, Chunks, and +Pages rather than to the size of the complete file. + +## Structural Invariants + +A conforming reader validates at least the following relationships before allocating or reading +an externally supplied length: + +- The head and tail magic strings are identical and the version byte is supported. +- `0 <= L <= F - 10` and the computed `file_metadata_pos` is after `meta_offset`. +- `meta_offset` is an in-file absolute offset whose byte is `0x02`. +- Every Chunk ends exactly `data_size` bytes after its Chunk Header. +- Every non-empty Page fits inside its Chunk; its stored payload contains exactly + `compressed_size` bytes. +- Metadata entry offsets are monotonic within a node and do not exceed `end_offset`. +- An aligned Value Page corresponds to the Time Page at the same ordinal position; its + `position_count` equals the number of positions in that Time Page. + +Length or offset violations are format errors. A reader must not use a marker byte found inside a +length-delimited Page payload as a top-level marker. + +The file contains exactly one top-level `0x02` separator that terminates the data section. It precedes all +metadata and is referenced by `meta_offset`. There is no separate separator or marker immediately +before `TsFileMetadata`; its position is determined by the length field at the end of the file. + +For a file of length `F` whose serialized `TsFileMetadata` has length `L`: + +```text +metadata_size_pos = F - 6 - 4 +file_metadata_pos = F - 6 - 4 - L +``` + +The magic bytes at both the beginning and the end of the file are: + +```text +54 73 46 69 6c 65 # "TsFile" +``` + +## Format Details + +- [Binary Representation](Binary-Representation.md) +- [Configurations](Configurations.md) +- [Extensibility](Extensibility.md) +- [Metadata](Metadata.md) +- [Types](Types.md) +- [Bloom Filter](Bloom-Filter.md) +- [Data Pages](DataPages/README.md) + - [Chunks](DataPages/Chunks.md) + - [Compression](DataPages/Compression.md) + - [Encodings](DataPages/Encodings.md) + - [Encryption](DataPages/Encryption.md) + - [Checksumming](DataPages/Checksumming.md) + - [Error Recovery](DataPages/Error-Recovery.md) +- [Nulls](Nulls.md) +- [Format Versions](Format-Versions.md) diff --git a/src/UserGuide/latest/FileFormat/Types.md b/src/UserGuide/latest/FileFormat/Types.md new file mode 100644 index 0000000000..b73dbb7a46 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Types.md @@ -0,0 +1,88 @@ + + +# Types + +Chunk Headers, Timeseries Metadata, and Table Schemas use a single-byte `data_type` identifier: + +| Value | Name | Physical or logical meaning | +| ---: | --- | --- | +| 0 | `BOOLEAN` | Boolean value | +| 1 | `INT32` | Signed 32-bit integer | +| 2 | `INT64` | Signed 64-bit integer | +| 3 | `FLOAT` | IEEE 754 binary32 | +| 4 | `DOUBLE` | IEEE 754 binary64 | +| 5 | `TEXT` | Length-prefixed byte string | +| 6 | `VECTOR` | Physical type marker used by an aligned Time Chunk | +| 7 | `UNKNOWN` | Unknown type marker | +| 8 | `TIMESTAMP` | Signed 64-bit integer; the upper layer defines the time unit | +| 9 | `DATE` | Signed 32-bit integer in decimal `YYYYMMDD` form | +| 10 | `BLOB` | Arbitrary byte string | +| 11 | `STRING` | UTF-8 string | +| 12 | `OBJECT` | Byte representation of an object value | + +For example, the physical `DATE` value for 2026-07-27 is the decimal integer `20260727`, not a +Unix epoch day. + +The data type determines the additional Statistics fields and the input type of the value +encoder. See [Encodings](DataPages/Encodings.md) for wire representations of values. + +## Physical and Logical Types + +The identifier determines the physical decoder. `TIMESTAMP` and `DATE` additionally carry a +logical interpretation: + +- `TIMESTAMP` has the same eight-byte physical value domain as `INT64`. The time unit is supplied + by the protocol or dataset using the file; TsFile v4 does not serialize that unit. +- `DATE` has the same integer encoder interface as `INT32`, but valid logical values use decimal + `YYYYMMDD` form. +- `TEXT` is a length-delimited byte string. `STRING` uses the same physical binary container but + has UTF-8 string semantics. `BLOB` is uninterpreted binary data. +- `OBJECT` is an opaque binary representation whose object contract must be agreed outside the + file format. + +`VECTOR` is a physical marker for the shared time column of aligned data. It is not a scalar value +type and does not define a standalone value-stream encoding. `UNKNOWN` is a sentinel, not a valid +type for a decodable measurement Page. + +## Type Consistency + +The same series type is repeated at different levels so a reader can enter the file through the +footer or through a sequential scan. These copies must agree: + +- A non-aligned or aligned Value Chunk Header uses the measurement's scalar `data_type`. +- Its Timeseries Metadata uses the same `data_type`. +- Statistics are decoded using that type and contain no independent type byte. +- An aligned Time Chunk uses `VECTOR`; its decoded values are signed 64-bit timestamps. + +Changing the interpretation of a previously assigned identifier is incompatible. A reader that +does not support a type identifier may still skip a complete Chunk using `data_size`, but it +cannot decode values or type-specific Statistics for that series. + +## Ordering and Comparison + +Integer and floating-point values use their numeric order for min/max Statistics. `STRING` +min/max values use binary lexicographic comparison of their serialized bytes. `TEXT` stores only +first and last values, while `BLOB` and `OBJECT` store no type-specific value Statistics. + +Type promotion and application-level casts are not encoded in TsFile. A reader can expose such +conversions, but they do not change the stored `data_type` or the bytes used to compute +Statistics. diff --git a/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md new file mode 100644 index 0000000000..d6d542c982 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md @@ -0,0 +1,106 @@ + + +# 二进制表示 + +所有偏移量都从文件首字节开始计算，所有长度均以字节为单位。除非单独说明，固定宽度 +整数和 IEEE 754 浮点数使用大端序，字符串使用 UTF-8。 + +## Fixed-width Values + +| 记号 | 字节数 | 含义 | +| --- | ---: | --- | +| `byte` | 1 | 8 位原始值 | +| `bool` | 1 | `0x00` 为 false，`0x01` 为 true | +| `int32` | 4 | 大端、有符号 32 位整数 | +| `int64` | 8 | 大端、有符号 64 位整数 | +| `float32` | 4 | 大端 IEEE 754 binary32 | +| `float64` | 8 | 大端 IEEE 754 binary64 | + +## Unsigned Variable-length Integer + +`UVarInt` 是无符号 LEB128 风格的 32 位变长整数。每个字节的低 7 位保存数据，最高位 +为 1 表示后面仍有字节，低位组先写。 + +```text +value = Σ ((byte[i] & 0x7f) << (7 * i)) +``` + +编码长度为 1 至 5 字节，并使用最短形式。 + +例如，十进制 `300` 拆分成从低到高的七位组 `0x2c` 和 `0x02`，线上字节为 `ac 02`。 +32 位字段的读取器最多读取五个字节，并检查移位或数值溢出。写入器只产生规范的最短表示， +不会写入冗长编码。 + +## Signed Variable-length Integer + +`VarInt` 先对有符号 32 位整数做 ZigZag 映射，再按 `UVarInt` 写入： + +```text +zigzag(n) = (n << 1) XOR (n >> 31) +``` + +| 原值 | 线上的无符号值 | 字节 | +| ---: | ---: | --- | +| 0 | 0 | `00` | +| -1 | 1 | `01` | +| 1 | 2 | `02` | +| -2 | 3 | `03` | + +## Strings and Byte Arrays + +| 记号 | 格式 | +| --- | --- | +| `VarString` | `VarInt byte_length` + `byte[byte_length]` | +| `String32` | `int32 byte_length` + `byte[byte_length]` | +| `Binary32` | `int32 byte_length` + `byte[byte_length]` | + +长度 `-1` 表示 null，长度 `0` 表示空字符串或空字节串。`VarString` 和 `String32` +保存 UTF-8；`Binary32` 保存原始字节。 + +长度统计编码后的字节数，而不是 Unicode code point 或字符数。例如，一个非 ASCII 字符的 +UTF-8 字节长度可能大于 1。对于允许 null 的字段，null 和空值具有不同线上表示，读取后 +必须保持区别。 + +## 边界与防御性解码 + +每个带长度字段都受更高层边界约束：字符串受所属记录约束，Page 受 Chunk 约束，Footer +对象受元数据 offset 或文件尾元数据长度约束。为长度 `N` 分配内存或创建切片前，读取器 +需要验证： + +```text +N >= 0 # 文档规定的 null 哨兵除外 +current_offset + N >= current_offset +current_offset + N <= enclosing_end +``` + +第二个比较用于检测整数回绕。读取器还应施加本地资源上限；结构上有效的长度不要求实现 +分配无限内存。 + +固定宽度值必须具有完整字节数。截断的 `int64`、没有终止字节的 VarInt、非 null 字段的 +负长度、要求字符串位置的无效 UTF-8，以及带长度记录结束后的尾随字节均属于格式错误。 + +## 偏移约定 + +元数据中的 offset 以有符号 64 位整数保存，但表示非负绝对字节位置。offset 指向所命名 +结构的第一个字节，例如 `offset_of_chunk_header` 指向 Chunk marker，`meta_offset` 指向 +`0x02` separator。长度不是隐式 offset；将 offset 与长度相加时必须检查溢出，并验证结果 +不超过文件或所属记录边界。 diff --git a/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md new file mode 100644 index 0000000000..feaa8a0f7a --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -0,0 +1,94 @@ + + +# 布隆过滤器 + +TsFile v4 在 `TsFileMetadata` 中保存文件级布隆过滤器，用于判断时间序列路径是否可能 +存在。布隆过滤器可以返回“肯定不存在”或“可能存在”，不能用于肯定某路径存在。 + +## File Format + +```text +BloomFilter := + UVarInt byte_length + byte bits[byte_length] + UVarInt bit_count? # byte_length > 0 时存在 + UVarInt hash_function_count? # byte_length > 0 时存在 +``` + +`byte_length = 0` 表示文件没有布隆过滤器，后两个字段不存在。 + +位编号 `i` 存储在 `bits[i / 8]` 的位 `i mod 8`，即每个字节从低位方向编号。尾部全零 +字节不写入文件，逻辑位数由 `bit_count` 保存。 + +## Filter Input + +过滤器输入是时间序列完整路径的 UTF-8 字节： + +```text +UTF8(join(DeviceID segments, ".") + "." + measurement_id) +``` + +null 设备片段在路径文本中表示为 ASCII `null`。 + +## Hashing + +哈希算法使用 MurmurHash3 x64 128 位变体。最终的两个 64 位结果相加，取低 32 位； +若结果为最小 32 位整数则使用 0，否则取绝对值并对 `bit_count` 取模。 + +哈希 seed 依次为： + +```text +5, 7, 11, 19, 31, 37, 43, 59 +``` + +实际使用的 seed 数量由 `hash_function_count` 指定，最大为 8。 + +## 查询过程 + +对一个精确路径进行查询时，读取器： + +1. 按相同的 `DeviceID` 分片和测点规则构造路径字节； +2. 使用前 `hash_function_count` 个 seed 计算位置； +3. 检查逻辑 `bit_count` 位向量中的对应位置； +4. 任意一位为 0 时返回“肯定不存在”，否则返回“可能存在”。 + +“可能存在”仍需访问元数据索引确认。只有查询端使用与写入器完全相同的路径规范化方式 +和 UTF-8 字节时，“肯定不存在”才可靠。 + +## 空间与概率 + +令插入路径数为 `n`、逻辑位数 `bit_count` 为 `m`、哈希函数数为 `k`，标准布隆过滤器的 +近似关系为： + +```text +false_positive_probability ≈ (1 - exp(-k * n / m))^k +optimal_k ≈ (m / n) * ln(2) +``` + +这些公式解释写入器在空间和准确率之间的权衡，不是额外的文件字段。读取时应以序列化的 +`bit_count` 和 `hash_function_count` 为准。 + +## 结构校验 + +当 `byte_length > 0` 时，`bit_count` 必须为正数，`hash_function_count` 必须在 `1..8` +范围内，且 `byte_length` 不得超过 `ceil(bit_count / 8)`。被省略的尾部零字节在逻辑上 +补零；如果位向量长度越过 `TsFileMetadata` 边界，读取器必须拒绝该结构。 diff --git a/src/zh/UserGuide/latest/FileFormat/Configurations.md b/src/zh/UserGuide/latest/FileFormat/Configurations.md new file mode 100644 index 0000000000..80edd9f3d8 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Configurations.md @@ -0,0 +1,129 @@ + + +# 配置 + +TsFile 的大多数配置属于写入策略。它们控制缓冲、Page 边界、Chunk Group 边界以及 +元数据索引的形状，但不会作为额外字段写入文件。读取器根据标记、长度字段、 +Chunk Header 和已序列化的元数据确定实际布局。 + +下列名称表示文件格式层面的概念。不同接口暴露的配置名称可能不同，但它们对磁盘字节的 +影响以及读取端是否需要原始配置是一致的。 + +| 概念配置 | 可观察的文件影响 | 配置值是否直接写入 | 读取器是否需要原始配置 | +| --- | --- | --- | --- | +| Chunk Group 大小 | 设备分组的刷出位置 | 否 | 否 | +| Page 大小目标 | Page 边界和 Statistics 粒度 | 否 | 否 | +| 每 Page 最大点数 | Page 边界 | 否 | 否 | +| 元数据索引扇出 | 索引树深度和节点宽度 | 否 | 否 | +| 布隆过滤器误判率 | 最终位数和哈希函数数 | 否；生成结果会写入 | 否 | +| 值编码 | Page 编码流 | 是，每个 Chunk 的 `encoding_type` | 否 | +| 压缩 | Page 磁盘负载 | 是，每个 Chunk 的 `compression_type` | 否 | +| 时间精度和时区 | `int64` 时间戳的解释 | 否 | 是，由上层协议提供 | +| 非对齐时间编码 | 每个非对齐 Page 的时间编码流 | 否 | 是 | +| 加密 | Page 磁盘负载和 Footer 属性 | 属性会写入；密钥上下文可能在外部 | 加密时需要 | + +“不直接写入”表示文件中没有以该阈值或比率命名的字段，但配置造成的布局结果仍可能被 +观察到。 + +## Chunk Group Size + +Chunk Group 大小表示写入器在刷出当前设备的各个 Chunk 之前允许缓冲的数据量。 +较大的 Chunk Group 可以减少标记与元数据开销，并有利于顺序 I/O，但需要更多写入内存； +较小的 Chunk Group 可以降低峰值内存，并更早地将数据写入输出。 + +该配置是刷出阈值，并不表示精确的磁盘占用大小。文件格式不规定默认值。 +Chunk Group 的实际边界由 `0x00` Chunk Group Header 标记以及下一个顶层标记确定。 + +## Data Page Size + +Data Page 大小是单个序列写入器在内存中累计数据量的目标值。较小的 Page 可以提供 +更细粒度的 Statistics 和数据跳过能力；较大的 Page 可以减少 Page Header 开销并可能 +提高压缩率，但需要更多缓冲空间，数据跳过粒度也更粗。 + +该阈值不等于精确的序列化 Page 大小。达到最大点数、显式刷出或需要维持对齐 Page +边界时，写入器都可以提前结束当前 Page。实际 Page 边界由 Page Header 中的长度确定。 +文件格式不规定默认的 Page 大小。 + +## Maximum Points per Page + +除 Page 大小目标外，写入器还可以限制一个 Page 中的逻辑位置数，以防大量紧凑值让 +单个 Page 包含过多数据点。参考写入配置使用 `10,000` 点。 + +对于对齐序列，Time Page 及其对应的各个 Value Page 必须覆盖相同的逻辑位置范围， +因此写入器会为这些对齐列使用共同的 Page 边界。 + +该上限不会作为独立字段写入文件。读取器使用 Page 边界；对于对齐 Value Page， +还会使用 `position_count`。 + +## Metadata Index Fanout + +元数据索引扇出限制一个内部元数据索引节点可以包含的子节点数量。较大的扇出形成 +层数更少但节点更大的树；较小的扇出形成层数更多但节点更小的树。该值必须至少为 `2`， +参考写入配置使用 `256`。 + +配置的上限不会写入文件。每个元数据索引节点都会保存节点类型、子节点和结束偏移， +因此读取器无需知道写入配置即可遍历最终生成的树。 + +## Bloom Filter False-positive Rate + +布隆过滤器误判率控制路径查找的空间与准确率权衡。误判率越低，需要的位数和哈希函数 +越多。标准写入器将该值限制在 `0.01` 到 `0.10` 之间，默认使用 `0.05`。 + +误判率本身不会写入文件。最终生成的过滤器字节、位数和哈希函数数量保存在 +`TsFileMetadata` 中，因此读取器不需要写入端的配置。详见 +[Bloom Filter](Bloom-Filter.md)。 + +## Value Encoding and Compression + +写入器可以按物理类型选择默认值编码，并选择默认压缩方式。这些默认值属于 Schema +与写入策略，不是读写端必须共享的配置： + +- Chunk Header 中的 `encoding_type` 标识值编码；对于对齐 Time Chunk，它标识时间编码。 +- Chunk Header 中的 `compression_type` 标识该 Chunk 内所有 Page 使用的压缩方式。 + +因此读取器直接使用 Chunk Header，不需要知道写入器的默认值。支持的标识符及其 +载荷格式见 [Encodings](DataPages/Encodings.md) 和 +[Compression](DataPages/Compression.md)。 + +## Time Precision and Time Zone + +物理时间戳是有符号 64 位整数。TsFile v4 不在文件中保存时间单位或时区；毫秒、微秒、 +纳秒等时间精度以及时区语义由使用该文件的上层协议确定。 + +## Non-aligned Time Encoding + +非对齐 Chunk Header 的 `encoding_type` 描述值流编码，不描述同一 Page 内的时间流编码。 +非对齐时间流编码由读写配置共同确定，默认值为 `TS_2DIFF`。 + +对齐 Time Chunk 不受这一限制；其时间编码保存在 Time Chunk Header 的 +`encoding_type` 中。 + +## Encryption + +Page 数据是否加密以及所用算法由 `TsFileMetadata` 的 `encryptLevel`、`encryptType` +和 `encryptKey` 属性描述。打开加密文件所需的密钥材料仍然保存在文件之外。 +详细格式见 [Encryption](DataPages/Encryption.md)。 + +## Implementation Settings + +内存分配、内存检查频率、批大小、并行执行、线程数、存储后端以及关闭文件时的同步 +策略都不定义 TsFile 字节。它们属于具体实现与部署范围，不属于本文件格式规约。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md new file mode 100644 index 0000000000..8226d8ce63 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md @@ -0,0 +1,49 @@ + + +# 校验和 + +TsFile v4 不保存通用的 Page 校验和、Chunk 校验和或文件校验和。 + +头尾魔数、长度字段、marker、offset 和结构边界可以发现部分截断或结构损坏，但不提供 +内容校验或密码学完整性保证。 + +## 结构校验能够发现什么 + +严格的读取器可以通过以下约束发现大量非法布局： + +- 头尾魔数和版本字节完整存在； +- Footer 长度与 `meta_offset` 能解析到合法边界； +- 每个 `data_size`、`compressed_size`、字符串长度和元数据列表长度都位于所属结构内； +- 解压结果恰好包含 `uncompressed_size` 字节； +- 解码元素数量、对齐位图、Statistics 和索引范围内部一致。 + +即使文件另有外部校验和，这些检查仍然必要。它们能够发现截断和许多意外改动，但某些 +字节替换不会改变长度与边界，因而仍可能不被发现。 + +## 静态存储与传输完整性 + +需要检测数据损坏的系统应为完整文件维护外部校验和或对象存储完整性字段。需要防止 +恶意修改时，应使用带认证的摘要或签名，并把密钥或可信摘要保存在 TsFile 之外。外部 +校验值必须覆盖完整文件的精确字节序列，包括 Footer 长度和尾魔数。 + +Page 负载加密是另一项独立能力。除非所选加密算法明确认证密文，否则仅加密既不会为 +TsFile 添加校验和，也不会验证未加密的 Header 与元数据。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md new file mode 100644 index 0000000000..840f8992f1 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -0,0 +1,172 @@ + + +# Chunk 与 Chunk Group + +TsFile 的数据区从偏移量 7 开始，由 marker 引导的 Chunk Group、Chunk 和 Operation +Index Range 组成。 + +顶层语法为： + +```text +DataSection := (ChunkGroup | OperationIndexRange)* Separator +ChunkGroup := ChunkGroupHeader Chunk* +Separator := byte(0x02) +``` + +`Chunk*` 组成的分组自身没有长度前缀。读取器必须根据每个 Chunk 的 `data_size` 完整消费 +该 Chunk，随后读取下一个顶层标记，才能确定 Chunk Group 已结束。 + +```text +File +└── Chunk Group 一个设备的一批数据 + ├── Non-aligned Chunk 自带时间流的一个测量列 + │ └── Page(s) + └── Aligned Chunk Set 多个测量列共享时间列 + ├── Time Chunk + │ └── Time Page(s) + └── Value Chunk(s) + └── Value Page(s) +``` + +## Markers + +| 值 | 名称 | 后续结构 | +| ---: | --- | --- | +| `0x00` | `CHUNK_GROUP_HEADER` | `DeviceID` | +| `0x01` | `CHUNK_HEADER` | 多页、非对齐 Chunk | +| `0x02` | `SEPARATOR` | 数据区结束，元数据区开始 | +| `0x03` | `VERSION` | v3 起不再使用，编号保留 | +| `0x04` | `OPERATION_INDEX_RANGE` | 两个 `int64` | +| `0x05` | `ONLY_ONE_PAGE_CHUNK_HEADER` | 单页、非对齐 Chunk | +| `0x41` | `VALUE_CHUNK_HEADER` | 多页、对齐 Value Chunk | +| `0x45` | `ONLY_ONE_PAGE_VALUE_CHUNK_HEADER` | 单页、对齐 Value Chunk | +| `0x81` | `TIME_CHUNK_HEADER` | 多页、对齐 Time Chunk | +| `0x85` | `ONLY_ONE_PAGE_TIME_CHUNK_HEADER` | 单页、对齐 Time Chunk | + +Chunk marker 的高两位表示列角色，低 6 位表示页数类别： + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0x01: more than one page +low 6 bits = 0x05: one page or no page +``` + +`data_size = 0` 的对齐 Value Chunk 可以没有 Page。 + +只有位于已知结构边界的字节才能解释为标记。编码或压缩后的 Page 载荷内部也可能出现相同 +字节值，但在载荷内部不具有结构含义。 + +## Chunk Group Header + +```text +ChunkGroupHeader := + byte marker = 0x00 + DeviceID device_id + +DeviceID := + UVarInt segment_count + VarString segment[segment_count] +``` + +设备标识是可空字符串片段组成的有序元组。序列化时删除末尾的 null 片段，设备标识不能 +全部由 null 片段组成。`segment_count = 0` 在读取时表示只包含一个空字符串片段的设备。 + +Chunk Group Header 之后的 Chunk 属于该设备，直到下一个 Chunk Group Header、Operation +Index Range 或 Separator。 + +`segment_count` 统计序列化元组元素数量，而不是 UTF-8 字节数。每个 segment 都有独立的 +长度。设备身份是有序元组本身；使用显示分隔符拼接 segment 不属于二进制格式，而且当 +segment 内含该分隔符时可能产生歧义。 + +## Chunk Header + +marker 是 Chunk Header 的第一个字段： + +```text +ChunkHeader := + byte chunk_type + VarString measurement_id + UVarInt data_size + byte data_type + byte compression_type + byte encoding_type +``` + +`data_size` 是 Chunk 中全部 Page Header 和已保存 Page 数据的总长度，不包括 Chunk +Header。读取器从 Chunk Header 结束处读取恰好 `data_size` 字节。 + +若 `H` 是 Chunk Header 后第一个字节的偏移，则下一结构从以下位置开始： + +```text +next_structure_offset = H + data_size +``` + +所有 Page Header 的序列化长度与相应 `compressed_size` 载荷长度之和必须等于 +`data_size`。因此，该值既是快速跳过 Chunk 的长度，也是校验 Page 不越界的边界。 + +对齐 Time Chunk 的 `measurement_id` 是空字符串，`data_type` 是 `VECTOR (0x06)`。 +对齐 Value Chunk 和非对齐 Chunk 的 `data_type` 是测量值类型。 + +对齐数据按 Chunk 连续存储，不按 Time Page、Value Page 交替存储。Time Chunk 保存完整 +的 Time Page 序列，每个 Value Chunk 保存自己的 Value Page 序列，Page 通过序号对应。 + +对齐 Chunk 集满足以下不变量： + +- Time Chunk 位于使用该时间列的 Value Chunk 之前。 +- 每个 Value Chunk 标识一个测量；Time Chunk 使用空 `measurement_id` 和物理类型标记 + `VECTOR (0x06)`。 +- Value Page `i` 覆盖 Time Page `i` 的逻辑位置。 +- 某个测量在该集合中完全没有数据时，可以不存在对应 Value Chunk；已存在的对齐 Value + Chunk 也可以具有 `data_size = 0`。 + +非对齐 Chunk 的每个逻辑数据点都包含时间戳和值，因此每个 Page 解码后的时间流和值流 +必须具有相同的元素数量。 + +## Chunk 扫描状态机 + +流式读取器或恢复读取器可以使用以下状态机扫描数据区： + +1. 从偏移 7 开始读取一个顶层标记。 +2. 遇到 `0x00` 时读取 DeviceID，并进入对应 Chunk Group。 +3. 遇到 Chunk 标记时读取 Header；可以解析 Page，也可以直接向后移动 `data_size` 字节。 +4. 遇到 `0x04` 时消费后续 16 字节 Operation Index Range。 +5. 遇到 `0x02` 时停止；下一个字节属于索引元数据区。 +6. 顶层标记无法识别，或声明边界超出可用字节时，判定文件不完整或损坏。 + +## Operation Index Range + +```text +OperationIndexRange := + byte marker = 0x04 + int64 min_operation_index + int64 max_operation_index +``` + +该结构记录最近一次写入批次涉及的最小和最大操作索引。Operation Index 是存储引擎的 +逻辑操作序号，不是时间戳、测量索引或文件偏移。它可用于 checkpoint、snapshot、backup +和恢复逻辑，以判断哪些逻辑写入已经进入文件。 + +对于通用写入器，该记录是可选的。没有 Operation Index 语义的写入器可以省略它，也可以 +写入占位范围。不使用该信息的读取器仍必须识别 `0x04` 并跳过完整的 16 字节载荷，保证在 +正确边界读取下一标记。一个文件可以包含多个此类记录，因为多个写入批次可以先后刷入同一 +文件。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md new file mode 100644 index 0000000000..e9ff7ab6d6 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -0,0 +1,74 @@ + + +# 压缩 + +每个 Chunk Header 使用单字节 `compression_type` 指定该 Chunk 中所有 Page 的压缩格式。 + +| 值 | 名称 | Page 数据格式 | +| ---: | --- | --- | +| 0 | `UNCOMPRESSED` | 原始字节 | +| 1 | `SNAPPY` | Snappy raw block | +| 2 | `GZIP` | 单个 gzip member | +| 3 | `LZO` | LZO block | +| 7 | `LZ4` | LZ4 raw block | +| 8 | `ZSTD` | Zstandard frame | +| 9 | `LZMA2` | XZ container 中的 LZMA2 流 | + +每个 Page 独立压缩，不跨 Page 共享压缩状态。Page Header 的 `uncompressed_size` 和 +`compressed_size` 分别保存压缩前和压缩后的负载长度。 + +这些数值是稳定的文件格式编号，不是某个压缩库枚举中的位置。v4 未分配编号 `4` 到 +`6`。`LZO` 作为历史编号保留；读取器未提供该编解码器时，不能解码相应 Chunk。 + +## Page 级帧边界 + +若 Page 起始偏移量为 `P`，其序列化 Page Header 占 `H` 字节，则磁盘负载恰好位于以下 +左闭右开区间： + +```text +[P + H, P + H + compressed_size) +``` + +使用 `UNCOMPRESSED` 时，`compressed_size` 等于 `uncompressed_size`，这些字节已经是编码 +后的 Page 负载。使用其他方式时，解压结果必须恰好包含 `uncompressed_size` 字节。结果 +更短、更长，或编解码器要求读取上述区间外的字节，都表示 Page 无效。 + +每个 Page 都是独立的压缩单元。因此，读取器可以先用 Page Statistics 排除不需要的 +Page，再直接前进 `compressed_size` 字节，而不必初始化解压器。随机访问也不依赖前一个 +Page 的字典或历史状态。 + +## 读取顺序与校验 + +读取器先读 Page Header，因而能在分配内存或解压前获得输入、输出边界： + +```text +读取 Page Header + -> 准确读取 compressed_size 字节 + -> 文件属性要求时解密 + -> 按 compression_type 解压 + -> 要求结果恰好为 uncompressed_size 字节 + -> 解码 Page 内各数据流 +``` + +实现应在分配内存前拒绝会导致偏移量运算溢出或超过资源限制的长度。不支持某个 +`compression_type` 时，读取器不能解码相应 Chunk，但仍可通过 `data_size` 跳过该 +Chunk；不能把未知方式当作 `UNCOMPRESSED`。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md new file mode 100644 index 0000000000..ea043c0486 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md @@ -0,0 +1,107 @@ + + +# 编码 + +Chunk Header 使用单字节 `encoding_type` 指定值流编码。对齐 Time Chunk 的该字段指定 +时间流编码；非对齐 Chunk 的该字段只指定值流编码。 + +| 值 | 名称 | +| ---: | --- | +| 0 | `PLAIN` | +| 1 | `DICTIONARY` | +| 2 | `RLE` | +| 3 | `DIFF` | +| 4 | `TS_2DIFF` | +| 5 | `BITMAP` | +| 6 | `GORILLA_V1` | +| 7 | `REGULAR` | +| 8 | `GORILLA` | +| 9 | `ZIGZAG` | +| 10 | `FREQ`（已弃用） | +| 11 | `CHIMP` | +| 12 | `SPRINTZ` | +| 13 | `RLBE` | +| 14 | `CAMEL` | + +类型与编码组合如下： + +| 数据类型 | 可用编码 | +| --- | --- | +| `BOOLEAN` | `PLAIN`, `RLE` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `FLOAT` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `DOUBLE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE`, `CAMEL` | +| `TEXT`, `STRING` | `PLAIN`, `DICTIONARY` | +| `BLOB`, `OBJECT` | `PLAIN` | + +## 编码类别 + +这些编号描述把逻辑值序列转换为字节的不同方式。它们是文件格式中的固定编号，不是 +某个实现内部枚举的序号。 + +| 类别 | 编码 | 通常利用的数据特征 | +| --- | --- | --- | +| 直接表示 | `PLAIN` | 不预测，各值独立序列化 | +| 字典 | `DICTIONARY` | 用字典引用替代重复的二进制值 | +| 游程与位打包 | `RLE`, `RLBE` | 重复值或较小的有效位宽 | +| 差分与预测 | `DIFF`, `TS_2DIFF`, `REGULAR`, `SPRINTZ` | 相邻值之间较小或较规律的变化 | +| 整数重映射 | `ZIGZAG` | 把绝对值较小的有符号整数映射为较小的无符号整数 | +| 基于异或的浮点编码 | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | 相邻浮点数具有稳定的前导位和尾随位 | +| 历史或专用编码 | `BITMAP`, `FREQ` | 保留的历史布局；`FREQ` 已弃用 | + +最合适的编码取决于数据分布。文件不记录写入器选择某种编码的原因；读取器必须同时 +根据 `data_type` 和 `encoding_type` 选择解码器。 + +## Plain Encoding + +`PLAIN` 流不保存元素个数，元素数量由 Page 上下文、时间流或 null 位图确定。各元素 +连续写入： + +| 数据类型 | 单个值的编码 | +| --- | --- | +| `BOOLEAN` | `0x00` 或 `0x01` | +| `INT32`, `DATE` | `VarInt` | +| `INT64`, `TIMESTAMP` | 大端序 `int64` | +| `FLOAT` | 大端序 `float32` | +| `DOUBLE` | 大端序 `float64` | +| `TEXT`, `STRING`, `BLOB`, `OBJECT` | `VarInt byte_length` + 原始字节 | + +`PLAIN` 二进制值的长度非负。非对齐值流不包含 null；对齐值流的 null 由 Value Page +位图表示。 + +其他编码编号决定各自的值流位布局，读取器不能把未知编码按 `PLAIN` 解码。 + +## 解码约束 + +编码状态只在一个解压后的 Page 流内有效。Page 解压完成后，读取器需要： + +1. 根据 Chunk 的 `data_type` 和 `encoding_type` 创建解码器； +2. 解码出 Page 上下文所要求的准确逻辑元素数量； +3. 验证解码过程既没有越过 Page 边界，也没有留下不完整的编码块； +4. 在处理下一个 Page 前丢弃当前解码状态。 + +非对齐 Page 的时间流使用文件元数据中 `time_encoder` 指定的独立编码， +`encoding_type` 只作用于值流。对齐 Time Page 中，`encoding_type` 作用于时间流；对齐 +Value Page 会先移除 null 位置再编码，因此值解码器必须恰好产生位图中置位的元素数。 + +读取器不支持某种编码时，不能解码对应 Chunk，但仍可借助 `data_size` 跳过其字节范围； +退回到另一种解码器并不合法。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md new file mode 100644 index 0000000000..7c628345b1 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -0,0 +1,72 @@ + + +# 加密 + +加密只作用于压缩后的 Page `stored_data`。Page Header、Chunk Header、Statistics、 +Metadata Index 和 File Metadata 不加密。 + +写入顺序为： + +```text +encoded page payload + -> compression + -> encryption + -> stored_data +``` + +读取顺序为先解密，再解压和解码。 + +## File Properties + +`TsFileMetadata` 的属性 Map 使用以下键描述加密： + +| 属性 | 含义 | +| --- | --- | +| `encryptLevel` | `0` 不加密；`1` 直接保存数据密钥；`2` 保存被外部主密钥加密的数据密钥 | +| `encryptType` | 加解密算法标识符 | +| `encryptKey` | 数据密钥或密钥封装结果的文本表示 | + +Page Header 没有单独的密文长度字段。加密结果长度与输入的压缩数据长度相同，文件中的 +`stored_data` 长度仍由 `compressed_size` 给出。 + +为保持兼容，缺少 `encryptLevel` 时按 `0` 处理。级别为 `1` 或 `2` 时，必须同时存在 +`encryptType` 和非空 `encryptKey`。级别未知或算法不可用时，加密的 Page 数据不可读； +读取器不能静默地把它当作未加密数据。 + +## 密钥解释 + +级别 `1` 的 `encryptKey` 表示供 `encryptType` 使用的数据密钥。级别 `2` 的该属性表示 +封装后的数据密钥：读取器先取得外部主密钥解密器并解封数据密钥，再使用 +`encryptType` 解密 Page 负载。密钥派生、主密钥保存、轮换和访问控制不属于文件格式。 + +## 安全边界 + +Page 加密保护编码后样本数据的机密性，但不会隐藏文件形状、设备和测点标识、类型、 +时间范围、值 Statistics、Schema、索引以及加密属性本身。应用需要把这些元数据视为 +可见信息。 + +TsFile v4 没有为 Page 或整个文件定义通用认证标签。密文被修改后能否检测出来取决于 +所选算法；[Checksumming](Checksumming.md) 中的结构检查和外部完整性建议仍然适用。 + +读取器应先解析加密属性和密钥材料，再分配 Page 缓冲区，并严格执行 +`读取 -> 解密 -> 解压 -> 解码`。解密失败、输出长度错误、算法未知或密钥缺失都属于 +对应数据的读取错误，不能退回明文处理。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md new file mode 100644 index 0000000000..290fb1a62e --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md @@ -0,0 +1,83 @@ + + +# 错误恢复 + +完成写入的 TsFile 包含头魔数、版本字节、`0x02` separator、Footer 元数据、4 字节 +`TsFileMetadata` 长度和尾魔数。没有 Footer 长度或尾魔数的文件是未完成文件。 + +恢复读取器可以从偏移量 7 开始按 marker 扫描数据区，并利用 Chunk Header 的 +`data_size` 跳过完整 Chunk。在 Chunk 内部，所有 Page 的序列化长度之和等于 +`data_size`。 + +扫描器不能把 Page 数据中出现的任意 `0x02` 当作 separator；只有在完整解析或跳过 +前一个 marker 所界定的结构后，才可以读取下一个 marker。 + +## 完成性检查 + +设文件大小为 `F`，区间 `[F - 10, F - 6)` 中保存的大端 Footer 长度为 `L`。普通的 +元数据优先读取开始前，完整文件必须满足： + +```text +bytes[F - 6, F) = "TsFile" +TsFileMetadata 区间 = [F - 10 - L, F - 10) +meta_offset = 顶层 0x02 separator 的偏移量 +meta_offset < F - 10 - L +``` + +头魔数和版本也必须有效，并且 `TsFileMetadata` 解码器必须恰好消费 `L` 字节。仅有合法的 +尾魔数并不能证明文件完整。 + +## 安全正向扫描 + +恢复扫描器可以识别数据区中最长的结构完整前缀： + +1. 校验头魔数和版本，把游标置于偏移量 `7`； +2. 仅在已知顶层边界读取一个 marker； +3. 遇到 Chunk Group Header（`0x00`）时，解码完整 `DeviceID` 并保存组上下文； +4. 遇到 Chunk marker 时，解码完整 Chunk Header，校验 `cursor + data_size`，再跳过该 + 区间或校验其中所有 Page 边界； +5. 遇到 Operation Index Range（`0x04`）时，准确消费两个大端 `int64`； +6. 遇到顶层 separator（`0x02`）时正常结束；字段或负载不完整时，在最后一个已知安全 + 边界停止。 + +最后安全边界是一个完整定界结构的末尾，不是“最后一个碰巧可读的字节”。恢复工具可以 +使用该边界之前的完整 Chunk 重建索引和新 Footer，但不能臆造或部分解码被截断的 Chunk。 + +检测到以下情况时，文件结构不完整或已损坏： + +- Chunk 的 `data_size` 超出文件或下一个结构边界。 +- Page 的长度字段超出 Chunk 边界。 +- Footer 长度无法定位到完整的 `TsFileMetadata`。 +- `meta_offset` 没有指向 `0x02` separator。 +- 索引 offset 位于文件之外或指入结构中间。 + +其他失败还包括：未知顶层 marker、未终止的 VarInt 或 VarString、计算结束偏移量时整数 +溢出、Page 解码结果数量错误，以及对齐 Time/Value Page 不匹配。 + +## 恢复能力边界 + +恢复只能重建结构，不能补回缺失信息。完整的压缩或加密 Page 仍需要其声明的编解码器， +必要时还需要外部密钥。由于没有文件级校验和，结构仍合法的字节替换可能无法与原始数据 +区分。Operation Index Range 可以帮助存储引擎把保留下来的前缀与外部操作日志关联， +但不能替代对 Chunk 和 Page 边界的校验。 + +TsFile v4 没有通用校验和；相关限制见 [Checksumming](Checksumming.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/README.md b/src/zh/UserGuide/latest/FileFormat/DataPages/README.md new file mode 100644 index 0000000000..d0a8e33a69 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/README.md @@ -0,0 +1,168 @@ + + +# 数据页 + +Chunk Header 之后连续保存一个或多个 Page。Page 之间没有填充字节；Chunk Header 的 +`data_size` 给出全部 Page Header 和 Page 数据的总长度。 + +## Page Header + +非空 Page 的格式为： + +```text +Page := + UVarInt uncompressed_size + UVarInt compressed_size + Statistics statistics? # 由 Chunk marker 决定 + byte stored_data[compressed_size] +``` + +`uncompressed_size` 是解密、解压后的 Page 负载大小，`compressed_size` 是文件中保存的 +Page 负载大小。当 `compression_type = UNCOMPRESSED` 时，两个值相等。 + +Chunk marker 的低 6 位决定 Page Header 是否包含 Statistics： + +- `0x01`：Chunk 有多页，每个非空 Page Header 包含 Statistics。 +- `0x05`：Chunk 只有一页或没有 Page，Page Header 不包含 Statistics。 + +Statistics 的二进制格式见 [Metadata](../Metadata.md#statistics)。 + +因此，无需检查 Page 载荷即可根据 Chunk marker 选择 Header 结构： + +| Chunk marker 低位 | Page 数量 | Page Statistics | +| ---: | --- | --- | +| `0x01` | 多于一个 | 每个非空 Page Header 中存在 | +| `0x05` | 零个或一个 | 不存在 | + +若 Page 起始偏移为 `P`，两个长度字段和可选 Statistics 的序列化总长度为 `h`，则磁盘 +载荷占用半开区间 `[P + h, P + h + compressed_size)`；如果还有下一个 Page，它从 +`P + h + compressed_size` 开始。该结束位置不得超过所属 Chunk 的边界。 + +解密并解压后，载荷必须恰好产生 `uncompressed_size` 字节。即使压缩库报告成功，实际输出 +长度不一致时读取器也必须拒绝该 Page。 + +## Page Encoding + +写入非空 Page 时，数据依次经过： + +```text +logical values + -> value encoding + -> assemble uncompressed page payload + -> compression + -> optional encryption + -> stored_data +``` + +读取顺序相反：先解密，再解压，最后执行值解码。 + +编码与压缩相互独立。编码把有类型的值转换成未压缩字节流，压缩把该结果视为普通字节。 +Page 边界会重置值编码器/解码器和压缩器/解压器，状态不会跨 Page 延续。 + +## Non-aligned Page + +非对齐 Page 的解压后负载为： + +```text +NonAlignedPagePayload := + UVarInt time_stream_size + byte encoded_time_stream[time_stream_size] + byte encoded_value_stream[remaining bytes] +``` + +一个非对齐 Page 同时保存时间和值。它没有独立的 Time Page 和 Value Page：Page 先保存 +完整时间流，再保存完整值流，而不是按 `timestamp, value` 逐点交替写入。两个流包含 +相同数量的元素，并按位置组成 `(timestamp, value)`。 + +设未压缩载荷大小为 `U`，序列化 `time_stream_size` 字段本身占 `v` 字节，则值流长度为： + +```text +value_stream_size = U - v - time_stream_size +``` + +结果必须非负。解码后，两个流的解码器都必须恰好到达各自边界，并产生相同的数据点数量。 +存在尾随字节、提前结束或数量不一致时，该 Page 格式错误。 + +非对齐 Chunk Header 只记录值流的 `encoding_type`。时间流编码见 +[Configurations](../Configurations.md#non-aligned-time-encoding)。 + +## Aligned Time Page + +对齐 Time Page 的解压后负载完全由时间编码流组成： + +```text +AlignedTimePagePayload := byte encoded_time_stream[remaining bytes] +``` + +时间编码由 Time Chunk Header 的 `encoding_type` 指定。 + +解码得到的时间戳数量就是该 Page 的逻辑位置数。所有对应对齐 Value Page 的 bitmap 宽度 +和 null 重建都由该数量决定。 + +## Aligned Value Page + +非空对齐 Value Page 的解压后负载为： + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +同一对齐集合中，第 `i` 个 Value Page 对应第 `i` 个 Time Page。null 位图和全 null +空 Page 的格式见 [Nulls](../Nulls.md)。 + +`position_count` 必须等于对应 Time Page 解码得到的时间戳数量。bitmap 必须恰好具有 +`ceil(position_count / 8)` 字节；最后一个字节中未使用的低位不表示逻辑位置。值解码器只 +对置位位置解码，并且必须恰好产生 bitmap 置位数个值。 + +## Page 选择与跳过 + +Page Statistics 允许读取器在读取磁盘 Page 数据前判断时间和值谓词。只有当 Statistics +能够证明没有数据点满足谓词时，才能跳过 Page。Statistics 是保守摘要：区间重叠只表示 +“可能匹配”，并不表示“一定匹配”。 + +单 Page Chunk 的 Page Header 不包含 Statistics。读取器可以在跳转到 Chunk 前使用文件尾 +中的 Chunk 或 Timeseries Statistics。某种数据类型的 Statistics 不包含查询所需摘要时， +读取器必须解码 Page 或依赖其他谓词，不能据此排除数据。 + +## 全 null 对齐 Value Page + +一个对齐值列在某个 Time Page 的全部位置上都可以为 null。该情况只写入一个无符号变长 +整数零： + +```text +EmptyAlignedValuePage := UVarInt(0) # uncompressed_size +``` + +该零之后没有 `compressed_size`、Page Statistics、bitmap 或磁盘载荷。位置数继承自对应 +Time Page。只有在对齐 Page 序列能够提供外部位置数时，才允许使用这种表示。 + +## Data Page Details + +- [Chunks](Chunks.md) +- [Compression](Compression.md) +- [Encodings](Encodings.md) +- [Encryption](Encryption.md) +- [Checksumming](Checksumming.md) +- [Error Recovery](Error-Recovery.md) diff --git a/src/zh/UserGuide/latest/FileFormat/Extensibility.md b/src/zh/UserGuide/latest/FileFormat/Extensibility.md new file mode 100644 index 0000000000..f8f16f4367 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Extensibility.md @@ -0,0 +1,61 @@ + + +# 扩展机制 + +TsFile v4 在以下位置使用已编号或自描述的扩展点： + +- 数据类型、编码和压缩方式使用单字节编号。 +- `TsFileMetadata` 末尾保存长度前缀的字符串属性 Map。 +- 文件版本字节用于选择完整的文件布局。 + +新数据类型、编码或压缩方式需要使用新的编号；已有编号的线上含义保持不变。 + +Footer 属性由键和值两个 `VarString` 组成。读取器可以在不改变其他 Footer 字段边界的 +情况下读取未识别属性。 + +数据区 marker 没有通用长度字段。读取器无法安全跳过未知 marker，因此新增无法由现有 +marker 表达的数据区结构需要新的文件版本或能够由现有结构完整界定的编码方式。 + +## 安全演进规则 + +只有旧读取器无需解释新负载就能确定其完整字节范围时，扩展才可以安全跳过。例如新的 +Footer 属性值，以及位于已知 `data_size` 的 Chunk 内的新编解码器。向没有长度边界的 +Header 直接追加字节并不可跳过，因为其后的所有字段都会发生位移。 + +在 v4 内，扩展需要： + +- 分配尚未使用的编号，且不复用任何已有值； +- 保持现有结构的字节序、长度解释和包含边界； +- 为新编码规定可用的物理数据类型； +- 为新编解码器规定输出长度、元素数量以及非法输入行为； +- 读取器不能解码该扩展时，必须明确报告能力缺失。 + +保留或未知编号不是默认值的别名。特别是，未知编码不等于 `PLAIN`，未知压缩方式不等于 +`UNCOMPRESSED`，未知 marker 也不表示填充字节。 + +## 属性与语义 + +长度前缀让未知 Footer 属性在结构上可读取，但不表示该属性的语义一定可忽略。若某个 +属性会改变现有字节的解释方式，就必须定义读取器如何声明和强制其支持能力。安全相关 +属性的语义无法执行时，必须按失败处理。 + +改变顶层结构语法、重新解释已分配编号，或让旧结构边界产生歧义的变更需要新的文件版本。 diff --git a/src/zh/UserGuide/latest/FileFormat/Format-Versions.md b/src/zh/UserGuide/latest/FileFormat/Format-Versions.md new file mode 100644 index 0000000000..8ea6c670d0 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Format-Versions.md @@ -0,0 +1,63 @@ + + +# 格式版本 + +TsFile 的文件版本位于头魔数之后： + +```text +6-byte magic string "TsFile" +1-byte format version +``` + +本文档描述版本字节 `0x04`。读取器在检查头魔数后，根据版本字节选择对应的文件布局。 + +v4 使用分片的 `DeviceID`，并在 `TsFileMetadata` 中保存按表划分的设备索引根和 +Table Schema。v3 使用单字符串设备标识和不同的 Footer 布局，不能使用 v4 的 +`DeviceID` 或 `TsFileMetadata` 结构解析。 + +## 读取器分派 + +必须在解释任何数据区 marker 之前处理版本字节。读取顺序为： + +```text +校验头魔数 -> 读取版本 -> 选择该版本的完整语法 -> 解析数据区与 Footer +``` + +只支持 v4、但读到其他版本字节的实现必须报告“不支持的版本”。即使某个 marker 或 +Footer 后缀看起来熟悉，也不能试探性地按 v4 解析，因为不同版本中结构的含义和宽度 +可能不同。 + +## 兼容性约束 + +兼容性在字节格式层面定义： + +- v4 读取器在支持目标数据所需全部编号的前提下，可以读取 v4 文件。 +- v4 写入器输出本文定义的 v4 语法，与其编程语言或内存类无关。 +- Footer 属性的键和值都有长度，可以读取并跳过未知属性。 +- 数据区 marker 没有统一的负载长度，因此通常无法跳过未知 marker。 +- v4 已分配的数据类型、编码、压缩方式、marker 和索引节点类型编号保持原有含义。 + +支持某个文件版本不代表支持所有可选编解码器或加密算法。缺少相应能力时，所属 Chunk +在结构上可能仍可跳过，但其中样本值不可解码。 + +项目发布系列与磁盘格式字节是两套独立版本。2.x 发布系列的产品可以读写格式字节 +`0x04`；TsFile Header 不保存项目发布版本标签。 diff --git a/src/zh/UserGuide/latest/FileFormat/Metadata.md b/src/zh/UserGuide/latest/FileFormat/Metadata.md new file mode 100644 index 0000000000..895b814262 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Metadata.md @@ -0,0 +1,275 @@ + + +# 元数据 + +TsFile 中的元数据分为两类： + +- 与数据一起保存的 Chunk Header、Page Header 和 Page Statistics。 +- 数据区之后的 Timeseries Metadata、Metadata Index 和 File Metadata。 + +Footer 元数据从 `SEPARATOR (0x02)` 之后开始。`TimeseriesMetadata` 范围与测量索引 +节点可以交错出现；索引中的 offset 和 `end_offset` 决定每个范围的实际边界。所有 +offset 都是从文件首字节开始计算的绝对偏移量。 + +元数据层级支持逐步细化的数据裁剪： + +| 层级 | 标识对象 | 主要用途 | +| --- | --- | --- | +| 文件元数据 | 表及设备索引根 | 进入正确的索引树 | +| 设备索引 | 设备 | 定位一个设备的测量索引 | +| 测量索引 | 测量名称区间 | 定位 Timeseries Metadata | +| Timeseries Metadata | 一个逻辑序列 | 选择 Chunk 偏移及序列 Statistics | +| Chunk Metadata | 一个物理 Chunk | 跳转到数据并利用 Chunk Statistics 裁剪 | +| Page Header | 一个 Page | 利用 Page Statistics 裁剪并读取载荷 | + +## Statistics + +Page、Chunk 和时间序列级 Statistics 使用相同的类型相关结构： + +```text +Statistics := + UVarInt count + int64 start_time + int64 end_time + ... type-specific fields +``` + +`count` 是该结构覆盖的值数。非对齐序列的值数等于点数；对齐值列的 `count` 是非 null +值数；对齐时间列的 `count` 是时间戳数。 + +类型专有字段按以下顺序紧随公共前缀： + +| 数据类型 | 类型专有字段 | +| --- | --- | +| `BOOLEAN` | `bool first`, `bool last`, `int64 true_count` | +| `INT32`, `DATE` | `int32 min`, `int32 max`, `int32 first`, `int32 last`, `int64 sum` | +| `INT64`, `TIMESTAMP` | `int64 min`, `int64 max`, `int64 first`, `int64 last`, `float64 sum` | +| `FLOAT` | `float32 min`, `float32 max`, `float32 first`, `float32 last`, `float64 sum` | +| `DOUBLE` | `float64 min`, `float64 max`, `float64 first`, `float64 last`, `float64 sum` | +| `TEXT` | `Binary32 first`, `Binary32 last` | +| `STRING` | `Binary32 first`, `Binary32 last`, `Binary32 min`, `Binary32 max` | +| `BLOB`, `OBJECT`, `VECTOR` | 无类型专有字段 | + +Statistics 自身不携带数据类型编号。解析 Statistics 时，数据类型来自 Chunk Header 或 +Timeseries Metadata。 + +`first` 和 `last` 表示值的先后顺序，不等同于最小值和最大值。对于按时间有序的序列， +它们对应覆盖区间首尾时间戳处的值。`start_time` 和 `end_time` 均包含在统计区间内。数值 +`sum` 用于聚合；其中 `INT64` 的 sum 以 `float64` 保存，数值较大时可能无法精确表示每个 +整数位。 + +Statistics 是保守的数据裁剪信息。查询时间范围与统计时间范围不相交，或者受支持的值摘要 +能够证明不存在匹配值时，读取器可以跳过该结构；区间重叠本身不能证明存在匹配值。 +`BLOB`、`OBJECT` 等没有最小/最大值字段的类型不能通过 Statistics 做值范围裁剪。 + +父层 Statistics 汇总该父层表示的全部值。序列包含多个 Chunk 时，series Statistics +覆盖各 Chunk Statistics 的并集；Chunk 包含多个 Page 时，Chunk Statistics 覆盖各 Page +Statistics 的并集。计数和时间边界必须与低层记录保持一致。 + +## Timeseries Metadata + +每条时间序列对应一个 `TimeseriesMetadata`： + +```text +TimeseriesMetadata := + byte timeseries_type + VarString measurement_id + byte data_type + UVarInt chunk_metadata_list_size + Statistics series_statistics + byte chunk_metadata_list[chunk_metadata_list_size] +``` + +`timeseries_type` 的高位表示列角色，低 6 位表示 Chunk Metadata 的布局： + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0: exactly one Chunk; ChunkMetadata omits Statistics +low 6 bits = 1: more than one Chunk; every ChunkMetadata contains Statistics +``` + +## Chunk Metadata + +当 `timeseries_type & 0x3f == 0` 时，时间序列只有一个 Chunk： + +```text +ChunkMetadata := int64 offset_of_chunk_header +``` + +该 Chunk 的 Statistics 直接使用 `series_statistics`。 + +当 `timeseries_type & 0x3f != 0` 时： + +```text +ChunkMetadata := + int64 offset_of_chunk_header + Statistics chunk_statistics +``` + +`offset_of_chunk_header` 指向 Chunk marker，即 Chunk Header 的第一个字节。 + +`chunk_metadata_list_size` 是字节边界，不是 Chunk 数量。读取器应先计算列表结束位置，再 +持续反序列化 Chunk Metadata，直到恰好到达该位置。提前结束、跨越边界或留下尾随字节均为 +格式错误。 + +## Metadata Index + +```text +MetadataIndexNode := + UVarInt child_count + MetadataEntry child[child_count] + int64 end_offset + byte node_type +``` + +`node_type` 决定节点层级、Entry 的编码和 Entry 指向的数据： + +| 值 | 名称 | Entry 编码 | Entry 指向 | +| ---: | --- | --- | --- | +| 0 | `INTERNAL_DEVICE` | `DeviceIndexEntry` | 子设备索引节点 | +| 1 | `LEAF_DEVICE` | `DeviceIndexEntry` | 设备的测量索引根节点 | +| 2 | `INTERNAL_MEASUREMENT` | `MeasurementIndexEntry` | 子测量索引节点 | +| 3 | `LEAF_MEASUREMENT` | `MeasurementIndexEntry` | 一段 Timeseries Metadata | + +Entry 格式为： + +```text +DeviceIndexEntry := + DeviceID first_device_id + int64 offset + +MeasurementIndexEntry := + VarString first_measurement_id + int64 offset +``` + +每个 Entry 的 `offset` 是对应范围的起点。终点是下一个 Entry 的 `offset`；最后一个 +Entry 的终点是节点的 `end_offset`。 + +```text +entry[0].offset <= entry[1].offset <= ... <= end_offset +``` + +Entry 的 key 是其覆盖范围内的第一个 key。设备索引按 `DeviceID` 片段逐段排序，null +小于非 null；短元组在公共前缀相同时排在长元组之前。测量索引按测量名词典序排序。 + +### 索引查找 + +在索引节点中查找键 `K` 时，选择 first key 小于或等于 `K` 的最后一个 Entry。所选字节 +区间从该 Entry 的 `offset` 开始，在下一 Entry 的 offset 处结束；最后一个 Entry 则在节点 +`end_offset` 处结束。内部节点区间包含下一级索引节点，叶子区间包含上表中对应的数据对象。 + +沿索引继续查找前，读取器需要验证节点： + +- `child_count` 与节点内完整 Entry 数量一致。 +- Entry key 按该节点的比较规则有序。 +- Entry offset 单调不减，每个引用区间长度非负。 +- 最后一个 Entry offset 不超过 `end_offset`。 +- 子节点层级与父节点的 node type 相符。 + +first-key 表示允许实现在节点内部对 Entry 使用二分查找，但文件格式不规定具体内存搜索 +算法。 + +## File Metadata + +`TsFileMetadata` 位于文件尾部长度字段之前： + +```text +TsFileMetadata := + UVarInt table_index_count + TableIndexRoot table_index[table_index_count] + UVarInt table_schema_count + NamedTableSchema table_schema[table_schema_count] + int64 meta_offset + BloomFilter bloom_filter + VarInt property_count + Property property[property_count] + +TableIndexRoot := + VarString table_name + MetadataIndexNode device_index_root + +NamedTableSchema := + VarString table_name + TableSchema schema + +Property := + VarString key + VarString value +``` + +`meta_offset` 指向数据区末尾的 `0x02` separator。`TsFileMetadata` 前没有 marker， +其起始位置由文件尾保存的 4 字节长度确定。 + +Map 项的物理顺序不参与文件语义。`table_index` 保存树模型和表模型的设备索引根； +`table_schema` 保存显式表模型的 schema。 + +文件尾四字节元数据长度位于 `TsFileMetadata` 之外。设文件长度为 `F`、元数据长度为 `L`， +读取器使用下式得到精确字节区间： + +```text +file_metadata_start = F - 6 - 4 - L +file_metadata_end = F - 6 - 4 # 半开区间结束位置 +``` + +读取器必须恰好消费 `L` 字节。该检查可以在接受尾部魔数之前发现 Schema 不匹配或变长字段 +损坏。 + +## Table Schema + +```text +TableSchema := + UVarInt column_count + TableColumn column[column_count] + +TableColumn := + MeasurementSchema schema + int32 column_category + +MeasurementSchema := + String32 column_name + byte data_type + byte encoding_type + byte compression_type + int32 property_count + SchemaProperty property[property_count] + +SchemaProperty := + String32 key + String32 value +``` + +`MeasurementSchema` 使用 `String32`，而不是 Footer 其他位置常用的 `VarString`。 + +列类别编号为： + +| 值 | 类别 | +| ---: | --- | +| 0 | `TAG` | +| 1 | `FIELD` | +| 2 | `ATTRIBUTE` | +| 3 | `TIME` | + +列顺序具有语义，因为表记录和对齐值列按位置引用 Schema。`TAG` 列参与设备身份，`FIELD` +列保存测量值，`ATTRIBUTE` 列描述实体，`TIME` 标识时间列。列类别不能替代 `data_type`； +解释一个列时必须同时使用这两个字段。 diff --git a/src/zh/UserGuide/latest/FileFormat/Nulls.md b/src/zh/UserGuide/latest/FileFormat/Nulls.md new file mode 100644 index 0000000000..833114aa5b --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Nulls.md @@ -0,0 +1,68 @@ + + +# 空值 + +非对齐时间序列只保存实际存在的 `(timestamp, value)` 点，Page 内没有 null 标记。 + +对齐值列通过 Value Page 的位图保存 null： + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +位置 `i` 对应 `present_bitmap[i / 8]` 的位 `7 - (i mod 8)`： + +- 位为 1：该位置有值。 +- 位为 0：该位置为 null。 + +值编码流只保存位图中为 1 的值，顺序与位置顺序一致。`position_count` 等于对应 +Time Page 的时间戳数量。 + +例如，`position_count = 10` 且位图为 `10110000 01000000` 时，位置 `0`、`2`、`3` 和 +`9` 有值。值解码器因此产生 4 个值并放到这些位置，其余 6 个位置为 null。最后一个 +字节中未使用的低位不对应逻辑位置，应当忽略。 + +非空 Value Page 满足以下约束： + +```text +bitmap_bytes = ceil(position_count / 8) +decoded_value_count = popcount(present_bitmap 的 position_count 个有效位) +position_count = 对应 Time Page 的 decoded_timestamp_count +``` + +任何不相等都表示结构损坏；读取器不能根据位图尾部的填充位或多余编码值推导新位置。 + +如果整个 Value Page 都是 null，Page 只写一个值为 0 的 `uncompressed_size`： + +```text +EmptyPage := UVarInt(0) +``` + +空 Page 不保存 `compressed_size`、Statistics、位图或值流。其位置数量由对应的 +Time Page 确定。 + +缺少 Value Chunk 与空 Value Page 的含义不同。缺少 Chunk 表示该 Chunk Group 中没有 +该测点的物理列；空 Page 表示该值列存在，并维持 Page 之间的位置对应关系，但在这个 +区间的所有位置均为 null。 diff --git a/src/zh/UserGuide/latest/FileFormat/README.md b/src/zh/UserGuide/latest/FileFormat/README.md new file mode 100644 index 0000000000..2b09d9a253 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/README.md @@ -0,0 +1,140 @@ + + +# 文件格式 + +本文描述 TsFile 第 4 版的语言无关二进制文件格式。文件版本字节为 `0x04`。 + + + +## 范围与术语 + +本规范定义独立实现之间交换完整 TsFile v4 所需的二进制字节。文件包含四个组织层级： + +- **Page** 是可以独立压缩和解码的最小数据单元。 +- **Chunk** 包含一个物理列的多个 Page，并确定这些 Page 的数据类型、编码和压缩方式。 +- **Chunk Group** 将一组连续 Chunk 关联到同一个设备标识符。 +- **文件**由顺序写入的数据区、索引以及文件级元数据组成。 + +“对齐”表示多个测量共享一个物理时间列；“非对齐”表示每个测量携带自己的时间流。 +对齐方式改变 Page 载荷的物理布局，但不改变时间戳和值本身的语义。 + +除非明确使用半开区间，本规范中的字节区间均包含首尾。绝对偏移从文件第一个字节开始 +计算。除非另有说明，序列化长度不包含保存该长度的字段自身。 + +## 物理区域 + +设文件长度为 `F`，文件尾四字节记录的 `TsFileMetadata` 长度为 `L`， +`meta_offset` 为 `TsFileMetadata` 内保存的值： + +| 区域 | 包含首尾的字节区间 | 定位方式 | +| --- | --- | --- | +| 头部魔数 | `0 … 5` | 固定位置 | +| 版本 | `6` | 固定位置 | +| 数据区 | `7 … meta_offset - 1` | 在分隔符前结束 | +| 分隔符 | `meta_offset` | 值为 `0x02`，由 `TsFileMetadata` 引用 | +| 索引元数据 | `meta_offset + 1 … F - 11 - L` | 由元数据偏移划分 | +| `TsFileMetadata` | `F - 10 - L … F - 11` | 根据文件尾长度反向定位 | +| 元数据长度 | `F - 10 … F - 7` | 相对文件尾的固定位置 | +| 尾部魔数 | `F - 6 … F - 1` | 相对文件尾的固定位置 | + +索引元数据区可以按实现选择的物理顺序保存 Timeseries Metadata 和 Metadata Index Node。 +它们之间的关系由绝对偏移确定，而不是由物理相邻关系确定。 + +## 写入模型 + +上例中，数据按设备分成多个 Chunk Group。一个 Chunk 保存一个测量列，每个 Chunk +包含一个或多个 Page。非对齐 Chunk 在同一个 Page 中保存时间流和值流；对齐数据由 +一个 Time Chunk 和若干 Value Chunk 共享时间列。 + +文件元数据位于数据之后，写入器可以先顺序写入所有 Chunk Group，再写入索引和 +Footer。读取器通常先读取文件尾，从 `TsFile Metadata size` 定位 +`TsFileMetadata`，再通过其中的索引查找所需 Chunk。 + +写入器可以选择 Page 大小、Chunk Group 刷新阈值、元数据索引扇出、编码和压缩方式。 +这些选择会影响性能和物理边界，但读取器只需根据文件中的标记、长度和元数据解析最终 +布局，不需要获得写入器配置。 + +## 元数据优先读取算法 + +随机访问读取器无需扫描数据区即可打开完整文件： + +1. 校验文件尾的六字节魔数。 +2. 在 `F - 10` 读取 `L`，计算 `file_metadata_pos = F - 10 - L`。 +3. 反序列化 `TsFileMetadata`，并验证 `meta_offset` 指向 `0x02`。 +4. 可以先用完整路径查询布隆过滤器。阴性结果可以确定路径不存在；阳性结果仍需查索引。 +5. 依次遍历表/设备索引和测量索引，定位目标 Timeseries Metadata 区间。 +6. 根据时间和值谓词选择 Chunk Metadata 中记录的 Chunk 偏移。 +7. 跳转到 Chunk Header，利用可用的 Page Statistics，仅读取和解码所需 Page。 + +因此，读取元数据的开销与选中的设备、测量、Chunk 和 Page 数量相关，而不是与整个文件 +大小线性相关。 + +## 结构不变量 + +读取外部提供的长度并进行内存分配之前，合规读取器至少需要验证以下关系： + +- 头部和尾部魔数相同，且版本字节受支持。 +- `0 <= L <= F - 10`，计算得到的 `file_metadata_pos` 位于 `meta_offset` 之后。 +- `meta_offset` 是文件内的绝对偏移，且对应字节为 `0x02`。 +- 每个 Chunk 恰好结束在其 Chunk Header 之后的 `data_size` 字节处。 +- 每个非空 Page 均位于所属 Chunk 内，磁盘载荷恰好包含 `compressed_size` 字节。 +- 同一索引节点内的条目偏移单调不减，且不超过 `end_offset`。 +- 对齐 Value Page 与相同序号的 Time Page 对应，其 `position_count` 等于该 Time Page 的 + 位置数。 + +长度或偏移不满足上述关系时即为格式错误。读取器不得把长度限定的 Page 载荷内部偶然出现 +的标记字节解释为顶层标记。 + +文件中只有一个用于结束数据区的顶层 `0x02` separator。它位于全部元数据之前，并由 +`meta_offset` 指向。`TsFileMetadata` 前没有单独的 separator 或 marker，它的位置由 +文件尾的长度字段确定。 + +设文件总长度为 `F`，`TsFileMetadata` 的序列化长度为 `L`： + +```text +metadata_size_pos = F - 6 - 4 +file_metadata_pos = F - 6 - 4 - L +``` + +文件头和文件尾的魔数字节均为： + +```text +54 73 46 69 6c 65 # "TsFile" +``` + +## Format Details + +- [Binary Representation](Binary-Representation.md) +- [Configurations](Configurations.md) +- [Extensibility](Extensibility.md) +- [Metadata](Metadata.md) +- [Types](Types.md) +- [Bloom Filter](Bloom-Filter.md) +- [Data Pages](DataPages/README.md) + - [Chunks](DataPages/Chunks.md) + - [Compression](DataPages/Compression.md) + - [Encodings](DataPages/Encodings.md) + - [Encryption](DataPages/Encryption.md) + - [Checksumming](DataPages/Checksumming.md) + - [Error Recovery](DataPages/Error-Recovery.md) +- [Nulls](Nulls.md) +- [Format Versions](Format-Versions.md) diff --git a/src/zh/UserGuide/latest/FileFormat/Types.md b/src/zh/UserGuide/latest/FileFormat/Types.md new file mode 100644 index 0000000000..5797dd605c --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Types.md @@ -0,0 +1,81 @@ + + +# 类型 + +Chunk Header、Timeseries Metadata 和 Table Schema 使用单字节 `data_type` 编号： + +| 值 | 名称 | 物理/逻辑含义 | +| ---: | --- | --- | +| 0 | `BOOLEAN` | 布尔值 | +| 1 | `INT32` | 有符号 32 位整数 | +| 2 | `INT64` | 有符号 64 位整数 | +| 3 | `FLOAT` | IEEE 754 binary32 | +| 4 | `DOUBLE` | IEEE 754 binary64 | +| 5 | `TEXT` | 长度前缀字节串 | +| 6 | `VECTOR` | 对齐 Time Chunk 使用的物理类型标记 | +| 7 | `UNKNOWN` | 未知类型标记 | +| 8 | `TIMESTAMP` | 有符号 64 位整数；时间单位由上层约定 | +| 9 | `DATE` | `YYYYMMDD` 十进制形式的有符号 32 位整数 | +| 10 | `BLOB` | 任意字节串 | +| 11 | `STRING` | UTF-8 字符串 | +| 12 | `OBJECT` | 对象值的字节表示 | + +`DATE` 的物理值示例：2026-07-27 写为十进制整数 `20260727`，不是 Unix epoch day。 + +数据类型决定 Statistics 的附加字段和值编码器的输入类型。具体线上值格式见 +[Encodings](DataPages/Encodings.md)。 + +## 物理类型与逻辑类型 + +类型编号决定物理解码器；`TIMESTAMP` 和 `DATE` 还具有额外的逻辑解释： + +- `TIMESTAMP` 与 `INT64` 具有相同的八字节物理值域。时间单位由使用该文件的协议或数据集 + 提供，TsFile v4 不序列化该单位。 +- `DATE` 与 `INT32` 使用相同的整数编码接口，但有效逻辑值采用十进制 `YYYYMMDD` 形式。 +- `TEXT` 是带长度的字节串。`STRING` 使用相同的物理二进制容器，但具有 UTF-8 字符串 + 语义；`BLOB` 是不解释的二进制数据。 +- `OBJECT` 是不透明的二进制表示，其对象契约必须在文件格式之外约定。 + +`VECTOR` 是对齐数据共享时间列使用的物理标记，不是标量值类型，也不定义独立值流编码。 +`UNKNOWN` 是哨兵值，不能作为可解码测量 Page 的有效类型。 + +## 类型一致性 + +同一序列的类型会在多个层级重复，使读取器既能从 Footer 进入，也能顺序扫描文件。这些副本 +必须一致： + +- 非对齐 Chunk 或对齐 Value Chunk Header 使用测量的标量 `data_type`。 +- 对应 Timeseries Metadata 使用相同的 `data_type`。 +- Statistics 根据该类型解析，自身不包含独立类型字节。 +- 对齐 Time Chunk 使用 `VECTOR`，解码值为有符号 64 位时间戳。 + +改变已经分配的类型编号含义属于不兼容变更。不支持某个类型编号的读取器仍可利用 +`data_size` 跳过完整 Chunk，但无法解码该序列的值或类型相关 Statistics。 + +## 排序与比较 + +整数和浮点值按数值顺序生成最小/最大 Statistics。`STRING` 的最小/最大值按序列化字节的 +二进制字典序比较。`TEXT` 只保存 first 和 last，`BLOB` 与 `OBJECT` 不保存类型专有的值 +Statistics。 + +类型提升和应用层转换不编码在 TsFile 中。读取器可以对外提供转换，但这不会改变磁盘 +`data_type`，也不会改变计算 Statistics 时使用的字节和值域。 From 0809473d390bc10c70077a1f9867d5f3d82f9fc2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 16:47:16 +0800 Subject: [PATCH 02/14] Refine TsFile v4 file format documentation --- src/.vuepress/components/tsFileStructure.ts | 190 ++++++++++-------- .../FileFormat/Binary-Representation.md | 39 ++-- .../latest/FileFormat/Bloom-Filter.md | 57 +++++- .../latest/FileFormat/Configurations.md | 36 ++-- .../FileFormat/DataPages/Checksumming.md | 43 ++-- .../latest/FileFormat/DataPages/Chunks.md | 68 ++++--- .../FileFormat/DataPages/Compression.md | 44 ++-- .../latest/FileFormat/DataPages/Encodings.md | 38 ++-- .../latest/FileFormat/DataPages/Encryption.md | 162 +++++++++++---- .../FileFormat/DataPages/Error-Recovery.md | 83 ++++---- .../latest/FileFormat/DataPages/README.md | 39 ++-- .../latest/FileFormat/Extensibility.md | 58 +++--- .../latest/FileFormat/Format-Versions.md | 49 ++--- src/UserGuide/latest/FileFormat/Metadata.md | 55 +++-- src/UserGuide/latest/FileFormat/Nulls.md | 9 +- src/UserGuide/latest/FileFormat/README.md | 175 ++++++++-------- src/UserGuide/latest/FileFormat/Types.md | 12 +- .../FileFormat/Binary-Representation.md | 51 +++-- .../latest/FileFormat/Bloom-Filter.md | 64 +++++- .../latest/FileFormat/Configurations.md | 77 ++++--- .../FileFormat/DataPages/Checksumming.md | 31 ++- .../latest/FileFormat/DataPages/Chunks.md | 74 ++++--- .../FileFormat/DataPages/Compression.md | 38 ++-- .../latest/FileFormat/DataPages/Encodings.md | 37 ++-- .../latest/FileFormat/DataPages/Encryption.md | 156 +++++++++++--- .../FileFormat/DataPages/Error-Recovery.md | 77 ++++--- .../latest/FileFormat/DataPages/README.md | 84 ++++---- .../latest/FileFormat/Extensibility.md | 48 ++--- .../latest/FileFormat/Format-Versions.md | 39 ++-- .../UserGuide/latest/FileFormat/Metadata.md | 72 ++++--- src/zh/UserGuide/latest/FileFormat/Nulls.md | 9 +- src/zh/UserGuide/latest/FileFormat/README.md | 153 +++++++------- src/zh/UserGuide/latest/FileFormat/Types.md | 21 +- 33 files changed, 1274 insertions(+), 914 deletions(-) diff --git a/src/.vuepress/components/tsFileStructure.ts b/src/.vuepress/components/tsFileStructure.ts index c179434960..94f6ee4f2f 100644 --- a/src/.vuepress/components/tsFileStructure.ts +++ b/src/.vuepress/components/tsFileStructure.ts @@ -50,17 +50,19 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'TsFile v4', description: - 'The complete physical file. All offsets are absolute byte offsets measured from the first byte. Readers normally locate the footer from the tail, traverse metadata, and then read only selected Chunks and Pages.', + 'Data is written in device groups, with indexes at the end. A reader usually opens the footer first, then seeks to the Chunks and Pages it needs.', offset: '0 … F - 1', size: 'F bytes', - layout: 'File Header | Data Section | Separator | Metadata | Footer', + layout: 'File Header | FileEncryptionHeader? | Data | Separator | Indexed Metadata | TsFileMetadata | Length | Tail Magic', + note: 'F is the file length, L is the TsFileMetadata length, and E is the total encryption-header length (0 when absent).', }, zh: { title: 'TsFile v4', - description: '完整的物理文件。所有偏移均为从文件第一个字节开始计算的绝对字节偏移。读取器通常先从文件尾定位 Footer、遍历元数据，再按需读取选中的 Chunk 和 Page。', + description: '数据按设备分组写入，索引放在文件尾部。读取器通常先查 Footer，再跳转到需要的 Chunk 和 Page。', offset: '0 … F - 1', size: 'F 字节', - layout: '文件头 | 数据区 | 分隔符 | 元数据 | 文件尾', + layout: '文件头 | FileEncryptionHeader? | 数据区 | 分隔符 | 索引元数据 | TsFileMetadata | 长度 | 尾魔数', + note: 'F 为文件长度，L 为 TsFileMetadata 长度，E 为可选加密头的总长度；无加密头时 E = 0。', }, }, children: [ @@ -71,14 +73,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'File Header', - description: 'A fixed seven-byte prefix identifying TsFile and format version 4. The version byte selects the complete grammar before any data marker is interpreted.', + description: 'The first seven bytes contain the TsFile magic string and format version. Read these before choosing how to parse the rest of the file.', offset: '0 … 6', size: '7 bytes', layout: 'magic[6] | version[1]', }, zh: { title: '文件头', - description: '固定的 7 字节前缀，用于标识 TsFile 以及第 4 版文件格式。解释任何数据区标记之前，必须先用版本字节选择完整语法。', + description: '文件最前面的七个字节保存 TsFile 魔数和格式版本。先读取这部分，再确定后续结构的解析方式。', offset: '0 … 6', size: '7 字节', layout: 'magic[6] | version[1]', @@ -91,14 +93,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Head Magic', - description: 'The UTF-8 bytes of the literal string "TsFile". They identify the container but are not a checksum or a content-integrity proof.', + description: 'The six bytes spelling "TsFile" identify the file format.', offset: '0 … 5', size: '6 bytes', layout: '54 73 46 69 6c 65', }, zh: { title: '头部魔数', - description: '字符串 “TsFile” 的 UTF-8 字节。它用于识别容器，但不是校验和，也不能证明内容完整。', + description: '由字符串“TsFile”对应的六个字节组成，用于识别文件格式。', offset: '0 … 5', size: '6 字节', layout: '54 73 46 69 6c 65', @@ -112,14 +114,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Format Version', - description: 'The one-byte on-disk format identifier. Value 0x04 selects this layout; it is independent of the project 2.x release label.', + description: 'Value 0x04 identifies format v4. This byte is separate from the project release number.', offset: '6', size: '1 byte', layout: '04', }, zh: { title: '格式版本', - description: '单字节磁盘格式标识。数值 0x04 选择本页面所述布局，它与项目的 2.x 发布版本标签相互独立。', + description: '数值 0x04 表示 v4 文件格式，与项目的发布版本号分开定义。', offset: '6', size: '1 字节', layout: '04', @@ -128,6 +130,29 @@ export const tsFileStructure: TsFileStructureNode = { }, ], }, + { + id: 'file-encryption-header', + kind: 'record', + link: 'DataPages/Encryption.html#file-encryption-header', + text: { + en: { + title: 'File Encryption Header (optional)', + description: 'An optional header at offset 7 selects Page AEAD. It identifies the provider, profile, file crypto ID, and wrapped data key.', + offset: '7 … 6 + E; absent when E = 0', + size: 'E = 1 + UVarInt-size(payload_size) + payload_size', + layout: '06 | payload_size | 01 | provider_id | profile_id | key_id | key_version | file_crypto_id_size | file_crypto_id | wrapped_data_key_size | wrapped_data_key', + note: 'The header selects the AEAD layout even though legacy footer encryptLevel is 0. E is its total serialized size; E = 0 when absent.', + }, + zh: { + title: '文件加密头（可选）', + description: '偏移 7 处的可选加密头用于选择 Page AEAD，记录 provider、profile、文件加密 ID 和封装数据密钥。', + offset: '7 … 6 + E；E = 0 时不存在', + size: 'E = 1 + UVarInt-size(payload_size) + payload_size', + layout: '06 | payload_size | 01 | provider_id | profile_id | key_id | key_version | file_crypto_id_size | file_crypto_id | wrapped_data_key_size | wrapped_data_key', + note: '即使旧版 Footer 的 encryptLevel 为 0，也以该头选择 AEAD 布局。E 是该头的总序列化长度，不存在时为 0。', + }, + }, + }, { id: 'data-section', kind: 'region', @@ -136,17 +161,17 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Data Section', description: - 'Marker-delimited Chunk Groups and optional Operation Index Ranges. Structures are written sequentially, and every marker is interpreted only at a known top-level boundary.', - offset: '7 … meta_offset - 1', + 'Chunk Groups and operation-index records appear here in write order. The section begins after any file encryption header and ends before the separator.', + offset: '7 + E … meta_offset - 1', size: 'variable', - layout: 'Chunk Group* | Operation Index Range*', + layout: '(ChunkGroup | OperationIndexRange)*', }, zh: { title: '数据区', - description: '由标记分隔的 Chunk Group 和可选 Operation Index Range。各结构顺序写入，只有位于已知顶层边界的字节才可解释为 marker。', - offset: '7 … meta_offset - 1', + description: '这里按写入顺序保存 Chunk Group 和操作索引记录，从可选文件加密头之后开始，到分隔符之前结束。', + offset: '7 + E … meta_offset - 1', size: '可变', - layout: 'Chunk Group* | Operation Index Range*', + layout: '(ChunkGroup | OperationIndexRange)*', }, }, children: [ @@ -158,14 +183,14 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Chunk Group', description: - 'A repeated device-scoped container. All following Chunks belong to its device until another top-level marker is encountered. The group has no total-length field of its own.', + 'A batch of Chunks for one device. Another Chunk continues the group; a new group, operation-index record, or separator ends it.', offset: 'chunk_group_offset', size: 'variable · repeated', layout: 'Chunk Group Header | Chunk*', }, zh: { - title: 'Chunk Group', - description: '按设备组织的重复容器。后续 Chunk 均属于该设备，直到遇到下一个顶层标记。Chunk Group 自身没有总长度字段。', + title: '设备分组（Chunk Group）', + description: '同一设备的一批 Chunk。后续 Chunk 标记表示分组继续，新的分组、操作索引记录或分隔符表示当前分组结束。', offset: 'chunk_group_offset', size: '可变 · 重复', layout: 'Chunk Group Header | Chunk*', @@ -179,14 +204,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Chunk Group Header', - description: 'Starts one device group and serializes the segmented DeviceID. The segment count is an element count, and every segment is a separate VarString.', + description: 'Begins a device group with its DeviceID. The identifier stores a segment count followed by separately length-prefixed string segments.', offset: 'chunk_group_offset', size: 'variable', layout: 'marker 0x00 | DeviceID', }, zh: { - title: 'Chunk Group Header', - description: '开始一个设备分组，并序列化分片 DeviceID。片段数表示元素数量，每个片段都是独立 VarString。', + title: 'Chunk Group 头部', + description: '开始一个设备分组，并保存 DeviceID：先记录片段数量，再依次记录各个带长度的字符串片段。', offset: 'chunk_group_offset', size: '可变', layout: 'marker 0x00 | DeviceID', @@ -201,14 +226,14 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Chunk Layout', description: - 'A device contains non-aligned Chunks, aligned Chunk sets, or both. Each Chunk stores one physical column.', + 'Non-aligned Chunks store time and values together. Aligned Chunk sets put shared timestamps in a Time Chunk and measurements in Value Chunks.', offset: 'after Chunk Group Header', size: 'variable · repeated', layout: 'Non-aligned Chunk | Aligned Chunk Set', }, zh: { title: 'Chunk 布局', - description: '一个设备可包含非对齐 Chunk、对齐 Chunk 集，或同时包含两者。每个 Chunk 保存一个物理列。', + description: '非对齐 Chunk 把时间和值放在一起；对齐 Chunk 集用一个 Time Chunk 保存共享时间戳，再用各个 Value Chunk 保存测量值。', offset: 'Chunk Group Header 之后', size: '可变 · 重复', layout: '非对齐 Chunk | 对齐 Chunk 集', @@ -223,14 +248,14 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Non-aligned Chunk', description: - 'Stores one measurement and its independent timestamp stream. Every Page contains both streams, and the decoded timestamp and value counts must match.', + 'Stores one measurement with its own timestamps. Each Page contains a time stream and a value stream with matching point counts.', offset: 'chunk_offset', size: 'header + data_size', layout: 'Chunk Header | Non-aligned Page*', }, zh: { title: '非对齐 Chunk', - description: '保存一个测量列及其独立时间戳流。每个 Page 同时包含时间流和值流，解码后的时间戳数和值数必须相等。', + description: '保存一个测量及其独立的时间戳。每个 Page 都有时间流和值流，两者解码后的数据点数量相同。', offset: 'chunk_offset', size: 'header + data_size', layout: 'Chunk Header | 非对齐 Page*', @@ -245,21 +270,21 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Chunk Header', description: - 'Describes the value column and gives the exact total byte length of all Page Headers and stored Page data. This boundary lets readers skip the whole Chunk without decoding its codec.', + 'Names the measurement and its codecs. data_size counts all Page Headers and stored bodies after this Header, so a reader can locate the next Chunk without decoding values.', offset: 'chunk_offset', size: 'variable', layout: - 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type', - note: 'For a non-aligned Chunk, encoding_type describes the value stream.', + 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type | AEAD suffix?', + note: 'encoding_type describes values; time encoding is external configuration (default TS_2DIFF). The AEAD suffix is int64 ordinal | int32 proof_size | proof[proof_size], outside data_size.', }, zh: { - title: 'Chunk Header', - description: '描述值列，并给出全部 Page Header 与磁盘 Page 数据的准确总字节数。读取器可以利用该边界跳过整个 Chunk，而无需调用其编解码器。', + title: 'Chunk 头部', + description: '记录测量名及编解码方式。data_size 统计头部之后全部 Page Header 和载荷的长度，读取器无需解码值就能定位下一个 Chunk。', offset: 'chunk_offset', size: '可变', layout: - 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type', - note: '对于非对齐 Chunk，encoding_type 描述的是值流。', + 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type | AEAD suffix?', + note: 'encoding_type 描述值流；时间编码来自外部配置，默认 TS_2DIFF。AEAD 后缀为 int64 序号 | int32 证明长度 | 证明字节，不计入 data_size。', }, }, }, @@ -271,21 +296,21 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Non-aligned Page', description: - 'The Page Header is followed by one stored Page payload. After decryption and decompression, a length-delimited time stream comes first and the value stream occupies the remaining bytes.', + 'After decryption and decompression, the payload starts with the time-stream length and time bytes. The value stream fills the rest of the payload.', offset: 'inside Chunk data_size', size: 'variable · repeated', layout: 'Page Header | time_stream_size | encoded_time_stream | encoded_value_stream', - note: 'Timestamps and values pair by position; they are not interleaved point by point.', + note: 'Time and values pair by position. Their codecs can differ; the time codec comes from shared configuration.', }, zh: { title: '非对齐 Page', - description: 'Page Header 后是一个磁盘 Page 载荷。解密并解压后，先存带长度的完整时间流，值流占据剩余全部字节。', + description: '解密、解压后的载荷先保存时间流长度和时间流字节，剩余部分保存值流。', offset: 'Chunk data_size 内部', size: '可变 · 重复', layout: 'Page Header | time_stream_size | encoded_time_stream | encoded_value_stream', - note: '时间戳和值按位置配对，并不是逐点交错存储。', + note: '时间戳和值按位置配对，可以使用不同编码；时间编码由共享配置提供。', }, }, }, @@ -299,14 +324,14 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Aligned Chunk Set', description: - 'Stores one shared time column followed by aligned measurement value columns. Chunks are contiguous rather than interleaved Page by Page, while equal Page ordinals define correspondence.', + 'One Time Chunk is followed by its Value Chunks. Page i in each Value Chunk corresponds to Page i in the Time Chunk; the set has no separate header.', offset: 'first aligned Chunk offset', size: 'variable · repeated', layout: 'Time Chunk | Value Chunk*', }, zh: { title: '对齐 Chunk 集', - description: '先保存一个共享时间列，再保存各个对齐测量值列。不同 Chunk 连续存放而不按 Page 交错；相同 Page 序号建立对应关系。', + description: '一个 Time Chunk 后跟若干 Value Chunk。每个 Value Chunk 的第 i 个 Page 对应 Time Chunk 的第 i 个 Page，集合本身没有额外头部。', offset: '第一个对齐 Chunk 的偏移', size: '可变 · 重复', layout: 'Time Chunk | Value Chunk*', @@ -320,14 +345,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Time Chunk', - description: 'Contains all Time Pages for the aligned set. Its Chunk Header records the time encoding, and each decoded Page establishes the logical positions shared by matching Value Pages.', + description: 'Stores the timestamps shared by the aligned measurements. Its Header names the time codec, and each Time Page defines positions for the corresponding Value Pages.', offset: 'time_chunk_offset', size: 'header + data_size', layout: 'Time Chunk Header | Time Page*', }, zh: { - title: 'Time Chunk', - description: '包含该对齐集合的全部 Time Page；其 Chunk Header 记录时间编码。每个解码后的 Page 建立对应 Value Page 共用的逻辑位置。', + title: '时间列（Time Chunk）', + description: '保存各对齐测量共享的时间戳。头部说明时间编码，每个 Time Page 确定对应 Value Page 使用的那组位置。', offset: 'time_chunk_offset', size: 'header + data_size', layout: 'Time Chunk Header | Time Page*', @@ -342,14 +367,14 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Value Chunk', description: - 'Contains one aligned measurement column. Value Page number i corresponds to Time Page number i; its bitmap maps present values and nulls onto those shared positions.', + 'Stores one aligned measurement. Its Page bitmaps place the stored values at the matching timestamps and leave other positions null.', offset: 'value_chunk_offset', size: 'header + data_size · repeated', layout: 'Value Chunk Header | Value Page*', }, zh: { - title: 'Value Chunk', - description: '包含一个对齐测量值列。编号为 i 的 Value Page 对应编号为 i 的 Time Page，并通过位图把实际值和 null 映射到共享位置。', + title: '测量列（Value Chunk）', + description: '保存一个对齐测量列，通过各 Page 的位图把值对应到时间戳，其余位置为 null。', offset: 'value_chunk_offset', size: 'header + data_size · 重复', layout: 'Value Chunk Header | Value Page*', @@ -369,15 +394,15 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Operation Index Range', - description: 'An optional storage-engine record containing two operation sequence numbers for checkpoint, snapshot, backup, or recovery coordination. They are not timestamps or file offsets.', - offset: 'between Chunk Groups', + description: 'Records the minimum and maximum operation sequence numbers for a flushed batch. A storage engine can use them to relate file data to checkpoints or its operation log.', + offset: 'top-level boundary', size: '17 bytes', layout: 'marker 0x04 | min_operation_index[8] | max_operation_index[8]', }, zh: { - title: 'Operation Index Range', - description: '可选的存储引擎顶层记录，包含两个操作序列号，用于协调检查点、快照、备份或恢复。它们不是时间戳，也不是文件偏移量。', - offset: 'Chunk Group 之间', + title: '操作索引范围', + description: '记录已刷出批次的最小和最大操作序号，供存储引擎把文件数据与检查点或操作日志关联。', + offset: '顶层结构边界', size: '17 字节', layout: 'marker 0x04 | min_operation_index[8] | max_operation_index[8]', }, @@ -392,14 +417,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Data/Metadata Separator', - description: 'The single top-level separator ending the data section. TsFileMetadata.meta_offset points to this byte; a matching 0x02 inside payload data has no marker meaning.', + description: 'Byte 0x02 ends the data section. meta_offset points here, and indexed metadata starts at the next byte.', offset: 'meta_offset', size: '1 byte', layout: '02', }, zh: { title: '数据/元数据分隔符', - description: '结束数据区的唯一顶层分隔符，TsFileMetadata.meta_offset 指向这个字节。负载内部出现相同的 0x02 不具有 marker 含义。', + description: '字节 0x02 结束数据区，meta_offset 指向此处，索引元数据从下一个字节开始。', offset: 'meta_offset', size: '1 字节', layout: '02', @@ -414,17 +439,17 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Indexed Metadata Section', description: - 'Timeseries Metadata and metadata index nodes. Their physical ranges can be interleaved and are delimited by absolute offsets; this is a graph of referenced byte ranges, not one self-delimiting list.', + 'Holds series metadata and index nodes. They can be interleaved, so follow their absolute offsets rather than assuming a fixed order.', offset: 'meta_offset + 1 … file_metadata_pos - 1', size: 'variable', - layout: 'TimeseriesMetadata* | MetadataIndexNode*', + layout: '(TimeseriesMetadata | MetadataIndexNode)*', }, zh: { title: '索引元数据区', - description: 'Timeseries Metadata 与元数据索引节点。两类记录的物理区间可以交错，并由绝对偏移划分；这里是由字节范围引用形成的图，而不是单一自定界列表。', + description: '保存时间序列元数据和索引节点。两者可以交错存放，读取时应跟随绝对偏移量，不依赖固定顺序。', offset: 'meta_offset + 1 … file_metadata_pos - 1', size: '可变', - layout: 'TimeseriesMetadata* | MetadataIndexNode*', + layout: '(TimeseriesMetadata | MetadataIndexNode)*', }, }, children: [ @@ -435,15 +460,15 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Timeseries Metadata', - description: 'One logical-series record containing aggregate Statistics and a byte-length-delimited Chunk Metadata list. Entries point back to physical Chunk Headers in the data section.', + description: 'Describes one series, with summary Statistics and a list of its Chunks. Each Chunk entry points to a Header in the data section.', offset: 'referenced by a measurement index entry', size: 'variable · repeated', layout: 'timeseries_type | measurement_id | data_type | chunk_metadata_list_size | Statistics | ChunkMetadata*', }, zh: { - title: 'Timeseries Metadata', - description: '每个逻辑序列对应一条记录，包含聚合 Statistics 和按字节长度定界的 Chunk Metadata 列表。各条目反向指向数据区的物理 Chunk Header。', + title: '时间序列元数据', + description: '描述一条时间序列，包含汇总统计信息和 Chunk 列表。列表中的每个条目指向数据区的一个 Chunk Header。', offset: '由测量索引条目引用', size: '可变 · 重复', layout: @@ -458,14 +483,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Metadata Index Node', - description: 'An ordered internal or leaf node used to locate devices, measurements, and metadata ranges. Each child offset starts a range whose end is the next child offset or end_offset.', + description: 'Locates devices, measurements, or further index nodes. An entry covers bytes from its offset up to, but not including, the next offset or end_offset.', offset: 'referenced by a parent node or TsFileMetadata', size: 'variable · repeated', layout: 'child_count | MetadataEntry* | end_offset | node_type', }, zh: { - title: 'Metadata Index Node', - description: '用于定位设备、测量列和元数据区间的有序内部节点或叶子节点。每个子项的范围从自身 offset 开始，到下一子项 offset 或 end_offset 结束。', + title: '元数据索引节点', + description: '用于查找设备、测量或下一级索引节点。每个条目的字节范围从自身偏移开始，到下一偏移或 end_offset 之前结束。', offset: '由父节点或 TsFileMetadata 引用', size: '可变 · 重复', layout: 'child_count | MetadataEntry* | end_offset | node_type', @@ -482,7 +507,7 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'TsFileMetadata', description: - 'The file-level index roots, table schemas, data-section offset, Bloom filter, and properties. Its start is derived from the trailing fixed-width length, so no preceding marker is needed.', + 'The entry point for metadata lookup: table index roots, schemas, separator offset, Bloom filter, and properties. Its start is calculated from the trailing length.', offset: 'file_metadata_pos = F - 6 - 4 - L', size: 'L bytes', layout: @@ -491,7 +516,7 @@ export const tsFileStructure: TsFileStructureNode = { }, zh: { title: 'TsFileMetadata', - description: '包含文件级索引根、表 Schema、数据区偏移、布隆过滤器以及文件属性。其起点由尾部定宽长度反向计算，因此前面无需 marker。', + description: '元数据查找的入口，保存表索引根、Schema、分隔符偏移、布隆过滤器和属性。通过文件尾的长度反向定位其起点。', offset: 'file_metadata_pos = F - 6 - 4 - L', size: 'L 字节', layout: 'table indexes | table schemas | meta_offset | BloomFilter | properties', @@ -505,13 +530,13 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Table Index Roots', - description: 'Named device-index roots for tree-model and table-model data. A root stores the entry range used to descend toward one device and then one measurement.', + description: 'Stores each table\'s device-index root. Follow a root to find a device, then its measurements.', offset: 'inside TsFileMetadata', size: 'variable · repeated', }, zh: { title: '表索引根', - description: '树模型与表模型数据使用的具名设备索引根。根节点保存向下定位某个设备、再定位某个测点所需的条目范围。', + description: '保存各表的设备索引根。从根节点查找设备，再查找设备下的测量。', offset: 'TsFileMetadata 内部', size: '可变 · 重复', }, @@ -524,13 +549,13 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Table Schemas', - description: 'Named schemas for explicit table-model data. Column names, physical types, and categories preserve a common column order.', + description: 'Describes table columns in their declared order, including names, types, and categories. FIELD Chunks are matched by measurement name.', offset: 'inside TsFileMetadata', size: 'variable · repeated', }, zh: { title: '表 Schema', - description: '显式表模型数据使用的具名 Schema。列名、物理类型和列类别共同保持同一个列顺序。', + description: '按声明顺序描述表的列，记录列名、类型和类别；FIELD Chunk 通过测量名与列对应。', offset: 'TsFileMetadata 内部', size: '可变 · 重复', }, @@ -542,14 +567,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'meta_offset', - description: 'Absolute offset of the top-level 0x02 separator at the end of the data section. The indexed metadata section begins at the following byte.', + description: 'The absolute offset of the separator. Add one to find the start of indexed metadata.', offset: 'inside TsFileMetadata', size: '8 bytes', layout: 'int64, big-endian', }, zh: { title: 'meta_offset', - description: '数据区末尾顶层 0x02 分隔符的绝对偏移；索引元数据区从它的下一个字节开始。', + description: '分隔符的绝对偏移量，加一就是索引元数据区的起点。', offset: 'TsFileMetadata 内部', size: '8 字节', layout: 'int64，大端序', @@ -563,37 +588,38 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Bloom Filter', - description: 'Optional membership filter used to reject missing full paths without index traversal. A negative answer is definitive; a positive answer still requires metadata lookup.', + description: 'Can rule out a missing path before an index lookup. Trust a negative result only when path encoding and hashing match the writer.', offset: 'inside TsFileMetadata', size: 'variable', - layout: 'filter bytes | bit count | hash-function count', + layout: 'byte_length | filter bytes | bit count | hash-function count', }, zh: { title: '布隆过滤器', - description: '可选的成员过滤器，无需遍历索引即可排除不存在的完整路径。否定结果是确定的，肯定结果仍需查询元数据确认。', + description: '可以在查索引之前排除不存在的路径，但要先确认路径编码和哈希方式与写入端一致。', offset: 'TsFileMetadata 内部', size: '可变', - layout: 'filter bytes | bit count | hash-function count', + layout: 'byte_length | filter bytes | bit count | hash-function count', }, }, }, { id: 'file-properties', kind: 'repeated', + link: 'Metadata.html#file-metadata', text: { en: { title: 'File Properties', - description: 'A length-delimited map of string keys and values, including encryption properties when present. Unknown properties remain structurally consumable.', + description: 'Maps UTF-8 keys to nullable binary values. Keep unknown values as raw bytes; only a property\'s own definition can say whether its value is text.', offset: 'end of TsFileMetadata', size: 'variable · repeated', - layout: 'property_count | (key | value)*', + layout: 'VarInt property_count | (VarString key | VarBinary value)*', }, zh: { title: '文件属性', - description: '按长度定界的字符串键值映射；启用加密时也包含加密属性。未知属性在结构上仍可读取和跳过。', + description: '把 UTF-8 键映射到可空的二进制值。未知值按原始字节保留，只有具体属性定义才能决定它是否应按文本解释。', offset: 'TsFileMetadata 尾部', size: '可变 · 重复', - layout: 'property_count | (key | value)*', + layout: 'VarInt property_count | (VarString key | VarBinary value)*', }, }, }, @@ -605,14 +631,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'TsFileMetadata Size', - description: 'The big-endian serialized byte length L of TsFileMetadata. With file size F, metadata occupies exactly [F - 10 - L, F - 10).', + description: 'Stores L, the byte length of TsFileMetadata. Subtract L and the ten trailing bytes from file size F to locate that metadata.', offset: 'F - 10 … F - 7', size: '4 bytes', layout: 'int32 L, big-endian', }, zh: { title: 'TsFileMetadata 长度', - description: 'TsFileMetadata 的大端序列化字节长度 L。文件大小为 F 时，元数据恰好位于 [F - 10 - L, F - 10)。', + description: '保存 TsFileMetadata 的字节长度 L。用文件大小 F 减去 L 和尾部十个字节，即可找到元数据起点。', offset: 'F - 10 … F - 7', size: '4 字节', layout: 'int32 L，大端序', @@ -625,14 +651,14 @@ export const tsFileStructure: TsFileStructureNode = { text: { en: { title: 'Tail Magic', - description: 'The same six-byte magic string as the file header. It supports completion checks but is not sufficient without validating the footer length and structure.', + description: 'The same six magic bytes as the header. Check this suffix together with the footer length and metadata structure before opening the file.', offset: 'F - 6 … F - 1', size: '6 bytes', layout: '54 73 46 69 6c 65', }, zh: { title: '尾部魔数', - description: '与文件头相同的 6 字节魔数字符串。它可用于完成性检查，但还必须同时校验 Footer 长度与结构。', + description: '与文件头相同的六个魔数字节。打开文件时，还要同时检查 Footer 长度和元数据结构。', offset: 'F - 6 … F - 1', size: '6 字节', layout: '54 73 46 69 6c 65', diff --git a/src/UserGuide/latest/FileFormat/Binary-Representation.md b/src/UserGuide/latest/FileFormat/Binary-Representation.md index a75c937d92..fc2c57d864 100644 --- a/src/UserGuide/latest/FileFormat/Binary-Representation.md +++ b/src/UserGuide/latest/FileFormat/Binary-Representation.md @@ -46,12 +46,9 @@ is written first. value = Σ ((byte[i] & 0x7f) << (7 * i)) ``` -The encoding occupies one to five bytes and uses the shortest form. - -For example, decimal `300` is split into low-order seven-bit groups `0x2c` and `0x02`, producing -`ac 02`. A reader must stop after at most five bytes for a 32-bit field and must detect a shift or -value overflow. Writers emit the canonical shortest representation; overlong representations are -not generated. +The encoding uses the shortest representation, from one to five bytes. For example, decimal +`300` splits into the seven-bit groups `0x2c` and `0x02`, so its encoded bytes are `ac 02`. +A 32-bit decoder rejects an unterminated sequence after five bytes and checks for overflow. ## Signed Variable-length Integer @@ -76,13 +73,17 @@ zigzag(n) = (n << 1) XOR (n >> 31) | `VarString` | `VarInt byte_length` + `byte[byte_length]` | | `String32` | `int32 byte_length` + `byte[byte_length]` | | `Binary32` | `int32 byte_length` + `byte[byte_length]` | +| `VarBinary` | `VarInt byte_length` + `byte[byte_length]` | -A length of `-1` represents null, and a length of `0` represents an empty string or byte array. -`VarString` and `String32` contain UTF-8; `Binary32` contains raw bytes. +For `VarString`, `String32`, and `VarBinary`, a length of `-1` represents null; a length of `0` +represents an empty string or byte array. `Binary32` does not accept a negative length. +`VarString` and `String32` contain UTF-8; `Binary32` and `VarBinary` contain raw bytes. +`VarBinary` uses a signed ZigZag length, unlike the unsigned lengths of the binary components in +[FileEncryptionHeader](DataPages/Encryption.md#file-encryption-header). -The length counts encoded bytes, not Unicode code points or characters. For example, a UTF-8 -string containing one non-ASCII character can have a byte length greater than one. Null and empty -are distinct wire values and must remain distinct in structures whose fields are nullable. +String lengths count bytes after UTF-8 encoding. For example, `温` occupies three bytes, so its +`VarString` length field is `VarInt(3)`, encoded as `06`. A nullable field preserves the distinction +between null and an empty value. ## Boundaries and Defensive Decoding @@ -96,9 +97,9 @@ current_offset + N >= current_offset current_offset + N <= enclosing_end ``` -The second comparison detects integer wraparound. A reader should additionally enforce a local -resource limit; a structurally valid length does not require an implementation to allocate an -unbounded amount of memory. +The second comparison detects wraparound when offset arithmetic uses fixed-width integers. +Check these bounds before allocating memory, and apply an implementation-specific size limit +as well: a length can fit within the file and still be too large for the reader to allocate. Fixed-width values require their complete byte width. A truncated `int64`, unterminated VarInt, negative non-null length, invalid UTF-8 where a string is required, or bytes left over after a @@ -109,5 +110,11 @@ length-delimited record are format errors. Offsets stored in metadata are signed 64-bit values but represent non-negative absolute byte positions. An offset points to the first byte of the named structure: for example, `offset_of_chunk_header` points to the Chunk marker, and `meta_offset` points to the `0x02` -separator. Lengths are not implicit offsets; arithmetic that combines an offset and length must -be checked for overflow and against the enclosing file or record boundary. +separator. To find the end of a structure, add its length to its start offset using the bounds checks above. + +## Reading Layout Examples + +In layout examples, `?` marks an optional field or structure, `*` allows zero or more repetitions, +and `[n]` means n elements. The surrounding description or a preceding length field determines +which case applies. `remaining bytes` means the unused bytes up to the enclosing payload boundary; +for a Page layout, that is the end of its decrypted, decompressed payload. diff --git a/src/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/UserGuide/latest/FileFormat/Bloom-Filter.md index 558f93059c..c773a95b90 100644 --- a/src/UserGuide/latest/FileFormat/Bloom-Filter.md +++ b/src/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -21,9 +21,10 @@ # Bloom Filter -TsFile v4 stores a file-level Bloom filter in `TsFileMetadata` to test whether a timeseries path -may exist. The filter can report that a path is definitely absent or possibly present. It cannot -confirm that a path exists. +The optional Bloom filter in `TsFileMetadata` can save an index lookup when a path is absent. +A clear bit rules the path out; all tested bits being set means only “possibly present,” so the +reader still checks the index. This result is reliable only when the query uses the writer's +path encoding and hash behavior, described below. ## File Format @@ -43,19 +44,41 @@ logical number of bits. ## Filter Input -The filter input is the UTF-8 representation of the complete timeseries path: +For a writer using UTF-8, the filter input is the byte representation of the complete timeseries path: ```text UTF8(join(DeviceID segments, ".") + "." + measurement_id) ``` -A null device segment is represented by the ASCII text `null` in the path. +A null device segment is represented by the ASCII text `null` in the path. The current Java +implementation calls `String.getBytes()` with the platform default charset; that charset is not +stored in the file. The examples below use UTF-8. A reader must match the writer's charset and +hashing behavior; if either is unknown, bypass the Bloom filter and use the metadata index. ## Hashing -Hashing uses the MurmurHash3 x64 128-bit variant. The final two 64-bit results are added and the -low 32 bits are used. If that result is the minimum 32-bit integer, the value is replaced with -zero; otherwise its absolute value is taken and reduced modulo `bit_count`. +TsFile uses the Java `Murmur128Hash` variant of MurmurHash3 x64 128-bit. Its complete 16-byte +blocks use the standard little-endian unsigned-byte assembly and mixing. Its incomplete final +block differs from standard MurmurHash3: **each tail byte is sign-extended from signed 8-bit to +64-bit before shifting**, and the shifted terms are XORed, not ORed. + +For a tail of `n = byte_length mod 16` bytes `t`, assemble its words as follows. All shifts and +arithmetic are modulo 64 bits; `signed64(b)` maps byte values `0x80..0xff` to `-128..-1`. + +```text +k1 = XOR(signed64(t[j]) << (8 * j)) for j = 0 .. min(n, 8) - 1 +k2 = XOR(signed64(t[j]) << (8 * (j - 8))) for j = 8 .. n - 1 +``` + +An empty XOR is zero. Apply the standard tail mixing to `k2` only when `n >= 9`, and to `k1` +only when `n >= 1`; then apply the standard MurmurHash3 length XOR and finalization. Add the +finalized `h1` and `h2` modulo 64 bits and interpret the low 32 bits as a signed integer `hash32`. +This signed-tail behavior must be preserved for existing files; substituting an ordinary +MurmurHash3 library can produce false negatives for non-ASCII paths. + +```text +bit_index = (0 if hash32 == -2147483648 else abs(hash32)) mod bit_count +``` The hash seeds, in order, are: @@ -65,6 +88,21 @@ The hash seeds, in order, are: `hash_function_count` selects the number of seeds used, with a maximum of 8. +### UTF-8 Test Vectors + +These vectors use `seed = 5` and `bit_count = 256`. The last row also exercises a complete +16-byte block before the signed tail. + +| Path | Signed `hash32` | Bit index | +| --- | ---: | ---: | +| `root.sg.d.s1` | -1267668613 | 133 | +| `root.设备.s1` | 1697017247 | 159 | +| `root.abcdefghijklmnop.设备.s1` | 866805081 | 89 | + +For `root.设备.s1`, a standard MurmurHash3 implementation followed by the same final sum instead +produces `1978455175`, selecting bit `135`. A reader using that bit can reject a path actually +present in the file. + ## Lookup Procedure For an exact path lookup, a reader: @@ -75,7 +113,8 @@ For an exact path lookup, a reader: 4. returns **definitely absent** if any tested bit is clear, otherwise **possibly present**. A positive result must still be checked in the metadata index. A negative result is reliable only -when the query uses exactly the same path canonicalization and UTF-8 bytes as the writer. +when the query uses the same path construction, charset, and signed-tail hash as the writer. +Otherwise, look up the metadata index directly. ## Space and Probability diff --git a/src/UserGuide/latest/FileFormat/Configurations.md b/src/UserGuide/latest/FileFormat/Configurations.md index f329c1b6c6..7d7d46397f 100644 --- a/src/UserGuide/latest/FileFormat/Configurations.md +++ b/src/UserGuide/latest/FileFormat/Configurations.md @@ -21,13 +21,12 @@ # Configurations -Most TsFile configurations are writer policies. They control buffering, Page boundaries, -Chunk Group boundaries, and the shape of metadata indexes, but they are not additional fields in -the file. A reader determines the actual layout from markers, length fields, Chunk Headers, and -serialized metadata. +Most settings affect how a writer groups and buffers data. A reader does not need the original +Page-size target or flush threshold: the resulting boundaries are recorded in the file. A few +settings, including timestamp units and non-aligned time encoding, still need to be agreed by +the applications exchanging the file. -The names below identify format-level concepts. Individual APIs may expose different setting names, -but the effect on bytes and the reader requirement are the same. +The table distinguishes those cases. Setting names in language-specific APIs may differ. | Conceptual setting | Observable file effect | Setting serialized directly? | Reader needs original setting? | | --- | --- | --- | --- | @@ -40,10 +39,9 @@ but the effect on bytes and the reader requirement are the same. | Compression | Stored Page payload | Yes, as `compression_type` in each Chunk | No | | Time precision and zone | Interpretation of `int64` timestamps | No | Yes, from the surrounding protocol | | Non-aligned time encoding | Encoded time stream in each non-aligned Page | No | Yes | -| Encryption | Stored Page payload and footer properties | Properties are serialized; key context may be external | Yes, when encrypted | +| Encryption | File encryption header, Chunk authentication, Page bodies, or legacy footer properties | Provider/profile or legacy properties are serialized; key context is external | Yes, when encrypted | -“Not serialized directly” means that the configured threshold or rate is absent as a named field. -Its result can still be visible in the physical layout. +For example, a Page-size target is not stored as a setting, but each Page records its actual size. ## Chunk Group Size @@ -53,8 +51,8 @@ I/O, but require more writer memory. Smaller Chunk Groups reduce peak memory and available to the output sooner. The configured size is a flush threshold, not an exact on-disk size. The format does not prescribe -a default. Chunk Group boundaries are identified by the `0x00` Chunk Group Header marker and the -next top-level marker. +a default. A group begins with a `0x00` Chunk Group Header and ends at the next group, operation-index +record, or separator. Individual Chunk markers continue the current group. ## Data Page Size @@ -81,9 +79,9 @@ aligned Value Page, `position_count`. ## Metadata Index Fanout -The metadata index fanout limits the number of children in one internal metadata index node. +Index fanout is the maximum number of children in an internal metadata index node. A larger fanout produces a shallower tree with larger nodes; a smaller fanout produces a deeper -tree with smaller nodes. The value must be at least `2`; the reference writer configuration uses +tree with smaller nodes. The value should be at least `2`; the reference writer configuration uses `256`. The configured limit is not serialized. Each metadata index node stores its node type, children, @@ -91,8 +89,8 @@ and end offset, so a reader traverses the resulting tree without knowing the wri ## Bloom Filter False-positive Rate -The Bloom filter false-positive rate controls the space/accuracy tradeoff for path lookup. A lower -rate uses more bits and hash functions. Standard writers constrain the value to the range `0.01` +A Bloom filter can report “possibly present” for a path that is absent. Lowering the target +false-positive rate reduces those extra lookups at the cost of more bits and hash functions. Standard writers constrain the value to the range `0.01` through `0.10` and use `0.05` by default. The rate itself is not serialized. The resulting filter bytes, bit count, and hash-function count @@ -129,9 +127,11 @@ encoding in `encoding_type`. ## Encryption -The `encryptLevel`, `encryptType`, and `encryptKey` properties in `TsFileMetadata` describe whether -Page data is encrypted and which algorithm is used. The key material required to open an encrypted -file remains external to the file. See [Encryption](DataPages/Encryption.md). +Encryption uses one of two descriptions: the optional file encryption header for Page AEAD, +or footer properties for legacy encryption. Both record how to interpret the Page bytes, while +key access remains external. The AEAD header is authoritative even though its legacy footer +properties say unencrypted. The layouts and mode-selection rules are in +[Encryption](DataPages/Encryption.md). ## Implementation Settings diff --git a/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md b/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md index 915e2b3af1..7732553721 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md @@ -21,36 +21,29 @@ # Checksumming -TsFile v4 does not store a general Page checksum, Chunk checksum, or file checksum. - -The leading and trailing magic strings, length fields, markers, offsets, and structure boundaries -can expose some truncation or structural corruption. They do not provide content verification or -a cryptographic integrity guarantee. +TsFile v4 has no general Page, Chunk, or whole-file checksum. A completed-looking file can still +contain damaged values, so the head and tail magic strings should not be used as content checksums. ## What Structural Validation Detects -A strict reader can detect many malformed layouts by requiring: - -- both magic strings and the version byte to be present; -- the footer length and `meta_offset` to resolve to valid boundaries; -- every `data_size`, `compressed_size`, string length, and metadata-list size to remain inside its - containing structure; -- decompression to produce exactly `uncompressed_size` bytes; -- decoded element counts, aligned bitmaps, Statistics, and index ranges to be internally - consistent. +Lengths, offsets, and counts help detect incomplete or malformed structures. A Page that extends +beyond its Chunk, a footer length outside the file, or a decompressed payload with the wrong size +can all be rejected. These checks are described in [Binary Representation](../Binary-Representation.md) +and [Data Pages](README.md). -These checks are necessary even when an external checksum exists. They can detect truncation and -many accidental changes, but a byte substitution may preserve every length and boundary and remain -undetected. +They do not detect every change. A modified value can occupy the same number of bytes and leave +all surrounding boundaries intact. Structural checks remain necessary even when a separate +checksum is available. ## Integrity at Rest or in Transit -Systems that need corruption detection should protect the complete file with an external checksum -or object-store integrity field. Systems that need protection against intentional modification -should use an authenticated digest or signature whose key or trusted value is stored outside the -TsFile. The external value must cover the exact complete-file byte sequence, including the footer -length and trailing magic string. +To detect changes anywhere in a file, maintain an external checksum or use the storage system's +integrity mechanism. If protection against intentional modification is required, use a keyed +authentication code or signature with a trusted key or reference stored outside the file. Cover +the complete byte sequence, including the footer length and tail magic. -Encryption of Page payloads is a separate feature. Unless the selected encryption algorithm -explicitly authenticates its ciphertext, encryption alone does not add a TsFile checksum or verify -the unencrypted headers and metadata. +Page AEAD provides a narrower guarantee: it authenticates non-empty Page bodies and their context, +and binds Chunk ordinals to file positions. It does not cover all visible headers, Statistics, +indexes, or footer metadata. Empty aligned Value Pages have no Page authentication tag, and +legacy encryption does not necessarily authenticate its ciphertext. See [Encryption](Encryption.md) +for the exact scope. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md index fe2be94f43..568f1155b3 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -21,19 +21,23 @@ # Chunks -The TsFile data section starts at offset 7 and consists of marker-delimited Chunk Groups, Chunks, -and Operation Index Ranges. +The TsFile data section consists of marker-delimited Chunk Groups, Chunks, and Operation Index +Ranges. It starts at offset `7` without a file encryption header, or at `7 + E` when a Page AEAD +[FileEncryptionHeader](Encryption.md#file-encryption-header) of total serialized size `E` occupies +offset `7`. That optional header occurs exactly once, before the data section. The top-level grammar is: ```text -DataSection := (ChunkGroup | OperationIndexRange)* Separator +FilePrefix := magic[6] version[1] FileEncryptionHeader? +DataSection := (ChunkGroup | OperationIndexRange)* ChunkGroup := ChunkGroupHeader Chunk* Separator := byte(0x02) ``` -`Chunk*` is not length-prefixed as a group. A reader recognizes the end of a Chunk Group only -after consuming each Chunk by its `data_size` and then reading the next top-level marker. +The `0x02` separator follows the data section; it is not part of a Chunk Group. A Chunk Group +has no total-length field. After reading each Chunk, the reader examines the next marker: another +Chunk continues the group, while a new group, operation-index record, or separator ends it. ```text File @@ -57,6 +61,7 @@ File | `0x03` | `VERSION` | Unused since v3; identifier reserved | | `0x04` | `OPERATION_INDEX_RANGE` | Two `int64` values | | `0x05` | `ONLY_ONE_PAGE_CHUNK_HEADER` | Single-page non-aligned Chunk | +| `0x06` | `ENCRYPTION_HEADER` | File encryption header at offset `7` only; not a Chunk | | `0x41` | `VALUE_CHUNK_HEADER` | Multi-page aligned Value Chunk | | `0x45` | `ONLY_ONE_PAGE_VALUE_CHUNK_HEADER` | Single-page aligned Value Chunk | | `0x81` | `TIME_CHUNK_HEADER` | Multi-page aligned Time Chunk | @@ -74,8 +79,8 @@ low 6 bits = 0x05: one Page or no Page An aligned Value Chunk with `data_size = 0` can contain no Page. -A marker is interpreted only at a known structure boundary. Marker values can occur inside an -encoded or compressed Page payload and have no structural meaning there. +Read markers only after consuming the preceding structure. A Page body can contain any byte +value, including bytes that look like markers. ## Chunk Group Header @@ -89,17 +94,16 @@ DeviceID := VarString segment[segment_count] ``` -A device identifier is an ordered tuple of nullable string segments. Trailing null segments are -removed during serialization, and a device identifier cannot consist entirely of null segments. -On read, `segment_count = 0` represents a device containing one empty-string segment. +A DeviceID is an ordered list of string segments; individual segments can be null. Serialization +removes trailing nulls, but at least one segment must be non-null. The special `segment_count = 0` +form is read as one empty-string segment. Chunks following a Chunk Group Header belong to that device until the next Chunk Group Header, Operation Index Range, or Separator. -`segment_count` counts serialized tuple elements, not UTF-8 bytes. Each segment is independently -length-delimited. Device identity is the ordered tuple itself; joining segments with a display -separator is not part of the binary format and can be ambiguous when a segment contains that -separator. +`segment_count` counts segments. Each segment has its own byte length. Keep the segments separate +when comparing device identities: joining them with dots for display can be ambiguous if a segment +itself contains a dot. ## Chunk Header @@ -113,8 +117,16 @@ ChunkHeader := byte data_type byte compression_type byte encoding_type + int64 chunk_ordinal? + int32 chunk_authentication_size? + byte chunk_authentication[chunk_authentication_size]? ``` +The final three fields appear only in Page AEAD files. The file encryption header selects this +suffix; the Chunk marker remains the same. The ordinal, proof format, and validation rules are +described in [AEAD Chunk Header](Encryption.md#aead-chunk-header). Like the rest of the Header, +the suffix is excluded from `data_size`. + `data_size` is the total size of all Page Headers and stored Page data in the Chunk. It does not include the Chunk Header. A reader consumes exactly `data_size` bytes after the Chunk Header. @@ -134,7 +146,8 @@ Aligned data is stored Chunk by Chunk, not as interleaved Time Pages and Value P Chunk stores the complete sequence of Time Pages, and each Value Chunk stores its own sequence of Value Pages. Pages correspond by ordinal position. -An aligned set has the following invariants: +An aligned Chunk set is one Time Chunk followed by its Value Chunks. It has no separate set +header. The relationships within a set are: - The Time Chunk appears before the Value Chunks that use it. - A Value Chunk identifies an individual measurement; the Time Chunk uses an empty @@ -151,10 +164,13 @@ elements. A streaming or recovery reader can process the data section with this state machine: -1. At offset 7, read one top-level marker. +1. Validate the prefix, inspect offset `7`, and consume the optional `0x06` encryption header. + Resolve its provider/profile before parsing Chunks; continue at `data_start = 7 + E` (`E = 0` + when absent). A later or repeated `0x06` is invalid. Read the first data-section marker there. 2. For `0x00`, read the DeviceID and enter that Chunk Group. -3. For a Chunk marker, read its Header and advance by exactly `data_size` bytes after optionally - parsing its Pages. +3. For a Chunk marker, read its complete Header, including the AEAD suffix when selected, and + verify the Chunk-position proof before using its identity. Advance by exactly `data_size` bytes + after optionally parsing its Pages. 4. For `0x04`, consume the following 16 bytes as an operation-index range. 5. For `0x02`, stop. The following byte belongs to the indexed metadata region. 6. Reject an unrecognized top-level marker or a structure whose declared boundary exceeds the @@ -169,13 +185,11 @@ OperationIndexRange := int64 max_operation_index ``` -This structure records the minimum and maximum operation indexes involved in the most recent -write batch. Operation indexes are storage-engine sequence numbers; they are not timestamps, -measurement indexes, or file offsets. They can support checkpoint, snapshot, backup, and recovery -logic by describing which logical operations have reached the file. +This optional record lets a storage engine associate a flushed batch with its operation log. +The two values are the minimum and maximum operation sequence numbers in that batch. They help +coordinate checkpoints, snapshots, backups, and recovery; they are unrelated to timestamps or +file offsets. -The record is optional for a general-purpose writer. A writer without operation-index semantics -may omit it or write a placeholder range. A reader that does not use this information still has -to recognize marker `0x04` and skip the complete 16-byte payload so that the next marker is read -at the correct boundary. Multiple records can occur because multiple write batches can be -flushed into one file. +A general-purpose writer can omit this record. A reader that does not use operation indexes +still consumes both `int64` fields before reading the next marker. Several records can appear +when several batches are flushed into the same file. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/UserGuide/latest/FileFormat/DataPages/Compression.md index 190bf995a7..e9d0afe511 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Compression.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -29,36 +29,42 @@ Page in that Chunk. | 0 | `UNCOMPRESSED` | Raw bytes | | 1 | `SNAPPY` | Snappy raw block | | 2 | `GZIP` | One gzip member | -| 3 | `LZO` | LZO block | +| 3 | `LZO` | LZO block (C++ implementation, enabled at build time) | +| 4 | `SDT` | Reserved; no codec | +| 5 | `PAA` | Reserved; no codec | +| 6 | `PLA` | Reserved; no codec | | 7 | `LZ4` | LZ4 raw block | | 8 | `ZSTD` | Zstandard frame | | 9 | `LZMA2` | LZMA2 stream in an XZ container | Each Page is compressed independently; compression state is not shared across Pages. The Page -Header's `uncompressed_size` and `compressed_size` record the payload lengths before and after -compression, respectively. +Header's `uncompressed_size` records the uncompressed payload length. `compressed_size` records +the stored body length after compression and any encryption, including Page AEAD framing and tags. The numeric values are stable wire identifiers rather than the ordinal positions of a compressor -library. Values `4` through `6` are not assigned by this version. `LZO` is retained as a historical -identifier; a reader that does not provide the codec cannot decode that Chunk. +library. Values `3` through `6` are legacy identifiers from earlier TsFile versions. The reference +Java implementation does not define or implement them and rejects these bytes while parsing a +Chunk Header. The C++ implementation retains them as `LZO`, `SDT`, `PAA`, and `PLA`; only `LZO` +has a codec (enabled at build time), while `SDT`, `PAA`, and `PLA` remain reserved without a codec. +A reader that does not provide the codec cannot decode the affected Chunk. ## Page-local Framing -If a Page begins at offset `P` and its serialized Page Header occupies `H` bytes, the stored payload -is exactly the half-open interval: +Let a Page start at `P` and its Header occupy `H` bytes. Read exactly `compressed_size` bytes +for its body: ```text [P + H, P + H + compressed_size) ``` -For `UNCOMPRESSED`, `compressed_size` equals `uncompressed_size` and the bytes are already the -encoded Page payload. For every other method, decompression must produce exactly -`uncompressed_size` bytes. A shorter result, a longer result, or a codec request for bytes outside -the interval makes the Page invalid. +This is the stored length, including any encryption framing. After decryption, `UNCOMPRESSED` +already contains the encoded Page payload; other methods need decompression. Either path must +produce exactly `uncompressed_size` bytes. With length-preserving encryption or no encryption, +`UNCOMPRESSED` has equal stored and uncompressed lengths. Page AEAD adds framing, so that equality +no longer holds. -Because each Page is an independent compression unit, a reader can use Page Statistics to reject a -Page and advance by `compressed_size` without initializing the codec. Random access does not need -the preceding Page's dictionary or history. +Pages do not share compression history. A reader can seek directly to a selected Page or skip +its body using the stored length, without decompressing earlier Pages. ## Read Pipeline and Validation @@ -68,13 +74,13 @@ decompressing: ```text read Page Header -> read exactly compressed_size bytes - -> decrypt when file properties require it + -> authenticate/decrypt according to the file encryption header or legacy properties -> decompress according to compression_type -> require exactly uncompressed_size bytes -> decode the Page streams ``` -Implementations should reject sizes that overflow offset arithmetic or exceed configured resource -limits before allocating memory. An unsupported `compression_type` prevents decoding the affected -Chunk, but the Chunk remains skippable through its `data_size`; it must not be interpreted as -`UNCOMPRESSED`. +Apply the length checks in [Binary Representation](../Binary-Representation.md) before allocating +buffers. An unavailable codec prevents decompression. A scanner can still use `data_size` to skip +the Chunk if it parses the raw Header fields, even when the codec registry rejects the identifier; +it must not treat that identifier as `UNCOMPRESSED`. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Encodings.md b/src/UserGuide/latest/FileFormat/DataPages/Encodings.md index 3eaf1190cf..49c9c2a552 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Encodings.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Encodings.md @@ -21,9 +21,9 @@ # Encodings -The single-byte `encoding_type` in a Chunk Header selects the value-stream encoding. In an -aligned Time Chunk, it selects the time-stream encoding. In a non-aligned Chunk, it selects only -the value-stream encoding. +Encoding turns a sequence of typed values into a byte stream before compression. The Chunk +Header records the selected codec in `encoding_type`. This field describes values in a Value or +non-aligned Chunk, and timestamps in an aligned Time Chunk. | Value | Name | | ---: | --- | @@ -43,7 +43,8 @@ the value-stream encoding. | 13 | `RLBE` | | 14 | `CAMEL` | -The following data-type and encoding combinations are defined: +The following combinations are supported by the reference value encoders. An assigned codec +number does not mean that every data type can use it: | Data type | Available encodings | | --- | --- | @@ -66,11 +67,11 @@ format identifiers, not implementation-specific enumeration ordinals. | Run length / bit packing | `RLE`, `RLBE` | Repeated values or small bit widths | | Delta and prediction | `DIFF`, `TS_2DIFF`, `REGULAR`, `SPRINTZ` | Small or regular changes between adjacent values | | Integer remapping | `ZIGZAG` | Small signed integers represented as small unsigned integers | -| XOR-based floating-point | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | Stable leading and trailing bits between adjacent floating-point values | +| XOR-based | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | Stable leading and trailing bits between adjacent floating-point values | | Legacy or specialized | `BITMAP`, `FREQ` | Reserved historical layouts; `FREQ` is deprecated | -The best encoding depends on the data distribution. The file does not record why a writer selected -an encoding, and a reader must select its decoder from both `data_type` and `encoding_type`. +Different distributions favor different encodings. Whatever the writer chooses, the reader +selects a decoder from the pair `(data_type, encoding_type)`. ## Plain Encoding @@ -94,18 +95,15 @@ decode an unknown encoding as `PLAIN`. ## Decoder Contract -Encoding state is scoped to one uncompressed Page stream. After Page decompression, a reader: +A decoder's state lasts for one Page stream. After decryption and decompression, select the codec, +decode within the stream boundary, and check that the number of values matches the Page context. +Start with fresh decoding state for the next Page. -1. creates a decoder for the Chunk's `data_type` and `encoding_type`; -2. decodes exactly the number of logical elements required by the Page context; -3. verifies that decoding neither reads past the Page boundary nor leaves an invalid partial block; -4. discards the decoder state before processing the next Page. +For aligned data, the Time Chunk Header names the time codec; an aligned Value Page decodes one +value for each set bitmap bit. For non-aligned data, the Chunk Header names only the value codec. +Its time codec comes from shared configuration, defaulting to `TS_2DIFF`; there is no serialized +`time_encoder` footer field. See [Configurations](../Configurations.md#non-aligned-time-encoding). -For a non-aligned Page, the time stream has its own encoding declared by `time_encoder` in the file -metadata; `encoding_type` applies to the value stream only. For an aligned Time Page, -`encoding_type` applies to the Page's time stream. For an aligned Value Page, null positions are -removed before value encoding, so the value decoder must produce exactly the number of set bits in -the bitmap. - -An unsupported encoding prevents decoding the affected Chunk, but its byte range can still be -skipped using `data_size`. Falling back to another decoder is not valid. +An unsupported encoding prevents value decoding. A scanner that parses the raw Header fields +can still locate the next Chunk using `data_size`, even if a higher-level codec registry rejects +the identifier. It must not substitute `PLAIN` or another decoder. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md index 218bf98736..76f73b432c 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Encryption.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -21,56 +21,148 @@ # Encryption -Encryption applies only to the compressed Page `stored_data`. Page Headers, Chunk Headers, -Statistics, Metadata Indexes, and File Metadata are not encrypted. +Encryption is applied after Page compression and reversed before decompression. Headers, +Statistics, schemas, and indexes remain visible. TsFile v4 supports two encryption mechanisms, +both under version byte `0x04`: -The write pipeline is: +| Mode | How the reader recognizes it | Chunk and Page layout | +| --- | --- | --- | +| Plaintext | No file encryption header; legacy level `0` or absent | Base Chunk Header; unencrypted Page body | +| Legacy encryption | No file encryption header; legacy level `1` or `2` | Base Chunk Header; algorithm-specific encrypted body | +| Page AEAD | `0x06` file encryption header at offset `7` | Extended Chunk Header; authenticated Page body defined by a profile | + +Check for the file encryption header before parsing Chunks. In an AEAD file, the legacy footer +properties still say unencrypted; the header takes precedence. + +The write and read paths are: + +```text +encoded page payload -> compression -> encryption -> stored_data +stored_data -> authenticated decryption when applicable -> decompression -> value decoding +``` + +## File Encryption Header + +An AEAD file puts one `FileEncryptionHeader` immediately after the seven-byte magic/version +prefix, before any Chunk Group or operation-index record: + +```text +FileEncryptionHeader := + byte marker = 0x06 + UVarInt payload_size + byte header_version = 0x01 + VarString provider_id + VarString profile_id + VarString key_id + VarString key_version + UVarInt file_crypto_id_size = 16 + byte file_crypto_id[16] + UVarInt wrapped_data_key_size + byte wrapped_data_key[wrapped_data_key_size] +``` + +`payload_size` counts from `header_version` through the wrapped key, excluding the marker and the +length field itself. Its allowed range is `1..1048576`, and parsing must end exactly at that +boundary. The four string identifiers must be non-null and non-blank. The file crypto ID is +exactly 16 bytes, and the wrapped data key is non-empty. These binary lengths use `UVarInt`, not +the nullable `VarBinary` used for footer properties. + +The **provider** identifies the encryption implementation or service. Its **profile** names a +fixed byte-format contract: Page framing, nonce and tag representation, size mapping, and Chunk +proof size. A provider can offer several profiles. The file stores these public identifiers and +the wrapped data key, rather than a Java implementation class or plaintext key. The reader needs +the matching provider/profile and access to the external key material. + +Header version `0x01` is the supported layout. Unknown versions, truncated fields, trailing bytes, +a second encryption header, or a header anywhere except offset `7` are errors. + +## AEAD Chunk Header + +The file encryption header also changes how every Chunk Header is parsed. The Chunk marker +stays the same, but three fields follow `encoding_type`: ```text -encoded page payload - -> compression - -> encryption - -> stored_data + int64 chunk_ordinal + int32 chunk_authentication_size + byte chunk_authentication[chunk_authentication_size] ``` -Reading decrypts before decompression and decoding. +`chunk_ordinal` is a non-negative identifier allocated uniquely across the file, including Time +and Value Chunks. The proof length must match the profile and lie in `1..1024`. These fields are +part of the Header, so none of them count towards `data_size`. -## File Properties +The proof binds the ordinal to the Chunk's absolute position. Verify it before using that +identity. Moving a Chunk changes the authenticated position and therefore requires a new proof. +Its associated data is the following big-endian sequence: -The property map in `TsFileMetadata` uses these keys to describe encryption: +```text +int32 domain_size = 20 +byte domain[20] = UTF8("TsFile/ChunkPosition") +int32 associated_data_version = 1 +byte file_crypto_id[16] +int64 absolute_chunk_marker_offset +int64 chunk_ordinal +``` + +## AEAD Page Body and Context + +For a non-empty Page, `compressed_size` records the entire encrypted body, including framing, +nonces, tags, and padding. It is larger than the compressed plaintext length, even with +`UNCOMPRESSED`. A profile may use fixed overhead or a size-dependent layout, so use its size +mapping to recover the candidate plaintext length. + +Page authentication also binds the body to its file, Chunk, and Page position. The associated +data is the exact big-endian sequence below. `SizedBytes` means an `int32` byte length followed +by those bytes; text is UTF-8. + +```text +int32 associated_data_version = 1 +SizedBytes provider_id +SizedBytes profile_id +SizedBytes key_id +SizedBytes key_version +SizedBytes file_crypto_id +int64 chunk_ordinal +int32 page_index +int32 uncompressed_size +int32 compressed_plaintext_size +``` + +`page_index` starts at zero for each physical Chunk and counts empty aligned Value Pages as well. +An all-null Value Page is still the single `UVarInt(0)` described in [Nulls](../Nulls.md); it has +no ciphertext or Page authentication tag. + +The candidate plaintext length becomes trusted only after authentication succeeds. Decryption +must return that length, and decompression must then return `uncompressed_size` bytes. An invalid +tag, wrong key, unsupported profile, or length mismatch is an error; do not retry as plaintext. + +## Legacy File Properties + +When there is no file encryption header, read the following `TsFileMetadata` properties: | Property | Meaning | | --- | --- | | `encryptLevel` | `0` for no encryption; `1` for a directly stored data key; `2` for a data key encrypted by an external master key | -| `encryptType` | Encryption and decryption algorithm identifier | +| `encryptType` | Legacy encryption/decryption algorithm identifier | | `encryptKey` | Text representation of the data key or key-wrapping result | -A Page Header has no separate ciphertext-length field. The encrypted result has the same length -as the compressed input, so `compressed_size` remains the length of `stored_data` in the file. +These property values contain text. At level `1`, the data key is stored directly. At level `2`, +an external master-key decryptor unwraps it before Page decryption. Missing `encryptLevel` means +`0`; levels `1` and `2` require `encryptType` and a non-empty `encryptKey`. Unknown levels and +unavailable algorithms are errors. -When `encryptLevel` is absent, it is interpreted as `0` for compatibility. For levels `1` and `2`, -both `encryptType` and a non-empty `encryptKey` are required. An unknown level or unavailable -algorithm makes the encrypted Page data unreadable; readers must not silently treat it as -unencrypted. +Legacy encryption uses the algorithm's encrypt/decrypt operations without the AEAD Chunk suffix +or Page context. `compressed_size` is still the actual stored output length; it equals compressed +plaintext length only for a length-preserving algorithm. -## Key Interpretation - -At level `1`, `encryptKey` represents the data key used with `encryptType`. At level `2`, it -represents a wrapped data key: the reader obtains the external master-key decryptor, unwraps the -data key, and then decrypts Page payloads with `encryptType`. Key derivation, master-key storage, -rotation, and access control are outside the file format. +AEAD writers set these legacy properties to level `0`, type +`org.apache.tsfile.encrypt.UNENCRYPTED`, and an empty key. The AEAD header supplies the real +mode and wrapped key, so those values must not be interpreted as a plaintext fallback. ## Security Scope -Page encryption protects the confidentiality of encoded sample data. It does not hide the file -shape, device and measurement identifiers, types, time ranges, value Statistics, schemas, indexes, -or the encryption properties themselves. Applications should treat this metadata as visible. - -TsFile v4 does not define a general authentication tag for Pages or for the file. Whether a -ciphertext modification is detected depends on the selected algorithm; the structural validation -and external integrity guidance in [Checksumming](Checksumming.md) still apply. - -Readers should resolve the encryption properties and key material before allocating Page buffers, -then apply the exact pipeline `read -> decrypt -> decompress -> decode`. A decryption failure, -wrong output length, unknown algorithm, or missing key is an error for the affected data and must -not trigger plaintext fallback. +Page AEAD authenticates non-empty Page bodies and their context, while Chunk proofs bind ordinals +to file positions. It does not authenticate every visible field or provide a whole-file digest. +Legacy encryption does not necessarily provide authentication. See [Checksumming](Checksumming.md) +for whole-file integrity and [Error Recovery](Error-Recovery.md) for retaining encryption context +when rebuilding a footer. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md b/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md index 2876e7cdb7..03a4274a44 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md @@ -21,21 +21,15 @@ # Error Recovery -A completed TsFile contains the leading magic string, version byte, `0x02` separator, footer -metadata, 4-byte `TsFileMetadata` length, and trailing magic string. A file without the footer -length or trailing magic string is incomplete. - -A recovery reader can scan the data section from offset 7 by markers and use the `data_size` in -each Chunk Header to skip a complete Chunk. Within a Chunk, the sum of the serialized Page lengths -equals `data_size`. - -A scanner cannot treat an arbitrary `0x02` byte in Page data as a separator. It may read the next -marker only after completely parsing or skipping the structure delimited by the preceding marker. +The footer is written after the data section. If writing stops before that happens, some complete +Chunks may still be present even though the file cannot be opened through its indexes. Recovery +finds those structures by scanning forward from the file header, then rebuilds the missing indexes +and footer from the data it can retain. ## Completion Check -Given file size `F` and the big-endian footer length `L` stored at `[F - 10, F - 6)`, a completed -file must satisfy all of the following before ordinary metadata-first reading begins: +For a file of size `F`, read the big-endian metadata length `L` from `[F - 10, F - 6)`. A completed +file has these relationships: ```text bytes[F - 6, F) = "TsFile" @@ -44,44 +38,45 @@ meta_offset = offset of the top-level 0x02 separator meta_offset < F - 10 - L ``` -The leading magic and version must also be valid, and the `TsFileMetadata` decoder must consume -exactly `L` bytes. A valid tail magic by itself does not prove that the file is complete. +Also check the leading magic and version, and parse exactly `L` bytes of `TsFileMetadata`. +Tail magic alone is insufficient: the footer length and its internal offsets must be usable. ## Safe Forward Scan -A recovery scanner can identify the longest structurally complete prefix of the data section: - -1. validate the leading magic and version and set the cursor to offset `7`; -2. at a known top-level boundary, read one marker; -3. for a Chunk Group Header (`0x00`), decode its complete `DeviceID` and remember the group context; -4. for a Chunk marker, decode the complete Chunk Header, validate `cursor + data_size`, then either - skip that range or validate all Page boundaries inside it; -5. for an Operation Index Range (`0x04`), consume exactly two big-endian `int64` values; -6. stop successfully at the top-level separator (`0x02`), or stop at the last known-good boundary - when a field or payload is incomplete. +A forward scan uses the same Chunk layout as an ordinary reader, but does not rely on the footer: -The last known-good boundary is the end of a complete delimited structure, not the last byte that -happened to be readable. A recovery tool may use the complete Chunks before that boundary to -rebuild indexes and a new footer; it must not invent or partially decode the truncated Chunk. +1. Check the magic and version. Inspect offset `7` and consume an optional `0x06` encryption + header. The data starts at `7 + E`, where `E` is the header's total size or zero when absent. +2. At a structure boundary, read a marker. For `0x00`, read the DeviceID and remember the group. +3. For a Chunk marker, read its complete Header, including the AEAD suffix when required. + Verify the Chunk proof before using its identity, then check that `data_size` fits in the + remaining file. Inspect the Pages or skip exactly that many bytes. +4. For `0x04`, consume the two operation-index `int64` values. +5. At the top-level `0x02` separator, stop scanning the data section. -The file is incomplete or corrupt when any of these conditions is detected: +Stop at the last complete structure if a field or body is truncated. An unknown marker, a second +or misplaced encryption header, an invalid length, or an authentication failure must be treated +as an error, not skipped by searching for another familiar byte. A `0x02` inside a Page body is +data and cannot identify the end of the data section. -- A Chunk's `data_size` extends beyond the file or the next structure boundary. -- A Page length extends beyond its Chunk boundary. -- The footer length cannot locate a complete `TsFileMetadata` structure. -- `meta_offset` does not point to the `0x02` separator. -- An index offset is outside the file or points into the middle of a structure. - -Other failures include an unknown top-level marker, an unterminated VarInt or VarString, integer -overflow while computing an end offset, a Page decoder that produces the wrong number of elements, -and an aligned Time/Value Page mismatch. +Retain only structures that have been read completely and checked. The readable part of a truncated +Chunk is not automatically a valid Chunk. The common bounds checks are in +[Binary Representation](../Binary-Representation.md), and Page boundaries are in [Data Pages](README.md). ## Recovery Limits -Recovery reconstructs structure, not missing information. A complete compressed or encrypted Page -still requires its declared codec and, where applicable, the external key. Without a file-level -checksum, a structurally valid byte substitution may be indistinguishable from original data. -Operation Index Range records can help a storage engine relate the salvaged prefix to its external -operation log, but their meaning does not replace Chunk and Page boundary validation. - -TsFile v4 has no general checksum. See [Checksumming](Checksumming.md) for the resulting limits. +Rebuilding an index cannot restore missing values or identify every change to otherwise valid +bytes. The retained Pages still need their codecs and, when encrypted, their key material. +Operation-index records can help a storage engine relate the retained data to its external log, +but they do not replace structure checks. + +For Page AEAD, preserve the original encryption header, file crypto ID, Chunk ordinals, and Page +indexes. Truncating the tail and replacing the footer can leave retained Chunks at their original +offsets. Moving a Chunk requires a new position proof; changing its file identity, ordinal, or +Page context also requires authenticating and re-encrypting the affected data as appropriate. +When appending, allocate ordinals above the highest retained ordinal. + +An authenticated recovery needs the matching provider and external keys. The legacy footer cannot +replace a damaged AEAD header, because its legacy encryption fields are deliberately set to +unencrypted. [Encryption](Encryption.md) describes that mode selection; [Checksumming](Checksumming.md) +explains which changes remain outside the format's integrity checks. diff --git a/src/UserGuide/latest/FileFormat/DataPages/README.md b/src/UserGuide/latest/FileFormat/DataPages/README.md index 97e5add290..04b1966651 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/README.md +++ b/src/UserGuide/latest/FileFormat/DataPages/README.md @@ -21,8 +21,9 @@ # Data Pages -One or more Pages follow a Chunk Header without padding between Pages. The Chunk Header's -`data_size` is the total byte length of all Page Headers and stored Page data in that Chunk. +A Chunk stores its Pages consecutively after the Chunk Header, without padding between them. +`data_size` counts all Page Headers and stored bodies together. An empty aligned Value Chunk +has `data_size = 0` and no Pages. ## Page Header @@ -38,22 +39,19 @@ Page := `uncompressed_size` is the size of the Page payload after decryption and decompression. `compressed_size` is the size of the Page payload stored in the file. The two values are equal -when `compression_type = UNCOMPRESSED`. +when `compression_type = UNCOMPRESSED` and no length-expanding encryption is used. For Page AEAD, +`compressed_size` includes the encrypted body framing and authentication overhead, even when +compression is disabled. See [Encryption](Encryption.md#aead-page-body-and-context). -The low six bits of the Chunk marker determine whether the Page Header contains Statistics: - -- `0x01`: the Chunk has multiple Pages; every non-empty Page Header contains Statistics. -- `0x05`: the Chunk has one Page or no Page; the Page Header does not contain Statistics. - -See [Metadata](../Metadata.md#statistics) for the binary format of Statistics. - -The Header shape can therefore be selected without inspecting the Page payload: +The low six bits of the Chunk marker determine whether Page Statistics are present: | Chunk marker low bits | Page count | Page Statistics | | ---: | --- | --- | | `0x01` | More than one | Present in every non-empty Page Header | | `0x05` | Zero or one | Absent | +The Statistics layout is defined in [Metadata](../Metadata.md#statistics). + If a Page starts at `P`, let `h` be the serialized size of its two length fields and optional Statistics. The stored payload occupies the half-open interval `[P + h, P + h + compressed_size)` and the next Page, if any, starts at `P + h + compressed_size`. That end position must not exceed @@ -69,7 +67,7 @@ When writing a non-empty Page, the data passes through these stages: ```text logical values - -> value encoding + -> encode time and/or values -> assemble uncompressed page payload -> compression -> optional encryption @@ -106,9 +104,8 @@ the value stream length is: value_stream_size = U - v - time_stream_size ``` -The result must be non-negative. After decoding, both stream decoders must reach their boundaries -and produce the same point count. A trailing byte, an early end, or unequal counts is a malformed -Page. +The length must be non-negative. Both streams are bounded by the Page payload and must decode +to the same number of points, with valid endings for their respective codecs. A non-aligned Chunk Header records only the value stream's `encoding_type`. See [Configurations](../Configurations.md#non-aligned-time-encoding) for the time-stream encoding. @@ -147,14 +144,12 @@ number of set bits. ## Page Selection and Skipping -Page Statistics allow a reader to evaluate time and value predicates before reading stored Page -data. A Page can be skipped only when its Statistics prove that no point can satisfy the -predicate. Statistics are conservative summaries: overlap means “possibly matches,” not -“definitely matches.” +Page Statistics can rule out a Page before its body is read. For example, a Page whose time +range misses the query range can be skipped. See [Statistics](../Metadata.md#statistics) for +which value summaries are available. -For a one-Page Chunk, the Page Header omits Statistics. The equivalent Chunk or Timeseries -Statistics in the footer can be used before seeking to the Chunk. If Statistics for a data type do -not contain the requested summary, the reader must decode the Page or rely on another predicate. +A one-Page Chunk omits Page Statistics, so use its Chunk or series Statistics from the footer. +If the available summary cannot rule out a match, read the Page and check its values. ## Empty Aligned Value Page diff --git a/src/UserGuide/latest/FileFormat/Extensibility.md b/src/UserGuide/latest/FileFormat/Extensibility.md index a3059220f4..4588bf499b 100644 --- a/src/UserGuide/latest/FileFormat/Extensibility.md +++ b/src/UserGuide/latest/FileFormat/Extensibility.md @@ -21,48 +21,36 @@ # Extensibility -TsFile v4 provides numbered or self-describing extension points in the following locations: +TsFile has three main ways to introduce new capabilities: assign a new type or codec identifier, +add a footer property, or define a new file layout. The choice depends on whether a reader can +still locate the next structure without understanding the new feature. -- Data types, encodings, and compression methods use single-byte identifiers. -- A length-delimited string property map is stored at the end of `TsFileMetadata`. -- The file version byte selects the complete file layout. - -A new data type, encoding, or compression method requires a new identifier. The wire meaning of -an existing identifier does not change. - -Each footer property consists of a `VarString` key and value. A reader can consume an -unrecognized property without changing the boundaries of the other footer fields. - -Data-section markers do not have a common length field. A reader cannot safely skip an unknown -marker. A new data-section structure that cannot be fully delimited by an existing marker -therefore requires either a new file version or an encoding contained within an existing -structure. +Footer properties are easy to delimit. Each has a `VarString` key and a nullable `VarBinary` +value, so a reader can preserve an unknown value without decoding it. A Chunk also has a known +byte length, which lets a scanner skip it even if it cannot decode the selected codec. +Data-section markers have no shared length field, so the same approach does not work for an +unknown marker. ## Safe Evolution Rules -An extension is safely skippable only when an older reader can determine its complete byte range -without interpreting the new payload. Examples are a new footer property value and a new codec -inside a Chunk whose `data_size` is known. Adding bytes to a non-length-delimited header is not -skippable because every following field would move. +A compatible extension keeps existing field meanings and boundaries intact: -Within v4, extension authors must: +- Give a new type or codec an unused identifier; keep the meaning of assigned identifiers. +- Define the supported data types, byte layout, output sizes, and error handling for a new codec. +- Keep the existing byte order and length conventions. +- Report an unsupported feature when it is needed to decode the requested data. -- allocate a previously unassigned identifier and never reuse an existing value; -- preserve byte order, length interpretation, and containment boundaries of existing structures; -- define which physical data types are valid for a new encoding; -- define output-size, element-count, and malformed-input behavior for new codecs; -- require an explicit capability error when a reader cannot decode the extension. +Adding bytes to a Header without a rule that selects the extended layout shifts the fields an +older reader expects. Such a change requires a new version or an explicit extension mechanism. -Reserved or unknown identifiers are not aliases for a default. In particular, an unknown encoding -is not `PLAIN`, an unknown compression method is not `UNCOMPRESSED`, and an unknown marker is not -padding. +The Page AEAD extension is an example of the latter. Its `0x06` file encryption header selects +extra Chunk Header fields and profile-specific Page framing while retaining version byte `0x04`. +Older v4 readers may reject those files. [Encryption](DataPages/Encryption.md) defines the layout +and required capabilities; knowing the base format version alone is not enough. ## Properties and Semantics -Length delimitation makes an unknown footer property structurally readable, but it does not make -the property's semantics optional. A property that changes how existing bytes must be interpreted -must define how readers advertise and enforce support. Security-critical properties must fail -closed when recognized semantics cannot be applied. - -Changes that alter the grammar of top-level structures, reinterpret assigned identifiers, or make -old boundaries ambiguous require a new format version. +Being able to skip a property does not mean its meaning can always be ignored. An informational +property may be optional, but a property that changes how Page bytes are decoded needs an explicit +support rule. If a reader cannot apply required encryption or another required interpretation, +it must report an error rather than decode the bytes with a default. diff --git a/src/UserGuide/latest/FileFormat/Format-Versions.md b/src/UserGuide/latest/FileFormat/Format-Versions.md index 45eb8cc0d1..42c17472f9 100644 --- a/src/UserGuide/latest/FileFormat/Format-Versions.md +++ b/src/UserGuide/latest/FileFormat/Format-Versions.md @@ -21,50 +21,41 @@ # Format Versions -The TsFile format version follows the leading magic string: +The format version is the byte immediately after the leading magic string: ```text 6-byte magic string "TsFile" 1-byte format version ``` -This documentation describes version byte `0x04`. After validating the leading magic string, a -reader selects the corresponding file layout from the version byte. +This guide describes `0x04`. The format byte and the project release number serve different +purposes: a release in the 2.x series can use file format v4. The project release label is not +stored in the file header. -Version 4 uses segmented `DeviceID` values and stores per-table device-index roots and Table -Schemas in `TsFileMetadata`. Version 3 uses a single-string device identifier and a different -footer layout; it cannot be parsed with the version 4 `DeviceID` or `TsFileMetadata` structures. +Version 4 uses segmented DeviceIDs and stores per-table device-index roots and Table Schemas +in `TsFileMetadata`. Version 3 uses a single-string device identifier and a different footer. +A reader needs the corresponding version's structures to parse either file correctly. ## Reader Dispatch -The version byte is interpreted before any data-section marker. A reader therefore follows this -order: +Read the version before interpreting data markers, then select the encryption mode: ```text -validate leading magic -> read version -> select complete version grammar -> parse data/footer +validate leading magic -> read version -> inspect optional encryption header + -> resolve encryption mode/profile -> parse the matching data/footer grammar ``` -An implementation that supports version 4 but not the observed byte must report an unsupported -version. It must not probe the file as v4 based on a familiar marker or footer suffix, because the -meaning and width of structures may differ between versions. +If the observed version is unsupported, report that before parsing the data. A familiar marker +or tail magic does not establish that the rest of the file uses the expected layout. ## Compatibility Contract -Compatibility is defined at the byte-format level: +Within a supported version, a reader also needs the types, codecs, and encryption features used +by the requested data. For example, v4 Page AEAD files have a `0x06` encryption header at offset +`7`, which selects an extended Chunk Header and a Page-body profile. The version byte stays +`0x04`; a v4 reader without this capability must report it as unsupported. -- A v4 reader may read a v4 file when it supports every identifier required by the selected data. -- A v4 writer emits the v4 grammar described by this documentation, independent of its programming - language or in-memory classes. -- Unknown footer properties can be consumed because each key and value is length-delimited. -- Unknown data-section markers cannot generally be skipped because markers have no common payload - length. -- Numeric identifiers already assigned to types, encodings, compression methods, markers, and - index-node kinds retain their meanings within v4. - -Support for a version does not imply support for every optional codec or encryption algorithm. -When such a feature is unavailable, the containing Chunk may remain structurally skippable, but -its sample values are not decodable. - -The project release series and the on-disk format byte are separate version spaces. A product in -the 2.x release series can read and write format byte `0x04`; the release label is not serialized -in the TsFile header. +Assigned identifiers keep their meanings. Unknown footer properties have length-delimited keys +and values, so their byte ranges can be consumed. Unknown data markers generally cannot be +skipped because they have no common payload length. [Extensibility](Extensibility.md) explains +these boundaries, and [Encryption](DataPages/Encryption.md) defines the AEAD layout. diff --git a/src/UserGuide/latest/FileFormat/Metadata.md b/src/UserGuide/latest/FileFormat/Metadata.md index 50d49e464d..7cdc5824a9 100644 --- a/src/UserGuide/latest/FileFormat/Metadata.md +++ b/src/UserGuide/latest/FileFormat/Metadata.md @@ -30,7 +30,8 @@ Footer metadata starts after `SEPARATOR (0x02)`. Ranges of `TimeseriesMetadata` index nodes may be interleaved. The `offset` and `end_offset` fields in the indexes delimit each range. Every offset is an absolute byte offset measured from the beginning of the file. -The metadata hierarchy supports progressively finer pruning: +A lookup follows the hierarchy below. Each level narrows the search, from a table to the Pages +that may contain matching values: | Level | Identifies | Primary use | | --- | --- | --- | @@ -78,15 +79,13 @@ they correspond to the values at the first and last covered timestamps. The `sta `end_time` fields are inclusive. Numeric `sum` fields are intended for aggregation; notably, an `INT64` sum is stored as `float64` and can lose integer precision for large magnitudes. -Statistics are conservative pruning metadata. A reader may skip a structure when its time range -does not intersect the query range, or when a supported value summary proves that no value can -match. It must not infer a match merely from overlapping ranges. Types without minimum/maximum -fields, such as `BLOB` and `OBJECT`, cannot support range pruning through Statistics. +Statistics help rule out data that cannot match a query. For example, a Page whose time range +falls entirely before the requested range can be skipped. Overlapping ranges only mean the Page +might match; the reader still needs to check its values. Types without minimum/maximum +fields, such as `BLOB`, `OBJECT`, and `TEXT`, cannot support range pruning through Statistics. -Statistics at a parent level summarize all values represented by that parent. For a series with -multiple Chunks, series Statistics cover the union of the Chunk Statistics. For a multi-Page -Chunk, Chunk Statistics cover the union of Page Statistics. Counts and time bounds must be -consistent with those lower-level records. +Chunk Statistics summarize all Pages in that Chunk; series Statistics summarize all of its +Chunks. Parent counts add the child counts, and parent time bounds cover the child ranges. ## Timeseries Metadata @@ -114,7 +113,7 @@ low 6 bits = 1: more than one Chunk; every ChunkMetadata contains Statistics ## Chunk Metadata -When `timeseries_type & 0x3f == 0`, the timeseries has exactly one Chunk: +When `(timeseries_type & 0x3f) == 0`, the timeseries has exactly one Chunk: ```text ChunkMetadata := int64 offset_of_chunk_header @@ -122,7 +121,7 @@ ChunkMetadata := int64 offset_of_chunk_header The Chunk uses `series_statistics` as its Statistics. -When `timeseries_type & 0x3f != 0`: +When `(timeseries_type & 0x3f) != 0`: ```text ChunkMetadata := @@ -132,9 +131,9 @@ ChunkMetadata := `offset_of_chunk_header` points to the Chunk marker, which is the first byte of the Chunk Header. -`chunk_metadata_list_size` is a byte boundary, not a Chunk count. A reader first computes the end -of the list, then deserializes Chunk Metadata records until it reaches exactly that end. Ending -early, crossing the boundary, or leaving trailing bytes is a format error. +`chunk_metadata_list_size` is the serialized byte length of the list. Add it to the list start +to find the end, then read Chunk Metadata records until that end is reached exactly. The field +does not give the number of Chunks. ## Metadata Index @@ -167,8 +166,9 @@ MeasurementIndexEntry := int64 offset ``` -The `offset` of an Entry is the start of its range. Its end is the `offset` of the next Entry; the -last Entry ends at the node's `end_offset`. +An Entry describes a half-open byte range `[offset, next_offset)`. The next Entry supplies +`next_offset`; for the last Entry, use the node's `end_offset`. The end byte belongs to the next +range, not the current one. ```text entry[0].offset <= entry[1].offset <= ... <= end_offset @@ -185,7 +185,10 @@ To find key `K` in an index node, select the last Entry whose first key is less or at the node's `end_offset` for the last Entry. An internal node range contains another index node; a leaf range contains the object named in the table above. -An index reader validates the node before following it: +If no Entry key is less than or equal to `K`, the key is absent from this node. A measurement +leaf can cover several Timeseries Metadata records; read that range to find the exact measurement. + +Before following a node, check that: - `child_count` is consistent with the number of complete Entries in the node. - Entry keys are sorted according to the node's key comparator. @@ -221,9 +224,14 @@ NamedTableSchema := Property := VarString key - VarString value + VarBinary value ``` +Property keys are UTF-8 strings; values are nullable raw byte arrays, not UTF-8 strings. +For example, value bytes `ff fe 00 80` serialize as `08 ff fe 00 80`: `08` is `VarInt(4)`. +A null value is `01` (`VarInt(-1)`), while an empty value is `00`. Preserve unknown values +byte-for-byte, and decode them as text only when the property contract requires it. + `meta_offset` points to the `0x02` separator at the end of the data section. There is no marker before `TsFileMetadata`; its starting position is determined by the 4-byte length stored at the end of the file. @@ -278,7 +286,10 @@ Column category identifiers are: | 2 | `ATTRIBUTE` | | 3 | `TIME` | -Column order is significant because table records and aligned value columns refer to the schema -positionally. `TAG` columns participate in device identity, `FIELD` columns hold measured values, -`ATTRIBUTE` columns describe entities, and `TIME` identifies the time column. The category does -not replace `data_type`; both fields are required to interpret a column. +The schema preserves column order. In particular, TAG column order determines the order of +DeviceID segments. FIELD Chunks are identified by `measurement_id`; their physical order does not +have to match the FIELD column order in the schema. + +`TAG` columns identify a device, `FIELD` columns hold measured values, `ATTRIBUTE` columns +describe an entity, and `TIME` identifies the time column. A column needs both its category and +its `data_type`: the category describes its role, while the type describes its values. diff --git a/src/UserGuide/latest/FileFormat/Nulls.md b/src/UserGuide/latest/FileFormat/Nulls.md index f7b9914e15..5779d5e7d3 100644 --- a/src/UserGuide/latest/FileFormat/Nulls.md +++ b/src/UserGuide/latest/FileFormat/Nulls.md @@ -60,12 +60,13 @@ bitmap bits or encoded values. If an entire Value Page is null, the Page consists only of an `uncompressed_size` equal to zero: ```text -EmptyPage := UVarInt(0) +EmptyAlignedValuePage := UVarInt(0) ``` An empty Page has no `compressed_size`, Statistics, bitmap, or value stream. Its position count is obtained from the corresponding Time Page. -An absent Value Chunk and an empty Value Page are different. An absent Chunk means that the -measurement has no physical column in that Chunk Group. An empty Page preserves the Page-to-Page -alignment of an existing value column while stating that every position in that interval is null. +An empty Value Page preserves its position in the Page sequence, so subsequent Value Pages still +match the correct Time Pages. An absent Value Chunk has no physical column for that aligned Chunk +set. A Value Chunk with `data_size = 0` is a third case: its Header exists, but it contains no +Pages. These cases must be distinguished when walking the aligned Chunk set. diff --git a/src/UserGuide/latest/FileFormat/README.md b/src/UserGuide/latest/FileFormat/README.md index 62f928339b..373598a84f 100644 --- a/src/UserGuide/latest/FileFormat/README.md +++ b/src/UserGuide/latest/FileFormat/README.md @@ -21,132 +21,131 @@ # File Format -This document describes the language-independent binary format of TsFile version 4. The file -version byte is `0x04`. +A TsFile stores time series data in columns. Writers append data in device groups, then write +the indexes at the end of the file. Readers usually work in the opposite direction: they open +the footer, look up a series, and seek to the Chunks that contain its values. - +This guide describes the v4 file layout, identified by version byte `0x04`, using the Java +reference implementation as of 2026-10-26. It covers the file structures and how they fit together. +The encoding chapter gives codec identifiers and the `PLAIN` layout; the bit layouts of the other +value codecs are outside its scope. ## Scope and Terminology -This specification defines the bytes required to exchange a completed TsFile v4 between -independent implementations. It distinguishes four levels of organization: +Data has three levels of organization within a file: -- A **Page** is the smallest independently compressed and decoded data unit. -- A **Chunk** contains the Pages of one physical column and fixes their data type, encoding, and - compression method. -- A **Chunk Group** associates a sequence of Chunks with one device identifier. -- A **file** contains the sequential data section followed by indexes and file-level metadata. +- A **Page** holds a batch of encoded values. Each Page is compressed independently. +- A **Chunk** holds the Pages of one physical column. Its header identifies the data type, + encoding, and compression method. An aligned Value Chunk can have `data_size = 0`; in that + case the Header exists but contains no Page. +- A **Chunk Group** holds a batch of Chunks for one device. The same device can appear in several + Chunk Groups as the writer flushes more data. -“Aligned” describes measurements that share one physical time column. “Non-aligned” describes a -measurement that carries its own time stream. Alignment changes the Page payload layout; it does -not change the meaning of a timestamp or value. +In **non-aligned** data, each measurement stores its own timestamps alongside its values. In +**aligned** data, a Time Chunk stores the timestamps shared by several Value Chunks. A bitmap in +each Value Page identifies the positions that have values. [Data Pages](DataPages/README.md) +explains both layouts. -All byte intervals in this specification are inclusive unless expressed as a half-open range. -An absolute offset is measured from the first byte of the file. A serialized length excludes the -field that stores that length unless stated otherwise. + ## Physical Regions -For a file of length `F`, let `L` be the four-byte length of `TsFileMetadata` and let -`meta_offset` be the value stored inside `TsFileMetadata`: +The file begins with the six-byte magic string `TsFile` and a one-byte version. An optional +file encryption header follows for Page AEAD files. The data section ends at a single `0x02` +separator; indexes and file metadata follow it. The last ten bytes hold the metadata length and +the trailing magic string. + +The table below uses these names: + +- `F`: total file length. +- `L`: serialized length of `TsFileMetadata`, stored in the four-byte trailing length field. +- `E`: total serialized length of the optional encryption header, or `0` when absent. +- `data_start = 7 + E`: start of the data section. +- `meta_offset`: separator offset, stored inside `TsFileMetadata`. | Region | Inclusive byte interval | How it is located | | --- | --- | --- | | Header magic | `0 … 5` | Fixed position | | Version | `6` | Fixed position | -| Data section | `7 … meta_offset - 1` | Ends immediately before the separator | +| Optional file encryption header | `7 … 6 + E` | Present only for Page AEAD; marker `0x06` | +| Data section | `data_start … meta_offset - 1` | Ends immediately before the separator | | Separator | `meta_offset` | `0x02`, referenced by `TsFileMetadata` | | Indexed metadata | `meta_offset + 1 … F - 11 - L` | Delimited by metadata offsets | | `TsFileMetadata` | `F - 10 - L … F - 11` | Located from the trailing length | | Metadata length | `F - 10 … F - 7` | Fixed position relative to the end | | Tail magic | `F - 6 … F - 1` | Fixed position relative to the end | -The indexed metadata interval can contain Timeseries Metadata and Metadata Index Nodes in an -implementation-dependent physical order. Their absolute offsets, rather than adjacency, define -their relationships. +All offsets are absolute, starting at the first byte of the file. The table uses inclusive +intervals; `[start, end)` elsewhere denotes a half-open interval. A length excludes its own +length field unless stated otherwise. The optional-header interval is absent when `E = 0`. + +Timeseries Metadata and index nodes can be interleaved in the indexed metadata section. Readers +follow their offsets rather than assuming a fixed physical order. `TsFileMetadata` has no marker +of its own: its length locates it from the end of the file. ## Write Model -The data is divided into Chunk Groups by device. A Chunk stores one measurement column and -contains one or more Pages. A non-aligned Chunk stores its time stream and value stream in the -same Page. Aligned data stores one shared time column in a Time Chunk and the measurement columns -in Value Chunks. +A writer appends Chunk Groups, then writes the indexes and footer when it finishes the file. +Page sizes, flush thresholds, and index fanout affect where structures begin and end, but readers +can recover those boundaries from headers and offsets. Timestamp units and non-aligned time +encoding need the shared context described in [Configurations](Configurations.md). -File metadata follows the data, so a writer can write all Chunk Groups sequentially before -writing the indexes and footer. A reader normally starts at the end of the file, uses the -`TsFile Metadata size` to locate `TsFileMetadata`, and follows its indexes to the required Chunks. +Both magic strings use the same bytes: -Writers may choose Page sizes, Chunk Group flush thresholds, metadata-index fanout, encodings, -and compression methods. Those choices affect performance and physical boundaries, but readers -derive the resulting layout entirely from the bytes in the file. +```text +54 73 46 69 6c 65 # "TsFile" +``` ## Metadata-first Read Algorithm -A random-access reader can open a completed file without scanning the data section: +A reader can locate the requested series without scanning the data section: -1. Verify the six-byte tail magic. -2. Read `L` at `F - 10` and compute `file_metadata_pos = F - 10 - L`. -3. Deserialize `TsFileMetadata`, then verify that `meta_offset` points to `0x02`. -4. Optionally test the complete path with the Bloom filter. A negative result proves absence; a - positive result requires index lookup. -5. Traverse the table/device index and then the measurement index to locate the requested - Timeseries Metadata range. -6. Read the Chunk Metadata offsets selected by the time and value predicates. -7. Seek to each Chunk Header, use Page Statistics when present, and decode only the required - Pages. +1. Check the leading magic and version. Inspect offset `7` for an optional + [file encryption header](DataPages/Encryption.md#file-encryption-header), which determines + how Chunk Headers and Page bodies are parsed. +2. Check the tail magic, read `L` at `F - 10`, and calculate `file_metadata_pos = F - 10 - L`. +3. Read `TsFileMetadata` and check that `meta_offset` points to the separator. +4. Optionally check the complete path in the [Bloom filter](Bloom-Filter.md). Use a negative + result only when the writer's charset, path construction, and hash behavior are known to + match. +5. Follow the table/device index, then the measurement index, to find the series metadata. +6. Use the series and Chunk Statistics to select Chunk offsets. Seek to those Chunks and use + any Page Statistics to narrow the selection further. +7. Read the selected Page bodies, decrypt when required, decompress, and decode their streams. -This path makes metadata size proportional to the selected devices, measurements, Chunks, and -Pages rather than to the size of the complete file. +This avoids reading the payloads of unrelated Pages. A single-Page Chunk omits Page Statistics, +so pruning at that level relies on the Chunk or series Statistics instead. ## Structural Invariants -A conforming reader validates at least the following relationships before allocating or reading -an externally supplied length: - -- The head and tail magic strings are identical and the version byte is supported. -- `0 <= L <= F - 10` and the computed `file_metadata_pos` is after `meta_offset`. -- `meta_offset` is an in-file absolute offset whose byte is `0x02`. -- Every Chunk ends exactly `data_size` bytes after its Chunk Header. -- Every non-empty Page fits inside its Chunk; its stored payload contains exactly - `compressed_size` bytes. -- Metadata entry offsets are monotonic within a node and do not exceed `end_offset`. -- An aligned Value Page corresponds to the Time Page at the same ordinal position; its - `position_count` equals the number of positions in that Time Page. +The main relationships to check when reading are: -Length or offset violations are format errors. A reader must not use a marker byte found inside a -length-delimited Page payload as a top-level marker. +- Both magic strings and the version are valid. +- A file encryption header, when present, appears once at offset `7`; its length and version + are valid. +- `0 <= L <= F - 10`, and `file_metadata_pos` lies after the separator. +- A Chunk's Pages occupy exactly `data_size` bytes after its complete Header. +- Each non-empty Page body occupies exactly `compressed_size` bytes within that Chunk. +- Index offsets are monotonic (non-decreasing) within a node and do not exceed `end_offset`. +- An aligned Value Page covers the same positions as the Time Page with the same ordinal; its + `position_count` equals the Time Page's position count. -The file contains exactly one top-level `0x02` separator that terminates the data section. It precedes all -metadata and is referenced by `meta_offset`. There is no separate separator or marker immediately -before `TsFileMetadata`; its position is determined by the length field at the end of the file. +A violation is a format error. -For a file of length `F` whose serialized `TsFileMetadata` has length `L`: - -```text -metadata_size_pos = F - 6 - 4 -file_metadata_pos = F - 6 - 4 - L -``` - -The magic bytes at both the beginning and the end of the file are: - -```text -54 73 46 69 6c 65 # "TsFile" -``` +[Binary Representation](Binary-Representation.md) describes the common length and offset checks. +[Encryption](DataPages/Encryption.md) adds the authentication checks for Page AEAD. Marker bytes +have meaning only at structure boundaries; the same bytes can occur inside a Page body. ## Format Details -- [Binary Representation](Binary-Representation.md) -- [Configurations](Configurations.md) -- [Extensibility](Extensibility.md) -- [Metadata](Metadata.md) -- [Types](Types.md) +For a first reading, start with [Binary Representation](Binary-Representation.md), +[Chunks](DataPages/Chunks.md), and [Data Pages](DataPages/README.md), then follow the metadata +lookup in [Metadata](Metadata.md). + +- [Types](Types.md) and [Encodings](DataPages/Encodings.md) +- [Nulls](Nulls.md) and [Compression](DataPages/Compression.md) - [Bloom Filter](Bloom-Filter.md) -- [Data Pages](DataPages/README.md) - - [Chunks](DataPages/Chunks.md) - - [Compression](DataPages/Compression.md) - - [Encodings](DataPages/Encodings.md) - - [Encryption](DataPages/Encryption.md) - - [Checksumming](DataPages/Checksumming.md) - - [Error Recovery](DataPages/Error-Recovery.md) -- [Nulls](Nulls.md) -- [Format Versions](Format-Versions.md) +- [Encryption](DataPages/Encryption.md) and [Checksumming](DataPages/Checksumming.md) +- [Error Recovery](DataPages/Error-Recovery.md) +- [Configurations](Configurations.md), [Format Versions](Format-Versions.md), and + [Extensibility](Extensibility.md) diff --git a/src/UserGuide/latest/FileFormat/Types.md b/src/UserGuide/latest/FileFormat/Types.md index b73dbb7a46..cb824f5c01 100644 --- a/src/UserGuide/latest/FileFormat/Types.md +++ b/src/UserGuide/latest/FileFormat/Types.md @@ -47,17 +47,17 @@ encoder. See [Encodings](DataPages/Encodings.md) for wire representations of val ## Physical and Logical Types -The identifier determines the physical decoder. `TIMESTAMP` and `DATE` additionally carry a -logical interpretation: +A data type describes the value domain; the encoding determines how those values become bytes. +`TIMESTAMP` and `DATE` also assign a meaning to the underlying integer: -- `TIMESTAMP` has the same eight-byte physical value domain as `INT64`. The time unit is supplied +- `TIMESTAMP` has the same signed 64-bit value domain as `INT64`. The time unit is supplied by the protocol or dataset using the file; TsFile v4 does not serialize that unit. - `DATE` has the same integer encoder interface as `INT32`, but valid logical values use decimal `YYYYMMDD` form. - `TEXT` is a length-delimited byte string. `STRING` uses the same physical binary container but has UTF-8 string semantics. `BLOB` is uninterpreted binary data. -- `OBJECT` is an opaque binary representation whose object contract must be agreed outside the - file format. +- `OBJECT` stores object bytes. The applications exchanging the file agree on how to reconstruct + an object from them. `VECTOR` is a physical marker for the shared time column of aligned data. It is not a scalar value type and does not define a standalone value-stream encoding. `UNKNOWN` is a sentinel, not a valid @@ -80,7 +80,7 @@ cannot decode values or type-specific Statistics for that series. ## Ordering and Comparison Integer and floating-point values use their numeric order for min/max Statistics. `STRING` -min/max values use binary lexicographic comparison of their serialized bytes. `TEXT` stores only +min/max values compare their raw value bytes lexicographically, excluding length prefixes. `TEXT` stores only first and last values, while `BLOB` and `OBJECT` store no type-specific value Statistics. Type promotion and application-level casts are not encoded in TsFile. A reader can expose such diff --git a/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md index d6d542c982..c3be0038b6 100644 --- a/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md +++ b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md @@ -24,7 +24,9 @@ 所有偏移量都从文件首字节开始计算，所有长度均以字节为单位。除非单独说明，固定宽度 整数和 IEEE 754 浮点数使用大端序，字符串使用 UTF-8。 -## Fixed-width Values +
+ +## 定长数值 | 记号 | 字节数 | 含义 | | --- | ---: | --- | @@ -35,7 +37,9 @@ | `float32` | 4 | 大端 IEEE 754 binary32 | | `float64` | 8 | 大端 IEEE 754 binary64 | -## Unsigned Variable-length Integer +
+ +## 无符号变长整数 `UVarInt` 是无符号 LEB128 风格的 32 位变长整数。每个字节的低 7 位保存数据，最高位 为 1 表示后面仍有字节，低位组先写。 @@ -44,13 +48,13 @@ value = Σ ((byte[i] & 0x7f) << (7 * i)) ``` -编码长度为 1 至 5 字节，并使用最短形式。 +编码使用最短表示，占 1 至 5 字节。例如，十进制 `300` 可以拆成从低到高的七位组 +`0x2c` 和 `0x02`，因此写成 `ac 02`。32 位解码器最多读取五个字节，同时检查数值是否 +溢出；五个字节后仍未结束的序列无效。 -例如，十进制 `300` 拆分成从低到高的七位组 `0x2c` 和 `0x02`，线上字节为 `ac 02`。 -32 位字段的读取器最多读取五个字节，并检查移位或数值溢出。写入器只产生规范的最短表示， -不会写入冗长编码。 +
-## Signed Variable-length Integer +## 有符号变长整数 `VarInt` 先对有符号 32 位整数做 ZigZag 映射，再按 `UVarInt` 写入： @@ -58,27 +62,33 @@ value = Σ ((byte[i] & 0x7f) << (7 * i)) zigzag(n) = (n << 1) XOR (n >> 31) ``` -| 原值 | 线上的无符号值 | 字节 | +| 原值 | 编码前的无符号值 | 字节 | | ---: | ---: | --- | | 0 | 0 | `00` | | -1 | 1 | `01` | | 1 | 2 | `02` | | -2 | 3 | `03` | -## Strings and Byte Arrays +
+ +## 字符串与字节数组 | 记号 | 格式 | | --- | --- | | `VarString` | `VarInt byte_length` + `byte[byte_length]` | | `String32` | `int32 byte_length` + `byte[byte_length]` | | `Binary32` | `int32 byte_length` + `byte[byte_length]` | +| `VarBinary` | `VarInt byte_length` + `byte[byte_length]` | -长度 `-1` 表示 null，长度 `0` 表示空字符串或空字节串。`VarString` 和 `String32` -保存 UTF-8；`Binary32` 保存原始字节。 +对于 `VarString`、`String32` 和 `VarBinary`，长度 `-1` 表示 null，长度 `0` 表示空字符串 +或空字节串；`Binary32` 不接受负长度。`VarString` 和 `String32` +保存 UTF-8；`Binary32` 和 `VarBinary` 保存原始字节。`VarBinary` 使用有符号 ZigZag +长度，不同于 [FileEncryptionHeader](DataPages/Encryption.md#file-encryption-header) +中二进制组件使用的无符号长度。 -长度统计编码后的字节数，而不是 Unicode code point 或字符数。例如，一个非 ASCII 字符的 -UTF-8 字节长度可能大于 1。对于允许 null 的字段，null 和空值具有不同线上表示，读取后 -必须保持区别。 +字符串长度按 UTF-8 编码后的字节数计算。例如，“温”占三个字节，其 `VarString` 长度字段 +为 `VarInt(3)`，编码为 `06`。允许 null 的字段还需要区分 null 和空值，不能在读取时将 +两者合并。 ## 边界与防御性解码 @@ -92,8 +102,8 @@ current_offset + N >= current_offset current_offset + N <= enclosing_end ``` -第二个比较用于检测整数回绕。读取器还应施加本地资源上限；结构上有效的长度不要求实现 -分配无限内存。 +使用定长整数计算偏移量时，第二个比较用于检测溢出回绕。应先检查这些边界，再分配内存， +并根据运行环境限制最大分配量：长度即使没有超出文件，也可能超过读取器能承受的范围。 固定宽度值必须具有完整字节数。截断的 `int64`、没有终止字节的 VarInt、非 null 字段的 负长度、要求字符串位置的无效 UTF-8，以及带长度记录结束后的尾随字节均属于格式错误。 @@ -102,5 +112,10 @@ current_offset + N <= enclosing_end 元数据中的 offset 以有符号 64 位整数保存，但表示非负绝对字节位置。offset 指向所命名 结构的第一个字节，例如 `offset_of_chunk_header` 指向 Chunk marker，`meta_offset` 指向 -`0x02` separator。长度不是隐式 offset；将 offset 与长度相加时必须检查溢出，并验证结果 -不超过文件或所属记录边界。 +`0x02` separator。计算结构末尾时，将起始 offset 与长度相加，并按上文检查溢出和所属结构边界。 + +## 如何阅读布局示例 + +结构示意中的 `?` 表示字段或结构可选，`*` 表示可以重复零次或多次，`[n]` 表示 n 个 +元素。具体是否出现、重复多少次，由旁边的说明或已读取的长度字段决定。`remaining bytes` +表示当前载荷边界内尚未使用的字节；对于 Page，这个边界指解密、解压后的载荷末尾。 diff --git a/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md index feaa8a0f7a..4511ad13c6 100644 --- a/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md +++ b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -21,10 +21,13 @@ # 布隆过滤器 -TsFile v4 在 `TsFileMetadata` 中保存文件级布隆过滤器，用于判断时间序列路径是否可能 -存在。布隆过滤器可以返回“肯定不存在”或“可能存在”，不能用于肯定某路径存在。 +`TsFileMetadata` 中可选的布隆过滤器用于省去不必要的索引查找。检查到某一位为零时， +可以排除该路径；检查的位全为一时，只能判断“可能存在”，仍需查索引。这一结论依赖 +查询端采用与写入端一致的路径编码和哈希方式，下面分别说明。 -## File Format +
+ +## 存储布局 ```text BloomFilter := @@ -39,20 +42,45 @@ BloomFilter := 位编号 `i` 存储在 `bits[i / 8]` 的位 `i mod 8`，即每个字节从低位方向编号。尾部全零 字节不写入文件，逻辑位数由 `bit_count` 保存。 -## Filter Input +
+ +## 路径如何转为字节 -过滤器输入是时间序列完整路径的 UTF-8 字节： +写入端使用 UTF-8 时，过滤器输入是时间序列完整路径的 UTF-8 字节： ```text UTF8(join(DeviceID segments, ".") + "." + measurement_id) ``` -null 设备片段在路径文本中表示为 ASCII `null`。 +null 设备片段在路径文本中表示为 ASCII `null`。当前 Java 实现调用无参数的 +`String.getBytes()`，使用平台默认字符集，且文件不保存该字符集。下列示例使用 UTF-8。 +读取器必须匹配写入端的字符集和哈希行为；无法确定时应绕过布隆过滤器，直接查询元数据 +索引。 + +
+ +## 哈希计算 -## Hashing +TsFile 使用 Java `Murmur128Hash` 所实现的 MurmurHash3 x64 128 位变体。完整的 16 字节块 +使用标准的小端无符号字节组装与混合步骤；不足 16 字节的尾块与标准算法不同：**每个尾部 +字节先从有符号 8 位符号扩展到 64 位，再移位；各项使用 XOR 而非 OR 合并**。 + +设尾块长度 `n = byte_length mod 16`，尾部字节为 `t`，按下式组装两个字。全部移位和运算 +限制在 64 位；`signed64(b)` 将字节 `0x80..0xff` 映射为 `-128..-1`。 + +```text +k1 = XOR(signed64(t[j]) << (8 * j)) for j = 0 .. min(n, 8) - 1 +k2 = XOR(signed64(t[j]) << (8 * (j - 8))) for j = 8 .. n - 1 +``` -哈希算法使用 MurmurHash3 x64 128 位变体。最终的两个 64 位结果相加，取低 32 位； -若结果为最小 32 位整数则使用 0，否则取绝对值并对 `bit_count` 取模。 +空 XOR 的结果为 0。仅当 `n >= 9` 时对 `k2` 做标准尾部混合，仅当 `n >= 1` 时对 `k1` +做标准尾部混合，再执行标准 MurmurHash3 的长度 XOR 与 finalization。将最终的 `h1` +和 `h2` 按 64 位相加，取低 32 位并解释为有符号整数 `hash32`。读取现有文件时必须保持 +这种有符号尾块行为；直接替换为标准 MurmurHash3 库可能让非 ASCII 路径出现假阴性。 + +```text +bit_index = (0 if hash32 == -2147483648 else abs(hash32)) mod bit_count +``` 哈希 seed 依次为： @@ -62,6 +90,20 @@ null 设备片段在路径文本中表示为 ASCII `null`。 实际使用的 seed 数量由 `hash_function_count` 指定，最大为 8。 +### UTF-8 测试向量 + +以下向量使用 `seed = 5` 和 `bit_count = 256`。最后一行还覆盖有符号尾块之前存在完整 +16 字节块的情况。 + +| 路径 | 有符号 `hash32` | 位编号 | +| --- | ---: | ---: | +| `root.sg.d.s1` | -1267668613 | 133 | +| `root.设备.s1` | 1697017247 | 159 | +| `root.abcdefghijklmnop.设备.s1` | 866805081 | 89 | + +对于 `root.设备.s1`，标准 MurmurHash3 在最终同样求和后得到 `1978455175`，对应位编号 +`135`。读取器若检查该位，可能把文件中实际存在的路径判为不存在。 + ## 查询过程 对一个精确路径进行查询时，读取器： @@ -71,8 +113,8 @@ null 设备片段在路径文本中表示为 ASCII `null`。 3. 检查逻辑 `bit_count` 位向量中的对应位置； 4. 任意一位为 0 时返回“肯定不存在”，否则返回“可能存在”。 -“可能存在”仍需访问元数据索引确认。只有查询端使用与写入器完全相同的路径规范化方式 -和 UTF-8 字节时，“肯定不存在”才可靠。 +“可能存在”仍需访问元数据索引确认。只有查询端使用与写入器完全相同的路径规范化方式、 +字符集及有符号尾块哈希时，“肯定不存在”才可靠。否则应跳过该优化并查询元数据索引。 ## 空间与概率 diff --git a/src/zh/UserGuide/latest/FileFormat/Configurations.md b/src/zh/UserGuide/latest/FileFormat/Configurations.md index 80edd9f3d8..61ee37109e 100644 --- a/src/zh/UserGuide/latest/FileFormat/Configurations.md +++ b/src/zh/UserGuide/latest/FileFormat/Configurations.md @@ -21,12 +21,11 @@ # 配置 -TsFile 的大多数配置属于写入策略。它们控制缓冲、Page 边界、Chunk Group 边界以及 -元数据索引的形状，但不会作为额外字段写入文件。读取器根据标记、长度字段、 -Chunk Header 和已序列化的元数据确定实际布局。 +多数配置影响写入器如何分组和缓冲数据。读取器无需知道原来的 Page 大小目标或刷出阈值， +因为最终形成的边界已经记录在文件中。时间单位、非对齐时间编码等少数信息则仍需由 +交换文件的应用共同约定。 -下列名称表示文件格式层面的概念。不同接口暴露的配置名称可能不同，但它们对磁盘字节的 -影响以及读取端是否需要原始配置是一致的。 +下表区分了这些情况。不同语言接口中的配置名称可能不同。 | 概念配置 | 可观察的文件影响 | 配置值是否直接写入 | 读取器是否需要原始配置 | | --- | --- | --- | --- | @@ -39,21 +38,25 @@ Chunk Header 和已序列化的元数据确定实际布局。 | 压缩 | Page 磁盘负载 | 是，每个 Chunk 的 `compression_type` | 否 | | 时间精度和时区 | `int64` 时间戳的解释 | 否 | 是，由上层协议提供 | | 非对齐时间编码 | 每个非对齐 Page 的时间编码流 | 否 | 是 | -| 加密 | Page 磁盘负载和 Footer 属性 | 属性会写入；密钥上下文可能在外部 | 加密时需要 | +| 加密 | 文件加密头、Chunk 认证、Page 载荷或旧版 Footer 属性 | provider/profile 或旧版属性会写入；密钥上下文在外部 | 加密时需要 | -“不直接写入”表示文件中没有以该阈值或比率命名的字段，但配置造成的布局结果仍可能被 -观察到。 +例如，文件不会保存“Page 大小目标”这个配置值，但会记录每个 Page 的实际长度。 -## Chunk Group Size +
+ +## Chunk Group 大小 Chunk Group 大小表示写入器在刷出当前设备的各个 Chunk 之前允许缓冲的数据量。 较大的 Chunk Group 可以减少标记与元数据开销，并有利于顺序 I/O，但需要更多写入内存； 较小的 Chunk Group 可以降低峰值内存，并更早地将数据写入输出。 该配置是刷出阈值，并不表示精确的磁盘占用大小。文件格式不规定默认值。 -Chunk Group 的实际边界由 `0x00` Chunk Group Header 标记以及下一个顶层标记确定。 +分组从 `0x00` Chunk Group Header 开始，到下一个分组、操作索引记录或分隔符结束； +中间的 Chunk 标记表示当前分组仍在继续。 + +
-## Data Page Size +## Page 大小 Data Page 大小是单个序列写入器在内存中累计数据量的目标值。较小的 Page 可以提供 更细粒度的 Statistics 和数据跳过能力；较大的 Page 可以减少 Page Header 开销并可能 @@ -63,7 +66,9 @@ Data Page 大小是单个序列写入器在内存中累计数据量的目标值 边界时，写入器都可以提前结束当前 Page。实际 Page 边界由 Page Header 中的长度确定。 文件格式不规定默认的 Page 大小。 -## Maximum Points per Page +
+ +## 每个 Page 的最大点数 除 Page 大小目标外，写入器还可以限制一个 Page 中的逻辑位置数，以防大量紧凑值让 单个 Page 包含过多数据点。参考写入配置使用 `10,000` 点。 @@ -74,25 +79,31 @@ Data Page 大小是单个序列写入器在内存中累计数据量的目标值 该上限不会作为独立字段写入文件。读取器使用 Page 边界；对于对齐 Value Page， 还会使用 `position_count`。 -## Metadata Index Fanout +
-元数据索引扇出限制一个内部元数据索引节点可以包含的子节点数量。较大的扇出形成 -层数更少但节点更大的树；较小的扇出形成层数更多但节点更小的树。该值必须至少为 `2`， +## 索引扇出 + +索引扇出就是一个内部索引节点最多能包含多少个子节点。较大的扇出形成 +层数更少但节点更大的树；较小的扇出形成层数更多但节点更小的树。该值应至少为 `2`， 参考写入配置使用 `256`。 配置的上限不会写入文件。每个元数据索引节点都会保存节点类型、子节点和结束偏移， 因此读取器无需知道写入配置即可遍历最终生成的树。 -## Bloom Filter False-positive Rate +
+ +## 布隆过滤器误判率 -布隆过滤器误判率控制路径查找的空间与准确率权衡。误判率越低，需要的位数和哈希函数 -越多。标准写入器将该值限制在 `0.01` 到 `0.10` 之间，默认使用 `0.05`。 +布隆过滤器有时会把不存在的路径判为“可能存在”，导致一次额外的索引查找。降低目标 +误判率可以减少这种情况，代价是使用更多位和哈希函数。标准写入器将该值限制在 `0.01` 到 `0.10` 之间，默认使用 `0.05`。 误判率本身不会写入文件。最终生成的过滤器字节、位数和哈希函数数量保存在 `TsFileMetadata` 中，因此读取器不需要写入端的配置。详见 -[Bloom Filter](Bloom-Filter.md)。 +[布隆过滤器](Bloom-Filter.md)。 -## Value Encoding and Compression +
+ +## 值编码与压缩 写入器可以按物理类型选择默认值编码，并选择默认压缩方式。这些默认值属于 Schema 与写入策略，不是读写端必须共享的配置： @@ -101,15 +112,19 @@ Data Page 大小是单个序列写入器在内存中累计数据量的目标值 - Chunk Header 中的 `compression_type` 标识该 Chunk 内所有 Page 使用的压缩方式。 因此读取器直接使用 Chunk Header，不需要知道写入器的默认值。支持的标识符及其 -载荷格式见 [Encodings](DataPages/Encodings.md) 和 -[Compression](DataPages/Compression.md)。 +载荷格式见 [编码](DataPages/Encodings.md) 和 +[压缩](DataPages/Compression.md)。 + +
-## Time Precision and Time Zone +## 时间精度与时区 物理时间戳是有符号 64 位整数。TsFile v4 不在文件中保存时间单位或时区；毫秒、微秒、 纳秒等时间精度以及时区语义由使用该文件的上层协议确定。 -## Non-aligned Time Encoding +
+ +## 非对齐时间编码 非对齐 Chunk Header 的 `encoding_type` 描述值流编码，不描述同一 Page 内的时间流编码。 非对齐时间流编码由读写配置共同确定，默认值为 `TS_2DIFF`。 @@ -117,13 +132,17 @@ Data Page 大小是单个序列写入器在内存中累计数据量的目标值 对齐 Time Chunk 不受这一限制；其时间编码保存在 Time Chunk Header 的 `encoding_type` 中。 -## Encryption +
+ +## 加密 + +加密方式有两处描述来源：Page AEAD 使用可选的文件加密头，旧版加密使用 Footer 属性。 +它们记录 Page 字节应如何解释，取得密钥仍需外部环境。AEAD 文件的旧版 Footer 属性虽然 +写为未加密状态，实际模式仍以文件加密头为准。布局和判断规则见[加密](DataPages/Encryption.md)。 -Page 数据是否加密以及所用算法由 `TsFileMetadata` 的 `encryptLevel`、`encryptType` -和 `encryptKey` 属性描述。打开加密文件所需的密钥材料仍然保存在文件之外。 -详细格式见 [Encryption](DataPages/Encryption.md)。 +
-## Implementation Settings +## 与格式无关的配置 内存分配、内存检查频率、批大小、并行执行、线程数、存储后端以及关闭文件时的同步 策略都不定义 TsFile 字节。它们属于具体实现与部署范围，不属于本文件格式规约。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md index 8226d8ce63..23a9adfbbd 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md @@ -21,29 +21,24 @@ # 校验和 -TsFile v4 不保存通用的 Page 校验和、Chunk 校验和或文件校验和。 - -头尾魔数、长度字段、marker、offset 和结构边界可以发现部分截断或结构损坏，但不提供 -内容校验或密码学完整性保证。 +TsFile v4 没有通用的 Page、Chunk 或完整文件校验和。一个看起来已经写完的文件，仍可能 +包含损坏的值，因此不能把头尾魔数当作内容校验和。 ## 结构校验能够发现什么 -严格的读取器可以通过以下约束发现大量非法布局： - -- 头尾魔数和版本字节完整存在； -- Footer 长度与 `meta_offset` 能解析到合法边界； -- 每个 `data_size`、`compressed_size`、字符串长度和元数据列表长度都位于所属结构内； -- 解压结果恰好包含 `uncompressed_size` 字节； -- 解码元素数量、对齐位图、Statistics 和索引范围内部一致。 +长度、偏移量和计数可以帮助发现不完整或格式错误的结构。例如，Page 超出 Chunk 边界、 +Footer 长度超出文件，或者解压结果长度不符，都可以判为错误。具体检查见 +[二进制表示](../Binary-Representation.md)和[数据页](README.md)。 -即使文件另有外部校验和，这些检查仍然必要。它们能够发现截断和许多意外改动，但某些 -字节替换不会改变长度与边界，因而仍可能不被发现。 +这些检查无法发现所有改动。一个值被修改后，字节数可能保持不变，周围的结构边界也可能 +仍然有效。即使另有校验和，读取时仍需要进行结构检查。 ## 静态存储与传输完整性 -需要检测数据损坏的系统应为完整文件维护外部校验和或对象存储完整性字段。需要防止 -恶意修改时，应使用带认证的摘要或签名，并把密钥或可信摘要保存在 TsFile 之外。外部 -校验值必须覆盖完整文件的精确字节序列，包括 Footer 长度和尾魔数。 +如果需要检测文件任意位置的变化，可以维护外部校验和，或使用存储系统提供的完整性检查。 +如果还需要防止恶意修改，则应使用带密钥的消息认证码或数字签名，并在文件之外保存可信 +密钥或参考值。校验范围应覆盖完整字节序列，包括 Footer 长度和尾部魔数。 -Page 负载加密是另一项独立能力。除非所选加密算法明确认证密文，否则仅加密既不会为 -TsFile 添加校验和，也不会验证未加密的 Header 与元数据。 +Page AEAD 的保护范围更具体：它认证非空 Page 载荷及上下文，并将 Chunk 序号绑定到文件 +位置，但不覆盖全部可见 Header、Statistics、索引或 Footer 元数据。空的对齐 Value Page +没有 Page 认证标签，旧版加密也不一定认证密文。具体范围见[加密](Encryption.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md index 840f8992f1..472b00f8bc 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -21,19 +21,23 @@ # Chunk 与 Chunk Group -TsFile 的数据区从偏移量 7 开始，由 marker 引导的 Chunk Group、Chunk 和 Operation -Index Range 组成。 +TsFile 数据区由 marker 引导的 Chunk Group、Chunk 和 Operation Index Range 组成。 +没有文件加密头时从偏移 `7` 开始；若偏移 `7` 处保存总序列化长度为 `E` 的 Page AEAD +[FileEncryptionHeader](Encryption.md#file-encryption-header)，数据区从 `7 + E` 开始。 +该可选加密头只允许出现一次，并位于数据区之前。 顶层语法为： ```text -DataSection := (ChunkGroup | OperationIndexRange)* Separator +FilePrefix := magic[6] version[1] FileEncryptionHeader? +DataSection := (ChunkGroup | OperationIndexRange)* ChunkGroup := ChunkGroupHeader Chunk* Separator := byte(0x02) ``` -`Chunk*` 组成的分组自身没有长度前缀。读取器必须根据每个 Chunk 的 `data_size` 完整消费 -该 Chunk，随后读取下一个顶层标记，才能确定 Chunk Group 已结束。 +数据区之后紧接 `0x02` 分隔符，它不属于任何 Chunk Group。Chunk Group 自身没有总长度 +字段。读取完一个 Chunk 后，要看下一个标记：如果仍是 Chunk，就继续读取；如果是新的 +Chunk Group、操作索引记录或分隔符，当前分组就结束了。 ```text File @@ -47,7 +51,9 @@ File └── Value Page(s) ``` -## Markers +
+ +## 标记字节 | 值 | 名称 | 后续结构 | | ---: | --- | --- | @@ -57,6 +63,7 @@ File | `0x03` | `VERSION` | v3 起不再使用，编号保留 | | `0x04` | `OPERATION_INDEX_RANGE` | 两个 `int64` | | `0x05` | `ONLY_ONE_PAGE_CHUNK_HEADER` | 单页、非对齐 Chunk | +| `0x06` | `ENCRYPTION_HEADER` | 仅位于偏移 `7` 的文件加密头；不属于 Chunk | | `0x41` | `VALUE_CHUNK_HEADER` | 多页、对齐 Value Chunk | | `0x45` | `ONLY_ONE_PAGE_VALUE_CHUNK_HEADER` | 单页、对齐 Value Chunk | | `0x81` | `TIME_CHUNK_HEADER` | 多页、对齐 Time Chunk | @@ -73,10 +80,12 @@ low 6 bits = 0x05: one page or no page `data_size = 0` 的对齐 Value Chunk 可以没有 Page。 -只有位于已知结构边界的字节才能解释为标记。编码或压缩后的 Page 载荷内部也可能出现相同 -字节值，但在载荷内部不具有结构含义。 +只有读完前一个结构，才能解释下一个标记。Page 载荷可以包含任意字节，其中看似标记的 +值也只是数据。 + +
-## Chunk Group Header +## Chunk Group 头部 ```text ChunkGroupHeader := @@ -88,17 +97,19 @@ DeviceID := VarString segment[segment_count] ``` -设备标识是可空字符串片段组成的有序元组。序列化时删除末尾的 null 片段，设备标识不能 -全部由 null 片段组成。`segment_count = 0` 在读取时表示只包含一个空字符串片段的设备。 +DeviceID 由一组有序的字符串片段组成，单个片段可以为 null。序列化时会去掉末尾的 null， +但至少要有一个非 null 片段。`segment_count = 0` 是一种特殊表示，读取后得到一个仅含 +空字符串片段的设备标识。 Chunk Group Header 之后的 Chunk 属于该设备，直到下一个 Chunk Group Header、Operation Index Range 或 Separator。 -`segment_count` 统计序列化元组元素数量，而不是 UTF-8 字节数。每个 segment 都有独立的 -长度。设备身份是有序元组本身；使用显示分隔符拼接 segment 不属于二进制格式，而且当 -segment 内含该分隔符时可能产生歧义。 +`segment_count` 统计片段数量，每个片段另有自己的字节长度。比较设备标识时应保留片段 +边界；用于显示的点号拼接可能产生歧义，因为片段自身也可以含点号。 + +
-## Chunk Header +## Chunk 头部 marker 是 Chunk Header 的第一个字段： @@ -110,8 +121,15 @@ ChunkHeader := byte data_type byte compression_type byte encoding_type + int64 chunk_ordinal? + int32 chunk_authentication_size? + byte chunk_authentication[chunk_authentication_size]? ``` +最后三个字段只出现在 Page AEAD 文件中，由文件加密头决定是否存在；Chunk marker +本身不变。序号、认证证明及校验规则见[AEAD Chunk 头部](Encryption.md#aead-chunk-header)。 +这些后缀字节和其他 Header 字段一样，不计入 `data_size`。 + `data_size` 是 Chunk 中全部 Page Header 和已保存 Page 数据的总长度，不包括 Chunk Header。读取器从 Chunk Header 结束处读取恰好 `data_size` 字节。 @@ -130,7 +148,8 @@ next_structure_offset = H + data_size 对齐数据按 Chunk 连续存储，不按 Time Page、Value Page 交替存储。Time Chunk 保存完整 的 Time Page 序列，每个 Value Chunk 保存自己的 Value Page 序列，Page 通过序号对应。 -对齐 Chunk 集满足以下不变量： +“对齐 Chunk 集”指一个 Time Chunk 及跟随它的 Value Chunk，并没有单独的集合头。 +它们之间有以下关系： - Time Chunk 位于使用该时间列的 Value Chunk 之前。 - 每个 Value Chunk 标识一个测量；Time Chunk 使用空 `measurement_id` 和物理类型标记 @@ -146,14 +165,19 @@ next_structure_offset = H + data_size 流式读取器或恢复读取器可以使用以下状态机扫描数据区： -1. 从偏移 7 开始读取一个顶层标记。 +1. 校验文件前缀，检查偏移 `7`，并消费可选的 `0x06` 加密头。在解析 Chunk 前确定 + provider/profile，再从 `data_start = 7 + E` 读取数据区第一个标记；无加密头时 `E = 0`。 + 后续或重复出现的 `0x06` 均无效。 2. 遇到 `0x00` 时读取 DeviceID，并进入对应 Chunk Group。 -3. 遇到 Chunk 标记时读取 Header；可以解析 Page，也可以直接向后移动 `data_size` 字节。 +3. 遇到 Chunk 标记时读取完整 Header，包括模式要求的 AEAD 后缀，并在使用 Chunk 身份 + 之前验证位置证明；可以解析 Page，也可以直接向后移动 `data_size` 字节。 4. 遇到 `0x04` 时消费后续 16 字节 Operation Index Range。 5. 遇到 `0x02` 时停止；下一个字节属于索引元数据区。 6. 顶层标记无法识别，或声明边界超出可用字节时，判定文件不完整或损坏。 -## Operation Index Range +
+ +## 操作索引范围 ```text OperationIndexRange := @@ -162,11 +186,9 @@ OperationIndexRange := int64 max_operation_index ``` -该结构记录最近一次写入批次涉及的最小和最大操作索引。Operation Index 是存储引擎的 -逻辑操作序号，不是时间戳、测量索引或文件偏移。它可用于 checkpoint、snapshot、backup -和恢复逻辑，以判断哪些逻辑写入已经进入文件。 +这条可选记录用于把已刷出的数据批次与存储引擎的操作日志关联起来。两个值分别是该批次 +涉及的最小和最大操作序号，可用于协调检查点、快照、备份和恢复，与时间戳或文件偏移量 +无关。 -对于通用写入器，该记录是可选的。没有 Operation Index 语义的写入器可以省略它，也可以 -写入占位范围。不使用该信息的读取器仍必须识别 `0x04` 并跳过完整的 16 字节载荷，保证在 -正确边界读取下一标记。一个文件可以包含多个此类记录，因为多个写入批次可以先后刷入同一 -文件。 +通用写入器可以省略这条记录。读取器即使不使用操作索引，也要完整读过两个 `int64`， +再读取下一个标记。一个文件可以先后刷入多个批次，因此也可以出现多条这样的记录。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md index e9ff7ab6d6..046978db0f 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -28,33 +28,39 @@ | 0 | `UNCOMPRESSED` | 原始字节 | | 1 | `SNAPPY` | Snappy raw block | | 2 | `GZIP` | 单个 gzip member | -| 3 | `LZO` | LZO block | +| 3 | `LZO` | LZO block（C++ 实现，编译期可选启用） | +| 4 | `SDT` | 保留编号，无编解码器 | +| 5 | `PAA` | 保留编号，无编解码器 | +| 6 | `PLA` | 保留编号，无编解码器 | | 7 | `LZ4` | LZ4 raw block | | 8 | `ZSTD` | Zstandard frame | | 9 | `LZMA2` | XZ container 中的 LZMA2 流 | 每个 Page 独立压缩，不跨 Page 共享压缩状态。Page Header 的 `uncompressed_size` 和 -`compressed_size` 分别保存压缩前和压缩后的负载长度。 +`compressed_size` 分别保存未压缩负载长度及压缩、加密后的磁盘载荷长度。Page AEAD 的 +磁盘长度还包含密文帧和认证标签。 -这些数值是稳定的文件格式编号，不是某个压缩库枚举中的位置。v4 未分配编号 `4` 到 -`6`。`LZO` 作为历史编号保留；读取器未提供该编解码器时，不能解码相应 Chunk。 +这些数值是稳定的文件格式编号，不是某个压缩库枚举中的位置。编号 `3` 到 `6` 是早期 +TsFile 版本留下的历史标识符。参考 Java 实现不定义也不实现它们，解析 Chunk Header 时 +会直接拒绝这些字节。C++ 实现保留 `LZO`、`SDT`、`PAA`、`PLA` 四个标识；其中只有 +`LZO` 有编解码器（编译期可选启用），`SDT`、`PAA`、`PLA` 仍是无编解码器的保留值。 +读取器未提供该编解码器时，不能解码相应 Chunk。 ## Page 级帧边界 -若 Page 起始偏移量为 `P`，其序列化 Page Header 占 `H` 字节，则磁盘负载恰好位于以下 -左闭右开区间： +设 Page 从偏移 `P` 开始，Header 占 `H` 字节，载荷就取后续恰好 `compressed_size` 字节： ```text [P + H, P + H + compressed_size) ``` -使用 `UNCOMPRESSED` 时，`compressed_size` 等于 `uncompressed_size`，这些字节已经是编码 -后的 Page 负载。使用其他方式时，解压结果必须恰好包含 `uncompressed_size` 字节。结果 -更短、更长，或编解码器要求读取上述区间外的字节，都表示 Page 无效。 +这里使用的是磁盘长度，包含加密帧信息。解密后，`UNCOMPRESSED` 已经是编码后的 Page +载荷；其他方式还需要解压。两种路径最终都应得到恰好 `uncompressed_size` 字节。 +不加密或加密保持长度时，`UNCOMPRESSED` 的这两个长度相等；Page AEAD 会增加帧开销， +因而不再相等。 -每个 Page 都是独立的压缩单元。因此，读取器可以先用 Page Statistics 排除不需要的 -Page，再直接前进 `compressed_size` 字节，而不必初始化解压器。随机访问也不依赖前一个 -Page 的字典或历史状态。 +Page 之间不共享压缩历史。读取器可以直接跳转到目标 Page，也可以按磁盘长度跳过它， +无需先解压前面的 Page。 ## 读取顺序与校验 @@ -63,12 +69,12 @@ Page 的字典或历史状态。 ```text 读取 Page Header -> 准确读取 compressed_size 字节 - -> 文件属性要求时解密 + -> 按文件加密头或旧版属性进行认证/解密 -> 按 compression_type 解压 -> 要求结果恰好为 uncompressed_size 字节 -> 解码 Page 内各数据流 ``` -实现应在分配内存前拒绝会导致偏移量运算溢出或超过资源限制的长度。不支持某个 -`compression_type` 时，读取器不能解码相应 Chunk，但仍可通过 `data_size` 跳过该 -Chunk；不能把未知方式当作 `UNCOMPRESSED`。 +分配缓冲区之前，按[二进制表示](../Binary-Representation.md)检查长度。编解码器不可用时， +无法解压该 Chunk；扫描器直接解析原始 Header 字段后，仍可利用 `data_size` 跳过它， +即使编解码器注册表拒绝这个编号。不能把该编号当作 `UNCOMPRESSED`。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md index ea043c0486..75e083cab8 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md @@ -21,8 +21,9 @@ # 编码 -Chunk Header 使用单字节 `encoding_type` 指定值流编码。对齐 Time Chunk 的该字段指定 -时间流编码；非对齐 Chunk 的该字段只指定值流编码。 +编码先把一串有类型的值转换为字节流，再交给压缩器处理。Chunk Header 的 +`encoding_type` 保存所选编码：在 Value Chunk 和非对齐 Chunk 中，它描述值流； +在对齐 Time Chunk 中，它描述时间戳流。 | 值 | 名称 | | ---: | --- | @@ -42,7 +43,7 @@ Chunk Header 使用单字节 `encoding_type` 指定值流编码。对齐 Time Ch | 13 | `RLBE` | | 14 | `CAMEL` | -类型与编码组合如下： +参考值编码器支持下列组合。一个编码已经分配了编号，并不表示所有数据类型都能使用它： | 数据类型 | 可用编码 | | --- | --- | @@ -65,13 +66,15 @@ Chunk Header 使用单字节 `encoding_type` 指定值流编码。对齐 Time Ch | 游程与位打包 | `RLE`, `RLBE` | 重复值或较小的有效位宽 | | 差分与预测 | `DIFF`, `TS_2DIFF`, `REGULAR`, `SPRINTZ` | 相邻值之间较小或较规律的变化 | | 整数重映射 | `ZIGZAG` | 把绝对值较小的有符号整数映射为较小的无符号整数 | -| 基于异或的浮点编码 | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | 相邻浮点数具有稳定的前导位和尾随位 | +| 基于异或的编码 | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | 相邻浮点数具有稳定的前导位和尾随位 | | 历史或专用编码 | `BITMAP`, `FREQ` | 保留的历史布局；`FREQ` 已弃用 | -最合适的编码取决于数据分布。文件不记录写入器选择某种编码的原因；读取器必须同时 -根据 `data_type` 和 `encoding_type` 选择解码器。 +不同的数据分布适合不同的编码。读取时，无论写入器为什么选择某种编码，都需要根据 +`(data_type, encoding_type)` 这一组合选择解码器。 -## Plain Encoding +
+ +## PLAIN 编码 `PLAIN` 流不保存元素个数，元素数量由 Page 上下文、时间流或 null 位图确定。各元素 连续写入： @@ -92,16 +95,14 @@ Chunk Header 使用单字节 `encoding_type` 指定值流编码。对齐 Time Ch ## 解码约束 -编码状态只在一个解压后的 Page 流内有效。Page 解压完成后，读取器需要： - -1. 根据 Chunk 的 `data_type` 和 `encoding_type` 创建解码器； -2. 解码出 Page 上下文所要求的准确逻辑元素数量； -3. 验证解码过程既没有越过 Page 边界，也没有留下不完整的编码块； -4. 在处理下一个 Page 前丢弃当前解码状态。 +解码状态只在一个 Page 流内有效。解密和解压之后，选择对应的解码器，在流的边界内读取， +并检查值的数量是否符合 Page 上下文。处理下一个 Page 时，重新开始解码状态。 -非对齐 Page 的时间流使用文件元数据中 `time_encoder` 指定的独立编码， -`encoding_type` 只作用于值流。对齐 Time Page 中，`encoding_type` 作用于时间流；对齐 -Value Page 会先移除 null 位置再编码，因此值解码器必须恰好产生位图中置位的元素数。 +对齐数据的时间编码来自 Time Chunk Header，Value Page 每遇到一个置位的位图位置就 +解码一个值。非对齐 Chunk Header 只说明值编码，时间编码来自共享配置，默认 +`TS_2DIFF`；Footer 中没有序列化的 `time_encoder` 字段。详见 +[配置](../Configurations.md#non-aligned-time-encoding)。 -读取器不支持某种编码时，不能解码对应 Chunk，但仍可借助 `data_size` 跳过其字节范围； -退回到另一种解码器并不合法。 +编码不受支持时，无法解码具体值。扫描器如果直接解析原始 Header 字段，仍能通过 +`data_size` 找到下一个 Chunk，即使上层编解码器注册表拒绝该编号。此时不能擅自改用 +`PLAIN` 或其他解码器。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md index 7c628345b1..dda8b442bc 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -21,52 +21,146 @@ # 加密 -加密只作用于压缩后的 Page `stored_data`。Page Header、Chunk Header、Statistics、 -Metadata Index 和 File Metadata 不加密。 +写入时先压缩 Page，再加密；读取时先解密，再解压。Header、统计信息、Schema 和索引 +仍然可见。TsFile v4 支持两种加密机制，格式版本字节都为 `0x04`： -写入顺序为： +| 模式 | 判断依据 | Chunk 和 Page 布局 | +| --- | --- | --- | +| 明文 | 没有文件加密头，旧版级别为 `0` 或不存在 | 基础 Chunk Header，Page 载荷不加密 | +| 旧版加密 | 没有文件加密头，旧版级别为 `1` 或 `2` | 基础 Chunk Header，按指定算法加密载荷 | +| Page AEAD | 偏移 `7` 处有 `0x06` 文件加密头 | 扩展 Chunk Header，按 profile 定义认证和加密 Page | + +解析 Chunk 之前，先检查文件加密头。AEAD 文件的旧版 Footer 属性仍写为未加密状态， +实际模式以加密头为准。 + +写入和读取的顺序为： + +```text +编码后的 Page 载荷 -> 压缩 -> 加密 -> stored_data +stored_data -> 按需认证解密 -> 解压 -> 值解码 +``` + +
+ +## 文件加密头 + +AEAD 文件在七字节魔数/版本前缀之后、任何 Chunk Group 或操作索引记录之前，保存一个 +`FileEncryptionHeader`： + +```text +FileEncryptionHeader := + byte marker = 0x06 + UVarInt payload_size + byte header_version = 0x01 + VarString provider_id + VarString profile_id + VarString key_id + VarString key_version + UVarInt file_crypto_id_size = 16 + byte file_crypto_id[16] + UVarInt wrapped_data_key_size + byte wrapped_data_key[wrapped_data_key_size] +``` + +`payload_size` 从 `header_version` 计到封装密钥结束，不包含标记和长度字段自身，取值范围 +为 `1..1048576`。解析必须恰好在该边界结束。四个字符串标识符均不得为 null 或空白， +文件加密 ID 恰好为 16 字节，封装数据密钥不得为空。这些二进制长度使用 `UVarInt`， +与 Footer 属性使用的可空 `VarBinary` 不同。 + +**Provider** 标识加密实现或服务，**profile** 则为一套固定的字节格式命名，其中包括 +Page 帧、nonce 和认证标签的表示、长度映射以及 Chunk 证明长度。一个 provider 可以 +提供多个 profile。文件保存这些公开标识和封装密钥，不依赖 Java 实现类名，也不保存 +AEAD 的明文密钥。读取端需要相应的 provider/profile，并能取得外部密钥材料。 + +当前支持的头版本为 `0x01`。未知版本、字段截断、尾随字节、重复的加密头，或加密头 +出现在偏移 `7` 以外的位置，均属于错误。 + +
+ +## AEAD Chunk 头部 + +文件加密头还决定了每个 Chunk Header 的解析方式。Chunk marker 保持不变，但在 +`encoding_type` 之后增加三个字段： ```text -encoded page payload - -> compression - -> encryption - -> stored_data + int64 chunk_ordinal + int32 chunk_authentication_size + byte chunk_authentication[chunk_authentication_size] ``` -读取顺序为先解密，再解压和解码。 +`chunk_ordinal` 是文件内唯一分配的非负标识，包括 Time Chunk 和 Value Chunk 在内。 +证明长度必须与 profile 声明一致，并位于 `1..1024`。这些字段属于 Header，均不计入 +`data_size`。 -## File Properties +认证证明把序号绑定到 Chunk 的绝对位置，使用该身份之前需要验证证明。移动 Chunk 会 +改变被认证的位置，因此必须重新生成证明。它使用的关联数据按大端序排列如下： -`TsFileMetadata` 的属性 Map 使用以下键描述加密： +```text +int32 domain_size = 20 +byte domain[20] = UTF8("TsFile/ChunkPosition") +int32 associated_data_version = 1 +byte file_crypto_id[16] +int64 absolute_chunk_marker_offset +int64 chunk_ordinal +``` + +
+ +## AEAD Page 载荷与关联数据 + +非空 Page 的 `compressed_size` 记录整个加密载荷的长度，包括帧信息、nonce、认证标签 +和填充。它大于压缩明文长度，使用 `UNCOMPRESSED` 时也是如此。Profile 可以采用固定 +开销，也可以采用随输入大小变化的布局，因此需要通过 profile 的长度映射取得候选明文 +长度。 + +Page 认证还把载荷绑定到对应的文件、Chunk 和 Page 位置。关联数据是下列字段按大端序 +精确拼接的结果。`SizedBytes` 表示 `int32` 字节长度加上相应字节，文本使用 UTF-8。 + +```text +int32 associated_data_version = 1 +SizedBytes provider_id +SizedBytes profile_id +SizedBytes key_id +SizedBytes key_version +SizedBytes file_crypto_id +int64 chunk_ordinal +int32 page_index +int32 uncompressed_size +int32 compressed_plaintext_size +``` + +`page_index` 在每个物理 Chunk 内从零开始，空的对齐 Value Page 也占一个序号。 +全 null Value Page 仍采用[空值](../Nulls.md)章节中的单个 `UVarInt(0)` 表示， +不保存密文或 Page 认证标签。 + +认证成功后，候选明文长度才可信。解密输出必须等于该长度，解压后还必须恰好得到 +`uncompressed_size` 字节。认证标签无效、密钥错误、profile 不受支持或长度不符时， +应报错，不能按明文重试。 + +
+ +## 旧版加密属性 + +没有文件加密头时，读取 `TsFileMetadata` 中的以下属性： | 属性 | 含义 | | --- | --- | | `encryptLevel` | `0` 不加密；`1` 直接保存数据密钥；`2` 保存被外部主密钥加密的数据密钥 | -| `encryptType` | 加解密算法标识符 | +| `encryptType` | 旧版加解密算法标识符 | | `encryptKey` | 数据密钥或密钥封装结果的文本表示 | -Page Header 没有单独的密文长度字段。加密结果长度与输入的压缩数据长度相同，文件中的 -`stored_data` 长度仍由 `compressed_size` 给出。 - -为保持兼容，缺少 `encryptLevel` 时按 `0` 处理。级别为 `1` 或 `2` 时，必须同时存在 -`encryptType` 和非空 `encryptKey`。级别未知或算法不可用时，加密的 Page 数据不可读； -读取器不能静默地把它当作未加密数据。 +这几个属性的值保存文本。级别 `1` 直接保存数据密钥；级别 `2` 先使用外部主密钥解密器 +解封密钥，再解密 Page。缺少 `encryptLevel` 时按 `0` 处理；级别 `1` 和 `2` 要求存在 +`encryptType` 和非空 `encryptKey`。未知级别或不可用算法均为错误。 -## 密钥解释 +旧版加密使用算法自身的加解密操作，没有 AEAD Chunk 后缀或 Page 上下文。 +`compressed_size` 仍表示实际磁盘输出长度，仅在算法保持长度时才等于压缩明文长度。 -级别 `1` 的 `encryptKey` 表示供 `encryptType` 使用的数据密钥。级别 `2` 的该属性表示 -封装后的数据密钥：读取器先取得外部主密钥解密器并解封数据密钥，再使用 -`encryptType` 解密 Page 负载。密钥派生、主密钥保存、轮换和访问控制不属于文件格式。 +AEAD 写入器把旧版属性设为级别 `0`、类型 `org.apache.tsfile.encrypt.UNENCRYPTED` +和空密钥，实际模式和封装密钥由文件加密头提供。这些旧版值不能用作回退到明文的依据。 ## 安全边界 -Page 加密保护编码后样本数据的机密性，但不会隐藏文件形状、设备和测点标识、类型、 -时间范围、值 Statistics、Schema、索引以及加密属性本身。应用需要把这些元数据视为 -可见信息。 - -TsFile v4 没有为 Page 或整个文件定义通用认证标签。密文被修改后能否检测出来取决于 -所选算法；[Checksumming](Checksumming.md) 中的结构检查和外部完整性建议仍然适用。 - -读取器应先解析加密属性和密钥材料，再分配 Page 缓冲区，并严格执行 -`读取 -> 解密 -> 解压 -> 解码`。解密失败、输出长度错误、算法未知或密钥缺失都属于 -对应数据的读取错误，不能退回明文处理。 +Page AEAD 认证非空 Page 载荷及其上下文，Chunk 证明将序号绑定到文件位置。它不认证 +全部可见字段，也不提供完整文件摘要。旧版加密不一定具有认证能力。完整文件的完整性 +检查见[校验和](Checksumming.md)，重建 Footer 时如何保留加密上下文见[错误恢复](Error-Recovery.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md index 290fb1a62e..c9ced30c8b 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md @@ -21,63 +21,56 @@ # 错误恢复 -完成写入的 TsFile 包含头魔数、版本字节、`0x02` separator、Footer 元数据、4 字节 -`TsFileMetadata` 长度和尾魔数。没有 Footer 长度或尾魔数的文件是未完成文件。 - -恢复读取器可以从偏移量 7 开始按 marker 扫描数据区，并利用 Chunk Header 的 -`data_size` 跳过完整 Chunk。在 Chunk 内部，所有 Page 的序列化长度之和等于 -`data_size`。 - -扫描器不能把 Page 数据中出现的任意 `0x02` 当作 separator；只有在完整解析或跳过 -前一个 marker 所界定的结构后，才可以读取下一个 marker。 +Footer 在数据区之后写入。如果写入提前中断，文件虽然无法通过索引正常打开，前面仍可能 +保留一些完整的 Chunk。恢复时从文件头向后扫描，找到可以保留的结构，再据此重建缺失的 +索引和 Footer。 ## 完成性检查 -设文件大小为 `F`，区间 `[F - 10, F - 6)` 中保存的大端 Footer 长度为 `L`。普通的 -元数据优先读取开始前，完整文件必须满足： +设文件大小为 `F`，从 `[F - 10, F - 6)` 读取大端元数据长度 `L`。完成写入的文件具有 +以下关系： ```text -bytes[F - 6, F) = "TsFile" -TsFileMetadata 区间 = [F - 10 - L, F - 10) -meta_offset = 顶层 0x02 separator 的偏移量 +bytes[F - 6, F) = "TsFile" +TsFileMetadata interval = [F - 10 - L, F - 10) +meta_offset = offset of the top-level 0x02 separator meta_offset < F - 10 - L ``` -头魔数和版本也必须有效，并且 `TsFileMetadata` 解码器必须恰好消费 `L` 字节。仅有合法的 -尾魔数并不能证明文件完整。 +还要校验头部魔数和版本，并恰好解析 `L` 字节的 `TsFileMetadata`。仅有尾部魔数不够， +Footer 长度和其中的偏移量也必须能正确使用。 ## 安全正向扫描 -恢复扫描器可以识别数据区中最长的结构完整前缀： - -1. 校验头魔数和版本，把游标置于偏移量 `7`； -2. 仅在已知顶层边界读取一个 marker； -3. 遇到 Chunk Group Header（`0x00`）时，解码完整 `DeviceID` 并保存组上下文； -4. 遇到 Chunk marker 时，解码完整 Chunk Header，校验 `cursor + data_size`，再跳过该 - 区间或校验其中所有 Page 边界； -5. 遇到 Operation Index Range（`0x04`）时，准确消费两个大端 `int64`； -6. 遇到顶层 separator（`0x02`）时正常结束；字段或负载不完整时，在最后一个已知安全 - 边界停止。 +正向扫描使用与普通读取器相同的 Chunk 布局，但不依赖 Footer： -最后安全边界是一个完整定界结构的末尾，不是“最后一个碰巧可读的字节”。恢复工具可以 -使用该边界之前的完整 Chunk 重建索引和新 Footer，但不能臆造或部分解码被截断的 Chunk。 +1. 校验魔数和版本，检查偏移 `7`，读取可选的 `0x06` 加密头。数据区从 `7 + E` 开始， + `E` 为该头的总长度；不存在时为零。 +2. 在结构边界读取标记。遇到 `0x00` 时读取 DeviceID，并记住当前分组。 +3. 遇到 Chunk 标记时读取完整 Header，包括需要的 AEAD 后缀。在使用 Chunk 身份之前 + 验证位置证明，再检查剩余文件是否容得下 `data_size` 字节。随后检查 Page，或按该 + 长度完整跳过。 +4. 遇到 `0x04` 时，读过两个操作索引 `int64`。 +5. 遇到顶层 `0x02` 分隔符时，停止扫描数据区。 -检测到以下情况时，文件结构不完整或已损坏： +字段或载荷被截断时，在最后一个完整结构处停止。未知标记、重复或位置错误的加密头、 +无效长度和认证失败都应作为错误处理，不能通过搜索下一个熟悉的字节来绕过。 +Page 载荷中的 `0x02` 是数据，不能据此确定数据区结束。 -- Chunk 的 `data_size` 超出文件或下一个结构边界。 -- Page 的长度字段超出 Chunk 边界。 -- Footer 长度无法定位到完整的 `TsFileMetadata`。 -- `meta_offset` 没有指向 `0x02` separator。 -- 索引 offset 位于文件之外或指入结构中间。 - -其他失败还包括：未知顶层 marker、未终止的 VarInt 或 VarString、计算结束偏移量时整数 -溢出、Page 解码结果数量错误，以及对齐 Time/Value Page 不匹配。 +只保留已经完整读取并检查过的结构。被截断 Chunk 中尚能读取的部分，不会自动成为有效 +Chunk。通用边界检查见[二进制表示](../Binary-Representation.md)，Page 边界见[数据页](README.md)。 ## 恢复能力边界 -恢复只能重建结构，不能补回缺失信息。完整的压缩或加密 Page 仍需要其声明的编解码器， -必要时还需要外部密钥。由于没有文件级校验和，结构仍合法的字节替换可能无法与原始数据 -区分。Operation Index Range 可以帮助存储引擎把保留下来的前缀与外部操作日志关联， -但不能替代对 Chunk 和 Page 边界的校验。 +重建索引不能补回缺失的值，也无法识别所有结构仍有效的字节改动。保留的 Page 仍需要 +相应的编解码器；已加密的数据还需要密钥。操作索引记录有助于把保留数据与外部日志关联， +但不能替代结构检查。 + +Page AEAD 恢复要保留原加密头、文件加密 ID、Chunk 序号和 Page 索引。仅截断文件尾并 +重写 Footer 时，可以保留 Chunk 的原始偏移。移动 Chunk 需要新的位置证明；改变文件 +身份、Chunk 序号或 Page 上下文时，还要按变化内容对相关数据重新认证和加密。追加写入 +应从已保留的最大 Chunk 序号之后继续分配。 -TsFile v4 没有通用校验和；相关限制见 [Checksumming](Checksumming.md)。 +认证恢复需要相应的 provider 和外部密钥。AEAD 文件的旧版 Footer 字段故意设为未加密， +因此不能用它代替损坏的加密头。模式判断见[加密](Encryption.md)，格式完整性检查仍有哪些 +限制，见[校验和](Checksumming.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/README.md b/src/zh/UserGuide/latest/FileFormat/DataPages/README.md index d0a8e33a69..f938ef9be0 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/README.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/README.md @@ -21,10 +21,13 @@ # 数据页 -Chunk Header 之后连续保存一个或多个 Page。Page 之间没有填充字节；Chunk Header 的 -`data_size` 给出全部 Page Header 和 Page 数据的总长度。 +一个 Chunk 的 Page 连续存放在 Chunk Header 之后，Page 之间没有填充字节。`data_size` +统计所有 Page Header 和载荷的总长度。空的对齐 Value Chunk 的 `data_size = 0`， +不包含任何 Page。 -## Page Header +
+ +## 页头（Page Header） 非空 Page 的格式为： @@ -37,22 +40,19 @@ Page := ``` `uncompressed_size` 是解密、解压后的 Page 负载大小，`compressed_size` 是文件中保存的 -Page 负载大小。当 `compression_type = UNCOMPRESSED` 时，两个值相等。 - -Chunk marker 的低 6 位决定 Page Header 是否包含 Statistics： - -- `0x01`：Chunk 有多页，每个非空 Page Header 包含 Statistics。 -- `0x05`：Chunk 只有一页或没有 Page，Page Header 不包含 Statistics。 +Page 负载大小。仅在 `compression_type = UNCOMPRESSED` 且加密不扩展长度时二者相等。 +Page AEAD 的 `compressed_size` 包含密文帧和认证开销，即使不压缩也可能更大，详见 +[加密](Encryption.md#aead-page-body-and-context)。 -Statistics 的二进制格式见 [Metadata](../Metadata.md#statistics)。 - -因此，无需检查 Page 载荷即可根据 Chunk marker 选择 Header 结构： +Chunk marker 的低 6 位决定页头中是否包含统计信息： | Chunk marker 低位 | Page 数量 | Page Statistics | | ---: | --- | --- | | `0x01` | 多于一个 | 每个非空 Page Header 中存在 | | `0x05` | 零个或一个 | 不存在 | +统计信息的字节布局见[元数据](../Metadata.md#statistics)。 + 若 Page 起始偏移为 `P`，两个长度字段和可选 Statistics 的序列化总长度为 `h`，则磁盘 载荷占用半开区间 `[P + h, P + h + compressed_size)`；如果还有下一个 Page，它从 `P + h + compressed_size` 开始。该结束位置不得超过所属 Chunk 的边界。 @@ -60,16 +60,18 @@ Statistics 的二进制格式见 [Metadata](../Metadata.md#statistics)。 解密并解压后，载荷必须恰好产生 `uncompressed_size` 字节。即使压缩库报告成功，实际输出 长度不一致时读取器也必须拒绝该 Page。 -## Page Encoding +
+ +## 写入与读取顺序 写入非空 Page 时，数据依次经过： ```text -logical values - -> value encoding - -> assemble uncompressed page payload - -> compression - -> optional encryption +时间戳和值 + -> 编码 + -> 组成未压缩的 Page 载荷 + -> 压缩 + -> 按需加密 -> stored_data ``` @@ -78,7 +80,9 @@ logical values 编码与压缩相互独立。编码把有类型的值转换成未压缩字节流，压缩把该结果视为普通字节。 Page 边界会重置值编码器/解码器和压缩器/解压器，状态不会跨 Page 延续。 -## Non-aligned Page +
+ +## 非对齐 Page 非对齐 Page 的解压后负载为： @@ -99,13 +103,15 @@ NonAlignedPagePayload := value_stream_size = U - v - time_stream_size ``` -结果必须非负。解码后，两个流的解码器都必须恰好到达各自边界，并产生相同的数据点数量。 -存在尾随字节、提前结束或数量不一致时，该 Page 格式错误。 +该长度必须非负。两个流都受 Page 载荷边界约束，必须按各自编码格式完整结束，并解出 +相同数量的数据点，才能将时间戳和值一一对应。 非对齐 Chunk Header 只记录值流的 `encoding_type`。时间流编码见 -[Configurations](../Configurations.md#non-aligned-time-encoding)。 +[配置](../Configurations.md#non-aligned-time-encoding)。 + +
-## Aligned Time Page +## 对齐 Time Page 对齐 Time Page 的解压后负载完全由时间编码流组成： @@ -118,7 +124,9 @@ AlignedTimePagePayload := byte encoded_time_stream[remaining bytes] 解码得到的时间戳数量就是该 Page 的逻辑位置数。所有对应对齐 Value Page 的 bitmap 宽度 和 null 重建都由该数量决定。 -## Aligned Value Page +
+ +## 对齐 Value Page 非空对齐 Value Page 的解压后负载为： @@ -130,7 +138,7 @@ AlignedValuePagePayload := ``` 同一对齐集合中，第 `i` 个 Value Page 对应第 `i` 个 Time Page。null 位图和全 null -空 Page 的格式见 [Nulls](../Nulls.md)。 +空 Page 的格式见 [空值](../Nulls.md)。 `position_count` 必须等于对应 Time Page 解码得到的时间戳数量。bitmap 必须恰好具有 `ceil(position_count / 8)` 字节；最后一个字节中未使用的低位不表示逻辑位置。值解码器只 @@ -138,13 +146,11 @@ AlignedValuePagePayload := ## Page 选择与跳过 -Page Statistics 允许读取器在读取磁盘 Page 数据前判断时间和值谓词。只有当 Statistics -能够证明没有数据点满足谓词时，才能跳过 Page。Statistics 是保守摘要：区间重叠只表示 -“可能匹配”，并不表示“一定匹配”。 +Page Statistics 可以在读取载荷之前排除不需要的 Page。例如，Page 的时间范围与查询 +范围不相交时，就可以直接跳过。各类型提供哪些值摘要，见[统计信息](../Metadata.md#statistics)。 -单 Page Chunk 的 Page Header 不包含 Statistics。读取器可以在跳转到 Chunk 前使用文件尾 -中的 Chunk 或 Timeseries Statistics。某种数据类型的 Statistics 不包含查询所需摘要时， -读取器必须解码 Page 或依赖其他谓词，不能据此排除数据。 +单 Page Chunk 不保存 Page Statistics，需要使用 Footer 中的 Chunk 或序列级统计信息。 +如果现有摘要无法排除匹配，就要读取 Page 并检查具体值。 ## 全 null 对齐 Value Page @@ -158,11 +164,13 @@ EmptyAlignedValuePage := UVarInt(0) # uncompressed_size 该零之后没有 `compressed_size`、Page Statistics、bitmap 或磁盘载荷。位置数继承自对应 Time Page。只有在对齐 Page 序列能够提供外部位置数时，才允许使用这种表示。 -## Data Page Details +
+ +## 相关章节 -- [Chunks](Chunks.md) -- [Compression](Compression.md) -- [Encodings](Encodings.md) -- [Encryption](Encryption.md) -- [Checksumming](Checksumming.md) -- [Error Recovery](Error-Recovery.md) +- [Chunk 与 Chunk Group](Chunks.md) +- [压缩](Compression.md) +- [编码](Encodings.md) +- [加密](Encryption.md) +- [校验和](Checksumming.md) +- [错误恢复](Error-Recovery.md) diff --git a/src/zh/UserGuide/latest/FileFormat/Extensibility.md b/src/zh/UserGuide/latest/FileFormat/Extensibility.md index f8f16f4367..badf64eb88 100644 --- a/src/zh/UserGuide/latest/FileFormat/Extensibility.md +++ b/src/zh/UserGuide/latest/FileFormat/Extensibility.md @@ -21,41 +21,33 @@ # 扩展机制 -TsFile v4 在以下位置使用已编号或自描述的扩展点： +TsFile 主要通过三种方式增加能力：分配新的类型或编解码器编号、增加 Footer 属性，或者 +定义新的文件布局。选择哪种方式，取决于读取器在不理解新能力时，是否仍能找到下一个 +结构的边界。 -- 数据类型、编码和压缩方式使用单字节编号。 -- `TsFileMetadata` 末尾保存长度前缀的字符串属性 Map。 -- 文件版本字节用于选择完整的文件布局。 - -新数据类型、编码或压缩方式需要使用新的编号；已有编号的线上含义保持不变。 - -Footer 属性由键和值两个 `VarString` 组成。读取器可以在不改变其他 Footer 字段边界的 -情况下读取未识别属性。 - -数据区 marker 没有通用长度字段。读取器无法安全跳过未知 marker，因此新增无法由现有 -marker 表达的数据区结构需要新的文件版本或能够由现有结构完整界定的编码方式。 +Footer 属性容易确定边界：每个属性由 `VarString` 键和可空的 `VarBinary` 值组成， +读取器无需解释值的内容，就能保留未知属性。Chunk 也有明确的字节长度，扫描器即使不支持 +其中的编解码器，仍可以跳过。数据区标记没有统一的长度字段，遇到未知标记时就不能套用 +同样的做法。 ## 安全演进规则 -只有旧读取器无需解释新负载就能确定其完整字节范围时，扩展才可以安全跳过。例如新的 -Footer 属性值，以及位于已知 `data_size` 的 Chunk 内的新编解码器。向没有长度边界的 -Header 直接追加字节并不可跳过，因为其后的所有字段都会发生位移。 +兼容的扩展应保留已有字段的含义和边界： -在 v4 内，扩展需要： +- 新类型和编解码器使用未分配的编号，已分配编号的含义保持不变。 +- 新编解码器说明支持的数据类型、字节布局、输出长度及错误处理方式。 +- 沿用现有的字节序和长度约定。 +- 解码目标数据必须使用、但读取器不支持的能力，应明确报错。 -- 分配尚未使用的编号，且不复用任何已有值； -- 保持现有结构的字节序、长度解释和包含边界； -- 为新编码规定可用的物理数据类型； -- 为新编解码器规定输出长度、元素数量以及非法输入行为； -- 读取器不能解码该扩展时，必须明确报告能力缺失。 +如果直接在 Header 中增加字节，却没有规则选择新布局，旧读取器预期的后续字段位置就会 +改变。这类修改需要新的版本号，或明确的扩展机制。 -保留或未知编号不是默认值的别名。特别是，未知编码不等于 `PLAIN`，未知压缩方式不等于 -`UNCOMPRESSED`，未知 marker 也不表示填充字节。 +Page AEAD 使用了后一种方式：`0x06` 文件加密头决定额外的 Chunk Header 字段和 +profile 指定的 Page 帧格式，文件版本字节仍为 `0x04`。旧 v4 读取器可能拒绝这些文件。 +[加密](DataPages/Encryption.md)说明了其布局和能力要求；仅支持基础格式版本还不够。 ## 属性与语义 -长度前缀让未知 Footer 属性在结构上可读取，但不表示该属性的语义一定可忽略。若某个 -属性会改变现有字节的解释方式，就必须定义读取器如何声明和强制其支持能力。安全相关 -属性的语义无法执行时，必须按失败处理。 - -改变顶层结构语法、重新解释已分配编号，或让旧结构边界产生歧义的变更需要新的文件版本。 +一个属性可以被跳过，不代表它的含义总能被忽略。描述性属性可以是可选的；如果属性会 +改变 Page 字节的解码方式，就需要明确规定读取器应如何支持。读取器无法执行必要的加密 +处理或其他解释规则时，应报告错误，不能按默认方式继续解码。 diff --git a/src/zh/UserGuide/latest/FileFormat/Format-Versions.md b/src/zh/UserGuide/latest/FileFormat/Format-Versions.md index 8ea6c670d0..fdd0db6e51 100644 --- a/src/zh/UserGuide/latest/FileFormat/Format-Versions.md +++ b/src/zh/UserGuide/latest/FileFormat/Format-Versions.md @@ -21,43 +21,38 @@ # 格式版本 -TsFile 的文件版本位于头魔数之后： +格式版本是头部魔数之后的一个字节： ```text 6-byte magic string "TsFile" 1-byte format version ``` -本文档描述版本字节 `0x04`。读取器在检查头魔数后，根据版本字节选择对应的文件布局。 +本文介绍 `0x04`。格式版本和项目发布版本含义不同：2.x 系列的一个发布版本可以使用 +v4 文件格式。文件头不保存项目的发布版本标签。 -v4 使用分片的 `DeviceID`，并在 `TsFileMetadata` 中保存按表划分的设备索引根和 -Table Schema。v3 使用单字符串设备标识和不同的 Footer 布局，不能使用 v4 的 -`DeviceID` 或 `TsFileMetadata` 结构解析。 +v4 使用分段 DeviceID，并在 `TsFileMetadata` 中保存按表划分的设备索引根和 Table Schema。 +v3 使用单字符串设备标识和不同的 Footer。读取这两种文件时，需要分别使用对应版本的 +结构定义。 ## 读取器分派 -必须在解释任何数据区 marker 之前处理版本字节。读取顺序为： +先读取版本，再解释数据标记并确定加密模式： ```text -校验头魔数 -> 读取版本 -> 选择该版本的完整语法 -> 解析数据区与 Footer +校验头部魔数 -> 读取版本 -> 检查可选加密头 + -> 确定加密模式/profile -> 按对应布局解析数据区和 Footer ``` -只支持 v4、但读到其他版本字节的实现必须报告“不支持的版本”。即使某个 marker 或 -Footer 后缀看起来熟悉，也不能试探性地按 v4 解析，因为不同版本中结构的含义和宽度 -可能不同。 +版本不受支持时，应在解析数据之前报告错误。某个标记或尾部魔数看起来熟悉，并不能说明 +文件其他部分也采用读取器预期的布局。 ## 兼容性约束 -兼容性在字节格式层面定义： +支持格式版本之后，还要支持目标数据使用的类型、编解码器和加密能力。例如，v4 的 +Page AEAD 文件在偏移 `7` 处有 `0x06` 加密头，用它选择扩展的 Chunk Header 和 Page +载荷 profile。版本字节仍为 `0x04`；不具备该能力的 v4 读取器应明确报告不支持。 -- v4 读取器在支持目标数据所需全部编号的前提下，可以读取 v4 文件。 -- v4 写入器输出本文定义的 v4 语法，与其编程语言或内存类无关。 -- Footer 属性的键和值都有长度，可以读取并跳过未知属性。 -- 数据区 marker 没有统一的负载长度，因此通常无法跳过未知 marker。 -- v4 已分配的数据类型、编码、压缩方式、marker 和索引节点类型编号保持原有含义。 - -支持某个文件版本不代表支持所有可选编解码器或加密算法。缺少相应能力时，所属 Chunk -在结构上可能仍可跳过，但其中样本值不可解码。 - -项目发布系列与磁盘格式字节是两套独立版本。2.x 发布系列的产品可以读写格式字节 -`0x04`；TsFile Header 不保存项目发布版本标签。 +已分配编号的含义保持不变。未知 Footer 属性的键和值都有长度，读取器可以读过相应 +字节区间；未知数据标记没有统一的载荷长度，通常不能安全跳过。这些边界见 +[扩展机制](Extensibility.md)，AEAD 布局见[加密](DataPages/Encryption.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/Metadata.md b/src/zh/UserGuide/latest/FileFormat/Metadata.md index 895b814262..06740978be 100644 --- a/src/zh/UserGuide/latest/FileFormat/Metadata.md +++ b/src/zh/UserGuide/latest/FileFormat/Metadata.md @@ -30,7 +30,7 @@ Footer 元数据从 `SEPARATOR (0x02)` 之后开始。`TimeseriesMetadata` 范 节点可以交错出现；索引中的 offset 和 `end_offset` 决定每个范围的实际边界。所有 offset 都是从文件首字节开始计算的绝对偏移量。 -元数据层级支持逐步细化的数据裁剪： +查找数据时，读取器沿下列层级逐步缩小范围，从表和设备定位到可能包含目标值的 Page： | 层级 | 标识对象 | 主要用途 | | --- | --- | --- | @@ -41,7 +41,9 @@ offset 都是从文件首字节开始计算的绝对偏移量。 | Chunk Metadata | 一个物理 Chunk | 跳转到数据并利用 Chunk Statistics 裁剪 | | Page Header | 一个 Page | 利用 Page Statistics 裁剪并读取载荷 | -## Statistics +
+ +## 统计信息 Page、Chunk 和时间序列级 Statistics 使用相同的类型相关结构： @@ -75,17 +77,18 @@ Timeseries Metadata。 `first` 和 `last` 表示值的先后顺序，不等同于最小值和最大值。对于按时间有序的序列， 它们对应覆盖区间首尾时间戳处的值。`start_time` 和 `end_time` 均包含在统计区间内。数值 `sum` 用于聚合；其中 `INT64` 的 sum 以 `float64` 保存，数值较大时可能无法精确表示每个 -整数位。 +整数值。 + +统计信息用于排除不可能匹配的数据。例如，Page 的时间范围完全早于查询范围时，可以 +跳过该 Page。范围有重叠只说明“可能匹配”，仍需读取数据检查具体值。 +`BLOB`、`OBJECT`、`TEXT` 等没有最小/最大值字段的类型不能通过 Statistics 做值范围裁剪。 -Statistics 是保守的数据裁剪信息。查询时间范围与统计时间范围不相交，或者受支持的值摘要 -能够证明不存在匹配值时，读取器可以跳过该结构；区间重叠本身不能证明存在匹配值。 -`BLOB`、`OBJECT` 等没有最小/最大值字段的类型不能通过 Statistics 做值范围裁剪。 +Chunk Statistics 汇总该 Chunk 的所有 Page，序列级 Statistics 再汇总该序列的所有 +Chunk。父层计数是各子层计数之和，时间范围也应覆盖各子层的时间范围。 -父层 Statistics 汇总该父层表示的全部值。序列包含多个 Chunk 时，series Statistics -覆盖各 Chunk Statistics 的并集；Chunk 包含多个 Page 时，Chunk Statistics 覆盖各 Page -Statistics 的并集。计数和时间边界必须与低层记录保持一致。 +
-## Timeseries Metadata +## 时间序列元数据 每条时间序列对应一个 `TimeseriesMetadata`： @@ -108,9 +111,11 @@ low 6 bits = 0: exactly one Chunk; ChunkMetadata omits Statistics low 6 bits = 1: more than one Chunk; every ChunkMetadata contains Statistics ``` -## Chunk Metadata +
+ +## Chunk 元数据 -当 `timeseries_type & 0x3f == 0` 时，时间序列只有一个 Chunk： +当 `(timeseries_type & 0x3f) == 0` 时，时间序列只有一个 Chunk： ```text ChunkMetadata := int64 offset_of_chunk_header @@ -118,7 +123,7 @@ ChunkMetadata := int64 offset_of_chunk_header 该 Chunk 的 Statistics 直接使用 `series_statistics`。 -当 `timeseries_type & 0x3f != 0` 时： +当 `(timeseries_type & 0x3f) != 0` 时： ```text ChunkMetadata := @@ -128,11 +133,12 @@ ChunkMetadata := `offset_of_chunk_header` 指向 Chunk marker，即 Chunk Header 的第一个字节。 -`chunk_metadata_list_size` 是字节边界，不是 Chunk 数量。读取器应先计算列表结束位置，再 -持续反序列化 Chunk Metadata，直到恰好到达该位置。提前结束、跨越边界或留下尾随字节均为 -格式错误。 +`chunk_metadata_list_size` 保存列表的序列化字节数，不是 Chunk 数量。将它加到列表起点 +即可得到结束位置，然后逐条读取 Chunk Metadata，直到恰好到达该位置。 -## Metadata Index +
+ +## 元数据索引 ```text MetadataIndexNode := @@ -163,8 +169,8 @@ MeasurementIndexEntry := int64 offset ``` -每个 Entry 的 `offset` 是对应范围的起点。终点是下一个 Entry 的 `offset`；最后一个 -Entry 的终点是节点的 `end_offset`。 +每个 Entry 描述半开字节区间 `[offset, next_offset)`。`next_offset` 取下一个 Entry 的 +偏移量；最后一个 Entry 则取节点的 `end_offset`。结束偏移处的字节属于下一个区间。 ```text entry[0].offset <= entry[1].offset <= ... <= end_offset @@ -179,7 +185,10 @@ Entry 的 key 是其覆盖范围内的第一个 key。设备索引按 `DeviceID` 区间从该 Entry 的 `offset` 开始，在下一 Entry 的 offset 处结束；最后一个 Entry 则在节点 `end_offset` 处结束。内部节点区间包含下一级索引节点，叶子区间包含上表中对应的数据对象。 -沿索引继续查找前，读取器需要验证节点： +如果没有 Entry 的键小于或等于 `K`，该节点中就不存在目标键。测量索引的一个叶子区间 +可以包含多条 Timeseries Metadata，需要在该区间中继续查找完整的测量名。 + +沿索引继续读取前，检查： - `child_count` 与节点内完整 Entry 数量一致。 - Entry key 按该节点的比较规则有序。 @@ -190,7 +199,9 @@ Entry 的 key 是其覆盖范围内的第一个 key。设备索引按 `DeviceID` first-key 表示允许实现在节点内部对 Entry 使用二分查找，但文件格式不规定具体内存搜索 算法。 -## File Metadata +
+ +## 文件元数据 `TsFileMetadata` 位于文件尾部长度字段之前： @@ -215,9 +226,14 @@ NamedTableSchema := Property := VarString key - VarString value + VarBinary value ``` +属性键是 UTF-8 字符串，值是可空的原始字节数组，不要求为 UTF-8。例如，值字节 +`ff fe 00 80` 序列化为 `08 ff fe 00 80`，其中 `08` 是 `VarInt(4)`。null 值为 +`01`（`VarInt(-1)`），空值为 `00`。未知属性值必须逐字节保留，仅在具体属性约定要求时 +才按文本解码。 + `meta_offset` 指向数据区末尾的 `0x02` separator。`TsFileMetadata` 前没有 marker， 其起始位置由文件尾保存的 4 字节长度确定。 @@ -235,7 +251,9 @@ file_metadata_end = F - 6 - 4 # 半开区间结束位置 读取器必须恰好消费 `L` 字节。该检查可以在接受尾部魔数之前发现 Schema 不匹配或变长字段 损坏。 -## Table Schema +
+ +## 表 Schema ```text TableSchema := @@ -270,6 +288,8 @@ SchemaProperty := | 2 | `ATTRIBUTE` | | 3 | `TIME` | -列顺序具有语义，因为表记录和对齐值列按位置引用 Schema。`TAG` 列参与设备身份，`FIELD` -列保存测量值，`ATTRIBUTE` 列描述实体，`TIME` 标识时间列。列类别不能替代 `data_type`； -解释一个列时必须同时使用这两个字段。 +Schema 保留列顺序，其中 TAG 列的顺序决定 DeviceID 片段的顺序。FIELD Chunk 通过 +`measurement_id` 标识测量列，其物理排列顺序不必与 Schema 中 FIELD 列的顺序相同。 + +`TAG` 列标识设备，`FIELD` 列保存测量值，`ATTRIBUTE` 列描述实体，`TIME` 标识时间列。 +列类别说明这一列的用途，`data_type` 说明它保存哪种值，两者共同决定列的含义。 diff --git a/src/zh/UserGuide/latest/FileFormat/Nulls.md b/src/zh/UserGuide/latest/FileFormat/Nulls.md index 833114aa5b..2ccdfaf767 100644 --- a/src/zh/UserGuide/latest/FileFormat/Nulls.md +++ b/src/zh/UserGuide/latest/FileFormat/Nulls.md @@ -57,12 +57,13 @@ position_count = 对应 Time Page 的 decoded_timestamp_count 如果整个 Value Page 都是 null，Page 只写一个值为 0 的 `uncompressed_size`： ```text -EmptyPage := UVarInt(0) +EmptyAlignedValuePage := UVarInt(0) ``` 空 Page 不保存 `compressed_size`、Statistics、位图或值流。其位置数量由对应的 Time Page 确定。 -缺少 Value Chunk 与空 Value Page 的含义不同。缺少 Chunk 表示该 Chunk Group 中没有 -该测点的物理列；空 Page 表示该值列存在，并维持 Page 之间的位置对应关系，但在这个 -区间的所有位置均为 null。 +空 Value Page 会保留它在 Page 序列中的位置，使后续 Value Page 仍能对应到正确的 +Time Page。缺少 Value Chunk 则表示这个对齐 Chunk 集中没有该测量的物理列。还有一种 +情况是 `data_size = 0` 的 Value Chunk：它有 Header，但不包含任何 Page。遍历对齐 +Chunk 集时，需要区分这三种情况。 diff --git a/src/zh/UserGuide/latest/FileFormat/README.md b/src/zh/UserGuide/latest/FileFormat/README.md index 2b09d9a253..e32f6f0499 100644 --- a/src/zh/UserGuide/latest/FileFormat/README.md +++ b/src/zh/UserGuide/latest/FileFormat/README.md @@ -21,120 +21,121 @@ # 文件格式 -本文描述 TsFile 第 4 版的语言无关二进制文件格式。文件版本字节为 `0x04`。 +TsFile 按列保存时序数据。写入时，数据按设备分组依次追加，索引则放在文件尾部。 +读取时通常反过来：先从文件尾找到索引，定位目标时间序列，再跳转到保存数据的 Chunk。 - +本文介绍版本字节为 `0x04` 的 TsFile v4 布局，以 2026-10-26 的 Java 参考实现为依据， +说明各结构的存储方式及相互关系。编码章节列出了编码编号，并展开 `PLAIN` 的字节布局； +其他值编码算法的完整位布局不在本文范围内。 ## 范围与术语 -本规范定义独立实现之间交换完整 TsFile v4 所需的二进制字节。文件包含四个组织层级： +文件中的数据分为三个层级： -- **Page** 是可以独立压缩和解码的最小数据单元。 -- **Chunk** 包含一个物理列的多个 Page，并确定这些 Page 的数据类型、编码和压缩方式。 -- **Chunk Group** 将一组连续 Chunk 关联到同一个设备标识符。 -- **文件**由顺序写入的数据区、索引以及文件级元数据组成。 +- **Page** 保存一批编码后的值，是独立压缩的单位。 +- **Chunk** 保存一个物理列的 Page，头部记录数据类型、编码和压缩方式。对齐的 + Value Chunk 的 `data_size` 可以为 `0`，此时 Header 仍存在，只是不含 Page。 +- **Chunk Group** 保存同一设备的一批 Chunk。随着数据分批刷出，一个设备可以在文件中 + 出现多次，对应多个 Chunk Group。 -“对齐”表示多个测量共享一个物理时间列；“非对齐”表示每个测量携带自己的时间流。 -对齐方式改变 Page 载荷的物理布局，但不改变时间戳和值本身的语义。 +**非对齐数据**的每个测量列各自保存时间戳和值。**对齐数据**把共享的时间戳放在 +Time Chunk 中，各测量列分别放在 Value Chunk 中，再用位图记录哪些位置有值。 +两种布局的细节见[数据页](DataPages/README.md)。 -除非明确使用半开区间，本规范中的字节区间均包含首尾。绝对偏移从文件第一个字节开始 -计算。除非另有说明，序列化长度不包含保存该长度的字段自身。 + ## 物理区域 -设文件长度为 `F`，文件尾四字节记录的 `TsFileMetadata` 长度为 `L`， -`meta_offset` 为 `TsFileMetadata` 内保存的值： +文件以六字节魔数 `TsFile` 和一字节版本号开头。Page AEAD 文件随后还有一个可选的 +文件加密头。数据区由唯一的 `0x02` 分隔符结束，之后是索引和文件元数据。 +最后十个字节保存元数据长度和尾部魔数。 + +下表使用以下记号： + +- `F`：文件总长度。 +- `L`：`TsFileMetadata` 的序列化长度，由文件尾的四字节长度字段保存。 +- `E`：可选文件加密头的总序列化长度；不存在时为 `0`。 +- `data_start = 7 + E`：数据区的起点。 +- `meta_offset`：分隔符的偏移量，保存在 `TsFileMetadata` 中。 | 区域 | 包含首尾的字节区间 | 定位方式 | | --- | --- | --- | | 头部魔数 | `0 … 5` | 固定位置 | | 版本 | `6` | 固定位置 | -| 数据区 | `7 … meta_offset - 1` | 在分隔符前结束 | +| 可选文件加密头 | `7 … 6 + E` | 仅 Page AEAD 存在，marker 为 `0x06` | +| 数据区 | `data_start … meta_offset - 1` | 在分隔符前结束 | | 分隔符 | `meta_offset` | 值为 `0x02`，由 `TsFileMetadata` 引用 | | 索引元数据 | `meta_offset + 1 … F - 11 - L` | 由元数据偏移划分 | | `TsFileMetadata` | `F - 10 - L … F - 11` | 根据文件尾长度反向定位 | | 元数据长度 | `F - 10 … F - 7` | 相对文件尾的固定位置 | | 尾部魔数 | `F - 6 … F - 1` | 相对文件尾的固定位置 | -索引元数据区可以按实现选择的物理顺序保存 Timeseries Metadata 和 Metadata Index Node。 -它们之间的关系由绝对偏移确定，而不是由物理相邻关系确定。 +偏移量从文件的第一个字节开始计算。上表的区间包含首尾；其他位置使用的 `[start, end)` +表示左闭右开区间。除非另有说明，长度不包含长度字段自身。`E = 0` 时，文件加密头对应 +的区间不存在。 + +索引元数据区中的 Timeseries Metadata 和索引节点可以交错存放，读取时应跟随偏移量， +不依赖它们的物理排列顺序。`TsFileMetadata` 自身没有标记，需要通过文件尾保存的长度 +反向定位。 ## 写入模型 -上例中，数据按设备分成多个 Chunk Group。一个 Chunk 保存一个测量列，每个 Chunk -包含一个或多个 Page。非对齐 Chunk 在同一个 Page 中保存时间流和值流；对齐数据由 -一个 Time Chunk 和若干 Value Chunk 共享时间列。 +写入器先追加各个 Chunk Group，结束文件时再写索引和 Footer。Page 大小、刷出阈值和 +索引扇出会影响结构边界，但这些边界都能从头部和偏移量中读出。时间单位和非对齐时间 +编码仍需读写双方约定，详见[配置](Configurations.md)。 -文件元数据位于数据之后，写入器可以先顺序写入所有 Chunk Group，再写入索引和 -Footer。读取器通常先读取文件尾，从 `TsFile Metadata size` 定位 -`TsFileMetadata`，再通过其中的索引查找所需 Chunk。 +头部和尾部魔数使用相同的字节： -写入器可以选择 Page 大小、Chunk Group 刷新阈值、元数据索引扇出、编码和压缩方式。 -这些选择会影响性能和物理边界，但读取器只需根据文件中的标记、长度和元数据解析最终 -布局，不需要获得写入器配置。 +```text +54 73 46 69 6c 65 # "TsFile" +``` ## 元数据优先读取算法 -随机访问读取器无需扫描数据区即可打开完整文件： +读取器无需扫描整个数据区，就能定位目标序列： -1. 校验文件尾的六字节魔数。 -2. 在 `F - 10` 读取 `L`，计算 `file_metadata_pos = F - 10 - L`。 -3. 反序列化 `TsFileMetadata`，并验证 `meta_offset` 指向 `0x02`。 -4. 可以先用完整路径查询布隆过滤器。阴性结果可以确定路径不存在；阳性结果仍需查索引。 -5. 依次遍历表/设备索引和测量索引，定位目标 Timeseries Metadata 区间。 -6. 根据时间和值谓词选择 Chunk Metadata 中记录的 Chunk 偏移。 -7. 跳转到 Chunk Header，利用可用的 Page Statistics，仅读取和解码所需 Page。 +1. 校验头部魔数和版本，检查偏移 `7` 处是否有[文件加密头](DataPages/Encryption.md#file-encryption-header)， + 据此确定 Chunk Header 和 Page 载荷的解析方式。 +2. 校验尾部魔数，在 `F - 10` 处读取 `L`，计算 `file_metadata_pos = F - 10 - L`。 +3. 读取 `TsFileMetadata`，检查 `meta_offset` 是否指向分隔符。 +4. 可以先用完整路径查询[布隆过滤器](Bloom-Filter.md)。只有确认字符集、路径构造规则和 + 哈希行为与写入端一致时，才能据此判断路径不存在。 +5. 依次查找表/设备索引、测量索引，定位目标序列的元数据。 +6. 根据序列和 Chunk 的统计信息选择 Chunk，跳转后再用可用的 Page 统计信息缩小范围。 +7. 读取选中 Page 的载荷，按需解密，然后解压和解码。 -因此，读取元数据的开销与选中的设备、测量、Chunk 和 Page 数量相关，而不是与整个文件 -大小线性相关。 +这样可以避免读取无关 Page 的数据。单 Page Chunk 不保存 Page Statistics，筛选时要使用 +Chunk 或序列级的统计信息。 ## 结构不变量 -读取外部提供的长度并进行内存分配之前，合规读取器至少需要验证以下关系： - -- 头部和尾部魔数相同，且版本字节受支持。 -- `0 <= L <= F - 10`，计算得到的 `file_metadata_pos` 位于 `meta_offset` 之后。 -- `meta_offset` 是文件内的绝对偏移，且对应字节为 `0x02`。 -- 每个 Chunk 恰好结束在其 Chunk Header 之后的 `data_size` 字节处。 -- 每个非空 Page 均位于所属 Chunk 内，磁盘载荷恰好包含 `compressed_size` 字节。 -- 同一索引节点内的条目偏移单调不减，且不超过 `end_offset`。 -- 对齐 Value Page 与相同序号的 Time Page 对应，其 `position_count` 等于该 Time Page 的 - 位置数。 +读取时需要检查的主要关系包括： -长度或偏移不满足上述关系时即为格式错误。读取器不得把长度限定的 Page 载荷内部偶然出现 -的标记字节解释为顶层标记。 +- 头尾魔数和版本有效。 +- 可选文件加密头只在偏移 `7` 处出现一次，且长度和版本有效。 +- `0 <= L <= F - 10`，且 `file_metadata_pos` 位于分隔符之后。 +- 一个 Chunk 的全部 Page 恰好占据完整 Chunk Header 之后的 `data_size` 字节。 +- 每个非空 Page 的载荷恰好占据 `compressed_size` 字节，且没有越过 Chunk 边界。 +- 同一索引节点内的偏移量单调不减，且不超过 `end_offset`。 +- 对齐 Value Page 和相同序号的 Time Page 覆盖同一组位置，且 `position_count` 等于该 + Time Page 的位置数。 -文件中只有一个用于结束数据区的顶层 `0x02` separator。它位于全部元数据之前，并由 -`meta_offset` 指向。`TsFileMetadata` 前没有单独的 separator 或 marker，它的位置由 -文件尾的长度字段确定。 +不满足这些关系即为格式错误。 -设文件总长度为 `F`，`TsFileMetadata` 的序列化长度为 `L`： +通用的长度和偏移量检查见[二进制表示](Binary-Representation.md)，Page AEAD 额外要求的 +认证检查见[加密](DataPages/Encryption.md)。标记字节只在结构边界处有含义，不能把 Page +载荷中碰巧相同的字节当成标记。 -```text -metadata_size_pos = F - 6 - 4 -file_metadata_pos = F - 6 - 4 - L -``` +
-文件头和文件尾的魔数字节均为： +## 阅读顺序与相关章节 -```text -54 73 46 69 6c 65 # "TsFile" -``` +初次阅读可以先看[二进制表示](Binary-Representation.md)、[Chunk 与 Chunk Group](DataPages/Chunks.md) +和[数据页](DataPages/README.md)，再通过[元数据](Metadata.md)了解索引查找过程。 -## Format Details - -- [Binary Representation](Binary-Representation.md) -- [Configurations](Configurations.md) -- [Extensibility](Extensibility.md) -- [Metadata](Metadata.md) -- [Types](Types.md) -- [Bloom Filter](Bloom-Filter.md) -- [Data Pages](DataPages/README.md) - - [Chunks](DataPages/Chunks.md) - - [Compression](DataPages/Compression.md) - - [Encodings](DataPages/Encodings.md) - - [Encryption](DataPages/Encryption.md) - - [Checksumming](DataPages/Checksumming.md) - - [Error Recovery](DataPages/Error-Recovery.md) -- [Nulls](Nulls.md) -- [Format Versions](Format-Versions.md) +- [数据类型](Types.md)与[编码](DataPages/Encodings.md) +- [空值](Nulls.md)与[压缩](DataPages/Compression.md) +- [布隆过滤器](Bloom-Filter.md) +- [加密](DataPages/Encryption.md)与[校验和](DataPages/Checksumming.md) +- [错误恢复](DataPages/Error-Recovery.md) +- [配置](Configurations.md)、[格式版本](Format-Versions.md)与[扩展机制](Extensibility.md) diff --git a/src/zh/UserGuide/latest/FileFormat/Types.md b/src/zh/UserGuide/latest/FileFormat/Types.md index 5797dd605c..97e68997ba 100644 --- a/src/zh/UserGuide/latest/FileFormat/Types.md +++ b/src/zh/UserGuide/latest/FileFormat/Types.md @@ -39,21 +39,22 @@ Chunk Header、Timeseries Metadata 和 Table Schema 使用单字节 `data_type` | 11 | `STRING` | UTF-8 字符串 | | 12 | `OBJECT` | 对象值的字节表示 | -`DATE` 的物理值示例：2026-07-27 写为十进制整数 `20260727`，不是 Unix epoch day。 +例如，日期 2026-07-27 用十进制整数 `20260727` 表示，并非从 Unix 纪元起累计的天数。 -数据类型决定 Statistics 的附加字段和值编码器的输入类型。具体线上值格式见 -[Encodings](DataPages/Encodings.md)。 +数据类型决定 Statistics 的附加字段和值编码器的输入类型。值的字节布局见 +[编码](DataPages/Encodings.md)。 ## 物理类型与逻辑类型 -类型编号决定物理解码器；`TIMESTAMP` 和 `DATE` 还具有额外的逻辑解释： +数据类型说明值的取值范围，编码方式决定这些值如何写成字节。`TIMESTAMP` 和 `DATE` +还为底层整数赋予了具体含义： -- `TIMESTAMP` 与 `INT64` 具有相同的八字节物理值域。时间单位由使用该文件的协议或数据集 - 提供，TsFile v4 不序列化该单位。 +- `TIMESTAMP` 与 `INT64` 具有相同的有符号 64 位取值范围，时间单位由使用该文件的协议 + 或数据集约定，TsFile v4 不保存这个单位。 - `DATE` 与 `INT32` 使用相同的整数编码接口，但有效逻辑值采用十进制 `YYYYMMDD` 形式。 - `TEXT` 是带长度的字节串。`STRING` 使用相同的物理二进制容器，但具有 UTF-8 字符串 语义；`BLOB` 是不解释的二进制数据。 -- `OBJECT` 是不透明的二进制表示，其对象契约必须在文件格式之外约定。 +- `OBJECT` 保存对象的字节表示，如何从这些字节还原对象，由交换文件的应用约定。 `VECTOR` 是对齐数据共享时间列使用的物理标记，不是标量值类型，也不定义独立值流编码。 `UNKNOWN` 是哨兵值，不能作为可解码测量 Page 的有效类型。 @@ -73,9 +74,9 @@ Chunk Header、Timeseries Metadata 和 Table Schema 使用单字节 `data_type` ## 排序与比较 -整数和浮点值按数值顺序生成最小/最大 Statistics。`STRING` 的最小/最大值按序列化字节的 -二进制字典序比较。`TEXT` 只保存 first 和 last，`BLOB` 与 `OBJECT` 不保存类型专有的值 -Statistics。 +整数和浮点值按数值顺序生成最小/最大 Statistics。`STRING` 的最小/最大值按原始值字节的 +字典序比较，不包含长度前缀。`TEXT` 只保存 first 和 last，`BLOB` 与 `OBJECT` 不保存 +类型专有的值统计信息。 类型提升和应用层转换不编码在 TsFile 中。读取器可以对外提供转换，但这不会改变磁盘 `data_type`，也不会改变计算 Statistics 时使用的字节和值域。 From 3c877767b3ae867f34e9c991d4a3a74e69f3cefe Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 17:43:15 +0800 Subject: [PATCH 03/14] Fix file format docs layout and copy --- .../components/TsFileStructureTree.vue | 29 ++++++++++++++++++- src/.vuepress/config.ts | 4 +++ src/.vuepress/styles/index.scss | 23 +++++++++++++++ src/UserGuide/latest/FileFormat/README.md | 3 +- src/zh/UserGuide/latest/FileFormat/README.md | 11 ++++--- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue index b221f0e166..cc2b0b69ea 100644 --- a/src/.vuepress/components/TsFileStructureTree.vue +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -468,8 +468,35 @@ function collapseAll(): void { @container (max-width: 38rem) { .tsfile-structure-toolbar { - align-items: flex-start; + align-items: stretch; flex-direction: column; } + + .tsfile-structure-actions { + order: 1; + width: 100%; + } + + .tsfile-structure-toolbar > p { + order: 2; + flex: 0 0 auto; + } +} + +@container (min-width: 46rem) { + .tsfile-structure-body { + display: grid; + grid-template-columns: minmax(0, 20rem) minmax(0, 1fr); + align-items: start; + } + + .tsfile-detail-panel { + position: sticky; + top: 5rem; + max-height: calc(100vh - 6rem); + overflow: auto; + border-top: 0; + border-left: 1px solid var(--vp-c-border); + } } diff --git a/src/.vuepress/config.ts b/src/.vuepress/config.ts index 0e926ee4dd..e65564fa2d 100644 --- a/src/.vuepress/config.ts +++ b/src/.vuepress/config.ts @@ -38,6 +38,10 @@ export default defineUserConfig({ }, }, + extendsMarkdown: (md) => { + md.options.breaks = false; + }, + theme, head: [ ['link', { rel: 'icon', href: '/favicon.ico' }], diff --git a/src/.vuepress/styles/index.scss b/src/.vuepress/styles/index.scss index 73df4fb6fd..7acdb0e840 100644 --- a/src/.vuepress/styles/index.scss +++ b/src/.vuepress/styles/index.scss @@ -18,6 +18,29 @@ */ // place your custom styles here +/* Technical documentation: do not insert automatic hyphenation. */ +.theme-hope-content, +.theme-hope-content p, +.theme-hope-content li { + hyphens: manual !important; +} + +/* The right TOC and print control overflow the layout below 1440px. */ +@media (max-width: 1439px) { + .vp-toc-placeholder > aside { + display: none; + } +} + +/* Keep tree controls tappable on narrow screens. */ +@media (max-width: 768px) { + .tsfile-structure-actions button, + .tsfile-structure .tsfile-tree-toggle, + .tsfile-structure .tsfile-tree-label { + min-height: 2.75rem !important; + } +} + #main-description { max-width: 55rem; font-weight: 400; diff --git a/src/UserGuide/latest/FileFormat/README.md b/src/UserGuide/latest/FileFormat/README.md index 373598a84f..70250c4f8b 100644 --- a/src/UserGuide/latest/FileFormat/README.md +++ b/src/UserGuide/latest/FileFormat/README.md @@ -36,8 +36,7 @@ Data has three levels of organization within a file: - A **Page** holds a batch of encoded values. Each Page is compressed independently. - A **Chunk** holds the Pages of one physical column. Its header identifies the data type, - encoding, and compression method. An aligned Value Chunk can have `data_size = 0`; in that - case the Header exists but contains no Page. + encoding, and compression method. - A **Chunk Group** holds a batch of Chunks for one device. The same device can appear in several Chunk Groups as the writer flushes more data. diff --git a/src/zh/UserGuide/latest/FileFormat/README.md b/src/zh/UserGuide/latest/FileFormat/README.md index e32f6f0499..e8f23a4523 100644 --- a/src/zh/UserGuide/latest/FileFormat/README.md +++ b/src/zh/UserGuide/latest/FileFormat/README.md @@ -32,10 +32,9 @@ TsFile 按列保存时序数据。写入时，数据按设备分组依次追加 文件中的数据分为三个层级： -- **Page** 保存一批编码后的值，是独立压缩的单位。 -- **Chunk** 保存一个物理列的 Page，头部记录数据类型、编码和压缩方式。对齐的 - Value Chunk 的 `data_size` 可以为 `0`，此时 Header 仍存在，只是不含 Page。 -- **Chunk Group** 保存同一设备的一批 Chunk。随着数据分批刷出，一个设备可以在文件中 +- **Page** 保存一组编码后的值，是独立压缩的单位。 +- **Chunk** 保存某个物理列的所有 Page，头部记录数据类型、编码和压缩方式。 +- **Chunk Group** 保存同一设备的一组 Chunk。随着数据分批刷出，一个设备可以在文件中 出现多次，对应多个 Chunk Group。 **非对齐数据**的每个测量列各自保存时间戳和值。**对齐数据**把共享的时间戳放在 @@ -70,7 +69,7 @@ Time Chunk 中，各测量列分别放在 Value Chunk 中，再用位图记录 | 元数据长度 | `F - 10 … F - 7` | 相对文件尾的固定位置 | | 尾部魔数 | `F - 6 … F - 1` | 相对文件尾的固定位置 | -偏移量从文件的第一个字节开始计算。上表的区间包含首尾；其他位置使用的 `[start, end)` +偏移量从文件的第一个字节开始计算。上表的区间包含首尾；本文其余位置使用的 `[start, end)` 表示左闭右开区间。除非另有说明，长度不包含长度字段自身。`E = 0` 时，文件加密头对应 的区间不存在。 @@ -80,7 +79,7 @@ Time Chunk 中，各测量列分别放在 Value Chunk 中，再用位图记录 ## 写入模型 -写入器先追加各个 Chunk Group，结束文件时再写索引和 Footer。Page 大小、刷出阈值和 +写入器先追加各个 Chunk Group，结束文件时再写索引和文件尾。Page 大小、刷出阈值和 索引扇出会影响结构边界，但这些边界都能从头部和偏移量中读出。时间单位和非对齐时间 编码仍需读写双方约定，详见[配置](Configurations.md)。 From ae1b6d13728bd504ce5d7f61b853de5b14ee46a9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 17:52:41 +0800 Subject: [PATCH 04/14] Preserve Chinese spacing when markdown breaks are disabled --- src/.vuepress/config.ts | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/.vuepress/config.ts b/src/.vuepress/config.ts index e65564fa2d..6435525878 100644 --- a/src/.vuepress/config.ts +++ b/src/.vuepress/config.ts @@ -40,6 +40,39 @@ export default defineUserConfig({ extendsMarkdown: (md) => { md.options.breaks = false; + + // With breaks disabled, hard-wrapped source lines become softbreaks. + // Remove softbreaks between CJK characters and punctuation while keeping + // spaces between Chinese text and Latin words or inline code. + md.core.ruler.push("collapse-cjk-spaces", (state) => { + const isCjk = (char) => + /[\u3000-\u303f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef]/.test( + char, + ); + state.tokens.forEach((token) => { + if (token.type !== "inline" || !token.children) { + return; + } + const children = token.children; + for (let index = 0; index < children.length; index += 1) { + if (children[index].type !== "softbreak") { + continue; + } + const previous = children[index - 1]; + const next = children[index + 1]; + const before = + previous && previous.type === "text" + ? previous.content.slice(-1) + : ""; + const after = + next && next.type === "text" ? next.content.slice(0, 1) : ""; + if (before && after && isCjk(before) && isCjk(after)) { + children.splice(index, 1); + index -= 1; + } + } + }); + }); }, theme, From b3f472cac25c5a4ce74cb8c80b9de557b360c565 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 18:10:52 +0800 Subject: [PATCH 05/14] Remove date and implementation-baseline wording --- .../components/TsFileStructureTree.vue | 76 +++++++++++-------- src/.vuepress/theme.ts | 3 +- .../latest/FileFormat/Bloom-Filter.md | 10 +-- .../latest/FileFormat/Configurations.md | 5 +- .../FileFormat/DataPages/Compression.md | 3 +- .../latest/FileFormat/DataPages/Encryption.md | 2 +- src/UserGuide/latest/FileFormat/README.md | 7 +- .../latest/FileFormat/Bloom-Filter.md | 9 +-- .../latest/FileFormat/Configurations.md | 4 +- .../FileFormat/DataPages/Compression.md | 2 +- .../latest/FileFormat/DataPages/Encryption.md | 2 +- src/zh/UserGuide/latest/FileFormat/README.md | 6 +- 12 files changed, 68 insertions(+), 61 deletions(-) diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue index cc2b0b69ea..c7a3ac9e00 100644 --- a/src/.vuepress/components/TsFileStructureTree.vue +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -161,36 +161,38 @@ function collapseAll(): void { @@ -367,6 +369,12 @@ function collapseAll(): void { } .tsfile-detail-panel { + padding: 0; + border-top: 1px solid var(--vp-c-border); + background: color-mix(in srgb, var(--tree-accent) 3%, var(--vp-c-bg-elv)); +} + +.tsfile-detail-inner { display: grid; grid-template-columns: repeat( auto-fit, @@ -374,8 +382,6 @@ function collapseAll(): void { ); gap: 1rem 1.5rem; padding: 1rem 1.1rem; - border-top: 1px solid var(--vp-c-border); - background: color-mix(in srgb, var(--tree-accent) 3%, var(--vp-c-bg-elv)); } .tsfile-detail-eyebrow { @@ -403,7 +409,7 @@ function collapseAll(): void { .tsfile-detail-panel dl { display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); + grid-template-columns: minmax(0, 1fr); gap: 0.55rem; margin: 0 0 0.7rem; } @@ -491,12 +497,16 @@ function collapseAll(): void { } .tsfile-detail-panel { + align-self: stretch; + border-top: 0; + border-left: 1px solid var(--vp-c-border); + } + + .tsfile-detail-inner { position: sticky; top: 5rem; max-height: calc(100vh - 6rem); overflow: auto; - border-top: 0; - border-left: 1px solid var(--vp-c-border); } } diff --git a/src/.vuepress/theme.ts b/src/.vuepress/theme.ts index bc4819c7fd..1511ab3a99 100644 --- a/src/.vuepress/theme.ts +++ b/src/.vuepress/theme.ts @@ -34,8 +34,9 @@ export default hopeTheme({ breadcrumb: false, darkmode: 'toggle', contributors: false, + pageInfo: false, - lastUpdated: true, + lastUpdated: false, navbarLayout: { start: ['Brand'], diff --git a/src/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/UserGuide/latest/FileFormat/Bloom-Filter.md index c773a95b90..a4fad8f62c 100644 --- a/src/UserGuide/latest/FileFormat/Bloom-Filter.md +++ b/src/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -50,14 +50,14 @@ For a writer using UTF-8, the filter input is the byte representation of the com UTF8(join(DeviceID segments, ".") + "." + measurement_id) ``` -A null device segment is represented by the ASCII text `null` in the path. The current Java -implementation calls `String.getBytes()` with the platform default charset; that charset is not -stored in the file. The examples below use UTF-8. A reader must match the writer's charset and -hashing behavior; if either is unknown, bypass the Bloom filter and use the metadata index. +A null device segment is represented by the ASCII text `null` in the path. The charset used to +build the path bytes is not stored in the file. The examples below use UTF-8. A reader must match +the writer's charset and hashing behavior; if either is unknown, bypass the Bloom filter and use +the metadata index. ## Hashing -TsFile uses the Java `Murmur128Hash` variant of MurmurHash3 x64 128-bit. Its complete 16-byte +TsFile uses the `Murmur128Hash` variant of MurmurHash3 x64 128-bit. Its complete 16-byte blocks use the standard little-endian unsigned-byte assembly and mixing. Its incomplete final block differs from standard MurmurHash3: **each tail byte is sign-extended from signed 8-bit to 64-bit before shifting**, and the shifted terms are XORed, not ORed. diff --git a/src/UserGuide/latest/FileFormat/Configurations.md b/src/UserGuide/latest/FileFormat/Configurations.md index 7d7d46397f..535b176a70 100644 --- a/src/UserGuide/latest/FileFormat/Configurations.md +++ b/src/UserGuide/latest/FileFormat/Configurations.md @@ -69,7 +69,7 @@ Page size. A writer may limit the number of logical positions in one Page independently of the Page size target. This prevents a Page containing many compact values from growing to an excessive number -of points. The reference writer configuration uses `10,000` points. +of points. A common writer setting is `10,000` points. For aligned series, the Time Page and its corresponding Value Pages must cover the same logical position range. A writer therefore applies a common boundary to the aligned columns. @@ -81,8 +81,7 @@ aligned Value Page, `position_count`. Index fanout is the maximum number of children in an internal metadata index node. A larger fanout produces a shallower tree with larger nodes; a smaller fanout produces a deeper -tree with smaller nodes. The value should be at least `2`; the reference writer configuration uses -`256`. +tree with smaller nodes. The value should be at least `2`; a common value is `256`. The configured limit is not serialized. Each metadata index node stores its node type, children, and end offset, so a reader traverses the resulting tree without knowing the writer setting. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/UserGuide/latest/FileFormat/DataPages/Compression.md index e9d0afe511..83332e14ee 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Compression.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -42,8 +42,7 @@ Header's `uncompressed_size` records the uncompressed payload length. `compresse the stored body length after compression and any encryption, including Page AEAD framing and tags. The numeric values are stable wire identifiers rather than the ordinal positions of a compressor -library. Values `3` through `6` are legacy identifiers from earlier TsFile versions. The reference -Java implementation does not define or implement them and rejects these bytes while parsing a +library. Values `3` through `6` are legacy identifiers from earlier TsFile versions. The Java implementation does not define or implement them and rejects these bytes while parsing a Chunk Header. The C++ implementation retains them as `LZO`, `SDT`, `PAA`, and `PLA`; only `LZO` has a codec (enabled at build time), while `SDT`, `PAA`, and `PLA` remain reserved without a codec. A reader that does not provide the codec cannot decode the affected Chunk. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md index 76f73b432c..3d0c13879a 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Encryption.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -70,7 +70,7 @@ the nullable `VarBinary` used for footer properties. The **provider** identifies the encryption implementation or service. Its **profile** names a fixed byte-format contract: Page framing, nonce and tag representation, size mapping, and Chunk proof size. A provider can offer several profiles. The file stores these public identifiers and -the wrapped data key, rather than a Java implementation class or plaintext key. The reader needs +the wrapped data key, rather than an implementation class name or the plaintext key. The reader needs the matching provider/profile and access to the external key material. Header version `0x01` is the supported layout. Unknown versions, truncated fields, trailing bytes, diff --git a/src/UserGuide/latest/FileFormat/README.md b/src/UserGuide/latest/FileFormat/README.md index 70250c4f8b..b32c2bedd3 100644 --- a/src/UserGuide/latest/FileFormat/README.md +++ b/src/UserGuide/latest/FileFormat/README.md @@ -25,10 +25,9 @@ A TsFile stores time series data in columns. Writers append data in device group the indexes at the end of the file. Readers usually work in the opposite direction: they open the footer, look up a series, and seek to the Chunks that contain its values. -This guide describes the v4 file layout, identified by version byte `0x04`, using the Java -reference implementation as of 2026-10-26. It covers the file structures and how they fit together. -The encoding chapter gives codec identifiers and the `PLAIN` layout; the bit layouts of the other -value codecs are outside its scope. +This guide describes the v4 file layout, identified by version byte `0x04`. It covers the file +structures and how they fit together. The encoding chapter gives codec identifiers and the +`PLAIN` layout; the bit layouts of the other value codecs are outside its scope. ## Scope and Terminology diff --git a/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md index 4511ad13c6..317988df43 100644 --- a/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md +++ b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -52,16 +52,15 @@ BloomFilter := UTF8(join(DeviceID segments, ".") + "." + measurement_id) ``` -null 设备片段在路径文本中表示为 ASCII `null`。当前 Java 实现调用无参数的 -`String.getBytes()`，使用平台默认字符集，且文件不保存该字符集。下列示例使用 UTF-8。 -读取器必须匹配写入端的字符集和哈希行为；无法确定时应绕过布隆过滤器，直接查询元数据 -索引。 +null 设备片段在路径文本中表示为 ASCII `null`。构建路径字节时使用的字符集不保存在 +文件中。下列示例使用 UTF-8。读取器必须匹配写入端的字符集和哈希行为；无法确定时应 +绕过布隆过滤器，直接查询元数据索引。
## 哈希计算 -TsFile 使用 Java `Murmur128Hash` 所实现的 MurmurHash3 x64 128 位变体。完整的 16 字节块 +TsFile 使用 `Murmur128Hash` 所实现的 MurmurHash3 x64 128 位变体。完整的 16 字节块 使用标准的小端无符号字节组装与混合步骤；不足 16 字节的尾块与标准算法不同：**每个尾部 字节先从有符号 8 位符号扩展到 64 位，再移位；各项使用 XOR 而非 OR 合并**。 diff --git a/src/zh/UserGuide/latest/FileFormat/Configurations.md b/src/zh/UserGuide/latest/FileFormat/Configurations.md index 61ee37109e..ec91fef329 100644 --- a/src/zh/UserGuide/latest/FileFormat/Configurations.md +++ b/src/zh/UserGuide/latest/FileFormat/Configurations.md @@ -71,7 +71,7 @@ Data Page 大小是单个序列写入器在内存中累计数据量的目标值 ## 每个 Page 的最大点数 除 Page 大小目标外，写入器还可以限制一个 Page 中的逻辑位置数，以防大量紧凑值让 -单个 Page 包含过多数据点。参考写入配置使用 `10,000` 点。 +单个 Page 包含过多数据点。常见取值为 `10,000` 点。 对于对齐序列，Time Page 及其对应的各个 Value Page 必须覆盖相同的逻辑位置范围， 因此写入器会为这些对齐列使用共同的 Page 边界。 @@ -85,7 +85,7 @@ Data Page 大小是单个序列写入器在内存中累计数据量的目标值 索引扇出就是一个内部索引节点最多能包含多少个子节点。较大的扇出形成 层数更少但节点更大的树；较小的扇出形成层数更多但节点更小的树。该值应至少为 `2`， -参考写入配置使用 `256`。 +常见取值为 `256`。 配置的上限不会写入文件。每个元数据索引节点都会保存节点类型、子节点和结束偏移， 因此读取器无需知道写入配置即可遍历最终生成的树。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md index 046978db0f..14e94bc900 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -41,7 +41,7 @@ 磁盘长度还包含密文帧和认证标签。 这些数值是稳定的文件格式编号，不是某个压缩库枚举中的位置。编号 `3` 到 `6` 是早期 -TsFile 版本留下的历史标识符。参考 Java 实现不定义也不实现它们，解析 Chunk Header 时 +TsFile 版本留下的历史标识符。Java 实现不定义也不实现它们，解析 Chunk Header 时 会直接拒绝这些字节。C++ 实现保留 `LZO`、`SDT`、`PAA`、`PLA` 四个标识；其中只有 `LZO` 有编解码器（编译期可选启用），`SDT`、`PAA`、`PLA` 仍是无编解码器的保留值。 读取器未提供该编解码器时，不能解码相应 Chunk。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md index dda8b442bc..aed9bdcb9a 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -69,7 +69,7 @@ FileEncryptionHeader := **Provider** 标识加密实现或服务，**profile** 则为一套固定的字节格式命名，其中包括 Page 帧、nonce 和认证标签的表示、长度映射以及 Chunk 证明长度。一个 provider 可以 -提供多个 profile。文件保存这些公开标识和封装密钥，不依赖 Java 实现类名，也不保存 +提供多个 profile。文件保存这些公开标识和封装密钥，不依赖实现类名，也不保存 AEAD 的明文密钥。读取端需要相应的 provider/profile，并能取得外部密钥材料。 当前支持的头版本为 `0x01`。未知版本、字段截断、尾随字节、重复的加密头，或加密头 diff --git a/src/zh/UserGuide/latest/FileFormat/README.md b/src/zh/UserGuide/latest/FileFormat/README.md index e8f23a4523..16e5256df6 100644 --- a/src/zh/UserGuide/latest/FileFormat/README.md +++ b/src/zh/UserGuide/latest/FileFormat/README.md @@ -24,9 +24,9 @@ TsFile 按列保存时序数据。写入时，数据按设备分组依次追加，索引则放在文件尾部。 读取时通常反过来：先从文件尾找到索引，定位目标时间序列，再跳转到保存数据的 Chunk。 -本文介绍版本字节为 `0x04` 的 TsFile v4 布局，以 2026-10-26 的 Java 参考实现为依据， -说明各结构的存储方式及相互关系。编码章节列出了编码编号，并展开 `PLAIN` 的字节布局； -其他值编码算法的完整位布局不在本文范围内。 +本文介绍版本字节为 `0x04` 的 TsFile v4 布局，说明各结构的存储方式及相互关系。 +编码章节列出了编码编号，并展开 `PLAIN` 的字节布局；其他值编码算法的完整位布局不在 +本文范围内。 ## 范围与术语 From f758c8645e12fb6ad0aba2b0b35fbf36203f6635 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 18:23:28 +0800 Subject: [PATCH 06/14] Keep File Format sidebar groups expanded on index pages --- src/.vuepress/sidebar/v2.x/en.ts | 4 ++++ src/.vuepress/sidebar/v2.x/zh.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/.vuepress/sidebar/v2.x/en.ts b/src/.vuepress/sidebar/v2.x/en.ts index 2514bb7122..336e624dad 100644 --- a/src/.vuepress/sidebar/v2.x/en.ts +++ b/src/.vuepress/sidebar/v2.x/en.ts @@ -36,6 +36,8 @@ export const enSidebar = { { text: 'File Format', collapsible: true, + expanded: true, + prefix: '/UserGuide/latest/FileFormat/', link: '/UserGuide/latest/FileFormat/', children: [ { text: 'Binary Representation', link: '/UserGuide/latest/FileFormat/Binary-Representation' }, @@ -47,6 +49,8 @@ export const enSidebar = { { text: 'Data Pages', collapsible: true, + expanded: true, + prefix: '/UserGuide/latest/FileFormat/DataPages/', link: '/UserGuide/latest/FileFormat/DataPages/', children: [ { text: 'Chunks', link: '/UserGuide/latest/FileFormat/DataPages/Chunks' }, diff --git a/src/.vuepress/sidebar/v2.x/zh.ts b/src/.vuepress/sidebar/v2.x/zh.ts index a0fe9fe165..1afb96b56e 100644 --- a/src/.vuepress/sidebar/v2.x/zh.ts +++ b/src/.vuepress/sidebar/v2.x/zh.ts @@ -36,6 +36,8 @@ export const zhSidebar = { { text: '文件格式', collapsible: true, + expanded: true, + prefix: '/zh/UserGuide/latest/FileFormat/', link: '/zh/UserGuide/latest/FileFormat/', children: [ { text: '二进制表示', link: '/zh/UserGuide/latest/FileFormat/Binary-Representation' }, @@ -47,6 +49,8 @@ export const zhSidebar = { { text: '数据页', collapsible: true, + expanded: true, + prefix: '/zh/UserGuide/latest/FileFormat/DataPages/', link: '/zh/UserGuide/latest/FileFormat/DataPages/', children: [ { text: '数据块', link: '/zh/UserGuide/latest/FileFormat/DataPages/Chunks' }, From 08d298c114ffa2368466645e180988dd96d35f5e Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 19:37:17 +0800 Subject: [PATCH 07/14] Render physical layout as a vertical flow --- .../components/TsFileStructureTree.vue | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue index c7a3ac9e00..9ae39a6a47 100644 --- a/src/.vuepress/components/TsFileStructureTree.vue +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -102,6 +102,27 @@ const selectedNode = computed(() => { const selectedText = computed(() => selectedNode.value.text[props.locale]); +function splitLayout(layout: string | undefined): string[] { + if (!layout) return []; + const parts: string[] = []; + let depth = 0; + let current = ''; + for (const char of layout) { + if (char === '(' || char === '[' || char === '{') depth += 1; + else if (char === ')' || char === ']' || char === '}') depth = Math.max(0, depth - 1); + if (char === '|' && depth === 0) { + if (current.trim()) parts.push(current.trim()); + current = ''; + } else { + current += char; + } + } + if (current.trim()) parts.push(current.trim()); + return parts; +} + +const selectedLayoutParts = computed(() => splitLayout(selectedText.value.layout)); + const detailsLink = computed(() => { if (!selectedNode.value.link) return undefined; const prefix = props.locale === 'zh' ? '/zh/UserGuide/latest/FileFormat/' : '/UserGuide/latest/FileFormat/'; @@ -191,7 +212,9 @@ function collapseAll(): void {
{{ ui.layout }} - {{ selectedText.layout }} +
+ {{ part }} +
@@ -450,6 +473,7 @@ function collapseAll(): void { } .tsfile-detail-layout code { + display: block; padding: 0; color: var(--vp-c-text); font-size: 0.86rem; @@ -458,6 +482,14 @@ function collapseAll(): void { background: transparent; } +.tsfile-layout-flow { + display: flex; + flex-direction: column; + gap: 0.3rem; + padding-left: 0.7rem; + border-left: 1px solid var(--vp-c-border); +} + .tsfile-detail-note { padding-left: 0.65rem; border-left: 2px solid var(--tree-accent); From fd373314827c489759bd846147cbb3e52e201879 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 19:43:11 +0800 Subject: [PATCH 08/14] Clarify where Chunk statistics are stored --- src/UserGuide/latest/FileFormat/DataPages/Chunks.md | 5 +++++ src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md index 568f1155b3..9b1b8796b1 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -127,6 +127,11 @@ suffix; the Chunk marker remains the same. The ordinal, proof format, and valida described in [AEAD Chunk Header](Encryption.md#aead-chunk-header). Like the rest of the Header, the suffix is excluded from `data_size`. +The Chunk Header contains no Statistics field. Page Statistics are stored in each non-empty Page +Header when a Chunk has multiple Pages; Chunk-level Statistics are stored with the Chunk Metadata +in the footer. A single-Page Chunk omits Page Statistics and relies on the Chunk or series +Statistics. See [Data Pages](README.md) and [Statistics](../Metadata.md#statistics). + `data_size` is the total size of all Page Headers and stored Page data in the Chunk. It does not include the Chunk Header. A reader consumes exactly `data_size` bytes after the Chunk Header. diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md index 472b00f8bc..e2bedf01a9 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -130,6 +130,11 @@ ChunkHeader := 本身不变。序号、认证证明及校验规则见[AEAD Chunk 头部](Encryption.md#aead-chunk-header)。 这些后缀字节和其他 Header 字段一样，不计入 `data_size`。 +Chunk Header 不包含 Statistics 字段。Chunk 包含多个 Page 时，每个非空 Page Header 保存 +Page Statistics；Chunk 级 Statistics 保存在文件尾的 Chunk Metadata 中。单 Page Chunk +不保存 Page Statistics，改用 Chunk 或序列级 Statistics。详见[数据页](README.md)和 +[统计信息](../Metadata.md#statistics)。 + `data_size` 是 Chunk 中全部 Page Header 和已保存 Page 数据的总长度，不包括 Chunk Header。读取器从 Chunk Header 结束处读取恰好 `data_size` 字节。 From 196c8fca3a3e311271d5fb7db0f7f5b0e860ef91 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 26 Oct 2026 19:52:42 +0800 Subject: [PATCH 09/14] Clarify Statistics adjacency to the Chunk Header --- src/UserGuide/latest/FileFormat/DataPages/Chunks.md | 10 ++++++---- src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md index 9b1b8796b1..d990af54eb 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -127,10 +127,12 @@ suffix; the Chunk marker remains the same. The ordinal, proof format, and valida described in [AEAD Chunk Header](Encryption.md#aead-chunk-header). Like the rest of the Header, the suffix is excluded from `data_size`. -The Chunk Header contains no Statistics field. Page Statistics are stored in each non-empty Page -Header when a Chunk has multiple Pages; Chunk-level Statistics are stored with the Chunk Metadata -in the footer. A single-Page Chunk omits Page Statistics and relies on the Chunk or series -Statistics. See [Data Pages](README.md) and [Statistics](../Metadata.md#statistics). +The Chunk Header contains no Statistics field. The first Page Header immediately follows the +Chunk Header. When that Page is non-empty and the Chunk has multiple Pages, its Statistics follows +the two length fields, so those Statistics bytes are physically adjacent to the Chunk Header but +belong to the Page Header. Chunk-level Statistics are stored with the Chunk Metadata in the footer; +a single-Page Chunk omits Page Statistics and relies on the Chunk or series Statistics. See +[Data Pages](README.md) and [Statistics](../Metadata.md#statistics). `data_size` is the total size of all Page Headers and stored Page data in the Chunk. It does not include the Chunk Header. A reader consumes exactly `data_size` bytes after the Chunk Header. diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md index e2bedf01a9..7e14005524 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -130,10 +130,11 @@ ChunkHeader := 本身不变。序号、认证证明及校验规则见[AEAD Chunk 头部](Encryption.md#aead-chunk-header)。 这些后缀字节和其他 Header 字段一样，不计入 `data_size`。 -Chunk Header 不包含 Statistics 字段。Chunk 包含多个 Page 时，每个非空 Page Header 保存 -Page Statistics；Chunk 级 Statistics 保存在文件尾的 Chunk Metadata 中。单 Page Chunk -不保存 Page Statistics，改用 Chunk 或序列级 Statistics。详见[数据页](README.md)和 -[统计信息](../Metadata.md#statistics)。 +Chunk Header 不包含 Statistics 字段。第一个 Page Header 紧跟在 Chunk Header 之后； +当该 Page 非空且 Chunk 包含多个 Page 时，其 Statistics 位于两个长度字段之后，因此这些 +Statistics 字节在物理上紧邻 Chunk Header，但属于 Page Header。Chunk 级 Statistics 保存在 +文件尾的 Chunk Metadata 中；单 Page Chunk 不保存 Page Statistics，改用 Chunk 或序列级 +Statistics。详见[数据页](README.md)和[统计信息](../Metadata.md#statistics)。 `data_size` 是 Chunk 中全部 Page Header 和已保存 Page 数据的总长度，不包括 Chunk Header。读取器从 Chunk Header 结束处读取恰好 `data_size` 字节。 From 12dc55bcba4d5bc2fe03d7b225766f9c8bc8656a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 28 Oct 2026 11:13:21 +0800 Subject: [PATCH 10/14] Expand and clarify file format documentation --- .../components/TsFileStructureTree.vue | 167 ++++++- src/.vuepress/components/tsFileStructure.ts | 445 ++++++++++++++++-- .../FileFormat/Binary-Representation.md | 6 +- .../latest/FileFormat/DataPages/Chunks.md | 7 +- .../FileFormat/Binary-Representation.md | 6 +- .../latest/FileFormat/DataPages/Chunks.md | 10 +- .../UserGuide/latest/FileFormat/Metadata.md | 2 +- src/zh/UserGuide/latest/FileFormat/README.md | 6 +- 8 files changed, 599 insertions(+), 50 deletions(-) diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue index 9ae39a6a47..5f8c9e2896 100644 --- a/src/.vuepress/components/TsFileStructureTree.vue +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -48,6 +48,7 @@ const messages = { offset: 'Offset', size: 'Size', layout: 'Physical layout', + exactSyntax: 'Exact syntax', details: 'Open format details', }, zh: { @@ -60,6 +61,7 @@ const messages = { offset: '偏移', size: '长度', layout: '物理布局', + exactSyntax: '精确语法', details: '查看格式详情', }, } as const; @@ -102,15 +104,23 @@ const selectedNode = computed(() => { const selectedText = computed(() => selectedNode.value.text[props.locale]); -function splitLayout(layout: string | undefined): string[] { - if (!layout) return []; +type LayoutRule = 'optional' | 'repeated' | 'one-or-more' | 'one-of'; + +interface LayoutRow { + text: string; + rule?: LayoutRule; + note?: string; + children?: LayoutRow[]; +} + +function splitTopLevel(value: string, separator: string): string[] { const parts: string[] = []; let depth = 0; let current = ''; - for (const char of layout) { + for (const char of value) { if (char === '(' || char === '[' || char === '{') depth += 1; else if (char === ')' || char === ']' || char === '}') depth = Math.max(0, depth - 1); - if (char === '|' && depth === 0) { + if (char === separator && depth === 0) { if (current.trim()) parts.push(current.trim()); current = ''; } else { @@ -121,7 +131,57 @@ function splitLayout(layout: string | undefined): string[] { return parts; } -const selectedLayoutParts = computed(() => splitLayout(selectedText.value.layout)); +function parseLayoutPart(part: string): LayoutRow { + let text = part.trim(); + let rule: LayoutRule | undefined; + + const groupMatch = text.match(/^\((.*)\)([*+?])$/); + if (groupMatch) { + const inner = groupMatch[1].trim(); + rule = + groupMatch[2] === '*' ? 'repeated' : groupMatch[2] === '+' ? 'one-or-more' : 'optional'; + const alternatives = splitTopLevel(inner, '|'); + if (alternatives.length > 1) { + const children = alternatives.map((item) => ({ text: item.trim() })); + const joiner = props.locale === 'zh' ? ' 或 ' : ' or '; + return { + text: children.map((child) => child.text).join(joiner), + rule, + children, + }; + } + return { text: inner, rule }; + } + + const suffixMatch = text.match(/^(.*?)([*+?])$/); + if (suffixMatch) { + text = suffixMatch[1].trim(); + rule = + suffixMatch[2] === '*' ? 'repeated' : suffixMatch[2] === '+' ? 'one-or-more' : 'optional'; + } + + return { text, rule }; +} + +function parseLayout(layout: string | undefined, notes: string[] | undefined): LayoutRow[] { + if (!layout) return []; + return splitTopLevel(layout, '|').map((part, index) => { + const row = parseLayoutPart(part); + if (notes?.[index]) row.note = notes[index]; + return row; + }); +} + +const selectedLayoutRows = computed( + () => selectedText.value.layoutRows ?? parseLayout(selectedText.value.layout, selectedText.value.layoutNotes), +); + +const ruleLabels = computed>(() => ({ + optional: props.locale === 'zh' ? '可选' : 'optional', + repeated: '0..N', + 'one-or-more': '1..N', + 'one-of': props.locale === 'zh' ? '二选一' : 'one of', +})); const detailsLink = computed(() => { if (!selectedNode.value.link) return undefined; @@ -213,8 +273,39 @@ function collapseAll(): void {
{{ ui.layout }}
- {{ part }} +
+
+ {{ row.text }} + + {{ ruleLabels[row.rule] }} + +
+ {{ row.note }} +
+
+
+ {{ child.text }} + + {{ ruleLabels[child.rule] }} + +
+ {{ child.note }} +
+
+
+
+ {{ ui.exactSyntax }} + {{ selectedText.layout }} +
@@ -485,11 +576,73 @@ function collapseAll(): void { .tsfile-layout-flow { display: flex; flex-direction: column; - gap: 0.3rem; + gap: 0.55rem; padding-left: 0.7rem; border-left: 1px solid var(--vp-c-border); } +.tsfile-layout-row { + display: flex; + flex-direction: column; + gap: 0.15rem; +} + +.tsfile-layout-head { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 0.45rem; +} + +.tsfile-layout-rule { + padding: 0.05rem 0.4rem; + border-radius: 999px; + color: var(--tree-accent); + font-size: 0.68rem; + font-weight: 700; + background: var(--tree-accent-bg); +} + +.tsfile-layout-note { + color: var(--vp-c-text-mute); + font-size: 0.78rem; + line-height: 1.45; +} + +.tsfile-layout-children { + display: flex; + flex-direction: column; + gap: 0.3rem; + margin: 0.1rem 0 0 0.7rem; + padding-left: 0.55rem; + border-left: 1px dashed var(--vp-c-border); +} + +.tsfile-layout-child { + display: flex; + flex-direction: column; + gap: 0.1rem; +} + +.tsfile-layout-grammar { + margin-top: 0.15rem; + color: var(--vp-c-text-mute); + font-size: 0.78rem; +} + +.tsfile-layout-grammar summary { + cursor: pointer; +} + +.tsfile-layout-grammar code { + padding: 0.45rem 0.55rem; + margin-top: 0.35rem; + border: 1px solid var(--vp-c-border); + border-radius: 5px; + white-space: pre-wrap; + background: var(--vp-c-bg); +} + .tsfile-detail-note { padding-left: 0.65rem; border-left: 2px solid var(--tree-accent); diff --git a/src/.vuepress/components/tsFileStructure.ts b/src/.vuepress/components/tsFileStructure.ts index 94f6ee4f2f..28c62357d1 100644 --- a/src/.vuepress/components/tsFileStructure.ts +++ b/src/.vuepress/components/tsFileStructure.ts @@ -26,12 +26,27 @@ export type TsFileStructureKind = | 'field' | 'choice'; +export type TsFileStructureLayoutRule = + | 'optional' + | 'repeated' + | 'one-or-more' + | 'one-of'; + +export interface TsFileStructureLayoutRow { + text: string; + rule?: TsFileStructureLayoutRule; + note?: string; + children?: TsFileStructureLayoutRow[]; +} + export interface TsFileStructureText { title: string; description: string; offset?: string; size?: string; layout?: string; + layoutNotes?: string[]; + layoutRows?: TsFileStructureLayoutRow[]; note?: string; } @@ -43,6 +58,75 @@ export interface TsFileStructureNode { children?: TsFileStructureNode[]; } +type PageDataText = Pick< + TsFileStructureText, + 'description' | 'layout' | 'layoutNotes' | 'note' +>; + +function pageHeaderNode(id: string): TsFileStructureNode { + return { + id, + kind: 'record', + link: 'DataPages/index.html#page-header', + text: { + en: { + title: 'Page Header', + description: + 'Sizes and optional Statistics for one Page. An empty aligned Value Page has no Header.', + offset: 'start of the Page', + size: 'header bytes', + layout: 'UVarInt uncompressed_size | UVarInt compressed_size | Statistics?', + layoutNotes: [ + 'Page payload length after decryption and decompression', + 'Stored payload length, including compression and encryption overhead', + 'Present for every non-empty Page in a multi-Page Chunk; absent for a single-Page Chunk', + ], + note: 'An empty aligned Value Page stores only UVarInt(0); it has no compressed_size, Statistics, or payload.', + }, + zh: { + title: '页头（Page Header）', + description: '一个 Page 的两个长度字段和可选 Statistics；空的对齐 Value Page 没有 Header。', + offset: 'Page 起始处', + size: 'Header 字节数', + layout: 'UVarInt uncompressed_size | UVarInt compressed_size | Statistics?', + layoutNotes: [ + '解密、解压后的 Page 载荷长度', + '磁盘载荷长度，包含压缩和加密开销', + '多 Page Chunk 的每个非空 Page 都有；单 Page Chunk 没有', + ], + note: '空的对齐 Value Page 只保存 UVarInt(0)，没有 compressed_size、Statistics 或载荷。', + }, + }, + }; +} + +function pageDataNode( + id: string, + link: string, + en: PageDataText, + zh: PageDataText, +): TsFileStructureNode { + return { + id, + kind: 'record', + link, + text: { + en: { + title: 'Page Data', + offset: 'after the Page Header', + size: 'compressed_size on disk / uncompressed_size decoded', + ...en, + }, + zh: { + title: '页数据（Page Data）', + offset: 'Page Header 之后', + size: '磁盘 compressed_size / 解码后 uncompressed_size', + ...zh, + }, + }, + }; +} + export const tsFileStructure: TsFileStructureNode = { id: 'tsfile', kind: 'file', @@ -69,7 +153,7 @@ export const tsFileStructure: TsFileStructureNode = { { id: 'file-header', kind: 'region', - link: 'Binary-Representation.html', + link: 'index.html#physical-regions', text: { en: { title: 'File Header', @@ -138,17 +222,43 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'File Encryption Header (optional)', description: 'An optional header at offset 7 selects Page AEAD. It identifies the provider, profile, file crypto ID, and wrapped data key.', - offset: '7 … 6 + E; absent when E = 0', + offset: '7 (when present)', size: 'E = 1 + UVarInt-size(payload_size) + payload_size', layout: '06 | payload_size | 01 | provider_id | profile_id | key_id | key_version | file_crypto_id_size | file_crypto_id | wrapped_data_key_size | wrapped_data_key', + layoutNotes: [ + 'Encryption header marker (ENCRYPTION_HEADER)', + 'UVarInt length of the following payload', + 'Header version (0x01)', + 'Encryption provider identifier', + 'Encryption profile identifier', + 'Key identifier', + 'Key version', + 'UVarInt file crypto ID length (must be 16)', + '16-byte file crypto ID', + 'UVarInt wrapped data key length', + 'Wrapped data key bytes', + ], note: 'The header selects the AEAD layout even though legacy footer encryptLevel is 0. E is its total serialized size; E = 0 when absent.', }, zh: { title: '文件加密头（可选）', description: '偏移 7 处的可选加密头用于选择 Page AEAD，记录 provider、profile、文件加密 ID 和封装数据密钥。', - offset: '7 … 6 + E；E = 0 时不存在', + offset: '7（存在时）', size: 'E = 1 + UVarInt-size(payload_size) + payload_size', layout: '06 | payload_size | 01 | provider_id | profile_id | key_id | key_version | file_crypto_id_size | file_crypto_id | wrapped_data_key_size | wrapped_data_key', + layoutNotes: [ + '加密头标记（ENCRYPTION_HEADER）', + '后续载荷的 UVarInt 长度', + '头部版本（0x01）', + '加密 provider 标识', + '加密 profile 标识', + '密钥标识', + '密钥版本', + '文件加密 ID 的 UVarInt 长度（必须为 16）', + '16 字节文件加密 ID', + '封装数据密钥的 UVarInt 长度', + '封装数据密钥字节', + ], note: '即使旧版 Footer 的 encryptLevel 为 0，也以该头选择 AEAD 布局。E 是该头的总序列化长度，不存在时为 0。', }, }, @@ -161,17 +271,55 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Data Section', description: - 'Chunk Groups and operation-index records appear here in write order. The section begins after any file encryption header and ends before the separator.', + 'Chunk Groups and operation-index records appear here in write order, grouped by flush batch. The section begins after any file encryption header and ends before the separator.', offset: '7 + E … meta_offset - 1', size: 'variable', - layout: '(ChunkGroup | OperationIndexRange)*', + layout: '(ChunkGroup+ OperationIndexRange?)*', + layoutRows: [ + { + text: 'Batch', + rule: 'repeated', + note: 'Zero or more flush batches.', + children: [ + { + text: 'ChunkGroup', + rule: 'one-or-more', + note: 'Each batch starts with one or more ChunkGroups.', + }, + { + text: 'OperationIndexRange', + rule: 'optional', + note: 'At most one record after each batch.', + }, + ], + }, + ], }, zh: { title: '数据区', - description: '这里按写入顺序保存 Chunk Group 和操作索引记录，从可选文件加密头之后开始，到分隔符之前结束。', + description: '这里按写入顺序保存 Chunk Group 和操作索引记录，并按刷出批次分组；从可选文件加密头之后开始，到分隔符之前结束。', offset: '7 + E … meta_offset - 1', size: '可变', - layout: '(ChunkGroup | OperationIndexRange)*', + layout: '(ChunkGroup+ OperationIndexRange?)*', + layoutRows: [ + { + text: '批次', + rule: 'repeated', + note: '零个或多个刷出批次。', + children: [ + { + text: 'ChunkGroup', + rule: 'one-or-more', + note: '每批以至少一个 ChunkGroup 开始。', + }, + { + text: 'OperationIndexRange', + rule: 'optional', + note: '每批之后最多一条。', + }, + ], + }, + ], }, }, children: [ @@ -210,7 +358,7 @@ export const tsFileStructure: TsFileStructureNode = { layout: 'marker 0x00 | DeviceID', }, zh: { - title: 'Chunk Group 头部', + title: '设备分组头部（Chunk Group Header）', description: '开始一个设备分组，并保存 DeviceID：先记录片段数量，再依次记录各个带长度的字符串片段。', offset: 'chunk_group_offset', size: '可变', @@ -232,7 +380,7 @@ export const tsFileStructure: TsFileStructureNode = { layout: 'Non-aligned Chunk | Aligned Chunk Set', }, zh: { - title: 'Chunk 布局', + title: '数据块布局（Chunk Layout）', description: '非对齐 Chunk 把时间和值放在一起；对齐 Chunk 集用一个 Time Chunk 保存共享时间戳，再用各个 Value Chunk 保存测量值。', offset: 'Chunk Group Header 之后', size: '可变 · 重复', @@ -250,15 +398,17 @@ export const tsFileStructure: TsFileStructureNode = { description: 'Stores one measurement with its own timestamps. Each Page contains a time stream and a value stream with matching point counts.', offset: 'chunk_offset', - size: 'header + data_size', + size: 'Chunk Header length + data_size', layout: 'Chunk Header | Non-aligned Page*', + note: 'data_size counts Page Headers and stored Page data; it excludes the Chunk Header. Add the full Header length (including an AEAD suffix) for the total span.', }, zh: { - title: '非对齐 Chunk', + title: '非对齐数据块（Non-aligned Chunk）', description: '保存一个测量及其独立的时间戳。每个 Page 都有时间流和值流，两者解码后的数据点数量相同。', offset: 'chunk_offset', - size: 'header + data_size', + size: 'Chunk Header 长度 + data_size', layout: 'Chunk Header | 非对齐 Page*', + note: 'data_size 统计 Page Header 和已保存 Page 数据的长度，不包含 Chunk Header；总长度需加上完整 Header 长度（含 AEAD 后缀）。', }, }, children: [ @@ -275,15 +425,33 @@ export const tsFileStructure: TsFileStructureNode = { size: 'variable', layout: 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type | AEAD suffix?', + layoutNotes: [ + 'Chunk role and Page-count marker', + 'Measurement identifier', + 'Byte length of Page Headers and stored Page data', + 'Value type', + 'Compression method', + 'Value encoding (time encoding for a Time Chunk)', + 'AEAD suffix: int64 ordinal, int32 proof length, and proof bytes', + ], note: 'encoding_type describes values; time encoding is external configuration (default TS_2DIFF). The AEAD suffix is int64 ordinal | int32 proof_size | proof[proof_size], outside data_size.', }, zh: { - title: 'Chunk 头部', + title: '数据块头部（Chunk Header）', description: '记录测量名及编解码方式。data_size 统计头部之后全部 Page Header 和载荷的长度，读取器无需解码值就能定位下一个 Chunk。', offset: 'chunk_offset', size: '可变', layout: 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type | AEAD suffix?', + layoutNotes: [ + 'Chunk 角色与 Page 数量标记', + '测量标识', + 'Page Header 与已保存 Page 数据的字节长度', + '值类型', + '压缩方式', + '值编码（Time Chunk 为时间编码）', + 'AEAD 后缀：int64 序号、int32 证明长度、证明字节', + ], note: 'encoding_type 描述值流；时间编码来自外部配置，默认 TS_2DIFF。AEAD 后缀为 int64 序号 | int32 证明长度 | 证明字节，不计入 data_size。', }, }, @@ -296,23 +464,49 @@ export const tsFileStructure: TsFileStructureNode = { en: { title: 'Non-aligned Page', description: - 'After decryption and decompression, the payload starts with the time-stream length and time bytes. The value stream fills the rest of the payload.', + 'One Page of a non-aligned Chunk: a Header followed by the encoded time and value streams.', offset: 'inside Chunk data_size', size: 'variable · repeated', - layout: - 'Page Header | time_stream_size | encoded_time_stream | encoded_value_stream', + layout: 'Page Header | Page Data', note: 'Time and values pair by position. Their codecs can differ; the time codec comes from shared configuration.', }, zh: { - title: '非对齐 Page', - description: '解密、解压后的载荷先保存时间流长度和时间流字节，剩余部分保存值流。', + title: '非对齐数据页（Non-aligned Page）', + description: '非对齐 Chunk 的一个 Page：Header 之后是编码后的时间流和值流。', offset: 'Chunk data_size 内部', size: '可变 · 重复', - layout: - 'Page Header | time_stream_size | encoded_time_stream | encoded_value_stream', + layout: 'Page Header | Page Data', note: '时间戳和值按位置配对，可以使用不同编码；时间编码由共享配置提供。', }, }, + children: [ + pageHeaderNode('non-aligned-page-header'), + pageDataNode( + 'non-aligned-page-data', + 'DataPages/index.html#non-aligned-page', + { + description: + 'Logical payload after decryption and decompression: the time stream comes first, then the value stream.', + layout: 'time_stream_size | encoded_time_stream | encoded_value_stream', + layoutNotes: [ + 'UVarInt length of the time stream', + 'Encoded time stream', + 'Encoded value stream; fills the remaining bytes', + ], + note: 'On disk the whole payload is stored_data[compressed_size], compressed and possibly encrypted.', + }, + { + description: '解密、解压后的逻辑载荷：先时间流，后值流。', + layout: 'time_stream_size | encoded_time_stream | encoded_value_stream', + layoutNotes: [ + '时间流的 UVarInt 长度', + '编码后的时间流', + '编码后的值流；占据剩余字节', + ], + note: '磁盘上整个载荷是 stored_data[compressed_size]，经过压缩并可能加密。', + }, + ), + ], }, ], }, @@ -330,7 +524,7 @@ export const tsFileStructure: TsFileStructureNode = { layout: 'Time Chunk | Value Chunk*', }, zh: { - title: '对齐 Chunk 集', + title: '对齐数据块集（Aligned Chunk Set）', description: '一个 Time Chunk 后跟若干 Value Chunk。每个 Value Chunk 的第 i 个 Page 对应 Time Chunk 的第 i 个 Page，集合本身没有额外头部。', offset: '第一个对齐 Chunk 的偏移', size: '可变 · 重复', @@ -347,17 +541,68 @@ export const tsFileStructure: TsFileStructureNode = { title: 'Time Chunk', description: 'Stores the timestamps shared by the aligned measurements. Its Header names the time codec, and each Time Page defines positions for the corresponding Value Pages.', offset: 'time_chunk_offset', - size: 'header + data_size', + size: 'Chunk Header length + data_size', layout: 'Time Chunk Header | Time Page*', + note: 'data_size counts Page Headers and stored Page data; it excludes the Chunk Header. Add the full Header length (including an AEAD suffix) for the total span.', }, zh: { title: '时间列（Time Chunk）', description: '保存各对齐测量共享的时间戳。头部说明时间编码，每个 Time Page 确定对应 Value Page 使用的那组位置。', offset: 'time_chunk_offset', - size: 'header + data_size', + size: 'Chunk Header 长度 + data_size', layout: 'Time Chunk Header | Time Page*', + note: 'data_size 统计 Page Header 和已保存 Page 数据的长度，不包含 Chunk Header；总长度需加上完整 Header 长度（含 AEAD 后缀）。', }, }, + children: [ + { + id: 'time-page', + kind: 'repeated', + link: 'DataPages/index.html#aligned-time-page', + text: { + en: { + title: 'Time Page', + description: 'One Page of encoded timestamps shared by the matching Value Pages.', + offset: 'inside Time Chunk data_size', + size: 'variable · repeated', + layout: 'Page Header | Page Data', + note: 'A Time Page has no bitmap; every position has a timestamp.', + }, + zh: { + title: '时间页（Time Page）', + description: '一个编码时间戳 Page，对应 Value Page 使用它的位置序列。', + offset: 'Time Chunk data_size 内部', + size: '可变 · 重复', + layout: 'Page Header | Page Data', + note: 'Time Page 没有位图；每个位置都有时间戳。', + }, + }, + children: [ + pageHeaderNode('time-page-header'), + pageDataNode( + 'time-page-data', + 'DataPages/index.html#aligned-time-page', + { + description: + 'Logical payload after decryption and decompression: the encoded time stream.', + layout: 'encoded_time_stream', + layoutNotes: [ + 'Encoded timestamps for the positions shared by the matching Value Pages', + ], + note: 'On disk this is stored_data[compressed_size].', + }, + { + description: '解密、解压后的逻辑载荷：编码后的时间流。', + layout: 'encoded_time_stream', + layoutNotes: [ + '对应 Value Page 共享位置的编码时间戳', + ], + note: '磁盘上是 stored_data[compressed_size]。', + }, + ), + ], + }, + ], }, { id: 'value-chunk', @@ -369,17 +614,75 @@ export const tsFileStructure: TsFileStructureNode = { description: 'Stores one aligned measurement. Its Page bitmaps place the stored values at the matching timestamps and leave other positions null.', offset: 'value_chunk_offset', - size: 'header + data_size · repeated', + size: 'Chunk Header length + data_size · repeated', layout: 'Value Chunk Header | Value Page*', + note: 'data_size counts Page Headers and stored Page data; it excludes the Chunk Header. Add the full Header length (including an AEAD suffix) for the total span.', }, zh: { - title: '测量列（Value Chunk）', - description: '保存一个对齐测量列，通过各 Page 的位图把值对应到时间戳，其余位置为 null。', + title: '测点列（Value Chunk）', + description: '保存一个对齐测点列，通过各 Page 的位图把值对应到时间戳，其余位置为 null。', offset: 'value_chunk_offset', - size: 'header + data_size · 重复', + size: 'Chunk Header 长度 + data_size · 重复', layout: 'Value Chunk Header | Value Page*', + note: 'data_size 统计 Page Header 和已保存 Page 数据的长度，不包含 Chunk Header；总长度需加上完整 Header 长度（含 AEAD 后缀）。', }, }, + children: [ + { + id: 'value-page', + kind: 'repeated', + link: 'DataPages/index.html#aligned-value-page', + text: { + en: { + title: 'Value Page', + description: 'One Page of values for an aligned measurement; its bitmap maps stored values onto positions from the matching Time Page.', + offset: 'inside Value Chunk data_size', + size: 'variable · repeated', + layout: 'Page Header | Page Data', + note: 'An all-null Value Page is only UVarInt(0); it has no Header or Data.', + }, + zh: { + title: '测点页（Value Page）', + description: '单个对齐测点的一个 Page；位图把值映射到对应 Time Page 的位置。', + offset: 'Value Chunk data_size 内部', + size: '可变 · 重复', + layout: 'Page Header | Page Data', + note: '全 null 的 Value Page 只保存 UVarInt(0)，没有 Header 或 Data。', + }, + }, + children: [ + pageHeaderNode('value-page-header'), + pageDataNode( + 'value-page-data', + 'DataPages/index.html#aligned-value-page', + { + description: + 'Logical payload after decryption and decompression: the position count, present bitmap, and non-null values.', + layout: + 'position_count | present_bitmap | encoded_non_null_values', + layoutNotes: [ + 'int32 position count, equal to the matching Time Page', + 'Bitmap with ceil(position_count / 8) bytes; a set bit means a value is present', + 'Encoded non-null values, in position order', + ], + note: 'On disk this is stored_data[compressed_size]; an all-null Value Page is only UVarInt(0).', + }, + { + description: + '解密、解压后的逻辑载荷：位置数、存在位图和非 null 值。', + layout: + 'position_count | present_bitmap | encoded_non_null_values', + layoutNotes: [ + 'int32 位置数，等于对应 Time Page 的位置数', + '位图长度为 ceil(position_count / 8)；置位表示该位置有值', + '按位置顺序编码的非 null 值', + ], + note: '磁盘上是 stored_data[compressed_size]；全 null 的 Value Page 只保存 UVarInt(0)。', + }, + ), + ], + }, + ], }, ], }, @@ -443,6 +746,9 @@ export const tsFileStructure: TsFileStructureNode = { offset: 'meta_offset + 1 … file_metadata_pos - 1', size: 'variable', layout: '(TimeseriesMetadata | MetadataIndexNode)*', + layoutNotes: [ + 'Zero or more entries; each entry is either a TimeseriesMetadata or a MetadataIndexNode. The two kinds may be interleaved.', + ], }, zh: { title: '索引元数据区', @@ -450,6 +756,9 @@ export const tsFileStructure: TsFileStructureNode = { offset: 'meta_offset + 1 … file_metadata_pos - 1', size: '可变', layout: '(TimeseriesMetadata | MetadataIndexNode)*', + layoutNotes: [ + '零个或多个条目；每条是 TimeseriesMetadata 或 MetadataIndexNode，两者可以交错出现。', + ], }, }, children: [ @@ -465,6 +774,14 @@ export const tsFileStructure: TsFileStructureNode = { size: 'variable · repeated', layout: 'timeseries_type | measurement_id | data_type | chunk_metadata_list_size | Statistics | ChunkMetadata*', + layoutNotes: [ + 'Column role and Chunk Metadata layout flags', + 'Measurement identifier', + 'Physical value type', + 'Byte length of the Chunk Metadata list', + 'Series-level Statistics', + 'Chunk Metadata records; one record per Chunk', + ], }, zh: { title: '时间序列元数据', @@ -473,8 +790,80 @@ export const tsFileStructure: TsFileStructureNode = { size: '可变 · 重复', layout: 'timeseries_type | measurement_id | data_type | chunk_metadata_list_size | Statistics | ChunkMetadata*', + layoutNotes: [ + '列角色与 Chunk Metadata 布局标志', + '测量标识', + '物理值类型', + 'Chunk Metadata 列表的字节长度', + '序列级 Statistics', + 'Chunk Metadata 记录；每个 Chunk 一条', + ], }, }, + children: [ + { + id: 'series-statistics', + kind: 'record', + link: 'Metadata.html#statistics', + text: { + en: { + title: 'Series Statistics', + description: 'Aggregated Statistics for the whole series. The common prefix is followed by fields selected by data_type.', + offset: 'inside Timeseries Metadata', + size: 'variable', + layout: 'UVarInt count | int64 start_time | int64 end_time | type-specific fields', + layoutNotes: [ + 'Number of values represented by this record', + 'Inclusive start time', + 'Inclusive end time', + 'Fields selected by data_type (min, max, first, last, sum, or none)', + ], + }, + zh: { + title: '统计信息（Statistics）', + description: '整个序列的汇总 Statistics；公共前缀之后是按 data_type 选择的类型字段。', + offset: '时间序列元数据内部', + size: '可变', + layout: 'UVarInt count | int64 start_time | int64 end_time | type-specific fields', + layoutNotes: [ + '该记录覆盖的值数量', + '包含首部的开始时间', + '包含尾部的结束时间', + '按 data_type 选择的字段（min / max / first / last / sum 或无）', + ], + }, + }, + }, + { + id: 'chunk-metadata', + kind: 'repeated', + link: 'Metadata.html#chunk-metadata', + text: { + en: { + title: 'Chunk Metadata', + description: 'One record per Chunk. It points to the Chunk Header in the data section; its Statistics are omitted when the series has only one Chunk.', + offset: 'referenced by a Chunk Metadata list entry', + size: 'variable · repeated', + layout: 'int64 offset_of_chunk_header | Statistics?', + layoutNotes: [ + 'Absolute offset of the Chunk marker', + 'Chunk-level Statistics; present only when the series has multiple Chunks', + ], + }, + zh: { + title: '数据块元数据（Chunk Metadata）', + description: '每个 Chunk 一条记录，指向数据区的 Chunk Header；序列只有一个 Chunk 时省略这里的 Statistics。', + offset: '由 Chunk Metadata 列表条目引用', + size: '可变 · 重复', + layout: 'int64 offset_of_chunk_header | Statistics?', + layoutNotes: [ + 'Chunk marker 的绝对偏移', + 'Chunk 级 Statistics；仅当序列包含多个 Chunk 时存在', + ], + }, + }, + }, + ], }, { id: 'metadata-index', @@ -554,7 +943,7 @@ export const tsFileStructure: TsFileStructureNode = { size: 'variable · repeated', }, zh: { - title: '表 Schema', + title: '表结构（Schema）', description: '按声明顺序描述表的列，记录列名、类型和类别；FIELD Chunk 通过测量名与列对应。', offset: 'TsFileMetadata 内部', size: '可变 · 重复', diff --git a/src/UserGuide/latest/FileFormat/Binary-Representation.md b/src/UserGuide/latest/FileFormat/Binary-Representation.md index fc2c57d864..6531d24b0d 100644 --- a/src/UserGuide/latest/FileFormat/Binary-Representation.md +++ b/src/UserGuide/latest/FileFormat/Binary-Representation.md @@ -114,7 +114,9 @@ separator. To find the end of a structure, add its length to its start offset us ## Reading Layout Examples -In layout examples, `?` marks an optional field or structure, `*` allows zero or more repetitions, -and `[n]` means n elements. The surrounding description or a preceding length field determines +In layout examples, `|` separates alternatives, parentheses group a sequence, `?` marks an +optional field or structure, `*` allows zero or more repetitions, `+` requires one or more +repetitions, and `[n]` means n elements. For example, `(A | B)*` means zero or more entries, each +of which is either A or B. The surrounding description or a preceding length field determines which case applies. `remaining bytes` means the unused bytes up to the enclosing payload boundary; for a Page layout, that is the end of its decrypted, decompressed payload. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md index d990af54eb..890ab75646 100644 --- a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -30,13 +30,14 @@ The top-level grammar is: ```text FilePrefix := magic[6] version[1] FileEncryptionHeader? -DataSection := (ChunkGroup | OperationIndexRange)* +DataSection := (ChunkGroup+ OperationIndexRange?)* ChunkGroup := ChunkGroupHeader Chunk* Separator := byte(0x02) ``` -The `0x02` separator follows the data section; it is not part of a Chunk Group. A Chunk Group -has no total-length field. After reading each Chunk, the reader examines the next marker: another +Each batch contains one or more ChunkGroups followed by an optional OperationIndexRange. The +`0x02` separator follows the data section; it is not part of a Chunk Group. A Chunk Group has no +total-length field. After reading each Chunk, the reader examines the next marker: another Chunk continues the group, while a new group, operation-index record, or separator ends it. ```text diff --git a/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md index c3be0038b6..52769eaf3f 100644 --- a/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md +++ b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md @@ -116,6 +116,8 @@ current_offset + N <= enclosing_end ## 如何阅读布局示例 -结构示意中的 `?` 表示字段或结构可选，`*` 表示可以重复零次或多次，`[n]` 表示 n 个 -元素。具体是否出现、重复多少次，由旁边的说明或已读取的长度字段决定。`remaining bytes` +结构示意中的 `|` 表示二选一，括号表示一个分组，`?` 表示字段或结构可选，`*` 表示 +可以重复零次或多次，`+` 表示至少出现一次，`[n]` 表示 n 个元素。例如 `(A | B)*` 表示 +零个或多个条目，每个条目是 A 或 B。具体是否出现、重复多少次，由旁边的说明或已读取的 +长度字段决定。`remaining bytes` 表示当前载荷边界内尚未使用的字节；对于 Page，这个边界指解密、解压后的载荷末尾。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md index 7e14005524..95063f8347 100644 --- a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -30,21 +30,21 @@ TsFile 数据区由 marker 引导的 Chunk Group、Chunk 和 Operation Index Ran ```text FilePrefix := magic[6] version[1] FileEncryptionHeader? -DataSection := (ChunkGroup | OperationIndexRange)* +DataSection := (ChunkGroup+ OperationIndexRange?)* ChunkGroup := ChunkGroupHeader Chunk* Separator := byte(0x02) ``` -数据区之后紧接 `0x02` 分隔符，它不属于任何 Chunk Group。Chunk Group 自身没有总长度 -字段。读取完一个 Chunk 后，要看下一个标记：如果仍是 Chunk，就继续读取；如果是新的 +每个批次包含一个或多个 ChunkGroup，随后可选一条 OperationIndexRange。数据区之后紧接 +`0x02` 分隔符，它不属于任何 Chunk Group。Chunk Group 自身没有总长度字段。读取完一个 Chunk 后，要看下一个标记：如果仍是 Chunk，就继续读取；如果是新的 Chunk Group、操作索引记录或分隔符，当前分组就结束了。 ```text File └── Chunk Group 一个设备的一批数据 - ├── Non-aligned Chunk 自带时间流的一个测量列 + ├── Non-aligned Chunk 自带时间流的一个测点列 │ └── Page(s) - └── Aligned Chunk Set 多个测量列共享时间列 + └── Aligned Chunk Set 多个测点列共享时间列 ├── Time Chunk │ └── Time Page(s) └── Value Chunk(s) diff --git a/src/zh/UserGuide/latest/FileFormat/Metadata.md b/src/zh/UserGuide/latest/FileFormat/Metadata.md index 06740978be..a521965949 100644 --- a/src/zh/UserGuide/latest/FileFormat/Metadata.md +++ b/src/zh/UserGuide/latest/FileFormat/Metadata.md @@ -289,7 +289,7 @@ SchemaProperty := | 3 | `TIME` | Schema 保留列顺序，其中 TAG 列的顺序决定 DeviceID 片段的顺序。FIELD Chunk 通过 -`measurement_id` 标识测量列，其物理排列顺序不必与 Schema 中 FIELD 列的顺序相同。 +`measurement_id` 标识测点列，其物理排列顺序不必与 Schema 中 FIELD 列的顺序相同。 `TAG` 列标识设备，`FIELD` 列保存测量值，`ATTRIBUTE` 列描述实体，`TIME` 标识时间列。 列类别说明这一列的用途，`data_type` 说明它保存哪种值，两者共同决定列的含义。 diff --git a/src/zh/UserGuide/latest/FileFormat/README.md b/src/zh/UserGuide/latest/FileFormat/README.md index 16e5256df6..fc584980bb 100644 --- a/src/zh/UserGuide/latest/FileFormat/README.md +++ b/src/zh/UserGuide/latest/FileFormat/README.md @@ -37,12 +37,14 @@ TsFile 按列保存时序数据。写入时，数据按设备分组依次追加 - **Chunk Group** 保存同一设备的一组 Chunk。随着数据分批刷出，一个设备可以在文件中 出现多次，对应多个 Chunk Group。 -**非对齐数据**的每个测量列各自保存时间戳和值。**对齐数据**把共享的时间戳放在 -Time Chunk 中，各测量列分别放在 Value Chunk 中，再用位图记录哪些位置有值。 +**非对齐数据**的每个测点列各自保存时间戳和值。**对齐数据**把共享的时间戳放在 +Time Chunk 中，各测点列分别放在 Value Chunk 中，再用位图记录哪些位置有值。 两种布局的细节见[数据页](DataPages/README.md)。 +
+ ## 物理区域 文件以六字节魔数 `TsFile` 和一字节版本号开头。Page AEAD 文件随后还有一个可选的 From 52b20ec5a4b845be7c16822e373d1679e996043f Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 28 Oct 2026 11:48:25 +0800 Subject: [PATCH 11/14] Expand table index root in the structure tree Break the metadata index root into its table-name prefix and device index root, and explain the lookup path down to Chunk Metadata. --- src/.vuepress/components/tsFileStructure.ts | 85 ++++++++++++++++++++- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/src/.vuepress/components/tsFileStructure.ts b/src/.vuepress/components/tsFileStructure.ts index 28c62357d1..f7a551e5e7 100644 --- a/src/.vuepress/components/tsFileStructure.ts +++ b/src/.vuepress/components/tsFileStructure.ts @@ -916,20 +916,99 @@ export const tsFileStructure: TsFileStructureNode = { { id: 'table-index-roots', kind: 'repeated', + link: 'Metadata.html#file-metadata', text: { en: { title: 'Table Index Roots', - description: 'Stores each table\'s device-index root. Follow a root to find a device, then its measurements.', + description: 'Entry point for each table\'s device-index tree. Look up the table root, descend to a device, then follow that device\'s measurement index to the Timeseries Metadata.', offset: 'inside TsFileMetadata', size: 'variable · repeated', + layout: 'VarString table_name | MetadataIndexNode device_index_root', + layoutNotes: [ + 'Table name: the actual table name, or the first DeviceID segment for tree-model data', + 'Root node of the table\'s device-level metadata index', + ], + note: 'Lookup path: table root -> device index -> measurement index -> Timeseries Metadata -> Chunk Metadata.', }, zh: { - title: '表索引根', - description: '保存各表的设备索引根。从根节点查找设备，再查找设备下的测量。', + title: '表索引根（Table Index Root）', + description: '每张表的设备索引树入口：先按表名找到根，再向下定位设备，最后通过设备的测量索引找到 Timeseries Metadata。', offset: 'TsFileMetadata 内部', size: '可变 · 重复', + layout: 'VarString table_name | MetadataIndexNode device_index_root', + layoutNotes: [ + '表名：表模型为实际表名；树模型为 DeviceID 的第一段', + '该表设备级元数据索引的根节点', + ], + note: '查找路径：表根 → 设备索引 → 测量索引 → Timeseries Metadata → Chunk Metadata。', }, }, + children: [ + { + id: 'device-index-root', + kind: 'record', + link: 'Metadata.html#metadata-index', + text: { + en: { + title: 'Device Index Root', + description: 'Root of the device-level index tree. Its entries locate a device and, through the entry offset, that device\'s measurement index.', + offset: 'device_index_root', + size: 'variable', + layout: 'child_count | DeviceIndexEntry* | end_offset | node_type', + layoutNotes: [ + 'Number of child entries', + 'Device entries: DeviceID plus an absolute offset', + 'End offset of this node\'s byte range', + 'LEAF_DEVICE points directly to measurement indexes; INTERNAL_DEVICE points to another device-index node', + ], + }, + zh: { + title: '设备索引根（Device Index Root）', + description: '设备级索引树的根；条目定位设备，并通过条目 offset 找到该设备的测量索引。', + offset: 'device_index_root', + size: '可变', + layout: 'child_count | DeviceIndexEntry* | end_offset | node_type', + layoutNotes: [ + '子条目数量', + '设备条目：DeviceID 加绝对偏移', + '该节点字节范围的结束偏移', + 'LEAF_DEVICE 直接指向测量索引；INTERNAL_DEVICE 指向下一级设备索引节点', + ], + }, + }, + children: [ + { + id: 'device-index-entry', + kind: 'repeated', + link: 'Metadata.html#metadata-index', + text: { + en: { + title: 'Device Index Entry', + description: 'One device entry in a device-index node. The entry key is the first DeviceID in its range.', + offset: 'inside a device-index node', + size: 'variable · repeated', + layout: 'DeviceID first_device_id | int64 offset', + layoutNotes: [ + 'First DeviceID covered by this entry', + 'Absolute offset to the measurement index root (LEAF_DEVICE) or to a child device-index node (INTERNAL_DEVICE)', + ], + }, + zh: { + title: '设备索引条目（Device Index Entry）', + description: '设备索引节点中的一个设备条目；条目 key 是它覆盖范围里的第一个 DeviceID。', + offset: '设备索引节点内部', + size: '可变 · 重复', + layout: 'DeviceID first_device_id | int64 offset', + layoutNotes: [ + '该条目覆盖范围里的第一个 DeviceID', + '绝对偏移：LEAF_DEVICE 指向测量索引根，INTERNAL_DEVICE 指向下一级设备索引节点', + ], + }, + }, + }, + ], + }, + ], }, { id: 'table-schemas', From 34144214fb7838b9cd3d9106eedc0de62b407397 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 28 Oct 2026 11:52:45 +0800 Subject: [PATCH 12/14] Drop the exact-syntax disclosure from the structure panel The parsed layout rows already carry the grammar, so the raw notation is redundant. --- .../components/TsFileStructureTree.vue | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue index 5f8c9e2896..7f521d4cb2 100644 --- a/src/.vuepress/components/TsFileStructureTree.vue +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -48,7 +48,6 @@ const messages = { offset: 'Offset', size: 'Size', layout: 'Physical layout', - exactSyntax: 'Exact syntax', details: 'Open format details', }, zh: { @@ -61,7 +60,6 @@ const messages = { offset: '偏移', size: '长度', layout: '物理布局', - exactSyntax: '精确语法', details: '查看格式详情', }, } as const; @@ -302,10 +300,6 @@ function collapseAll(): void { -
- {{ ui.exactSyntax }} - {{ selectedText.layout }} -
@@ -624,25 +618,6 @@ function collapseAll(): void { gap: 0.1rem; } -.tsfile-layout-grammar { - margin-top: 0.15rem; - color: var(--vp-c-text-mute); - font-size: 0.78rem; -} - -.tsfile-layout-grammar summary { - cursor: pointer; -} - -.tsfile-layout-grammar code { - padding: 0.45rem 0.55rem; - margin-top: 0.35rem; - border: 1px solid var(--vp-c-border); - border-radius: 5px; - white-space: pre-wrap; - background: var(--vp-c-bg); -} - .tsfile-detail-note { padding-left: 0.65rem; border-left: 2px solid var(--tree-accent); From 5c83c69df1c5cb5476083fbc55b584d730963dc5 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 28 Oct 2026 11:56:17 +0800 Subject: [PATCH 13/14] Show alternative layouts as an explicit choice Stacked rows read like a sequence. Mark alternative children and separate them with an 'or' divider, and make Chunk Layout spell out its two forms. --- .../components/TsFileStructureTree.vue | 38 +++++++++++++------ src/.vuepress/components/tsFileStructure.ts | 35 +++++++++++++++++ 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue index 7f521d4cb2..b12520f175 100644 --- a/src/.vuepress/components/TsFileStructureTree.vue +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -48,6 +48,7 @@ const messages = { offset: 'Offset', size: 'Size', layout: 'Physical layout', + or: 'or', details: 'Open format details', }, zh: { @@ -60,6 +61,7 @@ const messages = { offset: '偏移', size: '长度', layout: '物理布局', + or: '或', details: '查看格式详情', }, } as const; @@ -108,6 +110,7 @@ interface LayoutRow { text: string; rule?: LayoutRule; note?: string; + alternatives?: boolean; children?: LayoutRow[]; } @@ -145,6 +148,7 @@ function parseLayoutPart(part: string): LayoutRow { return { text: children.map((child) => child.text).join(joiner), rule, + alternatives: true, children, }; } @@ -284,19 +288,20 @@ function collapseAll(): void { {{ row.note }}
-
-
- {{ child.text }} - - {{ ruleLabels[child.rule] }} - +
@@ -618,6 +623,15 @@ function collapseAll(): void { gap: 0.1rem; } +.tsfile-layout-or { + padding-left: 0.1rem; + color: var(--tree-accent); + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; +} + .tsfile-detail-note { padding-left: 0.65rem; border-left: 2px solid var(--tree-accent); diff --git a/src/.vuepress/components/tsFileStructure.ts b/src/.vuepress/components/tsFileStructure.ts index f7a551e5e7..4727f62a8e 100644 --- a/src/.vuepress/components/tsFileStructure.ts +++ b/src/.vuepress/components/tsFileStructure.ts @@ -36,6 +36,7 @@ export interface TsFileStructureLayoutRow { text: string; rule?: TsFileStructureLayoutRule; note?: string; + alternatives?: boolean; children?: TsFileStructureLayoutRow[]; } @@ -378,6 +379,23 @@ export const tsFileStructure: TsFileStructureNode = { offset: 'after Chunk Group Header', size: 'variable · repeated', layout: 'Non-aligned Chunk | Aligned Chunk Set', + layoutRows: [ + { + text: 'Non-aligned Chunk or Aligned Chunk Set', + rule: 'one-of', + alternatives: true, + children: [ + { + text: 'Non-aligned Chunk', + note: 'Timestamps and values live in the same Chunk; every measurement carries its own time stream.', + }, + { + text: 'Aligned Chunk Set', + note: 'One Time Chunk holds the shared timestamps, and each Value Chunk holds one measurement.', + }, + ], + }, + ], }, zh: { title: '数据块布局（Chunk Layout）', @@ -385,6 +403,23 @@ export const tsFileStructure: TsFileStructureNode = { offset: 'Chunk Group Header 之后', size: '可变 · 重复', layout: '非对齐 Chunk | 对齐 Chunk 集', + layoutRows: [ + { + text: '非对齐 Chunk 或 对齐 Chunk 集', + rule: 'one-of', + alternatives: true, + children: [ + { + text: '非对齐 Chunk', + note: '时间和值在同一个 Chunk 内，每个测点各自带时间流。', + }, + { + text: '对齐 Chunk 集', + note: '一个 Time Chunk 保存共享时间戳，每个 Value Chunk 保存一个测点。', + }, + ], + }, + ], }, }, children: [ From 3c6c61a7e6edc1ce00ada52d4c0c4777dd1a0b6a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 28 Oct 2026 12:01:48 +0800 Subject: [PATCH 14/14] Drop the details link on the file-header node No section documents the file header on its own, so the link promised content that does not exist. The children still point at the version page. --- src/.vuepress/components/tsFileStructure.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/.vuepress/components/tsFileStructure.ts b/src/.vuepress/components/tsFileStructure.ts index 4727f62a8e..32063cdbf6 100644 --- a/src/.vuepress/components/tsFileStructure.ts +++ b/src/.vuepress/components/tsFileStructure.ts @@ -154,7 +154,6 @@ export const tsFileStructure: TsFileStructureNode = { { id: 'file-header', kind: 'region', - link: 'index.html#physical-regions', text: { en: { title: 'File Header',