diff --git a/src/.vuepress/client.ts b/src/.vuepress/client.ts index 8f2fd9f87..dabe83d29 100644 --- a/src/.vuepress/client.ts +++ b/src/.vuepress/client.ts @@ -20,11 +20,13 @@ import { defineDocSearchConfig } from '@vuepress/plugin-docsearch/client'; import { computed } from 'vue'; import { defineClientConfig, usePageData, useRouter, withBase } from 'vuepress/client'; import DocSearch from './components/DocSearch.vue'; +import TsFileStructureTree from './components/TsFileStructureTree.vue'; import { getDocVersion } from './utils/index.js'; export default defineClientConfig({ enhance: ({ app }) => { app.component('DocSearch', DocSearch); + app.component('TsFileStructureTree', TsFileStructureTree); }, setup() { const pageData = usePageData(); diff --git a/src/.vuepress/components/TsFileStructureNode.vue b/src/.vuepress/components/TsFileStructureNode.vue new file mode 100644 index 000000000..c9079afdf --- /dev/null +++ b/src/.vuepress/components/TsFileStructureNode.vue @@ -0,0 +1,107 @@ + + + + + diff --git a/src/.vuepress/components/TsFileStructureTree.vue b/src/.vuepress/components/TsFileStructureTree.vue new file mode 100644 index 000000000..b12520f17 --- /dev/null +++ b/src/.vuepress/components/TsFileStructureTree.vue @@ -0,0 +1,686 @@ + + + + + + + diff --git a/src/.vuepress/components/tsFileStructure.ts b/src/.vuepress/components/tsFileStructure.ts new file mode 100644 index 000000000..32063cdbf --- /dev/null +++ b/src/.vuepress/components/tsFileStructure.ts @@ -0,0 +1,1171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type TsFileStructureLocale = 'en' | 'zh'; + +export type TsFileStructureKind = + | 'file' + | 'region' + | 'repeated' + | 'record' + | 'field' + | 'choice'; + +export type TsFileStructureLayoutRule = + | 'optional' + | 'repeated' + | 'one-or-more' + | 'one-of'; + +export interface TsFileStructureLayoutRow { + text: string; + rule?: TsFileStructureLayoutRule; + note?: string; + alternatives?: boolean; + children?: TsFileStructureLayoutRow[]; +} + +export interface TsFileStructureText { + title: string; + description: string; + offset?: string; + size?: string; + layout?: string; + layoutNotes?: string[]; + layoutRows?: TsFileStructureLayoutRow[]; + note?: string; +} + +export interface TsFileStructureNode { + id: string; + kind: TsFileStructureKind; + text: Record; + link?: string; + children?: TsFileStructureNode[]; +} + +type PageDataText = Pick< + TsFileStructureText, + 'description' | 'layout' | 'layoutNotes' | 'note' +>; + +function pageHeaderNode(id: string): TsFileStructureNode { + return { + id, + kind: 'record', + link: 'DataPages/index.html#page-header', + text: { + en: { + title: 'Page Header', + description: + 'Sizes and optional Statistics for one Page. An empty aligned Value Page has no Header.', + offset: 'start of the Page', + size: 'header bytes', + layout: 'UVarInt uncompressed_size | UVarInt compressed_size | Statistics?', + layoutNotes: [ + 'Page payload length after decryption and decompression', + 'Stored payload length, including compression and encryption overhead', + 'Present for every non-empty Page in a multi-Page Chunk; absent for a single-Page Chunk', + ], + note: 'An empty aligned Value Page stores only UVarInt(0); it has no compressed_size, Statistics, or payload.', + }, + zh: { + title: '页头(Page Header)', + description: '一个 Page 的两个长度字段和可选 Statistics;空的对齐 Value Page 没有 Header。', + offset: 'Page 起始处', + size: 'Header 字节数', + layout: 'UVarInt uncompressed_size | UVarInt compressed_size | Statistics?', + layoutNotes: [ + '解密、解压后的 Page 载荷长度', + '磁盘载荷长度,包含压缩和加密开销', + '多 Page Chunk 的每个非空 Page 都有;单 Page Chunk 没有', + ], + note: '空的对齐 Value Page 只保存 UVarInt(0),没有 compressed_size、Statistics 或载荷。', + }, + }, + }; +} + +function pageDataNode( + id: string, + link: string, + en: PageDataText, + zh: PageDataText, +): TsFileStructureNode { + return { + id, + kind: 'record', + link, + text: { + en: { + title: 'Page Data', + offset: 'after the Page Header', + size: 'compressed_size on disk / uncompressed_size decoded', + ...en, + }, + zh: { + title: '页数据(Page Data)', + offset: 'Page Header 之后', + size: '磁盘 compressed_size / 解码后 uncompressed_size', + ...zh, + }, + }, + }; +} + +export const tsFileStructure: TsFileStructureNode = { + id: 'tsfile', + kind: 'file', + text: { + en: { + title: 'TsFile v4', + description: + 'Data is written in device groups, with indexes at the end. A reader usually opens the footer first, then seeks to the Chunks and Pages it needs.', + offset: '0 … F - 1', + size: 'F bytes', + layout: 'File Header | FileEncryptionHeader? | Data | Separator | Indexed Metadata | TsFileMetadata | Length | Tail Magic', + note: 'F is the file length, L is the TsFileMetadata length, and E is the total encryption-header length (0 when absent).', + }, + zh: { + title: 'TsFile v4', + description: '数据按设备分组写入,索引放在文件尾部。读取器通常先查 Footer,再跳转到需要的 Chunk 和 Page。', + offset: '0 … F - 1', + size: 'F 字节', + layout: '文件头 | FileEncryptionHeader? | 数据区 | 分隔符 | 索引元数据 | TsFileMetadata | 长度 | 尾魔数', + note: 'F 为文件长度,L 为 TsFileMetadata 长度,E 为可选加密头的总长度;无加密头时 E = 0。', + }, + }, + children: [ + { + id: 'file-header', + kind: 'region', + text: { + en: { + title: 'File Header', + description: 'The first seven bytes contain the TsFile magic string and format version. Read these before choosing how to parse the rest of the file.', + offset: '0 … 6', + size: '7 bytes', + layout: 'magic[6] | version[1]', + }, + zh: { + title: '文件头', + description: '文件最前面的七个字节保存 TsFile 魔数和格式版本。先读取这部分,再确定后续结构的解析方式。', + offset: '0 … 6', + size: '7 字节', + layout: 'magic[6] | version[1]', + }, + }, + children: [ + { + id: 'head-magic', + kind: 'field', + text: { + en: { + title: 'Head Magic', + description: 'The six bytes spelling "TsFile" identify the file format.', + offset: '0 … 5', + size: '6 bytes', + layout: '54 73 46 69 6c 65', + }, + zh: { + title: '头部魔数', + description: '由字符串“TsFile”对应的六个字节组成,用于识别文件格式。', + offset: '0 … 5', + size: '6 字节', + layout: '54 73 46 69 6c 65', + }, + }, + }, + { + id: 'version', + kind: 'field', + link: 'Format-Versions.html', + text: { + en: { + title: 'Format Version', + description: 'Value 0x04 identifies format v4. This byte is separate from the project release number.', + offset: '6', + size: '1 byte', + layout: '04', + }, + zh: { + title: '格式版本', + description: '数值 0x04 表示 v4 文件格式,与项目的发布版本号分开定义。', + offset: '6', + size: '1 字节', + layout: '04', + }, + }, + }, + ], + }, + { + id: 'file-encryption-header', + kind: 'record', + link: 'DataPages/Encryption.html#file-encryption-header', + text: { + en: { + title: 'File Encryption Header (optional)', + description: 'An optional header at offset 7 selects Page AEAD. It identifies the provider, profile, file crypto ID, and wrapped data key.', + offset: '7 (when present)', + size: 'E = 1 + UVarInt-size(payload_size) + payload_size', + layout: '06 | payload_size | 01 | provider_id | profile_id | key_id | key_version | file_crypto_id_size | file_crypto_id | wrapped_data_key_size | wrapped_data_key', + layoutNotes: [ + 'Encryption header marker (ENCRYPTION_HEADER)', + 'UVarInt length of the following payload', + 'Header version (0x01)', + 'Encryption provider identifier', + 'Encryption profile identifier', + 'Key identifier', + 'Key version', + 'UVarInt file crypto ID length (must be 16)', + '16-byte file crypto ID', + 'UVarInt wrapped data key length', + 'Wrapped data key bytes', + ], + note: 'The header selects the AEAD layout even though legacy footer encryptLevel is 0. E is its total serialized size; E = 0 when absent.', + }, + zh: { + title: '文件加密头(可选)', + description: '偏移 7 处的可选加密头用于选择 Page AEAD,记录 provider、profile、文件加密 ID 和封装数据密钥。', + offset: '7(存在时)', + size: 'E = 1 + UVarInt-size(payload_size) + payload_size', + layout: '06 | payload_size | 01 | provider_id | profile_id | key_id | key_version | file_crypto_id_size | file_crypto_id | wrapped_data_key_size | wrapped_data_key', + layoutNotes: [ + '加密头标记(ENCRYPTION_HEADER)', + '后续载荷的 UVarInt 长度', + '头部版本(0x01)', + '加密 provider 标识', + '加密 profile 标识', + '密钥标识', + '密钥版本', + '文件加密 ID 的 UVarInt 长度(必须为 16)', + '16 字节文件加密 ID', + '封装数据密钥的 UVarInt 长度', + '封装数据密钥字节', + ], + note: '即使旧版 Footer 的 encryptLevel 为 0,也以该头选择 AEAD 布局。E 是该头的总序列化长度,不存在时为 0。', + }, + }, + }, + { + id: 'data-section', + kind: 'region', + link: 'DataPages/Chunks.html', + text: { + en: { + title: 'Data Section', + description: + 'Chunk Groups and operation-index records appear here in write order, grouped by flush batch. The section begins after any file encryption header and ends before the separator.', + offset: '7 + E … meta_offset - 1', + size: 'variable', + layout: '(ChunkGroup+ OperationIndexRange?)*', + layoutRows: [ + { + text: 'Batch', + rule: 'repeated', + note: 'Zero or more flush batches.', + children: [ + { + text: 'ChunkGroup', + rule: 'one-or-more', + note: 'Each batch starts with one or more ChunkGroups.', + }, + { + text: 'OperationIndexRange', + rule: 'optional', + note: 'At most one record after each batch.', + }, + ], + }, + ], + }, + zh: { + title: '数据区', + description: '这里按写入顺序保存 Chunk Group 和操作索引记录,并按刷出批次分组;从可选文件加密头之后开始,到分隔符之前结束。', + offset: '7 + E … meta_offset - 1', + size: '可变', + layout: '(ChunkGroup+ OperationIndexRange?)*', + layoutRows: [ + { + text: '批次', + rule: 'repeated', + note: '零个或多个刷出批次。', + children: [ + { + text: 'ChunkGroup', + rule: 'one-or-more', + note: '每批以至少一个 ChunkGroup 开始。', + }, + { + text: 'OperationIndexRange', + rule: 'optional', + note: '每批之后最多一条。', + }, + ], + }, + ], + }, + }, + children: [ + { + id: 'chunk-group', + kind: 'repeated', + link: 'DataPages/Chunks.html#chunk-group-header', + text: { + en: { + title: 'Chunk Group', + description: + 'A batch of Chunks for one device. Another Chunk continues the group; a new group, operation-index record, or separator ends it.', + offset: 'chunk_group_offset', + size: 'variable · repeated', + layout: 'Chunk Group Header | Chunk*', + }, + zh: { + title: '设备分组(Chunk Group)', + description: '同一设备的一批 Chunk。后续 Chunk 标记表示分组继续,新的分组、操作索引记录或分隔符表示当前分组结束。', + offset: 'chunk_group_offset', + size: '可变 · 重复', + layout: 'Chunk Group Header | Chunk*', + }, + }, + children: [ + { + id: 'chunk-group-header', + kind: 'record', + link: 'DataPages/Chunks.html#chunk-group-header', + text: { + en: { + title: 'Chunk Group Header', + description: 'Begins a device group with its DeviceID. The identifier stores a segment count followed by separately length-prefixed string segments.', + offset: 'chunk_group_offset', + size: 'variable', + layout: 'marker 0x00 | DeviceID', + }, + zh: { + title: '设备分组头部(Chunk Group Header)', + description: '开始一个设备分组,并保存 DeviceID:先记录片段数量,再依次记录各个带长度的字符串片段。', + offset: 'chunk_group_offset', + size: '可变', + layout: 'marker 0x00 | DeviceID', + }, + }, + }, + { + id: 'chunk-layout', + kind: 'choice', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Chunk Layout', + description: + 'Non-aligned Chunks store time and values together. Aligned Chunk sets put shared timestamps in a Time Chunk and measurements in Value Chunks.', + offset: 'after Chunk Group Header', + size: 'variable · repeated', + layout: 'Non-aligned Chunk | Aligned Chunk Set', + layoutRows: [ + { + text: 'Non-aligned Chunk or Aligned Chunk Set', + rule: 'one-of', + alternatives: true, + children: [ + { + text: 'Non-aligned Chunk', + note: 'Timestamps and values live in the same Chunk; every measurement carries its own time stream.', + }, + { + text: 'Aligned Chunk Set', + note: 'One Time Chunk holds the shared timestamps, and each Value Chunk holds one measurement.', + }, + ], + }, + ], + }, + zh: { + title: '数据块布局(Chunk Layout)', + description: '非对齐 Chunk 把时间和值放在一起;对齐 Chunk 集用一个 Time Chunk 保存共享时间戳,再用各个 Value Chunk 保存测量值。', + offset: 'Chunk Group Header 之后', + size: '可变 · 重复', + layout: '非对齐 Chunk | 对齐 Chunk 集', + layoutRows: [ + { + text: '非对齐 Chunk 或 对齐 Chunk 集', + rule: 'one-of', + alternatives: true, + children: [ + { + text: '非对齐 Chunk', + note: '时间和值在同一个 Chunk 内,每个测点各自带时间流。', + }, + { + text: '对齐 Chunk 集', + note: '一个 Time Chunk 保存共享时间戳,每个 Value Chunk 保存一个测点。', + }, + ], + }, + ], + }, + }, + children: [ + { + id: 'non-aligned-chunk', + kind: 'repeated', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Non-aligned Chunk', + description: + 'Stores one measurement with its own timestamps. Each Page contains a time stream and a value stream with matching point counts.', + offset: 'chunk_offset', + size: 'Chunk Header length + data_size', + layout: 'Chunk Header | Non-aligned Page*', + note: 'data_size counts Page Headers and stored Page data; it excludes the Chunk Header. Add the full Header length (including an AEAD suffix) for the total span.', + }, + zh: { + title: '非对齐数据块(Non-aligned Chunk)', + description: '保存一个测量及其独立的时间戳。每个 Page 都有时间流和值流,两者解码后的数据点数量相同。', + offset: 'chunk_offset', + size: 'Chunk Header 长度 + data_size', + layout: 'Chunk Header | 非对齐 Page*', + note: 'data_size 统计 Page Header 和已保存 Page 数据的长度,不包含 Chunk Header;总长度需加上完整 Header 长度(含 AEAD 后缀)。', + }, + }, + children: [ + { + id: 'non-aligned-chunk-header', + kind: 'record', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Chunk Header', + description: + 'Names the measurement and its codecs. data_size counts all Page Headers and stored bodies after this Header, so a reader can locate the next Chunk without decoding values.', + offset: 'chunk_offset', + size: 'variable', + layout: + 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type | AEAD suffix?', + layoutNotes: [ + 'Chunk role and Page-count marker', + 'Measurement identifier', + 'Byte length of Page Headers and stored Page data', + 'Value type', + 'Compression method', + 'Value encoding (time encoding for a Time Chunk)', + 'AEAD suffix: int64 ordinal, int32 proof length, and proof bytes', + ], + note: 'encoding_type describes values; time encoding is external configuration (default TS_2DIFF). The AEAD suffix is int64 ordinal | int32 proof_size | proof[proof_size], outside data_size.', + }, + zh: { + title: '数据块头部(Chunk Header)', + description: '记录测量名及编解码方式。data_size 统计头部之后全部 Page Header 和载荷的长度,读取器无需解码值就能定位下一个 Chunk。', + offset: 'chunk_offset', + size: '可变', + layout: + 'chunk_type | measurement_id | data_size | data_type | compression_type | encoding_type | AEAD suffix?', + layoutNotes: [ + 'Chunk 角色与 Page 数量标记', + '测量标识', + 'Page Header 与已保存 Page 数据的字节长度', + '值类型', + '压缩方式', + '值编码(Time Chunk 为时间编码)', + 'AEAD 后缀:int64 序号、int32 证明长度、证明字节', + ], + note: 'encoding_type 描述值流;时间编码来自外部配置,默认 TS_2DIFF。AEAD 后缀为 int64 序号 | int32 证明长度 | 证明字节,不计入 data_size。', + }, + }, + }, + { + id: 'non-aligned-page', + kind: 'repeated', + link: 'DataPages/index.html#non-aligned-page', + text: { + en: { + title: 'Non-aligned Page', + description: + 'One Page of a non-aligned Chunk: a Header followed by the encoded time and value streams.', + offset: 'inside Chunk data_size', + size: 'variable · repeated', + layout: 'Page Header | Page Data', + note: 'Time and values pair by position. Their codecs can differ; the time codec comes from shared configuration.', + }, + zh: { + title: '非对齐数据页(Non-aligned Page)', + description: '非对齐 Chunk 的一个 Page:Header 之后是编码后的时间流和值流。', + offset: 'Chunk data_size 内部', + size: '可变 · 重复', + layout: 'Page Header | Page Data', + note: '时间戳和值按位置配对,可以使用不同编码;时间编码由共享配置提供。', + }, + }, + children: [ + pageHeaderNode('non-aligned-page-header'), + pageDataNode( + 'non-aligned-page-data', + 'DataPages/index.html#non-aligned-page', + { + description: + 'Logical payload after decryption and decompression: the time stream comes first, then the value stream.', + layout: 'time_stream_size | encoded_time_stream | encoded_value_stream', + layoutNotes: [ + 'UVarInt length of the time stream', + 'Encoded time stream', + 'Encoded value stream; fills the remaining bytes', + ], + note: 'On disk the whole payload is stored_data[compressed_size], compressed and possibly encrypted.', + }, + { + description: '解密、解压后的逻辑载荷:先时间流,后值流。', + layout: 'time_stream_size | encoded_time_stream | encoded_value_stream', + layoutNotes: [ + '时间流的 UVarInt 长度', + '编码后的时间流', + '编码后的值流;占据剩余字节', + ], + note: '磁盘上整个载荷是 stored_data[compressed_size],经过压缩并可能加密。', + }, + ), + ], + }, + ], + }, + { + id: 'aligned-chunk-set', + kind: 'repeated', + link: 'DataPages/Chunks.html#chunk-header', + text: { + en: { + title: 'Aligned Chunk Set', + description: + 'One Time Chunk is followed by its Value Chunks. Page i in each Value Chunk corresponds to Page i in the Time Chunk; the set has no separate header.', + offset: 'first aligned Chunk offset', + size: 'variable · repeated', + layout: 'Time Chunk | Value Chunk*', + }, + zh: { + title: '对齐数据块集(Aligned Chunk Set)', + description: '一个 Time Chunk 后跟若干 Value Chunk。每个 Value Chunk 的第 i 个 Page 对应 Time Chunk 的第 i 个 Page,集合本身没有额外头部。', + offset: '第一个对齐 Chunk 的偏移', + size: '可变 · 重复', + layout: 'Time Chunk | Value Chunk*', + }, + }, + children: [ + { + id: 'time-chunk', + kind: 'record', + link: 'DataPages/index.html#aligned-time-page', + text: { + en: { + title: 'Time Chunk', + description: 'Stores the timestamps shared by the aligned measurements. Its Header names the time codec, and each Time Page defines positions for the corresponding Value Pages.', + offset: 'time_chunk_offset', + size: 'Chunk Header length + data_size', + layout: 'Time Chunk Header | Time Page*', + note: 'data_size counts Page Headers and stored Page data; it excludes the Chunk Header. Add the full Header length (including an AEAD suffix) for the total span.', + }, + zh: { + title: '时间列(Time Chunk)', + description: '保存各对齐测量共享的时间戳。头部说明时间编码,每个 Time Page 确定对应 Value Page 使用的那组位置。', + offset: 'time_chunk_offset', + size: 'Chunk Header 长度 + data_size', + layout: 'Time Chunk Header | Time Page*', + note: 'data_size 统计 Page Header 和已保存 Page 数据的长度,不包含 Chunk Header;总长度需加上完整 Header 长度(含 AEAD 后缀)。', + }, + }, + children: [ + { + id: 'time-page', + kind: 'repeated', + link: 'DataPages/index.html#aligned-time-page', + text: { + en: { + title: 'Time Page', + description: 'One Page of encoded timestamps shared by the matching Value Pages.', + offset: 'inside Time Chunk data_size', + size: 'variable · repeated', + layout: 'Page Header | Page Data', + note: 'A Time Page has no bitmap; every position has a timestamp.', + }, + zh: { + title: '时间页(Time Page)', + description: '一个编码时间戳 Page,对应 Value Page 使用它的位置序列。', + offset: 'Time Chunk data_size 内部', + size: '可变 · 重复', + layout: 'Page Header | Page Data', + note: 'Time Page 没有位图;每个位置都有时间戳。', + }, + }, + children: [ + pageHeaderNode('time-page-header'), + pageDataNode( + 'time-page-data', + 'DataPages/index.html#aligned-time-page', + { + description: + 'Logical payload after decryption and decompression: the encoded time stream.', + layout: 'encoded_time_stream', + layoutNotes: [ + 'Encoded timestamps for the positions shared by the matching Value Pages', + ], + note: 'On disk this is stored_data[compressed_size].', + }, + { + description: '解密、解压后的逻辑载荷:编码后的时间流。', + layout: 'encoded_time_stream', + layoutNotes: [ + '对应 Value Page 共享位置的编码时间戳', + ], + note: '磁盘上是 stored_data[compressed_size]。', + }, + ), + ], + }, + ], + }, + { + id: 'value-chunk', + kind: 'repeated', + link: 'DataPages/index.html#aligned-value-page', + text: { + en: { + title: 'Value Chunk', + description: + 'Stores one aligned measurement. Its Page bitmaps place the stored values at the matching timestamps and leave other positions null.', + offset: 'value_chunk_offset', + size: 'Chunk Header length + data_size · repeated', + layout: 'Value Chunk Header | Value Page*', + note: 'data_size counts Page Headers and stored Page data; it excludes the Chunk Header. Add the full Header length (including an AEAD suffix) for the total span.', + }, + zh: { + title: '测点列(Value Chunk)', + description: '保存一个对齐测点列,通过各 Page 的位图把值对应到时间戳,其余位置为 null。', + offset: 'value_chunk_offset', + size: 'Chunk Header 长度 + data_size · 重复', + layout: 'Value Chunk Header | Value Page*', + note: 'data_size 统计 Page Header 和已保存 Page 数据的长度,不包含 Chunk Header;总长度需加上完整 Header 长度(含 AEAD 后缀)。', + }, + }, + children: [ + { + id: 'value-page', + kind: 'repeated', + link: 'DataPages/index.html#aligned-value-page', + text: { + en: { + title: 'Value Page', + description: 'One Page of values for an aligned measurement; its bitmap maps stored values onto positions from the matching Time Page.', + offset: 'inside Value Chunk data_size', + size: 'variable · repeated', + layout: 'Page Header | Page Data', + note: 'An all-null Value Page is only UVarInt(0); it has no Header or Data.', + }, + zh: { + title: '测点页(Value Page)', + description: '单个对齐测点的一个 Page;位图把值映射到对应 Time Page 的位置。', + offset: 'Value Chunk data_size 内部', + size: '可变 · 重复', + layout: 'Page Header | Page Data', + note: '全 null 的 Value Page 只保存 UVarInt(0),没有 Header 或 Data。', + }, + }, + children: [ + pageHeaderNode('value-page-header'), + pageDataNode( + 'value-page-data', + 'DataPages/index.html#aligned-value-page', + { + description: + 'Logical payload after decryption and decompression: the position count, present bitmap, and non-null values.', + layout: + 'position_count | present_bitmap | encoded_non_null_values', + layoutNotes: [ + 'int32 position count, equal to the matching Time Page', + 'Bitmap with ceil(position_count / 8) bytes; a set bit means a value is present', + 'Encoded non-null values, in position order', + ], + note: 'On disk this is stored_data[compressed_size]; an all-null Value Page is only UVarInt(0).', + }, + { + description: + '解密、解压后的逻辑载荷:位置数、存在位图和非 null 值。', + layout: + 'position_count | present_bitmap | encoded_non_null_values', + layoutNotes: [ + 'int32 位置数,等于对应 Time Page 的位置数', + '位图长度为 ceil(position_count / 8);置位表示该位置有值', + '按位置顺序编码的非 null 值', + ], + note: '磁盘上是 stored_data[compressed_size];全 null 的 Value Page 只保存 UVarInt(0)。', + }, + ), + ], + }, + ], + }, + ], + }, + ], + }, + ], + }, + { + id: 'operation-index-range', + kind: 'record', + link: 'DataPages/Chunks.html#operation-index-range', + text: { + en: { + title: 'Operation Index Range', + description: 'Records the minimum and maximum operation sequence numbers for a flushed batch. A storage engine can use them to relate file data to checkpoints or its operation log.', + offset: 'top-level boundary', + size: '17 bytes', + layout: 'marker 0x04 | min_operation_index[8] | max_operation_index[8]', + }, + zh: { + title: '操作索引范围', + description: '记录已刷出批次的最小和最大操作序号,供存储引擎把文件数据与检查点或操作日志关联。', + offset: '顶层结构边界', + size: '17 字节', + layout: 'marker 0x04 | min_operation_index[8] | max_operation_index[8]', + }, + }, + }, + ], + }, + { + id: 'separator', + kind: 'field', + link: 'DataPages/Chunks.html#markers', + text: { + en: { + title: 'Data/Metadata Separator', + description: 'Byte 0x02 ends the data section. meta_offset points here, and indexed metadata starts at the next byte.', + offset: 'meta_offset', + size: '1 byte', + layout: '02', + }, + zh: { + title: '数据/元数据分隔符', + description: '字节 0x02 结束数据区,meta_offset 指向此处,索引元数据从下一个字节开始。', + offset: 'meta_offset', + size: '1 字节', + layout: '02', + }, + }, + }, + { + id: 'metadata-section', + kind: 'region', + link: 'Metadata.html', + text: { + en: { + title: 'Indexed Metadata Section', + description: + 'Holds series metadata and index nodes. They can be interleaved, so follow their absolute offsets rather than assuming a fixed order.', + offset: 'meta_offset + 1 … file_metadata_pos - 1', + size: 'variable', + layout: '(TimeseriesMetadata | MetadataIndexNode)*', + layoutNotes: [ + 'Zero or more entries; each entry is either a TimeseriesMetadata or a MetadataIndexNode. The two kinds may be interleaved.', + ], + }, + zh: { + title: '索引元数据区', + description: '保存时间序列元数据和索引节点。两者可以交错存放,读取时应跟随绝对偏移量,不依赖固定顺序。', + offset: 'meta_offset + 1 … file_metadata_pos - 1', + size: '可变', + layout: '(TimeseriesMetadata | MetadataIndexNode)*', + layoutNotes: [ + '零个或多个条目;每条是 TimeseriesMetadata 或 MetadataIndexNode,两者可以交错出现。', + ], + }, + }, + children: [ + { + id: 'timeseries-metadata', + kind: 'repeated', + link: 'Metadata.html#timeseries-metadata', + text: { + en: { + title: 'Timeseries Metadata', + description: 'Describes one series, with summary Statistics and a list of its Chunks. Each Chunk entry points to a Header in the data section.', + offset: 'referenced by a measurement index entry', + size: 'variable · repeated', + layout: + 'timeseries_type | measurement_id | data_type | chunk_metadata_list_size | Statistics | ChunkMetadata*', + layoutNotes: [ + 'Column role and Chunk Metadata layout flags', + 'Measurement identifier', + 'Physical value type', + 'Byte length of the Chunk Metadata list', + 'Series-level Statistics', + 'Chunk Metadata records; one record per Chunk', + ], + }, + zh: { + title: '时间序列元数据', + description: '描述一条时间序列,包含汇总统计信息和 Chunk 列表。列表中的每个条目指向数据区的一个 Chunk Header。', + offset: '由测量索引条目引用', + size: '可变 · 重复', + layout: + 'timeseries_type | measurement_id | data_type | chunk_metadata_list_size | Statistics | ChunkMetadata*', + layoutNotes: [ + '列角色与 Chunk Metadata 布局标志', + '测量标识', + '物理值类型', + 'Chunk Metadata 列表的字节长度', + '序列级 Statistics', + 'Chunk Metadata 记录;每个 Chunk 一条', + ], + }, + }, + children: [ + { + id: 'series-statistics', + kind: 'record', + link: 'Metadata.html#statistics', + text: { + en: { + title: 'Series Statistics', + description: 'Aggregated Statistics for the whole series. The common prefix is followed by fields selected by data_type.', + offset: 'inside Timeseries Metadata', + size: 'variable', + layout: 'UVarInt count | int64 start_time | int64 end_time | type-specific fields', + layoutNotes: [ + 'Number of values represented by this record', + 'Inclusive start time', + 'Inclusive end time', + 'Fields selected by data_type (min, max, first, last, sum, or none)', + ], + }, + zh: { + title: '统计信息(Statistics)', + description: '整个序列的汇总 Statistics;公共前缀之后是按 data_type 选择的类型字段。', + offset: '时间序列元数据内部', + size: '可变', + layout: 'UVarInt count | int64 start_time | int64 end_time | type-specific fields', + layoutNotes: [ + '该记录覆盖的值数量', + '包含首部的开始时间', + '包含尾部的结束时间', + '按 data_type 选择的字段(min / max / first / last / sum 或无)', + ], + }, + }, + }, + { + id: 'chunk-metadata', + kind: 'repeated', + link: 'Metadata.html#chunk-metadata', + text: { + en: { + title: 'Chunk Metadata', + description: 'One record per Chunk. It points to the Chunk Header in the data section; its Statistics are omitted when the series has only one Chunk.', + offset: 'referenced by a Chunk Metadata list entry', + size: 'variable · repeated', + layout: 'int64 offset_of_chunk_header | Statistics?', + layoutNotes: [ + 'Absolute offset of the Chunk marker', + 'Chunk-level Statistics; present only when the series has multiple Chunks', + ], + }, + zh: { + title: '数据块元数据(Chunk Metadata)', + description: '每个 Chunk 一条记录,指向数据区的 Chunk Header;序列只有一个 Chunk 时省略这里的 Statistics。', + offset: '由 Chunk Metadata 列表条目引用', + size: '可变 · 重复', + layout: 'int64 offset_of_chunk_header | Statistics?', + layoutNotes: [ + 'Chunk marker 的绝对偏移', + 'Chunk 级 Statistics;仅当序列包含多个 Chunk 时存在', + ], + }, + }, + }, + ], + }, + { + id: 'metadata-index', + kind: 'repeated', + link: 'Metadata.html#metadata-index', + text: { + en: { + title: 'Metadata Index Node', + description: 'Locates devices, measurements, or further index nodes. An entry covers bytes from its offset up to, but not including, the next offset or end_offset.', + offset: 'referenced by a parent node or TsFileMetadata', + size: 'variable · repeated', + layout: 'child_count | MetadataEntry* | end_offset | node_type', + }, + zh: { + title: '元数据索引节点', + description: '用于查找设备、测量或下一级索引节点。每个条目的字节范围从自身偏移开始,到下一偏移或 end_offset 之前结束。', + offset: '由父节点或 TsFileMetadata 引用', + size: '可变 · 重复', + layout: 'child_count | MetadataEntry* | end_offset | node_type', + }, + }, + }, + ], + }, + { + id: 'file-metadata', + kind: 'record', + link: 'Metadata.html#file-metadata', + text: { + en: { + title: 'TsFileMetadata', + description: + 'The entry point for metadata lookup: table index roots, schemas, separator offset, Bloom filter, and properties. Its start is calculated from the trailing length.', + offset: 'file_metadata_pos = F - 6 - 4 - L', + size: 'L bytes', + layout: + 'table indexes | table schemas | meta_offset | BloomFilter | properties', + note: 'No separator or marker immediately precedes TsFileMetadata.', + }, + zh: { + title: 'TsFileMetadata', + description: '元数据查找的入口,保存表索引根、Schema、分隔符偏移、布隆过滤器和属性。通过文件尾的长度反向定位其起点。', + offset: 'file_metadata_pos = F - 6 - 4 - L', + size: 'L 字节', + layout: 'table indexes | table schemas | meta_offset | BloomFilter | properties', + note: 'TsFileMetadata 前面没有独立分隔符或标记。', + }, + }, + children: [ + { + id: 'table-index-roots', + kind: 'repeated', + link: 'Metadata.html#file-metadata', + text: { + en: { + title: 'Table Index Roots', + description: 'Entry point for each table\'s device-index tree. Look up the table root, descend to a device, then follow that device\'s measurement index to the Timeseries Metadata.', + offset: 'inside TsFileMetadata', + size: 'variable · repeated', + layout: 'VarString table_name | MetadataIndexNode device_index_root', + layoutNotes: [ + 'Table name: the actual table name, or the first DeviceID segment for tree-model data', + 'Root node of the table\'s device-level metadata index', + ], + note: 'Lookup path: table root -> device index -> measurement index -> Timeseries Metadata -> Chunk Metadata.', + }, + zh: { + title: '表索引根(Table Index Root)', + description: '每张表的设备索引树入口:先按表名找到根,再向下定位设备,最后通过设备的测量索引找到 Timeseries Metadata。', + offset: 'TsFileMetadata 内部', + size: '可变 · 重复', + layout: 'VarString table_name | MetadataIndexNode device_index_root', + layoutNotes: [ + '表名:表模型为实际表名;树模型为 DeviceID 的第一段', + '该表设备级元数据索引的根节点', + ], + note: '查找路径:表根 → 设备索引 → 测量索引 → Timeseries Metadata → Chunk Metadata。', + }, + }, + children: [ + { + id: 'device-index-root', + kind: 'record', + link: 'Metadata.html#metadata-index', + text: { + en: { + title: 'Device Index Root', + description: 'Root of the device-level index tree. Its entries locate a device and, through the entry offset, that device\'s measurement index.', + offset: 'device_index_root', + size: 'variable', + layout: 'child_count | DeviceIndexEntry* | end_offset | node_type', + layoutNotes: [ + 'Number of child entries', + 'Device entries: DeviceID plus an absolute offset', + 'End offset of this node\'s byte range', + 'LEAF_DEVICE points directly to measurement indexes; INTERNAL_DEVICE points to another device-index node', + ], + }, + zh: { + title: '设备索引根(Device Index Root)', + description: '设备级索引树的根;条目定位设备,并通过条目 offset 找到该设备的测量索引。', + offset: 'device_index_root', + size: '可变', + layout: 'child_count | DeviceIndexEntry* | end_offset | node_type', + layoutNotes: [ + '子条目数量', + '设备条目:DeviceID 加绝对偏移', + '该节点字节范围的结束偏移', + 'LEAF_DEVICE 直接指向测量索引;INTERNAL_DEVICE 指向下一级设备索引节点', + ], + }, + }, + children: [ + { + id: 'device-index-entry', + kind: 'repeated', + link: 'Metadata.html#metadata-index', + text: { + en: { + title: 'Device Index Entry', + description: 'One device entry in a device-index node. The entry key is the first DeviceID in its range.', + offset: 'inside a device-index node', + size: 'variable · repeated', + layout: 'DeviceID first_device_id | int64 offset', + layoutNotes: [ + 'First DeviceID covered by this entry', + 'Absolute offset to the measurement index root (LEAF_DEVICE) or to a child device-index node (INTERNAL_DEVICE)', + ], + }, + zh: { + title: '设备索引条目(Device Index Entry)', + description: '设备索引节点中的一个设备条目;条目 key 是它覆盖范围里的第一个 DeviceID。', + offset: '设备索引节点内部', + size: '可变 · 重复', + layout: 'DeviceID first_device_id | int64 offset', + layoutNotes: [ + '该条目覆盖范围里的第一个 DeviceID', + '绝对偏移:LEAF_DEVICE 指向测量索引根,INTERNAL_DEVICE 指向下一级设备索引节点', + ], + }, + }, + }, + ], + }, + ], + }, + { + id: 'table-schemas', + kind: 'repeated', + link: 'Metadata.html#table-schema', + text: { + en: { + title: 'Table Schemas', + description: 'Describes table columns in their declared order, including names, types, and categories. FIELD Chunks are matched by measurement name.', + offset: 'inside TsFileMetadata', + size: 'variable · repeated', + }, + zh: { + title: '表结构(Schema)', + description: '按声明顺序描述表的列,记录列名、类型和类别;FIELD Chunk 通过测量名与列对应。', + offset: 'TsFileMetadata 内部', + size: '可变 · 重复', + }, + }, + }, + { + id: 'meta-offset', + kind: 'field', + text: { + en: { + title: 'meta_offset', + description: 'The absolute offset of the separator. Add one to find the start of indexed metadata.', + offset: 'inside TsFileMetadata', + size: '8 bytes', + layout: 'int64, big-endian', + }, + zh: { + title: 'meta_offset', + description: '分隔符的绝对偏移量,加一就是索引元数据区的起点。', + offset: 'TsFileMetadata 内部', + size: '8 字节', + layout: 'int64,大端序', + }, + }, + }, + { + id: 'bloom-filter', + kind: 'record', + link: 'Bloom-Filter.html', + text: { + en: { + title: 'Bloom Filter', + description: 'Can rule out a missing path before an index lookup. Trust a negative result only when path encoding and hashing match the writer.', + offset: 'inside TsFileMetadata', + size: 'variable', + layout: 'byte_length | filter bytes | bit count | hash-function count', + }, + zh: { + title: '布隆过滤器', + description: '可以在查索引之前排除不存在的路径,但要先确认路径编码和哈希方式与写入端一致。', + offset: 'TsFileMetadata 内部', + size: '可变', + layout: 'byte_length | filter bytes | bit count | hash-function count', + }, + }, + }, + { + id: 'file-properties', + kind: 'repeated', + link: 'Metadata.html#file-metadata', + text: { + en: { + title: 'File Properties', + description: 'Maps UTF-8 keys to nullable binary values. Keep unknown values as raw bytes; only a property\'s own definition can say whether its value is text.', + offset: 'end of TsFileMetadata', + size: 'variable · repeated', + layout: 'VarInt property_count | (VarString key | VarBinary value)*', + }, + zh: { + title: '文件属性', + description: '把 UTF-8 键映射到可空的二进制值。未知值按原始字节保留,只有具体属性定义才能决定它是否应按文本解释。', + offset: 'TsFileMetadata 尾部', + size: '可变 · 重复', + layout: 'VarInt property_count | (VarString key | VarBinary value)*', + }, + }, + }, + ], + }, + { + id: 'metadata-size', + kind: 'field', + text: { + en: { + title: 'TsFileMetadata Size', + description: 'Stores L, the byte length of TsFileMetadata. Subtract L and the ten trailing bytes from file size F to locate that metadata.', + offset: 'F - 10 … F - 7', + size: '4 bytes', + layout: 'int32 L, big-endian', + }, + zh: { + title: 'TsFileMetadata 长度', + description: '保存 TsFileMetadata 的字节长度 L。用文件大小 F 减去 L 和尾部十个字节,即可找到元数据起点。', + offset: 'F - 10 … F - 7', + size: '4 字节', + layout: 'int32 L,大端序', + }, + }, + }, + { + id: 'tail-magic', + kind: 'field', + text: { + en: { + title: 'Tail Magic', + description: 'The same six magic bytes as the header. Check this suffix together with the footer length and metadata structure before opening the file.', + offset: 'F - 6 … F - 1', + size: '6 bytes', + layout: '54 73 46 69 6c 65', + }, + zh: { + title: '尾部魔数', + description: '与文件头相同的六个魔数字节。打开文件时,还要同时检查 Footer 长度和元数据结构。', + offset: 'F - 6 … F - 1', + size: '6 字节', + layout: '54 73 46 69 6c 65', + }, + }, + }, + ], +}; diff --git a/src/.vuepress/config.ts b/src/.vuepress/config.ts index 0e926ee4d..643552587 100644 --- a/src/.vuepress/config.ts +++ b/src/.vuepress/config.ts @@ -38,6 +38,43 @@ export default defineUserConfig({ }, }, + extendsMarkdown: (md) => { + md.options.breaks = false; + + // With breaks disabled, hard-wrapped source lines become softbreaks. + // Remove softbreaks between CJK characters and punctuation while keeping + // spaces between Chinese text and Latin words or inline code. + md.core.ruler.push("collapse-cjk-spaces", (state) => { + const isCjk = (char) => + /[\u3000-\u303f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef]/.test( + char, + ); + state.tokens.forEach((token) => { + if (token.type !== "inline" || !token.children) { + return; + } + const children = token.children; + for (let index = 0; index < children.length; index += 1) { + if (children[index].type !== "softbreak") { + continue; + } + const previous = children[index - 1]; + const next = children[index + 1]; + const before = + previous && previous.type === "text" + ? previous.content.slice(-1) + : ""; + const after = + next && next.type === "text" ? next.content.slice(0, 1) : ""; + if (before && after && isCjk(before) && isCjk(after)) { + children.splice(index, 1); + index -= 1; + } + } + }); + }); + }, + theme, head: [ ['link', { rel: 'icon', href: '/favicon.ico' }], diff --git a/src/.vuepress/sidebar/v2.x/en.ts b/src/.vuepress/sidebar/v2.x/en.ts index c7ae96653..336e624da 100644 --- a/src/.vuepress/sidebar/v2.x/en.ts +++ b/src/.vuepress/sidebar/v2.x/en.ts @@ -33,6 +33,38 @@ export const enSidebar = { collapsible: true, link: 'QuickStart/Data-Model', }, + { + text: 'File Format', + collapsible: true, + expanded: true, + prefix: '/UserGuide/latest/FileFormat/', + link: '/UserGuide/latest/FileFormat/', + children: [ + { text: 'Binary Representation', link: '/UserGuide/latest/FileFormat/Binary-Representation' }, + { text: 'Configurations', link: '/UserGuide/latest/FileFormat/Configurations' }, + { text: 'Extensibility', link: '/UserGuide/latest/FileFormat/Extensibility' }, + { text: 'Metadata', link: '/UserGuide/latest/FileFormat/Metadata' }, + { text: 'Types', link: '/UserGuide/latest/FileFormat/Types' }, + { text: 'Bloom Filter', link: '/UserGuide/latest/FileFormat/Bloom-Filter' }, + { + text: 'Data Pages', + collapsible: true, + expanded: true, + prefix: '/UserGuide/latest/FileFormat/DataPages/', + link: '/UserGuide/latest/FileFormat/DataPages/', + children: [ + { text: 'Chunks', link: '/UserGuide/latest/FileFormat/DataPages/Chunks' }, + { text: 'Compression', link: '/UserGuide/latest/FileFormat/DataPages/Compression' }, + { text: 'Encodings', link: '/UserGuide/latest/FileFormat/DataPages/Encodings' }, + { text: 'Encryption', link: '/UserGuide/latest/FileFormat/DataPages/Encryption' }, + { text: 'Checksumming', link: '/UserGuide/latest/FileFormat/DataPages/Checksumming' }, + { text: 'Error Recovery', link: '/UserGuide/latest/FileFormat/DataPages/Error-Recovery' }, + ], + }, + { text: 'Nulls', link: '/UserGuide/latest/FileFormat/Nulls' }, + { text: 'Format Versions', link: '/UserGuide/latest/FileFormat/Format-Versions' }, + ], + }, { text: 'Quick Start', collapsible: true, diff --git a/src/.vuepress/sidebar/v2.x/zh.ts b/src/.vuepress/sidebar/v2.x/zh.ts index f1a40e158..1afb96b56 100644 --- a/src/.vuepress/sidebar/v2.x/zh.ts +++ b/src/.vuepress/sidebar/v2.x/zh.ts @@ -33,6 +33,38 @@ export const zhSidebar = { collapsible: true, link: 'QuickStart/Data-Model', }, + { + text: '文件格式', + collapsible: true, + expanded: true, + prefix: '/zh/UserGuide/latest/FileFormat/', + link: '/zh/UserGuide/latest/FileFormat/', + children: [ + { text: '二进制表示', link: '/zh/UserGuide/latest/FileFormat/Binary-Representation' }, + { text: '配置', link: '/zh/UserGuide/latest/FileFormat/Configurations' }, + { text: '扩展机制', link: '/zh/UserGuide/latest/FileFormat/Extensibility' }, + { text: '元数据', link: '/zh/UserGuide/latest/FileFormat/Metadata' }, + { text: '类型', link: '/zh/UserGuide/latest/FileFormat/Types' }, + { text: '布隆过滤器', link: '/zh/UserGuide/latest/FileFormat/Bloom-Filter' }, + { + text: '数据页', + collapsible: true, + expanded: true, + prefix: '/zh/UserGuide/latest/FileFormat/DataPages/', + link: '/zh/UserGuide/latest/FileFormat/DataPages/', + children: [ + { text: '数据块', link: '/zh/UserGuide/latest/FileFormat/DataPages/Chunks' }, + { text: '压缩', link: '/zh/UserGuide/latest/FileFormat/DataPages/Compression' }, + { text: '编码', link: '/zh/UserGuide/latest/FileFormat/DataPages/Encodings' }, + { text: '加密', link: '/zh/UserGuide/latest/FileFormat/DataPages/Encryption' }, + { text: '校验和', link: '/zh/UserGuide/latest/FileFormat/DataPages/Checksumming' }, + { text: '错误恢复', link: '/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery' }, + ], + }, + { text: '空值', link: '/zh/UserGuide/latest/FileFormat/Nulls' }, + { text: '格式版本', link: '/zh/UserGuide/latest/FileFormat/Format-Versions' }, + ], + }, { text: '快速上手', collapsible: true, diff --git a/src/.vuepress/styles/index.scss b/src/.vuepress/styles/index.scss index 73df4fb6f..7acdb0e84 100644 --- a/src/.vuepress/styles/index.scss +++ b/src/.vuepress/styles/index.scss @@ -18,6 +18,29 @@ */ // place your custom styles here +/* Technical documentation: do not insert automatic hyphenation. */ +.theme-hope-content, +.theme-hope-content p, +.theme-hope-content li { + hyphens: manual !important; +} + +/* The right TOC and print control overflow the layout below 1440px. */ +@media (max-width: 1439px) { + .vp-toc-placeholder > aside { + display: none; + } +} + +/* Keep tree controls tappable on narrow screens. */ +@media (max-width: 768px) { + .tsfile-structure-actions button, + .tsfile-structure .tsfile-tree-toggle, + .tsfile-structure .tsfile-tree-label { + min-height: 2.75rem !important; + } +} + #main-description { max-width: 55rem; font-weight: 400; diff --git a/src/.vuepress/theme.ts b/src/.vuepress/theme.ts index bc4819c7f..1511ab3a9 100644 --- a/src/.vuepress/theme.ts +++ b/src/.vuepress/theme.ts @@ -34,8 +34,9 @@ export default hopeTheme({ breadcrumb: false, darkmode: 'toggle', contributors: false, + pageInfo: false, - lastUpdated: true, + lastUpdated: false, navbarLayout: { start: ['Brand'], diff --git a/src/UserGuide/latest/FileFormat/Binary-Representation.md b/src/UserGuide/latest/FileFormat/Binary-Representation.md new file mode 100644 index 000000000..6531d24b0 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Binary-Representation.md @@ -0,0 +1,122 @@ + + +# Binary Representation + +All offsets are measured from the first byte of the file, and all lengths are in bytes. Unless +otherwise specified, fixed-width integers and IEEE 754 floating-point values use big-endian byte +order. Strings use UTF-8. + +## Fixed-width Values + +| Notation | Bytes | Meaning | +| --- | ---: | --- | +| `byte` | 1 | An 8-bit raw value | +| `bool` | 1 | `0x00` for false and `0x01` for true | +| `int32` | 4 | Big-endian signed 32-bit integer | +| `int64` | 8 | Big-endian signed 64-bit integer | +| `float32` | 4 | Big-endian IEEE 754 binary32 | +| `float64` | 8 | Big-endian IEEE 754 binary64 | + +## Unsigned Variable-length Integer + +`UVarInt` is an unsigned LEB128-style variable-length 32-bit integer. The low seven bits of each +byte carry data. A set high bit indicates that another byte follows. The least-significant group +is written first. + +```text +value = Σ ((byte[i] & 0x7f) << (7 * i)) +``` + +The encoding uses the shortest representation, from one to five bytes. For example, decimal +`300` splits into the seven-bit groups `0x2c` and `0x02`, so its encoded bytes are `ac 02`. +A 32-bit decoder rejects an unterminated sequence after five bytes and checks for overflow. + +## Signed Variable-length Integer + +`VarInt` first maps a signed 32-bit integer with ZigZag encoding, then writes the result as a +`UVarInt`: + +```text +zigzag(n) = (n << 1) XOR (n >> 31) +``` + +| Original value | Unsigned wire value | Byte | +| ---: | ---: | --- | +| 0 | 0 | `00` | +| -1 | 1 | `01` | +| 1 | 2 | `02` | +| -2 | 3 | `03` | + +## Strings and Byte Arrays + +| Notation | Format | +| --- | --- | +| `VarString` | `VarInt byte_length` + `byte[byte_length]` | +| `String32` | `int32 byte_length` + `byte[byte_length]` | +| `Binary32` | `int32 byte_length` + `byte[byte_length]` | +| `VarBinary` | `VarInt byte_length` + `byte[byte_length]` | + +For `VarString`, `String32`, and `VarBinary`, a length of `-1` represents null; a length of `0` +represents an empty string or byte array. `Binary32` does not accept a negative length. +`VarString` and `String32` contain UTF-8; `Binary32` and `VarBinary` contain raw bytes. +`VarBinary` uses a signed ZigZag length, unlike the unsigned lengths of the binary components in +[FileEncryptionHeader](DataPages/Encryption.md#file-encryption-header). + +String lengths count bytes after UTF-8 encoding. For example, `温` occupies three bytes, so its +`VarString` length field is `VarInt(3)`, encoded as `06`. A nullable field preserves the distinction +between null and an empty value. + +## Boundaries and Defensive Decoding + +Every length-delimited field is contained by a higher-level boundary: a string by its enclosing +record, a Page by its Chunk, and footer objects by metadata offsets or the trailing metadata +length. Before allocating or slicing a buffer for length `N`, a reader validates: + +```text +N >= 0 # except the documented null sentinel +current_offset + N >= current_offset +current_offset + N <= enclosing_end +``` + +The second comparison detects wraparound when offset arithmetic uses fixed-width integers. +Check these bounds before allocating memory, and apply an implementation-specific size limit +as well: a length can fit within the file and still be too large for the reader to allocate. + +Fixed-width values require their complete byte width. A truncated `int64`, unterminated VarInt, +negative non-null length, invalid UTF-8 where a string is required, or bytes left over after a +length-delimited record are format errors. + +## Offset Conventions + +Offsets stored in metadata are signed 64-bit values but represent non-negative absolute byte +positions. An offset points to the first byte of the named structure: for example, +`offset_of_chunk_header` points to the Chunk marker, and `meta_offset` points to the `0x02` +separator. To find the end of a structure, add its length to its start offset using the bounds checks above. + +## Reading Layout Examples + +In layout examples, `|` separates alternatives, parentheses group a sequence, `?` marks an +optional field or structure, `*` allows zero or more repetitions, `+` requires one or more +repetitions, and `[n]` means n elements. For example, `(A | B)*` means zero or more entries, each +of which is either A or B. The surrounding description or a preceding length field determines +which case applies. `remaining bytes` means the unused bytes up to the enclosing payload boundary; +for a Page layout, that is the end of its decrypted, decompressed payload. diff --git a/src/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/UserGuide/latest/FileFormat/Bloom-Filter.md new file mode 100644 index 000000000..a4fad8f62 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -0,0 +1,137 @@ + + +# Bloom Filter + +The optional Bloom filter in `TsFileMetadata` can save an index lookup when a path is absent. +A clear bit rules the path out; all tested bits being set means only “possibly present,” so the +reader still checks the index. This result is reliable only when the query uses the writer's +path encoding and hash behavior, described below. + +## File Format + +```text +BloomFilter := + UVarInt byte_length + byte bits[byte_length] + UVarInt bit_count? # present when byte_length > 0 + UVarInt hash_function_count? # present when byte_length > 0 +``` + +When `byte_length = 0`, the file has no Bloom filter and the final two fields are absent. + +Bit `i` is stored at bit `i mod 8` in `bits[i / 8]`; bits within each byte are numbered from the +least-significant bit. Trailing all-zero bytes are omitted from the file. `bit_count` records the +logical number of bits. + +## Filter Input + +For a writer using UTF-8, the filter input is the byte representation of the complete timeseries path: + +```text +UTF8(join(DeviceID segments, ".") + "." + measurement_id) +``` + +A null device segment is represented by the ASCII text `null` in the path. The charset used to +build the path bytes is not stored in the file. The examples below use UTF-8. A reader must match +the writer's charset and hashing behavior; if either is unknown, bypass the Bloom filter and use +the metadata index. + +## Hashing + +TsFile uses the `Murmur128Hash` variant of MurmurHash3 x64 128-bit. Its complete 16-byte +blocks use the standard little-endian unsigned-byte assembly and mixing. Its incomplete final +block differs from standard MurmurHash3: **each tail byte is sign-extended from signed 8-bit to +64-bit before shifting**, and the shifted terms are XORed, not ORed. + +For a tail of `n = byte_length mod 16` bytes `t`, assemble its words as follows. All shifts and +arithmetic are modulo 64 bits; `signed64(b)` maps byte values `0x80..0xff` to `-128..-1`. + +```text +k1 = XOR(signed64(t[j]) << (8 * j)) for j = 0 .. min(n, 8) - 1 +k2 = XOR(signed64(t[j]) << (8 * (j - 8))) for j = 8 .. n - 1 +``` + +An empty XOR is zero. Apply the standard tail mixing to `k2` only when `n >= 9`, and to `k1` +only when `n >= 1`; then apply the standard MurmurHash3 length XOR and finalization. Add the +finalized `h1` and `h2` modulo 64 bits and interpret the low 32 bits as a signed integer `hash32`. +This signed-tail behavior must be preserved for existing files; substituting an ordinary +MurmurHash3 library can produce false negatives for non-ASCII paths. + +```text +bit_index = (0 if hash32 == -2147483648 else abs(hash32)) mod bit_count +``` + +The hash seeds, in order, are: + +```text +5, 7, 11, 19, 31, 37, 43, 59 +``` + +`hash_function_count` selects the number of seeds used, with a maximum of 8. + +### UTF-8 Test Vectors + +These vectors use `seed = 5` and `bit_count = 256`. The last row also exercises a complete +16-byte block before the signed tail. + +| Path | Signed `hash32` | Bit index | +| --- | ---: | ---: | +| `root.sg.d.s1` | -1267668613 | 133 | +| `root.设备.s1` | 1697017247 | 159 | +| `root.abcdefghijklmnop.设备.s1` | 866805081 | 89 | + +For `root.设备.s1`, a standard MurmurHash3 implementation followed by the same final sum instead +produces `1978455175`, selecting bit `135`. A reader using that bit can reject a path actually +present in the file. + +## Lookup Procedure + +For an exact path lookup, a reader: + +1. constructs the path bytes using the same `DeviceID` segment and measurement rules; +2. evaluates the first `hash_function_count` seeds; +3. tests the corresponding positions in the logical `bit_count`-bit vector; +4. returns **definitely absent** if any tested bit is clear, otherwise **possibly present**. + +A positive result must still be checked in the metadata index. A negative result is reliable only +when the query uses the same path construction, charset, and signed-tail hash as the writer. +Otherwise, look up the metadata index directly. + +## Space and Probability + +Let `n` be the number of inserted paths, `m` the logical `bit_count`, and `k` the +`hash_function_count`. The usual Bloom-filter approximation is: + +```text +false_positive_probability ≈ (1 - exp(-k * n / m))^k +optimal_k ≈ (m / n) * ln(2) +``` + +These equations explain the writer's space/accuracy tradeoff; they are not additional file fields. +The serialized `bit_count` and `hash_function_count` are authoritative for reading. + +## Validation + +When `byte_length > 0`, `bit_count` must be positive, `hash_function_count` must be in the range +`1..8`, and `byte_length` must not exceed `ceil(bit_count / 8)`. Omitted trailing zero bytes are +logically reconstructed as zero. The reader rejects a bit-vector length that crosses the +`TsFileMetadata` boundary. diff --git a/src/UserGuide/latest/FileFormat/Configurations.md b/src/UserGuide/latest/FileFormat/Configurations.md new file mode 100644 index 000000000..535b176a7 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Configurations.md @@ -0,0 +1,139 @@ + + +# Configurations + +Most settings affect how a writer groups and buffers data. A reader does not need the original +Page-size target or flush threshold: the resulting boundaries are recorded in the file. A few +settings, including timestamp units and non-aligned time encoding, still need to be agreed by +the applications exchanging the file. + +The table distinguishes those cases. Setting names in language-specific APIs may differ. + +| Conceptual setting | Observable file effect | Setting serialized directly? | Reader needs original setting? | +| --- | --- | --- | --- | +| Chunk Group size | Where a device group is flushed | No | No | +| Page size target | Page boundaries and Statistics granularity | No | No | +| Maximum points per Page | Page boundaries | No | No | +| Metadata index fanout | Index-tree depth and node widths | No | No | +| Bloom false-positive rate | Generated bit count and hash-function count | No; generated values are serialized | No | +| Value encoding | Encoded Page stream | Yes, as `encoding_type` in each Chunk | No | +| Compression | Stored Page payload | Yes, as `compression_type` in each Chunk | No | +| Time precision and zone | Interpretation of `int64` timestamps | No | Yes, from the surrounding protocol | +| Non-aligned time encoding | Encoded time stream in each non-aligned Page | No | Yes | +| Encryption | File encryption header, Chunk authentication, Page bodies, or legacy footer properties | Provider/profile or legacy properties are serialized; key context is external | Yes, when encrypted | + +For example, a Page-size target is not stored as a setting, but each Page records its actual size. + +## Chunk Group Size + +The Chunk Group size is the amount of data that a writer buffers before it flushes the Chunks of +the current device. Larger Chunk Groups reduce marker and metadata overhead and favor sequential +I/O, but require more writer memory. Smaller Chunk Groups reduce peak memory and make data +available to the output sooner. + +The configured size is a flush threshold, not an exact on-disk size. The format does not prescribe +a default. A group begins with a `0x00` Chunk Group Header and ends at the next group, operation-index +record, or separator. Individual Chunk markers continue the current group. + +## Data Page Size + +The Data Page size is a target for the in-memory data accumulated by a series writer. Smaller +Pages provide finer-grained Statistics and skipping. Larger Pages reduce Page Header overhead and +can improve compression, at the cost of more buffering and coarser skipping. + +The threshold is not an exact serialized Page size. A writer can finish a Page earlier because of +the maximum point count, an explicit flush, or aligned-page boundary requirements. The actual +Page boundary is determined by the Page Header lengths. The format does not prescribe a default +Page size. + +## Maximum Points per Page + +A writer may limit the number of logical positions in one Page independently of the Page size +target. This prevents a Page containing many compact values from growing to an excessive number +of points. A common writer setting is `10,000` points. + +For aligned series, the Time Page and its corresponding Value Pages must cover the same logical +position range. A writer therefore applies a common boundary to the aligned columns. + +The configured limit is not stored as a separate field. Readers use Page boundaries and, for an +aligned Value Page, `position_count`. + +## Metadata Index Fanout + +Index fanout is the maximum number of children in an internal metadata index node. +A larger fanout produces a shallower tree with larger nodes; a smaller fanout produces a deeper +tree with smaller nodes. The value should be at least `2`; a common value is `256`. + +The configured limit is not serialized. Each metadata index node stores its node type, children, +and end offset, so a reader traverses the resulting tree without knowing the writer setting. + +## Bloom Filter False-positive Rate + +A Bloom filter can report “possibly present” for a path that is absent. Lowering the target +false-positive rate reduces those extra lookups at the cost of more bits and hash functions. Standard writers constrain the value to the range `0.01` +through `0.10` and use `0.05` by default. + +The rate itself is not serialized. The resulting filter bytes, bit count, and hash-function count +are stored in `TsFileMetadata`, so a reader does not need the writer setting. See +[Bloom Filter](Bloom-Filter.md). + +## Value Encoding and Compression + +A writer may select a default value encoding by physical type and a default compression method. +These defaults are schema and writer policies rather than shared reader configuration: + +- `encoding_type` in the Chunk Header identifies the value encoding, or the time encoding for an + aligned Time Chunk. +- `compression_type` in the Chunk Header identifies how every Page in the Chunk is compressed. + +Consequently, a reader uses the Chunk Header and does not need the writer's defaults. Supported +identifiers and their payload formats are defined in [Encodings](DataPages/Encodings.md) and +[Compression](DataPages/Compression.md). + +## Time Precision and Time Zone + +A physical timestamp is a signed 64-bit integer. TsFile v4 does not record a time unit or time +zone in the file. The protocol that uses the file defines whether the precision is milliseconds, +microseconds, nanoseconds, or another unit, as well as any time-zone semantics. + +## Non-aligned Time Encoding + +The `encoding_type` in a non-aligned Chunk Header describes the value stream, not the time stream +in the same Page. Readers and writers determine the non-aligned time-stream encoding from shared +configuration. The default is `TS_2DIFF`. + +This restriction does not apply to an aligned Time Chunk. Its Chunk Header records the time +encoding in `encoding_type`. + +## Encryption + +Encryption uses one of two descriptions: the optional file encryption header for Page AEAD, +or footer properties for legacy encryption. Both record how to interpret the Page bytes, while +key access remains external. The AEAD header is authoritative even though its legacy footer +properties say unencrypted. The layouts and mode-selection rules are in +[Encryption](DataPages/Encryption.md). + +## Implementation Settings + +Settings for memory allocation, memory-check cadence, batch size, parallel execution, thread +count, storage backend, and close-time synchronization do not define TsFile bytes. They are +implementation and deployment concerns and are outside this file-format specification. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md b/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md new file mode 100644 index 000000000..773255372 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Checksumming.md @@ -0,0 +1,49 @@ + + +# Checksumming + +TsFile v4 has no general Page, Chunk, or whole-file checksum. A completed-looking file can still +contain damaged values, so the head and tail magic strings should not be used as content checksums. + +## What Structural Validation Detects + +Lengths, offsets, and counts help detect incomplete or malformed structures. A Page that extends +beyond its Chunk, a footer length outside the file, or a decompressed payload with the wrong size +can all be rejected. These checks are described in [Binary Representation](../Binary-Representation.md) +and [Data Pages](README.md). + +They do not detect every change. A modified value can occupy the same number of bytes and leave +all surrounding boundaries intact. Structural checks remain necessary even when a separate +checksum is available. + +## Integrity at Rest or in Transit + +To detect changes anywhere in a file, maintain an external checksum or use the storage system's +integrity mechanism. If protection against intentional modification is required, use a keyed +authentication code or signature with a trusted key or reference stored outside the file. Cover +the complete byte sequence, including the footer length and tail magic. + +Page AEAD provides a narrower guarantee: it authenticates non-empty Page bodies and their context, +and binds Chunk ordinals to file positions. It does not cover all visible headers, Statistics, +indexes, or footer metadata. Empty aligned Value Pages have no Page authentication tag, and +legacy encryption does not necessarily authenticate its ciphertext. See [Encryption](Encryption.md) +for the exact scope. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md new file mode 100644 index 000000000..890ab7564 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -0,0 +1,203 @@ + + +# Chunks + +The TsFile data section consists of marker-delimited Chunk Groups, Chunks, and Operation Index +Ranges. It starts at offset `7` without a file encryption header, or at `7 + E` when a Page AEAD +[FileEncryptionHeader](Encryption.md#file-encryption-header) of total serialized size `E` occupies +offset `7`. That optional header occurs exactly once, before the data section. + +The top-level grammar is: + +```text +FilePrefix := magic[6] version[1] FileEncryptionHeader? +DataSection := (ChunkGroup+ OperationIndexRange?)* +ChunkGroup := ChunkGroupHeader Chunk* +Separator := byte(0x02) +``` + +Each batch contains one or more ChunkGroups followed by an optional OperationIndexRange. The +`0x02` separator follows the data section; it is not part of a Chunk Group. A Chunk Group has no +total-length field. After reading each Chunk, the reader examines the next marker: another +Chunk continues the group, while a new group, operation-index record, or separator ends it. + +```text +File +└── Chunk Group a batch of data for one device + ├── Non-aligned Chunk one measurement column with its own time stream + │ └── Page(s) + └── Aligned Chunk Set measurement columns sharing a time column + ├── Time Chunk + │ └── Time Page(s) + └── Value Chunk(s) + └── Value Page(s) +``` + +## Markers + +| Value | Name | Following structure | +| ---: | --- | --- | +| `0x00` | `CHUNK_GROUP_HEADER` | `DeviceID` | +| `0x01` | `CHUNK_HEADER` | Multi-page non-aligned Chunk | +| `0x02` | `SEPARATOR` | End of data section and start of metadata section | +| `0x03` | `VERSION` | Unused since v3; identifier reserved | +| `0x04` | `OPERATION_INDEX_RANGE` | Two `int64` values | +| `0x05` | `ONLY_ONE_PAGE_CHUNK_HEADER` | Single-page non-aligned Chunk | +| `0x06` | `ENCRYPTION_HEADER` | File encryption header at offset `7` only; not a Chunk | +| `0x41` | `VALUE_CHUNK_HEADER` | Multi-page aligned Value Chunk | +| `0x45` | `ONLY_ONE_PAGE_VALUE_CHUNK_HEADER` | Single-page aligned Value Chunk | +| `0x81` | `TIME_CHUNK_HEADER` | Multi-page aligned Time Chunk | +| `0x85` | `ONLY_ONE_PAGE_TIME_CHUNK_HEADER` | Single-page aligned Time Chunk | + +The high two bits of a Chunk marker identify the column role. The low six bits identify the Page +count category: + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0x01: more than one Page +low 6 bits = 0x05: one Page or no Page +``` + +An aligned Value Chunk with `data_size = 0` can contain no Page. + +Read markers only after consuming the preceding structure. A Page body can contain any byte +value, including bytes that look like markers. + +## Chunk Group Header + +```text +ChunkGroupHeader := + byte marker = 0x00 + DeviceID device_id + +DeviceID := + UVarInt segment_count + VarString segment[segment_count] +``` + +A DeviceID is an ordered list of string segments; individual segments can be null. Serialization +removes trailing nulls, but at least one segment must be non-null. The special `segment_count = 0` +form is read as one empty-string segment. + +Chunks following a Chunk Group Header belong to that device until the next Chunk Group Header, +Operation Index Range, or Separator. + +`segment_count` counts segments. Each segment has its own byte length. Keep the segments separate +when comparing device identities: joining them with dots for display can be ambiguous if a segment +itself contains a dot. + +## Chunk Header + +The marker is the first field of the Chunk Header: + +```text +ChunkHeader := + byte chunk_type + VarString measurement_id + UVarInt data_size + byte data_type + byte compression_type + byte encoding_type + int64 chunk_ordinal? + int32 chunk_authentication_size? + byte chunk_authentication[chunk_authentication_size]? +``` + +The final three fields appear only in Page AEAD files. The file encryption header selects this +suffix; the Chunk marker remains the same. The ordinal, proof format, and validation rules are +described in [AEAD Chunk Header](Encryption.md#aead-chunk-header). Like the rest of the Header, +the suffix is excluded from `data_size`. + +The Chunk Header contains no Statistics field. The first Page Header immediately follows the +Chunk Header. When that Page is non-empty and the Chunk has multiple Pages, its Statistics follows +the two length fields, so those Statistics bytes are physically adjacent to the Chunk Header but +belong to the Page Header. Chunk-level Statistics are stored with the Chunk Metadata in the footer; +a single-Page Chunk omits Page Statistics and relies on the Chunk or series Statistics. See +[Data Pages](README.md) and [Statistics](../Metadata.md#statistics). + +`data_size` is the total size of all Page Headers and stored Page data in the Chunk. It does not +include the Chunk Header. A reader consumes exactly `data_size` bytes after the Chunk Header. + +If `H` is the offset immediately after the Chunk Header, then the next structure begins at: + +```text +next_structure_offset = H + data_size +``` + +The sum of every serialized Page Header and its `compressed_size` bytes must equal `data_size`. +The value is therefore both a skip length and a containment boundary for validating Pages. + +For an aligned Time Chunk, `measurement_id` is an empty string and `data_type` is `VECTOR (0x06)`. +For an aligned Value Chunk or non-aligned Chunk, `data_type` is the measurement value type. + +Aligned data is stored Chunk by Chunk, not as interleaved Time Pages and Value Pages. The Time +Chunk stores the complete sequence of Time Pages, and each Value Chunk stores its own sequence of +Value Pages. Pages correspond by ordinal position. + +An aligned Chunk set is one Time Chunk followed by its Value Chunks. It has no separate set +header. The relationships within a set are: + +- The Time Chunk appears before the Value Chunks that use it. +- A Value Chunk identifies an individual measurement; the Time Chunk uses an empty + `measurement_id` and `VECTOR (0x06)` as its physical type marker. +- Value Page `i` covers the logical positions of Time Page `i`. +- A Value Chunk may be absent when the measurement has no data in the set. An existing aligned + Value Chunk can also have `data_size = 0`. + +For a non-aligned Chunk, every logical point carries a timestamp and a value. Consequently, the +decoded time stream and decoded value stream of each Page must contain the same number of +elements. + +## Chunk Scan State Machine + +A streaming or recovery reader can process the data section with this state machine: + +1. Validate the prefix, inspect offset `7`, and consume the optional `0x06` encryption header. + Resolve its provider/profile before parsing Chunks; continue at `data_start = 7 + E` (`E = 0` + when absent). A later or repeated `0x06` is invalid. Read the first data-section marker there. +2. For `0x00`, read the DeviceID and enter that Chunk Group. +3. For a Chunk marker, read its complete Header, including the AEAD suffix when selected, and + verify the Chunk-position proof before using its identity. Advance by exactly `data_size` bytes + after optionally parsing its Pages. +4. For `0x04`, consume the following 16 bytes as an operation-index range. +5. For `0x02`, stop. The following byte belongs to the indexed metadata region. +6. Reject an unrecognized top-level marker or a structure whose declared boundary exceeds the + available bytes. + +## Operation Index Range + +```text +OperationIndexRange := + byte marker = 0x04 + int64 min_operation_index + int64 max_operation_index +``` + +This optional record lets a storage engine associate a flushed batch with its operation log. +The two values are the minimum and maximum operation sequence numbers in that batch. They help +coordinate checkpoints, snapshots, backups, and recovery; they are unrelated to timestamps or +file offsets. + +A general-purpose writer can omit this record. A reader that does not use operation indexes +still consumes both `int64` fields before reading the next marker. Several records can appear +when several batches are flushed into the same file. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/UserGuide/latest/FileFormat/DataPages/Compression.md new file mode 100644 index 000000000..83332e14e --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -0,0 +1,85 @@ + + +# Compression + +The single-byte `compression_type` in a Chunk Header selects the compression format for every +Page in that Chunk. + +| Value | Name | Page data format | +| ---: | --- | --- | +| 0 | `UNCOMPRESSED` | Raw bytes | +| 1 | `SNAPPY` | Snappy raw block | +| 2 | `GZIP` | One gzip member | +| 3 | `LZO` | LZO block (C++ implementation, enabled at build time) | +| 4 | `SDT` | Reserved; no codec | +| 5 | `PAA` | Reserved; no codec | +| 6 | `PLA` | Reserved; no codec | +| 7 | `LZ4` | LZ4 raw block | +| 8 | `ZSTD` | Zstandard frame | +| 9 | `LZMA2` | LZMA2 stream in an XZ container | + +Each Page is compressed independently; compression state is not shared across Pages. The Page +Header's `uncompressed_size` records the uncompressed payload length. `compressed_size` records +the stored body length after compression and any encryption, including Page AEAD framing and tags. + +The numeric values are stable wire identifiers rather than the ordinal positions of a compressor +library. Values `3` through `6` are legacy identifiers from earlier TsFile versions. The Java implementation does not define or implement them and rejects these bytes while parsing a +Chunk Header. The C++ implementation retains them as `LZO`, `SDT`, `PAA`, and `PLA`; only `LZO` +has a codec (enabled at build time), while `SDT`, `PAA`, and `PLA` remain reserved without a codec. +A reader that does not provide the codec cannot decode the affected Chunk. + +## Page-local Framing + +Let a Page start at `P` and its Header occupy `H` bytes. Read exactly `compressed_size` bytes +for its body: + +```text +[P + H, P + H + compressed_size) +``` + +This is the stored length, including any encryption framing. After decryption, `UNCOMPRESSED` +already contains the encoded Page payload; other methods need decompression. Either path must +produce exactly `uncompressed_size` bytes. With length-preserving encryption or no encryption, +`UNCOMPRESSED` has equal stored and uncompressed lengths. Page AEAD adds framing, so that equality +no longer holds. + +Pages do not share compression history. A reader can seek directly to a selected Page or skip +its body using the stored length, without decompressing earlier Pages. + +## Read Pipeline and Validation + +The Page Header is read before the payload, so the reader knows both bounds before allocating or +decompressing: + +```text +read Page Header + -> read exactly compressed_size bytes + -> authenticate/decrypt according to the file encryption header or legacy properties + -> decompress according to compression_type + -> require exactly uncompressed_size bytes + -> decode the Page streams +``` + +Apply the length checks in [Binary Representation](../Binary-Representation.md) before allocating +buffers. An unavailable codec prevents decompression. A scanner can still use `data_size` to skip +the Chunk if it parses the raw Header fields, even when the codec registry rejects the identifier; +it must not treat that identifier as `UNCOMPRESSED`. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Encodings.md b/src/UserGuide/latest/FileFormat/DataPages/Encodings.md new file mode 100644 index 000000000..49c9c2a55 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Encodings.md @@ -0,0 +1,109 @@ + + +# Encodings + +Encoding turns a sequence of typed values into a byte stream before compression. The Chunk +Header records the selected codec in `encoding_type`. This field describes values in a Value or +non-aligned Chunk, and timestamps in an aligned Time Chunk. + +| Value | Name | +| ---: | --- | +| 0 | `PLAIN` | +| 1 | `DICTIONARY` | +| 2 | `RLE` | +| 3 | `DIFF` | +| 4 | `TS_2DIFF` | +| 5 | `BITMAP` | +| 6 | `GORILLA_V1` | +| 7 | `REGULAR` | +| 8 | `GORILLA` | +| 9 | `ZIGZAG` | +| 10 | `FREQ` (deprecated) | +| 11 | `CHIMP` | +| 12 | `SPRINTZ` | +| 13 | `RLBE` | +| 14 | `CAMEL` | + +The following combinations are supported by the reference value encoders. An assigned codec +number does not mean that every data type can use it: + +| Data type | Available encodings | +| --- | --- | +| `BOOLEAN` | `PLAIN`, `RLE` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `FLOAT` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `DOUBLE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE`, `CAMEL` | +| `TEXT`, `STRING` | `PLAIN`, `DICTIONARY` | +| `BLOB`, `OBJECT` | `PLAIN` | + +## Encoding Families + +The identifiers describe different ways to transform a logical sequence into bytes. They are wire +format identifiers, not implementation-specific enumeration ordinals. + +| Family | Encodings | Data property typically exploited | +| --- | --- | --- | +| Literal | `PLAIN` | No prediction; values are serialized independently | +| Dictionary | `DICTIONARY` | Repeated binary values are replaced by dictionary references | +| Run length / bit packing | `RLE`, `RLBE` | Repeated values or small bit widths | +| Delta and prediction | `DIFF`, `TS_2DIFF`, `REGULAR`, `SPRINTZ` | Small or regular changes between adjacent values | +| Integer remapping | `ZIGZAG` | Small signed integers represented as small unsigned integers | +| XOR-based | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | Stable leading and trailing bits between adjacent floating-point values | +| Legacy or specialized | `BITMAP`, `FREQ` | Reserved historical layouts; `FREQ` is deprecated | + +Different distributions favor different encodings. Whatever the writer chooses, the reader +selects a decoder from the pair `(data_type, encoding_type)`. + +## Plain Encoding + +A `PLAIN` stream does not store an element count. The Page context, time stream, or null bitmap +determines the number of elements. Values are written consecutively: + +| Data type | Encoding of one value | +| --- | --- | +| `BOOLEAN` | `0x00` or `0x01` | +| `INT32`, `DATE` | `VarInt` | +| `INT64`, `TIMESTAMP` | Big-endian `int64` | +| `FLOAT` | Big-endian `float32` | +| `DOUBLE` | Big-endian `float64` | +| `TEXT`, `STRING`, `BLOB`, `OBJECT` | `VarInt byte_length` + raw bytes | + +The length of a `PLAIN` binary value is non-negative. A non-aligned value stream does not contain +nulls; the Value Page bitmap represents nulls in an aligned value stream. + +The other encoding identifiers select their respective value-stream bit layouts. A reader cannot +decode an unknown encoding as `PLAIN`. + +## Decoder Contract + +A decoder's state lasts for one Page stream. After decryption and decompression, select the codec, +decode within the stream boundary, and check that the number of values matches the Page context. +Start with fresh decoding state for the next Page. + +For aligned data, the Time Chunk Header names the time codec; an aligned Value Page decodes one +value for each set bitmap bit. For non-aligned data, the Chunk Header names only the value codec. +Its time codec comes from shared configuration, defaulting to `TS_2DIFF`; there is no serialized +`time_encoder` footer field. See [Configurations](../Configurations.md#non-aligned-time-encoding). + +An unsupported encoding prevents value decoding. A scanner that parses the raw Header fields +can still locate the next Chunk using `data_size`, even if a higher-level codec registry rejects +the identifier. It must not substitute `PLAIN` or another decoder. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md new file mode 100644 index 000000000..3d0c13879 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -0,0 +1,168 @@ + + +# Encryption + +Encryption is applied after Page compression and reversed before decompression. Headers, +Statistics, schemas, and indexes remain visible. TsFile v4 supports two encryption mechanisms, +both under version byte `0x04`: + +| Mode | How the reader recognizes it | Chunk and Page layout | +| --- | --- | --- | +| Plaintext | No file encryption header; legacy level `0` or absent | Base Chunk Header; unencrypted Page body | +| Legacy encryption | No file encryption header; legacy level `1` or `2` | Base Chunk Header; algorithm-specific encrypted body | +| Page AEAD | `0x06` file encryption header at offset `7` | Extended Chunk Header; authenticated Page body defined by a profile | + +Check for the file encryption header before parsing Chunks. In an AEAD file, the legacy footer +properties still say unencrypted; the header takes precedence. + +The write and read paths are: + +```text +encoded page payload -> compression -> encryption -> stored_data +stored_data -> authenticated decryption when applicable -> decompression -> value decoding +``` + +## File Encryption Header + +An AEAD file puts one `FileEncryptionHeader` immediately after the seven-byte magic/version +prefix, before any Chunk Group or operation-index record: + +```text +FileEncryptionHeader := + byte marker = 0x06 + UVarInt payload_size + byte header_version = 0x01 + VarString provider_id + VarString profile_id + VarString key_id + VarString key_version + UVarInt file_crypto_id_size = 16 + byte file_crypto_id[16] + UVarInt wrapped_data_key_size + byte wrapped_data_key[wrapped_data_key_size] +``` + +`payload_size` counts from `header_version` through the wrapped key, excluding the marker and the +length field itself. Its allowed range is `1..1048576`, and parsing must end exactly at that +boundary. The four string identifiers must be non-null and non-blank. The file crypto ID is +exactly 16 bytes, and the wrapped data key is non-empty. These binary lengths use `UVarInt`, not +the nullable `VarBinary` used for footer properties. + +The **provider** identifies the encryption implementation or service. Its **profile** names a +fixed byte-format contract: Page framing, nonce and tag representation, size mapping, and Chunk +proof size. A provider can offer several profiles. The file stores these public identifiers and +the wrapped data key, rather than an implementation class name or the plaintext key. The reader needs +the matching provider/profile and access to the external key material. + +Header version `0x01` is the supported layout. Unknown versions, truncated fields, trailing bytes, +a second encryption header, or a header anywhere except offset `7` are errors. + +## AEAD Chunk Header + +The file encryption header also changes how every Chunk Header is parsed. The Chunk marker +stays the same, but three fields follow `encoding_type`: + +```text + int64 chunk_ordinal + int32 chunk_authentication_size + byte chunk_authentication[chunk_authentication_size] +``` + +`chunk_ordinal` is a non-negative identifier allocated uniquely across the file, including Time +and Value Chunks. The proof length must match the profile and lie in `1..1024`. These fields are +part of the Header, so none of them count towards `data_size`. + +The proof binds the ordinal to the Chunk's absolute position. Verify it before using that +identity. Moving a Chunk changes the authenticated position and therefore requires a new proof. +Its associated data is the following big-endian sequence: + +```text +int32 domain_size = 20 +byte domain[20] = UTF8("TsFile/ChunkPosition") +int32 associated_data_version = 1 +byte file_crypto_id[16] +int64 absolute_chunk_marker_offset +int64 chunk_ordinal +``` + +## AEAD Page Body and Context + +For a non-empty Page, `compressed_size` records the entire encrypted body, including framing, +nonces, tags, and padding. It is larger than the compressed plaintext length, even with +`UNCOMPRESSED`. A profile may use fixed overhead or a size-dependent layout, so use its size +mapping to recover the candidate plaintext length. + +Page authentication also binds the body to its file, Chunk, and Page position. The associated +data is the exact big-endian sequence below. `SizedBytes` means an `int32` byte length followed +by those bytes; text is UTF-8. + +```text +int32 associated_data_version = 1 +SizedBytes provider_id +SizedBytes profile_id +SizedBytes key_id +SizedBytes key_version +SizedBytes file_crypto_id +int64 chunk_ordinal +int32 page_index +int32 uncompressed_size +int32 compressed_plaintext_size +``` + +`page_index` starts at zero for each physical Chunk and counts empty aligned Value Pages as well. +An all-null Value Page is still the single `UVarInt(0)` described in [Nulls](../Nulls.md); it has +no ciphertext or Page authentication tag. + +The candidate plaintext length becomes trusted only after authentication succeeds. Decryption +must return that length, and decompression must then return `uncompressed_size` bytes. An invalid +tag, wrong key, unsupported profile, or length mismatch is an error; do not retry as plaintext. + +## Legacy File Properties + +When there is no file encryption header, read the following `TsFileMetadata` properties: + +| Property | Meaning | +| --- | --- | +| `encryptLevel` | `0` for no encryption; `1` for a directly stored data key; `2` for a data key encrypted by an external master key | +| `encryptType` | Legacy encryption/decryption algorithm identifier | +| `encryptKey` | Text representation of the data key or key-wrapping result | + +These property values contain text. At level `1`, the data key is stored directly. At level `2`, +an external master-key decryptor unwraps it before Page decryption. Missing `encryptLevel` means +`0`; levels `1` and `2` require `encryptType` and a non-empty `encryptKey`. Unknown levels and +unavailable algorithms are errors. + +Legacy encryption uses the algorithm's encrypt/decrypt operations without the AEAD Chunk suffix +or Page context. `compressed_size` is still the actual stored output length; it equals compressed +plaintext length only for a length-preserving algorithm. + +AEAD writers set these legacy properties to level `0`, type +`org.apache.tsfile.encrypt.UNENCRYPTED`, and an empty key. The AEAD header supplies the real +mode and wrapped key, so those values must not be interpreted as a plaintext fallback. + +## Security Scope + +Page AEAD authenticates non-empty Page bodies and their context, while Chunk proofs bind ordinals +to file positions. It does not authenticate every visible field or provide a whole-file digest. +Legacy encryption does not necessarily provide authentication. See [Checksumming](Checksumming.md) +for whole-file integrity and [Error Recovery](Error-Recovery.md) for retaining encryption context +when rebuilding a footer. diff --git a/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md b/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md new file mode 100644 index 000000000..03a4274a4 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md @@ -0,0 +1,82 @@ + + +# Error Recovery + +The footer is written after the data section. If writing stops before that happens, some complete +Chunks may still be present even though the file cannot be opened through its indexes. Recovery +finds those structures by scanning forward from the file header, then rebuilds the missing indexes +and footer from the data it can retain. + +## Completion Check + +For a file of size `F`, read the big-endian metadata length `L` from `[F - 10, F - 6)`. A completed +file has these relationships: + +```text +bytes[F - 6, F) = "TsFile" +TsFileMetadata interval = [F - 10 - L, F - 10) +meta_offset = offset of the top-level 0x02 separator +meta_offset < F - 10 - L +``` + +Also check the leading magic and version, and parse exactly `L` bytes of `TsFileMetadata`. +Tail magic alone is insufficient: the footer length and its internal offsets must be usable. + +## Safe Forward Scan + +A forward scan uses the same Chunk layout as an ordinary reader, but does not rely on the footer: + +1. Check the magic and version. Inspect offset `7` and consume an optional `0x06` encryption + header. The data starts at `7 + E`, where `E` is the header's total size or zero when absent. +2. At a structure boundary, read a marker. For `0x00`, read the DeviceID and remember the group. +3. For a Chunk marker, read its complete Header, including the AEAD suffix when required. + Verify the Chunk proof before using its identity, then check that `data_size` fits in the + remaining file. Inspect the Pages or skip exactly that many bytes. +4. For `0x04`, consume the two operation-index `int64` values. +5. At the top-level `0x02` separator, stop scanning the data section. + +Stop at the last complete structure if a field or body is truncated. An unknown marker, a second +or misplaced encryption header, an invalid length, or an authentication failure must be treated +as an error, not skipped by searching for another familiar byte. A `0x02` inside a Page body is +data and cannot identify the end of the data section. + +Retain only structures that have been read completely and checked. The readable part of a truncated +Chunk is not automatically a valid Chunk. The common bounds checks are in +[Binary Representation](../Binary-Representation.md), and Page boundaries are in [Data Pages](README.md). + +## Recovery Limits + +Rebuilding an index cannot restore missing values or identify every change to otherwise valid +bytes. The retained Pages still need their codecs and, when encrypted, their key material. +Operation-index records can help a storage engine relate the retained data to its external log, +but they do not replace structure checks. + +For Page AEAD, preserve the original encryption header, file crypto ID, Chunk ordinals, and Page +indexes. Truncating the tail and replacing the footer can leave retained Chunks at their original +offsets. Moving a Chunk requires a new position proof; changing its file identity, ordinal, or +Page context also requires authenticating and re-encrypting the affected data as appropriate. +When appending, allocate ordinals above the highest retained ordinal. + +An authenticated recovery needs the matching provider and external keys. The legacy footer cannot +replace a damaged AEAD header, because its legacy encryption fields are deliberately set to +unencrypted. [Encryption](Encryption.md) describes that mode selection; [Checksumming](Checksumming.md) +explains which changes remain outside the format's integrity checks. diff --git a/src/UserGuide/latest/FileFormat/DataPages/README.md b/src/UserGuide/latest/FileFormat/DataPages/README.md new file mode 100644 index 000000000..04b196665 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/DataPages/README.md @@ -0,0 +1,174 @@ + + +# Data Pages + +A Chunk stores its Pages consecutively after the Chunk Header, without padding between them. +`data_size` counts all Page Headers and stored bodies together. An empty aligned Value Chunk +has `data_size = 0` and no Pages. + +## Page Header + +A non-empty Page has the following format: + +```text +Page := + UVarInt uncompressed_size + UVarInt compressed_size + Statistics statistics? # determined by the Chunk marker + byte stored_data[compressed_size] +``` + +`uncompressed_size` is the size of the Page payload after decryption and decompression. +`compressed_size` is the size of the Page payload stored in the file. The two values are equal +when `compression_type = UNCOMPRESSED` and no length-expanding encryption is used. For Page AEAD, +`compressed_size` includes the encrypted body framing and authentication overhead, even when +compression is disabled. See [Encryption](Encryption.md#aead-page-body-and-context). + +The low six bits of the Chunk marker determine whether Page Statistics are present: + +| Chunk marker low bits | Page count | Page Statistics | +| ---: | --- | --- | +| `0x01` | More than one | Present in every non-empty Page Header | +| `0x05` | Zero or one | Absent | + +The Statistics layout is defined in [Metadata](../Metadata.md#statistics). + +If a Page starts at `P`, let `h` be the serialized size of its two length fields and optional +Statistics. The stored payload occupies the half-open interval `[P + h, P + h + compressed_size)` +and the next Page, if any, starts at `P + h + compressed_size`. That end position must not exceed +the enclosing Chunk boundary. + +After decryption and decompression, the payload must contain exactly `uncompressed_size` bytes. +A decoder rejects a payload whose produced size differs, even if the compression library reports +successful termination. + +## Page Encoding + +When writing a non-empty Page, the data passes through these stages: + +```text +logical values + -> encode time and/or values + -> assemble uncompressed page payload + -> compression + -> optional encryption + -> stored_data +``` + +Reading applies the inverse order: decrypt, decompress, then decode values. + +Encoding and compression are independent. The encoding converts typed values into an +uncompressed byte stream; compression treats that stream as bytes. Page boundaries reset both +the value encoder/decoder and the compressor/decompressor, so no state is carried from one Page +to the next. + +## Non-aligned Page + +The uncompressed payload of a non-aligned Page is: + +```text +NonAlignedPagePayload := + UVarInt time_stream_size + byte encoded_time_stream[time_stream_size] + byte encoded_value_stream[remaining bytes] +``` + +A non-aligned Page stores both time and values. It does not use a separate Time Page followed by +a Value Page. The complete time stream is stored first, followed by the complete value stream; +`timestamp, value` pairs are not interleaved point by point. The two streams contain the same +number of elements and pair by position to form `(timestamp, value)` points. + +For an uncompressed payload of size `U` and a serialized `time_stream_size` field of `v` bytes, +the value stream length is: + +```text +value_stream_size = U - v - time_stream_size +``` + +The length must be non-negative. Both streams are bounded by the Page payload and must decode +to the same number of points, with valid endings for their respective codecs. + +A non-aligned Chunk Header records only the value stream's `encoding_type`. See +[Configurations](../Configurations.md#non-aligned-time-encoding) for the time-stream encoding. + +## Aligned Time Page + +The uncompressed payload of an aligned Time Page consists entirely of the encoded time stream: + +```text +AlignedTimePagePayload := byte encoded_time_stream[remaining bytes] +``` + +The `encoding_type` in the Time Chunk Header identifies the time encoding. + +The number of decoded timestamps is the logical position count of this Page. That count controls +the bitmap width and null reconstruction of every corresponding aligned Value Page. + +## Aligned Value Page + +The uncompressed payload of a non-empty aligned Value Page is: + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +Within one aligned set, Value Page number `i` corresponds to Time Page number `i`. See +[Nulls](../Nulls.md) for the null bitmap and the all-null empty Page representation. + +`position_count` must equal the decoded timestamp count of the corresponding Time Page. The +bitmap has exactly `ceil(position_count / 8)` bytes; unused low-order bits in its last byte do not +describe positions. The value decoder is invoked only for set bits and must produce exactly the +number of set bits. + +## Page Selection and Skipping + +Page Statistics can rule out a Page before its body is read. For example, a Page whose time +range misses the query range can be skipped. See [Statistics](../Metadata.md#statistics) for +which value summaries are available. + +A one-Page Chunk omits Page Statistics, so use its Chunk or series Statistics from the footer. +If the available summary cannot rule out a match, read the Page and check its values. + +## Empty Aligned Value Page + +An aligned value column can be null for every position in a Time Page. This case is encoded as a +single unsigned-varint zero: + +```text +EmptyAlignedValuePage := UVarInt(0) # uncompressed_size +``` + +There is no `compressed_size`, Page Statistics, bitmap, or stored payload after that zero. The +position count is inherited from the corresponding Time Page. This representation is valid only +where the aligned Page sequence supplies that external position count. + +## Data Page Details + +- [Chunks](Chunks.md) +- [Compression](Compression.md) +- [Encodings](Encodings.md) +- [Encryption](Encryption.md) +- [Checksumming](Checksumming.md) +- [Error Recovery](Error-Recovery.md) diff --git a/src/UserGuide/latest/FileFormat/Extensibility.md b/src/UserGuide/latest/FileFormat/Extensibility.md new file mode 100644 index 000000000..4588bf499 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Extensibility.md @@ -0,0 +1,56 @@ + + +# Extensibility + +TsFile has three main ways to introduce new capabilities: assign a new type or codec identifier, +add a footer property, or define a new file layout. The choice depends on whether a reader can +still locate the next structure without understanding the new feature. + +Footer properties are easy to delimit. Each has a `VarString` key and a nullable `VarBinary` +value, so a reader can preserve an unknown value without decoding it. A Chunk also has a known +byte length, which lets a scanner skip it even if it cannot decode the selected codec. +Data-section markers have no shared length field, so the same approach does not work for an +unknown marker. + +## Safe Evolution Rules + +A compatible extension keeps existing field meanings and boundaries intact: + +- Give a new type or codec an unused identifier; keep the meaning of assigned identifiers. +- Define the supported data types, byte layout, output sizes, and error handling for a new codec. +- Keep the existing byte order and length conventions. +- Report an unsupported feature when it is needed to decode the requested data. + +Adding bytes to a Header without a rule that selects the extended layout shifts the fields an +older reader expects. Such a change requires a new version or an explicit extension mechanism. + +The Page AEAD extension is an example of the latter. Its `0x06` file encryption header selects +extra Chunk Header fields and profile-specific Page framing while retaining version byte `0x04`. +Older v4 readers may reject those files. [Encryption](DataPages/Encryption.md) defines the layout +and required capabilities; knowing the base format version alone is not enough. + +## Properties and Semantics + +Being able to skip a property does not mean its meaning can always be ignored. An informational +property may be optional, but a property that changes how Page bytes are decoded needs an explicit +support rule. If a reader cannot apply required encryption or another required interpretation, +it must report an error rather than decode the bytes with a default. diff --git a/src/UserGuide/latest/FileFormat/Format-Versions.md b/src/UserGuide/latest/FileFormat/Format-Versions.md new file mode 100644 index 000000000..42c17472f --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Format-Versions.md @@ -0,0 +1,61 @@ + + +# Format Versions + +The format version is the byte immediately after the leading magic string: + +```text +6-byte magic string "TsFile" +1-byte format version +``` + +This guide describes `0x04`. The format byte and the project release number serve different +purposes: a release in the 2.x series can use file format v4. The project release label is not +stored in the file header. + +Version 4 uses segmented DeviceIDs and stores per-table device-index roots and Table Schemas +in `TsFileMetadata`. Version 3 uses a single-string device identifier and a different footer. +A reader needs the corresponding version's structures to parse either file correctly. + +## Reader Dispatch + +Read the version before interpreting data markers, then select the encryption mode: + +```text +validate leading magic -> read version -> inspect optional encryption header + -> resolve encryption mode/profile -> parse the matching data/footer grammar +``` + +If the observed version is unsupported, report that before parsing the data. A familiar marker +or tail magic does not establish that the rest of the file uses the expected layout. + +## Compatibility Contract + +Within a supported version, a reader also needs the types, codecs, and encryption features used +by the requested data. For example, v4 Page AEAD files have a `0x06` encryption header at offset +`7`, which selects an extended Chunk Header and a Page-body profile. The version byte stays +`0x04`; a v4 reader without this capability must report it as unsupported. + +Assigned identifiers keep their meanings. Unknown footer properties have length-delimited keys +and values, so their byte ranges can be consumed. Unknown data markers generally cannot be +skipped because they have no common payload length. [Extensibility](Extensibility.md) explains +these boundaries, and [Encryption](DataPages/Encryption.md) defines the AEAD layout. diff --git a/src/UserGuide/latest/FileFormat/Metadata.md b/src/UserGuide/latest/FileFormat/Metadata.md new file mode 100644 index 000000000..7cdc5824a --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Metadata.md @@ -0,0 +1,295 @@ + + +# Metadata + +TsFile metadata falls into two categories: + +- Chunk Headers, Page Headers, and Page Statistics stored with the data. +- Timeseries Metadata, Metadata Indexes, and File Metadata stored after the data section. + +Footer metadata starts after `SEPARATOR (0x02)`. Ranges of `TimeseriesMetadata` and measurement +index nodes may be interleaved. The `offset` and `end_offset` fields in the indexes delimit each +range. Every offset is an absolute byte offset measured from the beginning of the file. + +A lookup follows the hierarchy below. Each level narrows the search, from a table to the Pages +that may contain matching values: + +| Level | Identifies | Primary use | +| --- | --- | --- | +| File metadata | Tables and device-index roots | Enter the correct index tree | +| Device index | Devices | Locate one device's measurement index | +| Measurement index | Measurement-name ranges | Locate Timeseries Metadata | +| Timeseries Metadata | One logical series | Select Chunk offsets and series Statistics | +| Chunk Metadata | One physical Chunk | Seek to data and prune by Chunk Statistics | +| Page Header | One Page | Prune by Page Statistics and read its payload | + +## Statistics + +Page-, Chunk-, and timeseries-level Statistics use the same type-specific structure: + +```text +Statistics := + UVarInt count + int64 start_time + int64 end_time + ... type-specific fields +``` + +`count` is the number of values covered by the structure. It equals the number of points in a +non-aligned series, the number of non-null values in an aligned value column, or the number of +timestamps in an aligned time column. + +Type-specific fields immediately follow the common prefix in the order shown below: + +| Data type | Type-specific fields | +| --- | --- | +| `BOOLEAN` | `bool first`, `bool last`, `int64 true_count` | +| `INT32`, `DATE` | `int32 min`, `int32 max`, `int32 first`, `int32 last`, `int64 sum` | +| `INT64`, `TIMESTAMP` | `int64 min`, `int64 max`, `int64 first`, `int64 last`, `float64 sum` | +| `FLOAT` | `float32 min`, `float32 max`, `float32 first`, `float32 last`, `float64 sum` | +| `DOUBLE` | `float64 min`, `float64 max`, `float64 first`, `float64 last`, `float64 sum` | +| `TEXT` | `Binary32 first`, `Binary32 last` | +| `STRING` | `Binary32 first`, `Binary32 last`, `Binary32 min`, `Binary32 max` | +| `BLOB`, `OBJECT`, `VECTOR` | No type-specific fields | + +Statistics do not carry a data type identifier. The type used to parse Statistics comes from the +Chunk Header or Timeseries Metadata. + +`first` and `last` describe value order, not the minimum and maximum. For an ordered timeseries, +they correspond to the values at the first and last covered timestamps. The `start_time` and +`end_time` fields are inclusive. Numeric `sum` fields are intended for aggregation; notably, an +`INT64` sum is stored as `float64` and can lose integer precision for large magnitudes. + +Statistics help rule out data that cannot match a query. For example, a Page whose time range +falls entirely before the requested range can be skipped. Overlapping ranges only mean the Page +might match; the reader still needs to check its values. Types without minimum/maximum +fields, such as `BLOB`, `OBJECT`, and `TEXT`, cannot support range pruning through Statistics. + +Chunk Statistics summarize all Pages in that Chunk; series Statistics summarize all of its +Chunks. Parent counts add the child counts, and parent time bounds cover the child ranges. + +## Timeseries Metadata + +Each timeseries has one `TimeseriesMetadata` structure: + +```text +TimeseriesMetadata := + byte timeseries_type + VarString measurement_id + byte data_type + UVarInt chunk_metadata_list_size + Statistics series_statistics + byte chunk_metadata_list[chunk_metadata_list_size] +``` + +The high bits of `timeseries_type` identify the column role, and the low six bits identify the +Chunk Metadata layout: + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0: exactly one Chunk; ChunkMetadata omits Statistics +low 6 bits = 1: more than one Chunk; every ChunkMetadata contains Statistics +``` + +## Chunk Metadata + +When `(timeseries_type & 0x3f) == 0`, the timeseries has exactly one Chunk: + +```text +ChunkMetadata := int64 offset_of_chunk_header +``` + +The Chunk uses `series_statistics` as its Statistics. + +When `(timeseries_type & 0x3f) != 0`: + +```text +ChunkMetadata := + int64 offset_of_chunk_header + Statistics chunk_statistics +``` + +`offset_of_chunk_header` points to the Chunk marker, which is the first byte of the Chunk Header. + +`chunk_metadata_list_size` is the serialized byte length of the list. Add it to the list start +to find the end, then read Chunk Metadata records until that end is reached exactly. The field +does not give the number of Chunks. + +## Metadata Index + +```text +MetadataIndexNode := + UVarInt child_count + MetadataEntry child[child_count] + int64 end_offset + byte node_type +``` + +`node_type` determines the node level, Entry encoding, and referenced data: + +| Value | Name | Entry encoding | Entry refers to | +| ---: | --- | --- | --- | +| 0 | `INTERNAL_DEVICE` | `DeviceIndexEntry` | A child device-index node | +| 1 | `LEAF_DEVICE` | `DeviceIndexEntry` | The measurement-index root of a device | +| 2 | `INTERNAL_MEASUREMENT` | `MeasurementIndexEntry` | A child measurement-index node | +| 3 | `LEAF_MEASUREMENT` | `MeasurementIndexEntry` | A range of Timeseries Metadata | + +The Entry structures are: + +```text +DeviceIndexEntry := + DeviceID first_device_id + int64 offset + +MeasurementIndexEntry := + VarString first_measurement_id + int64 offset +``` + +An Entry describes a half-open byte range `[offset, next_offset)`. The next Entry supplies +`next_offset`; for the last Entry, use the node's `end_offset`. The end byte belongs to the next +range, not the current one. + +```text +entry[0].offset <= entry[1].offset <= ... <= end_offset +``` + +An Entry key is the first key in the range it covers. Device indexes compare `DeviceID` values +segment by segment, with null before non-null and a shorter tuple before a longer tuple when their +common prefixes are equal. Measurement indexes use lexicographic measurement-name order. + +### Index Lookup + +To find key `K` in an index node, select the last Entry whose first key is less than or equal to +`K`. The selected byte range begins at that Entry's `offset` and ends at the next Entry's offset, +or at the node's `end_offset` for the last Entry. An internal node range contains another index +node; a leaf range contains the object named in the table above. + +If no Entry key is less than or equal to `K`, the key is absent from this node. A measurement +leaf can cover several Timeseries Metadata records; read that range to find the exact measurement. + +Before following a node, check that: + +- `child_count` is consistent with the number of complete Entries in the node. +- Entry keys are sorted according to the node's key comparator. +- Entry offsets are monotonic and each referenced range is non-negative. +- The last Entry offset does not exceed `end_offset`. +- Child nodes have the level expected by the parent node type. + +The first-key representation allows an implementation to binary-search Entries within a node. +The file format does not require a particular in-memory search algorithm. + +## File Metadata + +`TsFileMetadata` immediately precedes the file's trailing length field: + +```text +TsFileMetadata := + UVarInt table_index_count + TableIndexRoot table_index[table_index_count] + UVarInt table_schema_count + NamedTableSchema table_schema[table_schema_count] + int64 meta_offset + BloomFilter bloom_filter + VarInt property_count + Property property[property_count] + +TableIndexRoot := + VarString table_name + MetadataIndexNode device_index_root + +NamedTableSchema := + VarString table_name + TableSchema schema + +Property := + VarString key + VarBinary value +``` + +Property keys are UTF-8 strings; values are nullable raw byte arrays, not UTF-8 strings. +For example, value bytes `ff fe 00 80` serialize as `08 ff fe 00 80`: `08` is `VarInt(4)`. +A null value is `01` (`VarInt(-1)`), while an empty value is `00`. Preserve unknown values +byte-for-byte, and decode them as text only when the property contract requires it. + +`meta_offset` points to the `0x02` separator at the end of the data section. There is no marker +before `TsFileMetadata`; its starting position is determined by the 4-byte length stored at the +end of the file. + +The physical order of map entries has no file-format meaning. `table_index` contains device-index +roots for both tree-model and table-model data. `table_schema` contains schemas for the explicit +table model. + +The trailing four-byte metadata length is outside `TsFileMetadata`. For a file length `F` and +metadata length `L`, a reader obtains the exact byte interval using: + +```text +file_metadata_start = F - 6 - 4 - L +file_metadata_end = F - 6 - 4 # exclusive +``` + +The reader must consume exactly `L` bytes. This check detects a schema mismatch or malformed +length-delimited field before the trailing magic is accepted as the file terminator. + +## Table Schema + +```text +TableSchema := + UVarInt column_count + TableColumn column[column_count] + +TableColumn := + MeasurementSchema schema + int32 column_category + +MeasurementSchema := + String32 column_name + byte data_type + byte encoding_type + byte compression_type + int32 property_count + SchemaProperty property[property_count] + +SchemaProperty := + String32 key + String32 value +``` + +`MeasurementSchema` uses `String32`, rather than the `VarString` used by most other footer fields. + +Column category identifiers are: + +| Value | Category | +| ---: | --- | +| 0 | `TAG` | +| 1 | `FIELD` | +| 2 | `ATTRIBUTE` | +| 3 | `TIME` | + +The schema preserves column order. In particular, TAG column order determines the order of +DeviceID segments. FIELD Chunks are identified by `measurement_id`; their physical order does not +have to match the FIELD column order in the schema. + +`TAG` columns identify a device, `FIELD` columns hold measured values, `ATTRIBUTE` columns +describe an entity, and `TIME` identifies the time column. A column needs both its category and +its `data_type`: the category describes its role, while the type describes its values. diff --git a/src/UserGuide/latest/FileFormat/Nulls.md b/src/UserGuide/latest/FileFormat/Nulls.md new file mode 100644 index 000000000..5779d5e7d --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Nulls.md @@ -0,0 +1,72 @@ + + +# Nulls + +A non-aligned timeseries stores only the `(timestamp, value)` points that exist. Its Pages do not +contain null markers. + +An aligned value column records nulls in the bitmap of each Value Page: + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +Position `i` corresponds to bit `7 - (i mod 8)` of `present_bitmap[i / 8]`: + +- A set bit means that the position has a value. +- A clear bit means that the position is null. + +The encoded value stream contains only values whose bitmap bits are set, in position order. +`position_count` equals the number of timestamps in the corresponding Time Page. + +For example, with `position_count = 10`, bitmap bytes `10110000 01000000` indicate values at +positions `0`, `2`, `3`, and `9`. The value decoder therefore produces four values and places them +at those positions; the remaining six positions are null. Unused low-order bits in the final byte +are not logical positions and are ignored. + +For a non-empty Value Page the following invariants hold: + +```text +bitmap_bytes = ceil(position_count / 8) +decoded_value_count = popcount(present_bitmap over position_count bits) +position_count = decoded_timestamp_count of the corresponding Time Page +``` + +A mismatch is structural corruption; the reader must not infer additional positions from trailing +bitmap bits or encoded values. + +If an entire Value Page is null, the Page consists only of an `uncompressed_size` equal to zero: + +```text +EmptyAlignedValuePage := UVarInt(0) +``` + +An empty Page has no `compressed_size`, Statistics, bitmap, or value stream. Its position count is +obtained from the corresponding Time Page. + +An empty Value Page preserves its position in the Page sequence, so subsequent Value Pages still +match the correct Time Pages. An absent Value Chunk has no physical column for that aligned Chunk +set. A Value Chunk with `data_size = 0` is a third case: its Header exists, but it contains no +Pages. These cases must be distinguished when walking the aligned Chunk set. diff --git a/src/UserGuide/latest/FileFormat/README.md b/src/UserGuide/latest/FileFormat/README.md new file mode 100644 index 000000000..b32c2bedd --- /dev/null +++ b/src/UserGuide/latest/FileFormat/README.md @@ -0,0 +1,149 @@ + + +# File Format + +A TsFile stores time series data in columns. Writers append data in device groups, then write +the indexes at the end of the file. Readers usually work in the opposite direction: they open +the footer, look up a series, and seek to the Chunks that contain its values. + +This guide describes the v4 file layout, identified by version byte `0x04`. It covers the file +structures and how they fit together. The encoding chapter gives codec identifiers and the +`PLAIN` layout; the bit layouts of the other value codecs are outside its scope. + +## Scope and Terminology + +Data has three levels of organization within a file: + +- A **Page** holds a batch of encoded values. Each Page is compressed independently. +- A **Chunk** holds the Pages of one physical column. Its header identifies the data type, + encoding, and compression method. +- A **Chunk Group** holds a batch of Chunks for one device. The same device can appear in several + Chunk Groups as the writer flushes more data. + +In **non-aligned** data, each measurement stores its own timestamps alongside its values. In +**aligned** data, a Time Chunk stores the timestamps shared by several Value Chunks. A bitmap in +each Value Page identifies the positions that have values. [Data Pages](DataPages/README.md) +explains both layouts. + + + +## Physical Regions + +The file begins with the six-byte magic string `TsFile` and a one-byte version. An optional +file encryption header follows for Page AEAD files. The data section ends at a single `0x02` +separator; indexes and file metadata follow it. The last ten bytes hold the metadata length and +the trailing magic string. + +The table below uses these names: + +- `F`: total file length. +- `L`: serialized length of `TsFileMetadata`, stored in the four-byte trailing length field. +- `E`: total serialized length of the optional encryption header, or `0` when absent. +- `data_start = 7 + E`: start of the data section. +- `meta_offset`: separator offset, stored inside `TsFileMetadata`. + +| Region | Inclusive byte interval | How it is located | +| --- | --- | --- | +| Header magic | `0 … 5` | Fixed position | +| Version | `6` | Fixed position | +| Optional file encryption header | `7 … 6 + E` | Present only for Page AEAD; marker `0x06` | +| Data section | `data_start … meta_offset - 1` | Ends immediately before the separator | +| Separator | `meta_offset` | `0x02`, referenced by `TsFileMetadata` | +| Indexed metadata | `meta_offset + 1 … F - 11 - L` | Delimited by metadata offsets | +| `TsFileMetadata` | `F - 10 - L … F - 11` | Located from the trailing length | +| Metadata length | `F - 10 … F - 7` | Fixed position relative to the end | +| Tail magic | `F - 6 … F - 1` | Fixed position relative to the end | + +All offsets are absolute, starting at the first byte of the file. The table uses inclusive +intervals; `[start, end)` elsewhere denotes a half-open interval. A length excludes its own +length field unless stated otherwise. The optional-header interval is absent when `E = 0`. + +Timeseries Metadata and index nodes can be interleaved in the indexed metadata section. Readers +follow their offsets rather than assuming a fixed physical order. `TsFileMetadata` has no marker +of its own: its length locates it from the end of the file. + +## Write Model + +A writer appends Chunk Groups, then writes the indexes and footer when it finishes the file. +Page sizes, flush thresholds, and index fanout affect where structures begin and end, but readers +can recover those boundaries from headers and offsets. Timestamp units and non-aligned time +encoding need the shared context described in [Configurations](Configurations.md). + +Both magic strings use the same bytes: + +```text +54 73 46 69 6c 65 # "TsFile" +``` + +## Metadata-first Read Algorithm + +A reader can locate the requested series without scanning the data section: + +1. Check the leading magic and version. Inspect offset `7` for an optional + [file encryption header](DataPages/Encryption.md#file-encryption-header), which determines + how Chunk Headers and Page bodies are parsed. +2. Check the tail magic, read `L` at `F - 10`, and calculate `file_metadata_pos = F - 10 - L`. +3. Read `TsFileMetadata` and check that `meta_offset` points to the separator. +4. Optionally check the complete path in the [Bloom filter](Bloom-Filter.md). Use a negative + result only when the writer's charset, path construction, and hash behavior are known to + match. +5. Follow the table/device index, then the measurement index, to find the series metadata. +6. Use the series and Chunk Statistics to select Chunk offsets. Seek to those Chunks and use + any Page Statistics to narrow the selection further. +7. Read the selected Page bodies, decrypt when required, decompress, and decode their streams. + +This avoids reading the payloads of unrelated Pages. A single-Page Chunk omits Page Statistics, +so pruning at that level relies on the Chunk or series Statistics instead. + +## Structural Invariants + +The main relationships to check when reading are: + +- Both magic strings and the version are valid. +- A file encryption header, when present, appears once at offset `7`; its length and version + are valid. +- `0 <= L <= F - 10`, and `file_metadata_pos` lies after the separator. +- A Chunk's Pages occupy exactly `data_size` bytes after its complete Header. +- Each non-empty Page body occupies exactly `compressed_size` bytes within that Chunk. +- Index offsets are monotonic (non-decreasing) within a node and do not exceed `end_offset`. +- An aligned Value Page covers the same positions as the Time Page with the same ordinal; its + `position_count` equals the Time Page's position count. + +A violation is a format error. + +[Binary Representation](Binary-Representation.md) describes the common length and offset checks. +[Encryption](DataPages/Encryption.md) adds the authentication checks for Page AEAD. Marker bytes +have meaning only at structure boundaries; the same bytes can occur inside a Page body. + +## Format Details + +For a first reading, start with [Binary Representation](Binary-Representation.md), +[Chunks](DataPages/Chunks.md), and [Data Pages](DataPages/README.md), then follow the metadata +lookup in [Metadata](Metadata.md). + +- [Types](Types.md) and [Encodings](DataPages/Encodings.md) +- [Nulls](Nulls.md) and [Compression](DataPages/Compression.md) +- [Bloom Filter](Bloom-Filter.md) +- [Encryption](DataPages/Encryption.md) and [Checksumming](DataPages/Checksumming.md) +- [Error Recovery](DataPages/Error-Recovery.md) +- [Configurations](Configurations.md), [Format Versions](Format-Versions.md), and + [Extensibility](Extensibility.md) diff --git a/src/UserGuide/latest/FileFormat/Types.md b/src/UserGuide/latest/FileFormat/Types.md new file mode 100644 index 000000000..cb824f5c0 --- /dev/null +++ b/src/UserGuide/latest/FileFormat/Types.md @@ -0,0 +1,88 @@ + + +# Types + +Chunk Headers, Timeseries Metadata, and Table Schemas use a single-byte `data_type` identifier: + +| Value | Name | Physical or logical meaning | +| ---: | --- | --- | +| 0 | `BOOLEAN` | Boolean value | +| 1 | `INT32` | Signed 32-bit integer | +| 2 | `INT64` | Signed 64-bit integer | +| 3 | `FLOAT` | IEEE 754 binary32 | +| 4 | `DOUBLE` | IEEE 754 binary64 | +| 5 | `TEXT` | Length-prefixed byte string | +| 6 | `VECTOR` | Physical type marker used by an aligned Time Chunk | +| 7 | `UNKNOWN` | Unknown type marker | +| 8 | `TIMESTAMP` | Signed 64-bit integer; the upper layer defines the time unit | +| 9 | `DATE` | Signed 32-bit integer in decimal `YYYYMMDD` form | +| 10 | `BLOB` | Arbitrary byte string | +| 11 | `STRING` | UTF-8 string | +| 12 | `OBJECT` | Byte representation of an object value | + +For example, the physical `DATE` value for 2026-07-27 is the decimal integer `20260727`, not a +Unix epoch day. + +The data type determines the additional Statistics fields and the input type of the value +encoder. See [Encodings](DataPages/Encodings.md) for wire representations of values. + +## Physical and Logical Types + +A data type describes the value domain; the encoding determines how those values become bytes. +`TIMESTAMP` and `DATE` also assign a meaning to the underlying integer: + +- `TIMESTAMP` has the same signed 64-bit value domain as `INT64`. The time unit is supplied + by the protocol or dataset using the file; TsFile v4 does not serialize that unit. +- `DATE` has the same integer encoder interface as `INT32`, but valid logical values use decimal + `YYYYMMDD` form. +- `TEXT` is a length-delimited byte string. `STRING` uses the same physical binary container but + has UTF-8 string semantics. `BLOB` is uninterpreted binary data. +- `OBJECT` stores object bytes. The applications exchanging the file agree on how to reconstruct + an object from them. + +`VECTOR` is a physical marker for the shared time column of aligned data. It is not a scalar value +type and does not define a standalone value-stream encoding. `UNKNOWN` is a sentinel, not a valid +type for a decodable measurement Page. + +## Type Consistency + +The same series type is repeated at different levels so a reader can enter the file through the +footer or through a sequential scan. These copies must agree: + +- A non-aligned or aligned Value Chunk Header uses the measurement's scalar `data_type`. +- Its Timeseries Metadata uses the same `data_type`. +- Statistics are decoded using that type and contain no independent type byte. +- An aligned Time Chunk uses `VECTOR`; its decoded values are signed 64-bit timestamps. + +Changing the interpretation of a previously assigned identifier is incompatible. A reader that +does not support a type identifier may still skip a complete Chunk using `data_size`, but it +cannot decode values or type-specific Statistics for that series. + +## Ordering and Comparison + +Integer and floating-point values use their numeric order for min/max Statistics. `STRING` +min/max values compare their raw value bytes lexicographically, excluding length prefixes. `TEXT` stores only +first and last values, while `BLOB` and `OBJECT` store no type-specific value Statistics. + +Type promotion and application-level casts are not encoded in TsFile. A reader can expose such +conversions, but they do not change the stored `data_type` or the bytes used to compute +Statistics. diff --git a/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md new file mode 100644 index 000000000..52769eaf3 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Binary-Representation.md @@ -0,0 +1,123 @@ + + +# 二进制表示 + +所有偏移量都从文件首字节开始计算,所有长度均以字节为单位。除非单独说明,固定宽度 +整数和 IEEE 754 浮点数使用大端序,字符串使用 UTF-8。 + +
+ +## 定长数值 + +| 记号 | 字节数 | 含义 | +| --- | ---: | --- | +| `byte` | 1 | 8 位原始值 | +| `bool` | 1 | `0x00` 为 false,`0x01` 为 true | +| `int32` | 4 | 大端、有符号 32 位整数 | +| `int64` | 8 | 大端、有符号 64 位整数 | +| `float32` | 4 | 大端 IEEE 754 binary32 | +| `float64` | 8 | 大端 IEEE 754 binary64 | + +
+ +## 无符号变长整数 + +`UVarInt` 是无符号 LEB128 风格的 32 位变长整数。每个字节的低 7 位保存数据,最高位 +为 1 表示后面仍有字节,低位组先写。 + +```text +value = Σ ((byte[i] & 0x7f) << (7 * i)) +``` + +编码使用最短表示,占 1 至 5 字节。例如,十进制 `300` 可以拆成从低到高的七位组 +`0x2c` 和 `0x02`,因此写成 `ac 02`。32 位解码器最多读取五个字节,同时检查数值是否 +溢出;五个字节后仍未结束的序列无效。 + +
+ +## 有符号变长整数 + +`VarInt` 先对有符号 32 位整数做 ZigZag 映射,再按 `UVarInt` 写入: + +```text +zigzag(n) = (n << 1) XOR (n >> 31) +``` + +| 原值 | 编码前的无符号值 | 字节 | +| ---: | ---: | --- | +| 0 | 0 | `00` | +| -1 | 1 | `01` | +| 1 | 2 | `02` | +| -2 | 3 | `03` | + +
+ +## 字符串与字节数组 + +| 记号 | 格式 | +| --- | --- | +| `VarString` | `VarInt byte_length` + `byte[byte_length]` | +| `String32` | `int32 byte_length` + `byte[byte_length]` | +| `Binary32` | `int32 byte_length` + `byte[byte_length]` | +| `VarBinary` | `VarInt byte_length` + `byte[byte_length]` | + +对于 `VarString`、`String32` 和 `VarBinary`,长度 `-1` 表示 null,长度 `0` 表示空字符串 +或空字节串;`Binary32` 不接受负长度。`VarString` 和 `String32` +保存 UTF-8;`Binary32` 和 `VarBinary` 保存原始字节。`VarBinary` 使用有符号 ZigZag +长度,不同于 [FileEncryptionHeader](DataPages/Encryption.md#file-encryption-header) +中二进制组件使用的无符号长度。 + +字符串长度按 UTF-8 编码后的字节数计算。例如,“温”占三个字节,其 `VarString` 长度字段 +为 `VarInt(3)`,编码为 `06`。允许 null 的字段还需要区分 null 和空值,不能在读取时将 +两者合并。 + +## 边界与防御性解码 + +每个带长度字段都受更高层边界约束:字符串受所属记录约束,Page 受 Chunk 约束,Footer +对象受元数据 offset 或文件尾元数据长度约束。为长度 `N` 分配内存或创建切片前,读取器 +需要验证: + +```text +N >= 0 # 文档规定的 null 哨兵除外 +current_offset + N >= current_offset +current_offset + N <= enclosing_end +``` + +使用定长整数计算偏移量时,第二个比较用于检测溢出回绕。应先检查这些边界,再分配内存, +并根据运行环境限制最大分配量:长度即使没有超出文件,也可能超过读取器能承受的范围。 + +固定宽度值必须具有完整字节数。截断的 `int64`、没有终止字节的 VarInt、非 null 字段的 +负长度、要求字符串位置的无效 UTF-8,以及带长度记录结束后的尾随字节均属于格式错误。 + +## 偏移约定 + +元数据中的 offset 以有符号 64 位整数保存,但表示非负绝对字节位置。offset 指向所命名 +结构的第一个字节,例如 `offset_of_chunk_header` 指向 Chunk marker,`meta_offset` 指向 +`0x02` separator。计算结构末尾时,将起始 offset 与长度相加,并按上文检查溢出和所属结构边界。 + +## 如何阅读布局示例 + +结构示意中的 `|` 表示二选一,括号表示一个分组,`?` 表示字段或结构可选,`*` 表示 +可以重复零次或多次,`+` 表示至少出现一次,`[n]` 表示 n 个元素。例如 `(A | B)*` 表示 +零个或多个条目,每个条目是 A 或 B。具体是否出现、重复多少次,由旁边的说明或已读取的 +长度字段决定。`remaining bytes` +表示当前载荷边界内尚未使用的字节;对于 Page,这个边界指解密、解压后的载荷末尾。 diff --git a/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md new file mode 100644 index 000000000..317988df4 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Bloom-Filter.md @@ -0,0 +1,135 @@ + + +# 布隆过滤器 + +`TsFileMetadata` 中可选的布隆过滤器用于省去不必要的索引查找。检查到某一位为零时, +可以排除该路径;检查的位全为一时,只能判断“可能存在”,仍需查索引。这一结论依赖 +查询端采用与写入端一致的路径编码和哈希方式,下面分别说明。 + +
+ +## 存储布局 + +```text +BloomFilter := + UVarInt byte_length + byte bits[byte_length] + UVarInt bit_count? # byte_length > 0 时存在 + UVarInt hash_function_count? # byte_length > 0 时存在 +``` + +`byte_length = 0` 表示文件没有布隆过滤器,后两个字段不存在。 + +位编号 `i` 存储在 `bits[i / 8]` 的位 `i mod 8`,即每个字节从低位方向编号。尾部全零 +字节不写入文件,逻辑位数由 `bit_count` 保存。 + +
+ +## 路径如何转为字节 + +写入端使用 UTF-8 时,过滤器输入是时间序列完整路径的 UTF-8 字节: + +```text +UTF8(join(DeviceID segments, ".") + "." + measurement_id) +``` + +null 设备片段在路径文本中表示为 ASCII `null`。构建路径字节时使用的字符集不保存在 +文件中。下列示例使用 UTF-8。读取器必须匹配写入端的字符集和哈希行为;无法确定时应 +绕过布隆过滤器,直接查询元数据索引。 + +
+ +## 哈希计算 + +TsFile 使用 `Murmur128Hash` 所实现的 MurmurHash3 x64 128 位变体。完整的 16 字节块 +使用标准的小端无符号字节组装与混合步骤;不足 16 字节的尾块与标准算法不同:**每个尾部 +字节先从有符号 8 位符号扩展到 64 位,再移位;各项使用 XOR 而非 OR 合并**。 + +设尾块长度 `n = byte_length mod 16`,尾部字节为 `t`,按下式组装两个字。全部移位和运算 +限制在 64 位;`signed64(b)` 将字节 `0x80..0xff` 映射为 `-128..-1`。 + +```text +k1 = XOR(signed64(t[j]) << (8 * j)) for j = 0 .. min(n, 8) - 1 +k2 = XOR(signed64(t[j]) << (8 * (j - 8))) for j = 8 .. n - 1 +``` + +空 XOR 的结果为 0。仅当 `n >= 9` 时对 `k2` 做标准尾部混合,仅当 `n >= 1` 时对 `k1` +做标准尾部混合,再执行标准 MurmurHash3 的长度 XOR 与 finalization。将最终的 `h1` +和 `h2` 按 64 位相加,取低 32 位并解释为有符号整数 `hash32`。读取现有文件时必须保持 +这种有符号尾块行为;直接替换为标准 MurmurHash3 库可能让非 ASCII 路径出现假阴性。 + +```text +bit_index = (0 if hash32 == -2147483648 else abs(hash32)) mod bit_count +``` + +哈希 seed 依次为: + +```text +5, 7, 11, 19, 31, 37, 43, 59 +``` + +实际使用的 seed 数量由 `hash_function_count` 指定,最大为 8。 + +### UTF-8 测试向量 + +以下向量使用 `seed = 5` 和 `bit_count = 256`。最后一行还覆盖有符号尾块之前存在完整 +16 字节块的情况。 + +| 路径 | 有符号 `hash32` | 位编号 | +| --- | ---: | ---: | +| `root.sg.d.s1` | -1267668613 | 133 | +| `root.设备.s1` | 1697017247 | 159 | +| `root.abcdefghijklmnop.设备.s1` | 866805081 | 89 | + +对于 `root.设备.s1`,标准 MurmurHash3 在最终同样求和后得到 `1978455175`,对应位编号 +`135`。读取器若检查该位,可能把文件中实际存在的路径判为不存在。 + +## 查询过程 + +对一个精确路径进行查询时,读取器: + +1. 按相同的 `DeviceID` 分片和测点规则构造路径字节; +2. 使用前 `hash_function_count` 个 seed 计算位置; +3. 检查逻辑 `bit_count` 位向量中的对应位置; +4. 任意一位为 0 时返回“肯定不存在”,否则返回“可能存在”。 + +“可能存在”仍需访问元数据索引确认。只有查询端使用与写入器完全相同的路径规范化方式、 +字符集及有符号尾块哈希时,“肯定不存在”才可靠。否则应跳过该优化并查询元数据索引。 + +## 空间与概率 + +令插入路径数为 `n`、逻辑位数 `bit_count` 为 `m`、哈希函数数为 `k`,标准布隆过滤器的 +近似关系为: + +```text +false_positive_probability ≈ (1 - exp(-k * n / m))^k +optimal_k ≈ (m / n) * ln(2) +``` + +这些公式解释写入器在空间和准确率之间的权衡,不是额外的文件字段。读取时应以序列化的 +`bit_count` 和 `hash_function_count` 为准。 + +## 结构校验 + +当 `byte_length > 0` 时,`bit_count` 必须为正数,`hash_function_count` 必须在 `1..8` +范围内,且 `byte_length` 不得超过 `ceil(bit_count / 8)`。被省略的尾部零字节在逻辑上 +补零;如果位向量长度越过 `TsFileMetadata` 边界,读取器必须拒绝该结构。 diff --git a/src/zh/UserGuide/latest/FileFormat/Configurations.md b/src/zh/UserGuide/latest/FileFormat/Configurations.md new file mode 100644 index 000000000..ec91fef32 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Configurations.md @@ -0,0 +1,148 @@ + + +# 配置 + +多数配置影响写入器如何分组和缓冲数据。读取器无需知道原来的 Page 大小目标或刷出阈值, +因为最终形成的边界已经记录在文件中。时间单位、非对齐时间编码等少数信息则仍需由 +交换文件的应用共同约定。 + +下表区分了这些情况。不同语言接口中的配置名称可能不同。 + +| 概念配置 | 可观察的文件影响 | 配置值是否直接写入 | 读取器是否需要原始配置 | +| --- | --- | --- | --- | +| Chunk Group 大小 | 设备分组的刷出位置 | 否 | 否 | +| Page 大小目标 | Page 边界和 Statistics 粒度 | 否 | 否 | +| 每 Page 最大点数 | Page 边界 | 否 | 否 | +| 元数据索引扇出 | 索引树深度和节点宽度 | 否 | 否 | +| 布隆过滤器误判率 | 最终位数和哈希函数数 | 否;生成结果会写入 | 否 | +| 值编码 | Page 编码流 | 是,每个 Chunk 的 `encoding_type` | 否 | +| 压缩 | Page 磁盘负载 | 是,每个 Chunk 的 `compression_type` | 否 | +| 时间精度和时区 | `int64` 时间戳的解释 | 否 | 是,由上层协议提供 | +| 非对齐时间编码 | 每个非对齐 Page 的时间编码流 | 否 | 是 | +| 加密 | 文件加密头、Chunk 认证、Page 载荷或旧版 Footer 属性 | provider/profile 或旧版属性会写入;密钥上下文在外部 | 加密时需要 | + +例如,文件不会保存“Page 大小目标”这个配置值,但会记录每个 Page 的实际长度。 + +
+ +## Chunk Group 大小 + +Chunk Group 大小表示写入器在刷出当前设备的各个 Chunk 之前允许缓冲的数据量。 +较大的 Chunk Group 可以减少标记与元数据开销,并有利于顺序 I/O,但需要更多写入内存; +较小的 Chunk Group 可以降低峰值内存,并更早地将数据写入输出。 + +该配置是刷出阈值,并不表示精确的磁盘占用大小。文件格式不规定默认值。 +分组从 `0x00` Chunk Group Header 开始,到下一个分组、操作索引记录或分隔符结束; +中间的 Chunk 标记表示当前分组仍在继续。 + +
+ +## Page 大小 + +Data Page 大小是单个序列写入器在内存中累计数据量的目标值。较小的 Page 可以提供 +更细粒度的 Statistics 和数据跳过能力;较大的 Page 可以减少 Page Header 开销并可能 +提高压缩率,但需要更多缓冲空间,数据跳过粒度也更粗。 + +该阈值不等于精确的序列化 Page 大小。达到最大点数、显式刷出或需要维持对齐 Page +边界时,写入器都可以提前结束当前 Page。实际 Page 边界由 Page Header 中的长度确定。 +文件格式不规定默认的 Page 大小。 + +
+ +## 每个 Page 的最大点数 + +除 Page 大小目标外,写入器还可以限制一个 Page 中的逻辑位置数,以防大量紧凑值让 +单个 Page 包含过多数据点。常见取值为 `10,000` 点。 + +对于对齐序列,Time Page 及其对应的各个 Value Page 必须覆盖相同的逻辑位置范围, +因此写入器会为这些对齐列使用共同的 Page 边界。 + +该上限不会作为独立字段写入文件。读取器使用 Page 边界;对于对齐 Value Page, +还会使用 `position_count`。 + +
+ +## 索引扇出 + +索引扇出就是一个内部索引节点最多能包含多少个子节点。较大的扇出形成 +层数更少但节点更大的树;较小的扇出形成层数更多但节点更小的树。该值应至少为 `2`, +常见取值为 `256`。 + +配置的上限不会写入文件。每个元数据索引节点都会保存节点类型、子节点和结束偏移, +因此读取器无需知道写入配置即可遍历最终生成的树。 + +
+ +## 布隆过滤器误判率 + +布隆过滤器有时会把不存在的路径判为“可能存在”,导致一次额外的索引查找。降低目标 +误判率可以减少这种情况,代价是使用更多位和哈希函数。标准写入器将该值限制在 `0.01` 到 `0.10` 之间,默认使用 `0.05`。 + +误判率本身不会写入文件。最终生成的过滤器字节、位数和哈希函数数量保存在 +`TsFileMetadata` 中,因此读取器不需要写入端的配置。详见 +[布隆过滤器](Bloom-Filter.md)。 + +
+ +## 值编码与压缩 + +写入器可以按物理类型选择默认值编码,并选择默认压缩方式。这些默认值属于 Schema +与写入策略,不是读写端必须共享的配置: + +- Chunk Header 中的 `encoding_type` 标识值编码;对于对齐 Time Chunk,它标识时间编码。 +- Chunk Header 中的 `compression_type` 标识该 Chunk 内所有 Page 使用的压缩方式。 + +因此读取器直接使用 Chunk Header,不需要知道写入器的默认值。支持的标识符及其 +载荷格式见 [编码](DataPages/Encodings.md) 和 +[压缩](DataPages/Compression.md)。 + +
+ +## 时间精度与时区 + +物理时间戳是有符号 64 位整数。TsFile v4 不在文件中保存时间单位或时区;毫秒、微秒、 +纳秒等时间精度以及时区语义由使用该文件的上层协议确定。 + +
+ +## 非对齐时间编码 + +非对齐 Chunk Header 的 `encoding_type` 描述值流编码,不描述同一 Page 内的时间流编码。 +非对齐时间流编码由读写配置共同确定,默认值为 `TS_2DIFF`。 + +对齐 Time Chunk 不受这一限制;其时间编码保存在 Time Chunk Header 的 +`encoding_type` 中。 + +
+ +## 加密 + +加密方式有两处描述来源:Page AEAD 使用可选的文件加密头,旧版加密使用 Footer 属性。 +它们记录 Page 字节应如何解释,取得密钥仍需外部环境。AEAD 文件的旧版 Footer 属性虽然 +写为未加密状态,实际模式仍以文件加密头为准。布局和判断规则见[加密](DataPages/Encryption.md)。 + +
+ +## 与格式无关的配置 + +内存分配、内存检查频率、批大小、并行执行、线程数、存储后端以及关闭文件时的同步 +策略都不定义 TsFile 字节。它们属于具体实现与部署范围,不属于本文件格式规约。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md new file mode 100644 index 000000000..23a9adfbb --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Checksumming.md @@ -0,0 +1,44 @@ + + +# 校验和 + +TsFile v4 没有通用的 Page、Chunk 或完整文件校验和。一个看起来已经写完的文件,仍可能 +包含损坏的值,因此不能把头尾魔数当作内容校验和。 + +## 结构校验能够发现什么 + +长度、偏移量和计数可以帮助发现不完整或格式错误的结构。例如,Page 超出 Chunk 边界、 +Footer 长度超出文件,或者解压结果长度不符,都可以判为错误。具体检查见 +[二进制表示](../Binary-Representation.md)和[数据页](README.md)。 + +这些检查无法发现所有改动。一个值被修改后,字节数可能保持不变,周围的结构边界也可能 +仍然有效。即使另有校验和,读取时仍需要进行结构检查。 + +## 静态存储与传输完整性 + +如果需要检测文件任意位置的变化,可以维护外部校验和,或使用存储系统提供的完整性检查。 +如果还需要防止恶意修改,则应使用带密钥的消息认证码或数字签名,并在文件之外保存可信 +密钥或参考值。校验范围应覆盖完整字节序列,包括 Footer 长度和尾部魔数。 + +Page AEAD 的保护范围更具体:它认证非空 Page 载荷及上下文,并将 Chunk 序号绑定到文件 +位置,但不覆盖全部可见 Header、Statistics、索引或 Footer 元数据。空的对齐 Value Page +没有 Page 认证标签,旧版加密也不一定认证密文。具体范围见[加密](Encryption.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md new file mode 100644 index 000000000..95063f834 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Chunks.md @@ -0,0 +1,200 @@ + + +# Chunk 与 Chunk Group + +TsFile 数据区由 marker 引导的 Chunk Group、Chunk 和 Operation Index Range 组成。 +没有文件加密头时从偏移 `7` 开始;若偏移 `7` 处保存总序列化长度为 `E` 的 Page AEAD +[FileEncryptionHeader](Encryption.md#file-encryption-header),数据区从 `7 + E` 开始。 +该可选加密头只允许出现一次,并位于数据区之前。 + +顶层语法为: + +```text +FilePrefix := magic[6] version[1] FileEncryptionHeader? +DataSection := (ChunkGroup+ OperationIndexRange?)* +ChunkGroup := ChunkGroupHeader Chunk* +Separator := byte(0x02) +``` + +每个批次包含一个或多个 ChunkGroup,随后可选一条 OperationIndexRange。数据区之后紧接 +`0x02` 分隔符,它不属于任何 Chunk Group。Chunk Group 自身没有总长度字段。读取完一个 Chunk 后,要看下一个标记:如果仍是 Chunk,就继续读取;如果是新的 +Chunk Group、操作索引记录或分隔符,当前分组就结束了。 + +```text +File +└── Chunk Group 一个设备的一批数据 + ├── Non-aligned Chunk 自带时间流的一个测点列 + │ └── Page(s) + └── Aligned Chunk Set 多个测点列共享时间列 + ├── Time Chunk + │ └── Time Page(s) + └── Value Chunk(s) + └── Value Page(s) +``` + +
+ +## 标记字节 + +| 值 | 名称 | 后续结构 | +| ---: | --- | --- | +| `0x00` | `CHUNK_GROUP_HEADER` | `DeviceID` | +| `0x01` | `CHUNK_HEADER` | 多页、非对齐 Chunk | +| `0x02` | `SEPARATOR` | 数据区结束,元数据区开始 | +| `0x03` | `VERSION` | v3 起不再使用,编号保留 | +| `0x04` | `OPERATION_INDEX_RANGE` | 两个 `int64` | +| `0x05` | `ONLY_ONE_PAGE_CHUNK_HEADER` | 单页、非对齐 Chunk | +| `0x06` | `ENCRYPTION_HEADER` | 仅位于偏移 `7` 的文件加密头;不属于 Chunk | +| `0x41` | `VALUE_CHUNK_HEADER` | 多页、对齐 Value Chunk | +| `0x45` | `ONLY_ONE_PAGE_VALUE_CHUNK_HEADER` | 单页、对齐 Value Chunk | +| `0x81` | `TIME_CHUNK_HEADER` | 多页、对齐 Time Chunk | +| `0x85` | `ONLY_ONE_PAGE_TIME_CHUNK_HEADER` | 单页、对齐 Time Chunk | + +Chunk marker 的高两位表示列角色,低 6 位表示页数类别: + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0x01: more than one page +low 6 bits = 0x05: one page or no page +``` + +`data_size = 0` 的对齐 Value Chunk 可以没有 Page。 + +只有读完前一个结构,才能解释下一个标记。Page 载荷可以包含任意字节,其中看似标记的 +值也只是数据。 + +
+ +## Chunk Group 头部 + +```text +ChunkGroupHeader := + byte marker = 0x00 + DeviceID device_id + +DeviceID := + UVarInt segment_count + VarString segment[segment_count] +``` + +DeviceID 由一组有序的字符串片段组成,单个片段可以为 null。序列化时会去掉末尾的 null, +但至少要有一个非 null 片段。`segment_count = 0` 是一种特殊表示,读取后得到一个仅含 +空字符串片段的设备标识。 + +Chunk Group Header 之后的 Chunk 属于该设备,直到下一个 Chunk Group Header、Operation +Index Range 或 Separator。 + +`segment_count` 统计片段数量,每个片段另有自己的字节长度。比较设备标识时应保留片段 +边界;用于显示的点号拼接可能产生歧义,因为片段自身也可以含点号。 + +
+ +## Chunk 头部 + +marker 是 Chunk Header 的第一个字段: + +```text +ChunkHeader := + byte chunk_type + VarString measurement_id + UVarInt data_size + byte data_type + byte compression_type + byte encoding_type + int64 chunk_ordinal? + int32 chunk_authentication_size? + byte chunk_authentication[chunk_authentication_size]? +``` + +最后三个字段只出现在 Page AEAD 文件中,由文件加密头决定是否存在;Chunk marker +本身不变。序号、认证证明及校验规则见[AEAD Chunk 头部](Encryption.md#aead-chunk-header)。 +这些后缀字节和其他 Header 字段一样,不计入 `data_size`。 + +Chunk Header 不包含 Statistics 字段。第一个 Page Header 紧跟在 Chunk Header 之后; +当该 Page 非空且 Chunk 包含多个 Page 时,其 Statistics 位于两个长度字段之后,因此这些 +Statistics 字节在物理上紧邻 Chunk Header,但属于 Page Header。Chunk 级 Statistics 保存在 +文件尾的 Chunk Metadata 中;单 Page Chunk 不保存 Page Statistics,改用 Chunk 或序列级 +Statistics。详见[数据页](README.md)和[统计信息](../Metadata.md#statistics)。 + +`data_size` 是 Chunk 中全部 Page Header 和已保存 Page 数据的总长度,不包括 Chunk +Header。读取器从 Chunk Header 结束处读取恰好 `data_size` 字节。 + +若 `H` 是 Chunk Header 后第一个字节的偏移,则下一结构从以下位置开始: + +```text +next_structure_offset = H + data_size +``` + +所有 Page Header 的序列化长度与相应 `compressed_size` 载荷长度之和必须等于 +`data_size`。因此,该值既是快速跳过 Chunk 的长度,也是校验 Page 不越界的边界。 + +对齐 Time Chunk 的 `measurement_id` 是空字符串,`data_type` 是 `VECTOR (0x06)`。 +对齐 Value Chunk 和非对齐 Chunk 的 `data_type` 是测量值类型。 + +对齐数据按 Chunk 连续存储,不按 Time Page、Value Page 交替存储。Time Chunk 保存完整 +的 Time Page 序列,每个 Value Chunk 保存自己的 Value Page 序列,Page 通过序号对应。 + +“对齐 Chunk 集”指一个 Time Chunk 及跟随它的 Value Chunk,并没有单独的集合头。 +它们之间有以下关系: + +- Time Chunk 位于使用该时间列的 Value Chunk 之前。 +- 每个 Value Chunk 标识一个测量;Time Chunk 使用空 `measurement_id` 和物理类型标记 + `VECTOR (0x06)`。 +- Value Page `i` 覆盖 Time Page `i` 的逻辑位置。 +- 某个测量在该集合中完全没有数据时,可以不存在对应 Value Chunk;已存在的对齐 Value + Chunk 也可以具有 `data_size = 0`。 + +非对齐 Chunk 的每个逻辑数据点都包含时间戳和值,因此每个 Page 解码后的时间流和值流 +必须具有相同的元素数量。 + +## Chunk 扫描状态机 + +流式读取器或恢复读取器可以使用以下状态机扫描数据区: + +1. 校验文件前缀,检查偏移 `7`,并消费可选的 `0x06` 加密头。在解析 Chunk 前确定 + provider/profile,再从 `data_start = 7 + E` 读取数据区第一个标记;无加密头时 `E = 0`。 + 后续或重复出现的 `0x06` 均无效。 +2. 遇到 `0x00` 时读取 DeviceID,并进入对应 Chunk Group。 +3. 遇到 Chunk 标记时读取完整 Header,包括模式要求的 AEAD 后缀,并在使用 Chunk 身份 + 之前验证位置证明;可以解析 Page,也可以直接向后移动 `data_size` 字节。 +4. 遇到 `0x04` 时消费后续 16 字节 Operation Index Range。 +5. 遇到 `0x02` 时停止;下一个字节属于索引元数据区。 +6. 顶层标记无法识别,或声明边界超出可用字节时,判定文件不完整或损坏。 + +
+ +## 操作索引范围 + +```text +OperationIndexRange := + byte marker = 0x04 + int64 min_operation_index + int64 max_operation_index +``` + +这条可选记录用于把已刷出的数据批次与存储引擎的操作日志关联起来。两个值分别是该批次 +涉及的最小和最大操作序号,可用于协调检查点、快照、备份和恢复,与时间戳或文件偏移量 +无关。 + +通用写入器可以省略这条记录。读取器即使不使用操作索引,也要完整读过两个 `int64`, +再读取下一个标记。一个文件可以先后刷入多个批次,因此也可以出现多条这样的记录。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md new file mode 100644 index 000000000..14e94bc90 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Compression.md @@ -0,0 +1,80 @@ + + +# 压缩 + +每个 Chunk Header 使用单字节 `compression_type` 指定该 Chunk 中所有 Page 的压缩格式。 + +| 值 | 名称 | Page 数据格式 | +| ---: | --- | --- | +| 0 | `UNCOMPRESSED` | 原始字节 | +| 1 | `SNAPPY` | Snappy raw block | +| 2 | `GZIP` | 单个 gzip member | +| 3 | `LZO` | LZO block(C++ 实现,编译期可选启用) | +| 4 | `SDT` | 保留编号,无编解码器 | +| 5 | `PAA` | 保留编号,无编解码器 | +| 6 | `PLA` | 保留编号,无编解码器 | +| 7 | `LZ4` | LZ4 raw block | +| 8 | `ZSTD` | Zstandard frame | +| 9 | `LZMA2` | XZ container 中的 LZMA2 流 | + +每个 Page 独立压缩,不跨 Page 共享压缩状态。Page Header 的 `uncompressed_size` 和 +`compressed_size` 分别保存未压缩负载长度及压缩、加密后的磁盘载荷长度。Page AEAD 的 +磁盘长度还包含密文帧和认证标签。 + +这些数值是稳定的文件格式编号,不是某个压缩库枚举中的位置。编号 `3` 到 `6` 是早期 +TsFile 版本留下的历史标识符。Java 实现不定义也不实现它们,解析 Chunk Header 时 +会直接拒绝这些字节。C++ 实现保留 `LZO`、`SDT`、`PAA`、`PLA` 四个标识;其中只有 +`LZO` 有编解码器(编译期可选启用),`SDT`、`PAA`、`PLA` 仍是无编解码器的保留值。 +读取器未提供该编解码器时,不能解码相应 Chunk。 + +## Page 级帧边界 + +设 Page 从偏移 `P` 开始,Header 占 `H` 字节,载荷就取后续恰好 `compressed_size` 字节: + +```text +[P + H, P + H + compressed_size) +``` + +这里使用的是磁盘长度,包含加密帧信息。解密后,`UNCOMPRESSED` 已经是编码后的 Page +载荷;其他方式还需要解压。两种路径最终都应得到恰好 `uncompressed_size` 字节。 +不加密或加密保持长度时,`UNCOMPRESSED` 的这两个长度相等;Page AEAD 会增加帧开销, +因而不再相等。 + +Page 之间不共享压缩历史。读取器可以直接跳转到目标 Page,也可以按磁盘长度跳过它, +无需先解压前面的 Page。 + +## 读取顺序与校验 + +读取器先读 Page Header,因而能在分配内存或解压前获得输入、输出边界: + +```text +读取 Page Header + -> 准确读取 compressed_size 字节 + -> 按文件加密头或旧版属性进行认证/解密 + -> 按 compression_type 解压 + -> 要求结果恰好为 uncompressed_size 字节 + -> 解码 Page 内各数据流 +``` + +分配缓冲区之前,按[二进制表示](../Binary-Representation.md)检查长度。编解码器不可用时, +无法解压该 Chunk;扫描器直接解析原始 Header 字段后,仍可利用 `data_size` 跳过它, +即使编解码器注册表拒绝这个编号。不能把该编号当作 `UNCOMPRESSED`。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md new file mode 100644 index 000000000..75e083cab --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Encodings.md @@ -0,0 +1,108 @@ + + +# 编码 + +编码先把一串有类型的值转换为字节流,再交给压缩器处理。Chunk Header 的 +`encoding_type` 保存所选编码:在 Value Chunk 和非对齐 Chunk 中,它描述值流; +在对齐 Time Chunk 中,它描述时间戳流。 + +| 值 | 名称 | +| ---: | --- | +| 0 | `PLAIN` | +| 1 | `DICTIONARY` | +| 2 | `RLE` | +| 3 | `DIFF` | +| 4 | `TS_2DIFF` | +| 5 | `BITMAP` | +| 6 | `GORILLA_V1` | +| 7 | `REGULAR` | +| 8 | `GORILLA` | +| 9 | `ZIGZAG` | +| 10 | `FREQ`(已弃用) | +| 11 | `CHIMP` | +| 12 | `SPRINTZ` | +| 13 | `RLBE` | +| 14 | `CAMEL` | + +参考值编码器支持下列组合。一个编码已经分配了编号,并不表示所有数据类型都能使用它: + +| 数据类型 | 可用编码 | +| --- | --- | +| `BOOLEAN` | `PLAIN`, `RLE` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `FLOAT` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE` | +| `DOUBLE` | `PLAIN`, `RLE`, `TS_2DIFF`, `GORILLA_V1`, `GORILLA`, `CHIMP`, `SPRINTZ`, `RLBE`, `CAMEL` | +| `TEXT`, `STRING` | `PLAIN`, `DICTIONARY` | +| `BLOB`, `OBJECT` | `PLAIN` | + +## 编码类别 + +这些编号描述把逻辑值序列转换为字节的不同方式。它们是文件格式中的固定编号,不是 +某个实现内部枚举的序号。 + +| 类别 | 编码 | 通常利用的数据特征 | +| --- | --- | --- | +| 直接表示 | `PLAIN` | 不预测,各值独立序列化 | +| 字典 | `DICTIONARY` | 用字典引用替代重复的二进制值 | +| 游程与位打包 | `RLE`, `RLBE` | 重复值或较小的有效位宽 | +| 差分与预测 | `DIFF`, `TS_2DIFF`, `REGULAR`, `SPRINTZ` | 相邻值之间较小或较规律的变化 | +| 整数重映射 | `ZIGZAG` | 把绝对值较小的有符号整数映射为较小的无符号整数 | +| 基于异或的编码 | `GORILLA_V1`, `GORILLA`, `CHIMP`, `CAMEL` | 相邻浮点数具有稳定的前导位和尾随位 | +| 历史或专用编码 | `BITMAP`, `FREQ` | 保留的历史布局;`FREQ` 已弃用 | + +不同的数据分布适合不同的编码。读取时,无论写入器为什么选择某种编码,都需要根据 +`(data_type, encoding_type)` 这一组合选择解码器。 + +
+ +## PLAIN 编码 + +`PLAIN` 流不保存元素个数,元素数量由 Page 上下文、时间流或 null 位图确定。各元素 +连续写入: + +| 数据类型 | 单个值的编码 | +| --- | --- | +| `BOOLEAN` | `0x00` 或 `0x01` | +| `INT32`, `DATE` | `VarInt` | +| `INT64`, `TIMESTAMP` | 大端序 `int64` | +| `FLOAT` | 大端序 `float32` | +| `DOUBLE` | 大端序 `float64` | +| `TEXT`, `STRING`, `BLOB`, `OBJECT` | `VarInt byte_length` + 原始字节 | + +`PLAIN` 二进制值的长度非负。非对齐值流不包含 null;对齐值流的 null 由 Value Page +位图表示。 + +其他编码编号决定各自的值流位布局,读取器不能把未知编码按 `PLAIN` 解码。 + +## 解码约束 + +解码状态只在一个 Page 流内有效。解密和解压之后,选择对应的解码器,在流的边界内读取, +并检查值的数量是否符合 Page 上下文。处理下一个 Page 时,重新开始解码状态。 + +对齐数据的时间编码来自 Time Chunk Header,Value Page 每遇到一个置位的位图位置就 +解码一个值。非对齐 Chunk Header 只说明值编码,时间编码来自共享配置,默认 +`TS_2DIFF`;Footer 中没有序列化的 `time_encoder` 字段。详见 +[配置](../Configurations.md#non-aligned-time-encoding)。 + +编码不受支持时,无法解码具体值。扫描器如果直接解析原始 Header 字段,仍能通过 +`data_size` 找到下一个 Chunk,即使上层编解码器注册表拒绝该编号。此时不能擅自改用 +`PLAIN` 或其他解码器。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md new file mode 100644 index 000000000..aed9bdcb9 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Encryption.md @@ -0,0 +1,166 @@ + + +# 加密 + +写入时先压缩 Page,再加密;读取时先解密,再解压。Header、统计信息、Schema 和索引 +仍然可见。TsFile v4 支持两种加密机制,格式版本字节都为 `0x04`: + +| 模式 | 判断依据 | Chunk 和 Page 布局 | +| --- | --- | --- | +| 明文 | 没有文件加密头,旧版级别为 `0` 或不存在 | 基础 Chunk Header,Page 载荷不加密 | +| 旧版加密 | 没有文件加密头,旧版级别为 `1` 或 `2` | 基础 Chunk Header,按指定算法加密载荷 | +| Page AEAD | 偏移 `7` 处有 `0x06` 文件加密头 | 扩展 Chunk Header,按 profile 定义认证和加密 Page | + +解析 Chunk 之前,先检查文件加密头。AEAD 文件的旧版 Footer 属性仍写为未加密状态, +实际模式以加密头为准。 + +写入和读取的顺序为: + +```text +编码后的 Page 载荷 -> 压缩 -> 加密 -> stored_data +stored_data -> 按需认证解密 -> 解压 -> 值解码 +``` + +
+ +## 文件加密头 + +AEAD 文件在七字节魔数/版本前缀之后、任何 Chunk Group 或操作索引记录之前,保存一个 +`FileEncryptionHeader`: + +```text +FileEncryptionHeader := + byte marker = 0x06 + UVarInt payload_size + byte header_version = 0x01 + VarString provider_id + VarString profile_id + VarString key_id + VarString key_version + UVarInt file_crypto_id_size = 16 + byte file_crypto_id[16] + UVarInt wrapped_data_key_size + byte wrapped_data_key[wrapped_data_key_size] +``` + +`payload_size` 从 `header_version` 计到封装密钥结束,不包含标记和长度字段自身,取值范围 +为 `1..1048576`。解析必须恰好在该边界结束。四个字符串标识符均不得为 null 或空白, +文件加密 ID 恰好为 16 字节,封装数据密钥不得为空。这些二进制长度使用 `UVarInt`, +与 Footer 属性使用的可空 `VarBinary` 不同。 + +**Provider** 标识加密实现或服务,**profile** 则为一套固定的字节格式命名,其中包括 +Page 帧、nonce 和认证标签的表示、长度映射以及 Chunk 证明长度。一个 provider 可以 +提供多个 profile。文件保存这些公开标识和封装密钥,不依赖实现类名,也不保存 +AEAD 的明文密钥。读取端需要相应的 provider/profile,并能取得外部密钥材料。 + +当前支持的头版本为 `0x01`。未知版本、字段截断、尾随字节、重复的加密头,或加密头 +出现在偏移 `7` 以外的位置,均属于错误。 + +
+ +## AEAD Chunk 头部 + +文件加密头还决定了每个 Chunk Header 的解析方式。Chunk marker 保持不变,但在 +`encoding_type` 之后增加三个字段: + +```text + int64 chunk_ordinal + int32 chunk_authentication_size + byte chunk_authentication[chunk_authentication_size] +``` + +`chunk_ordinal` 是文件内唯一分配的非负标识,包括 Time Chunk 和 Value Chunk 在内。 +证明长度必须与 profile 声明一致,并位于 `1..1024`。这些字段属于 Header,均不计入 +`data_size`。 + +认证证明把序号绑定到 Chunk 的绝对位置,使用该身份之前需要验证证明。移动 Chunk 会 +改变被认证的位置,因此必须重新生成证明。它使用的关联数据按大端序排列如下: + +```text +int32 domain_size = 20 +byte domain[20] = UTF8("TsFile/ChunkPosition") +int32 associated_data_version = 1 +byte file_crypto_id[16] +int64 absolute_chunk_marker_offset +int64 chunk_ordinal +``` + +
+ +## AEAD Page 载荷与关联数据 + +非空 Page 的 `compressed_size` 记录整个加密载荷的长度,包括帧信息、nonce、认证标签 +和填充。它大于压缩明文长度,使用 `UNCOMPRESSED` 时也是如此。Profile 可以采用固定 +开销,也可以采用随输入大小变化的布局,因此需要通过 profile 的长度映射取得候选明文 +长度。 + +Page 认证还把载荷绑定到对应的文件、Chunk 和 Page 位置。关联数据是下列字段按大端序 +精确拼接的结果。`SizedBytes` 表示 `int32` 字节长度加上相应字节,文本使用 UTF-8。 + +```text +int32 associated_data_version = 1 +SizedBytes provider_id +SizedBytes profile_id +SizedBytes key_id +SizedBytes key_version +SizedBytes file_crypto_id +int64 chunk_ordinal +int32 page_index +int32 uncompressed_size +int32 compressed_plaintext_size +``` + +`page_index` 在每个物理 Chunk 内从零开始,空的对齐 Value Page 也占一个序号。 +全 null Value Page 仍采用[空值](../Nulls.md)章节中的单个 `UVarInt(0)` 表示, +不保存密文或 Page 认证标签。 + +认证成功后,候选明文长度才可信。解密输出必须等于该长度,解压后还必须恰好得到 +`uncompressed_size` 字节。认证标签无效、密钥错误、profile 不受支持或长度不符时, +应报错,不能按明文重试。 + +
+ +## 旧版加密属性 + +没有文件加密头时,读取 `TsFileMetadata` 中的以下属性: + +| 属性 | 含义 | +| --- | --- | +| `encryptLevel` | `0` 不加密;`1` 直接保存数据密钥;`2` 保存被外部主密钥加密的数据密钥 | +| `encryptType` | 旧版加解密算法标识符 | +| `encryptKey` | 数据密钥或密钥封装结果的文本表示 | + +这几个属性的值保存文本。级别 `1` 直接保存数据密钥;级别 `2` 先使用外部主密钥解密器 +解封密钥,再解密 Page。缺少 `encryptLevel` 时按 `0` 处理;级别 `1` 和 `2` 要求存在 +`encryptType` 和非空 `encryptKey`。未知级别或不可用算法均为错误。 + +旧版加密使用算法自身的加解密操作,没有 AEAD Chunk 后缀或 Page 上下文。 +`compressed_size` 仍表示实际磁盘输出长度,仅在算法保持长度时才等于压缩明文长度。 + +AEAD 写入器把旧版属性设为级别 `0`、类型 `org.apache.tsfile.encrypt.UNENCRYPTED` +和空密钥,实际模式和封装密钥由文件加密头提供。这些旧版值不能用作回退到明文的依据。 + +## 安全边界 + +Page AEAD 认证非空 Page 载荷及其上下文,Chunk 证明将序号绑定到文件位置。它不认证 +全部可见字段,也不提供完整文件摘要。旧版加密不一定具有认证能力。完整文件的完整性 +检查见[校验和](Checksumming.md),重建 Footer 时如何保留加密上下文见[错误恢复](Error-Recovery.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md b/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md new file mode 100644 index 000000000..c9ced30c8 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/Error-Recovery.md @@ -0,0 +1,76 @@ + + +# 错误恢复 + +Footer 在数据区之后写入。如果写入提前中断,文件虽然无法通过索引正常打开,前面仍可能 +保留一些完整的 Chunk。恢复时从文件头向后扫描,找到可以保留的结构,再据此重建缺失的 +索引和 Footer。 + +## 完成性检查 + +设文件大小为 `F`,从 `[F - 10, F - 6)` 读取大端元数据长度 `L`。完成写入的文件具有 +以下关系: + +```text +bytes[F - 6, F) = "TsFile" +TsFileMetadata interval = [F - 10 - L, F - 10) +meta_offset = offset of the top-level 0x02 separator +meta_offset < F - 10 - L +``` + +还要校验头部魔数和版本,并恰好解析 `L` 字节的 `TsFileMetadata`。仅有尾部魔数不够, +Footer 长度和其中的偏移量也必须能正确使用。 + +## 安全正向扫描 + +正向扫描使用与普通读取器相同的 Chunk 布局,但不依赖 Footer: + +1. 校验魔数和版本,检查偏移 `7`,读取可选的 `0x06` 加密头。数据区从 `7 + E` 开始, + `E` 为该头的总长度;不存在时为零。 +2. 在结构边界读取标记。遇到 `0x00` 时读取 DeviceID,并记住当前分组。 +3. 遇到 Chunk 标记时读取完整 Header,包括需要的 AEAD 后缀。在使用 Chunk 身份之前 + 验证位置证明,再检查剩余文件是否容得下 `data_size` 字节。随后检查 Page,或按该 + 长度完整跳过。 +4. 遇到 `0x04` 时,读过两个操作索引 `int64`。 +5. 遇到顶层 `0x02` 分隔符时,停止扫描数据区。 + +字段或载荷被截断时,在最后一个完整结构处停止。未知标记、重复或位置错误的加密头、 +无效长度和认证失败都应作为错误处理,不能通过搜索下一个熟悉的字节来绕过。 +Page 载荷中的 `0x02` 是数据,不能据此确定数据区结束。 + +只保留已经完整读取并检查过的结构。被截断 Chunk 中尚能读取的部分,不会自动成为有效 +Chunk。通用边界检查见[二进制表示](../Binary-Representation.md),Page 边界见[数据页](README.md)。 + +## 恢复能力边界 + +重建索引不能补回缺失的值,也无法识别所有结构仍有效的字节改动。保留的 Page 仍需要 +相应的编解码器;已加密的数据还需要密钥。操作索引记录有助于把保留数据与外部日志关联, +但不能替代结构检查。 + +Page AEAD 恢复要保留原加密头、文件加密 ID、Chunk 序号和 Page 索引。仅截断文件尾并 +重写 Footer 时,可以保留 Chunk 的原始偏移。移动 Chunk 需要新的位置证明;改变文件 +身份、Chunk 序号或 Page 上下文时,还要按变化内容对相关数据重新认证和加密。追加写入 +应从已保留的最大 Chunk 序号之后继续分配。 + +认证恢复需要相应的 provider 和外部密钥。AEAD 文件的旧版 Footer 字段故意设为未加密, +因此不能用它代替损坏的加密头。模式判断见[加密](Encryption.md),格式完整性检查仍有哪些 +限制,见[校验和](Checksumming.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/DataPages/README.md b/src/zh/UserGuide/latest/FileFormat/DataPages/README.md new file mode 100644 index 000000000..f938ef9be --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/DataPages/README.md @@ -0,0 +1,176 @@ + + +# 数据页 + +一个 Chunk 的 Page 连续存放在 Chunk Header 之后,Page 之间没有填充字节。`data_size` +统计所有 Page Header 和载荷的总长度。空的对齐 Value Chunk 的 `data_size = 0`, +不包含任何 Page。 + + + +## 页头(Page Header) + +非空 Page 的格式为: + +```text +Page := + UVarInt uncompressed_size + UVarInt compressed_size + Statistics statistics? # 由 Chunk marker 决定 + byte stored_data[compressed_size] +``` + +`uncompressed_size` 是解密、解压后的 Page 负载大小,`compressed_size` 是文件中保存的 +Page 负载大小。仅在 `compression_type = UNCOMPRESSED` 且加密不扩展长度时二者相等。 +Page AEAD 的 `compressed_size` 包含密文帧和认证开销,即使不压缩也可能更大,详见 +[加密](Encryption.md#aead-page-body-and-context)。 + +Chunk marker 的低 6 位决定页头中是否包含统计信息: + +| Chunk marker 低位 | Page 数量 | Page Statistics | +| ---: | --- | --- | +| `0x01` | 多于一个 | 每个非空 Page Header 中存在 | +| `0x05` | 零个或一个 | 不存在 | + +统计信息的字节布局见[元数据](../Metadata.md#statistics)。 + +若 Page 起始偏移为 `P`,两个长度字段和可选 Statistics 的序列化总长度为 `h`,则磁盘 +载荷占用半开区间 `[P + h, P + h + compressed_size)`;如果还有下一个 Page,它从 +`P + h + compressed_size` 开始。该结束位置不得超过所属 Chunk 的边界。 + +解密并解压后,载荷必须恰好产生 `uncompressed_size` 字节。即使压缩库报告成功,实际输出 +长度不一致时读取器也必须拒绝该 Page。 + +
+ +## 写入与读取顺序 + +写入非空 Page 时,数据依次经过: + +```text +时间戳和值 + -> 编码 + -> 组成未压缩的 Page 载荷 + -> 压缩 + -> 按需加密 + -> stored_data +``` + +读取顺序相反:先解密,再解压,最后执行值解码。 + +编码与压缩相互独立。编码把有类型的值转换成未压缩字节流,压缩把该结果视为普通字节。 +Page 边界会重置值编码器/解码器和压缩器/解压器,状态不会跨 Page 延续。 + +
+ +## 非对齐 Page + +非对齐 Page 的解压后负载为: + +```text +NonAlignedPagePayload := + UVarInt time_stream_size + byte encoded_time_stream[time_stream_size] + byte encoded_value_stream[remaining bytes] +``` + +一个非对齐 Page 同时保存时间和值。它没有独立的 Time Page 和 Value Page:Page 先保存 +完整时间流,再保存完整值流,而不是按 `timestamp, value` 逐点交替写入。两个流包含 +相同数量的元素,并按位置组成 `(timestamp, value)`。 + +设未压缩载荷大小为 `U`,序列化 `time_stream_size` 字段本身占 `v` 字节,则值流长度为: + +```text +value_stream_size = U - v - time_stream_size +``` + +该长度必须非负。两个流都受 Page 载荷边界约束,必须按各自编码格式完整结束,并解出 +相同数量的数据点,才能将时间戳和值一一对应。 + +非对齐 Chunk Header 只记录值流的 `encoding_type`。时间流编码见 +[配置](../Configurations.md#non-aligned-time-encoding)。 + +
+ +## 对齐 Time Page + +对齐 Time Page 的解压后负载完全由时间编码流组成: + +```text +AlignedTimePagePayload := byte encoded_time_stream[remaining bytes] +``` + +时间编码由 Time Chunk Header 的 `encoding_type` 指定。 + +解码得到的时间戳数量就是该 Page 的逻辑位置数。所有对应对齐 Value Page 的 bitmap 宽度 +和 null 重建都由该数量决定。 + +
+ +## 对齐 Value Page + +非空对齐 Value Page 的解压后负载为: + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +同一对齐集合中,第 `i` 个 Value Page 对应第 `i` 个 Time Page。null 位图和全 null +空 Page 的格式见 [空值](../Nulls.md)。 + +`position_count` 必须等于对应 Time Page 解码得到的时间戳数量。bitmap 必须恰好具有 +`ceil(position_count / 8)` 字节;最后一个字节中未使用的低位不表示逻辑位置。值解码器只 +对置位位置解码,并且必须恰好产生 bitmap 置位数个值。 + +## Page 选择与跳过 + +Page Statistics 可以在读取载荷之前排除不需要的 Page。例如,Page 的时间范围与查询 +范围不相交时,就可以直接跳过。各类型提供哪些值摘要,见[统计信息](../Metadata.md#statistics)。 + +单 Page Chunk 不保存 Page Statistics,需要使用 Footer 中的 Chunk 或序列级统计信息。 +如果现有摘要无法排除匹配,就要读取 Page 并检查具体值。 + +## 全 null 对齐 Value Page + +一个对齐值列在某个 Time Page 的全部位置上都可以为 null。该情况只写入一个无符号变长 +整数零: + +```text +EmptyAlignedValuePage := UVarInt(0) # uncompressed_size +``` + +该零之后没有 `compressed_size`、Page Statistics、bitmap 或磁盘载荷。位置数继承自对应 +Time Page。只有在对齐 Page 序列能够提供外部位置数时,才允许使用这种表示。 + +
+ +## 相关章节 + +- [Chunk 与 Chunk Group](Chunks.md) +- [压缩](Compression.md) +- [编码](Encodings.md) +- [加密](Encryption.md) +- [校验和](Checksumming.md) +- [错误恢复](Error-Recovery.md) diff --git a/src/zh/UserGuide/latest/FileFormat/Extensibility.md b/src/zh/UserGuide/latest/FileFormat/Extensibility.md new file mode 100644 index 000000000..badf64eb8 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Extensibility.md @@ -0,0 +1,53 @@ + + +# 扩展机制 + +TsFile 主要通过三种方式增加能力:分配新的类型或编解码器编号、增加 Footer 属性,或者 +定义新的文件布局。选择哪种方式,取决于读取器在不理解新能力时,是否仍能找到下一个 +结构的边界。 + +Footer 属性容易确定边界:每个属性由 `VarString` 键和可空的 `VarBinary` 值组成, +读取器无需解释值的内容,就能保留未知属性。Chunk 也有明确的字节长度,扫描器即使不支持 +其中的编解码器,仍可以跳过。数据区标记没有统一的长度字段,遇到未知标记时就不能套用 +同样的做法。 + +## 安全演进规则 + +兼容的扩展应保留已有字段的含义和边界: + +- 新类型和编解码器使用未分配的编号,已分配编号的含义保持不变。 +- 新编解码器说明支持的数据类型、字节布局、输出长度及错误处理方式。 +- 沿用现有的字节序和长度约定。 +- 解码目标数据必须使用、但读取器不支持的能力,应明确报错。 + +如果直接在 Header 中增加字节,却没有规则选择新布局,旧读取器预期的后续字段位置就会 +改变。这类修改需要新的版本号,或明确的扩展机制。 + +Page AEAD 使用了后一种方式:`0x06` 文件加密头决定额外的 Chunk Header 字段和 +profile 指定的 Page 帧格式,文件版本字节仍为 `0x04`。旧 v4 读取器可能拒绝这些文件。 +[加密](DataPages/Encryption.md)说明了其布局和能力要求;仅支持基础格式版本还不够。 + +## 属性与语义 + +一个属性可以被跳过,不代表它的含义总能被忽略。描述性属性可以是可选的;如果属性会 +改变 Page 字节的解码方式,就需要明确规定读取器应如何支持。读取器无法执行必要的加密 +处理或其他解释规则时,应报告错误,不能按默认方式继续解码。 diff --git a/src/zh/UserGuide/latest/FileFormat/Format-Versions.md b/src/zh/UserGuide/latest/FileFormat/Format-Versions.md new file mode 100644 index 000000000..fdd0db6e5 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Format-Versions.md @@ -0,0 +1,58 @@ + + +# 格式版本 + +格式版本是头部魔数之后的一个字节: + +```text +6-byte magic string "TsFile" +1-byte format version +``` + +本文介绍 `0x04`。格式版本和项目发布版本含义不同:2.x 系列的一个发布版本可以使用 +v4 文件格式。文件头不保存项目的发布版本标签。 + +v4 使用分段 DeviceID,并在 `TsFileMetadata` 中保存按表划分的设备索引根和 Table Schema。 +v3 使用单字符串设备标识和不同的 Footer。读取这两种文件时,需要分别使用对应版本的 +结构定义。 + +## 读取器分派 + +先读取版本,再解释数据标记并确定加密模式: + +```text +校验头部魔数 -> 读取版本 -> 检查可选加密头 + -> 确定加密模式/profile -> 按对应布局解析数据区和 Footer +``` + +版本不受支持时,应在解析数据之前报告错误。某个标记或尾部魔数看起来熟悉,并不能说明 +文件其他部分也采用读取器预期的布局。 + +## 兼容性约束 + +支持格式版本之后,还要支持目标数据使用的类型、编解码器和加密能力。例如,v4 的 +Page AEAD 文件在偏移 `7` 处有 `0x06` 加密头,用它选择扩展的 Chunk Header 和 Page +载荷 profile。版本字节仍为 `0x04`;不具备该能力的 v4 读取器应明确报告不支持。 + +已分配编号的含义保持不变。未知 Footer 属性的键和值都有长度,读取器可以读过相应 +字节区间;未知数据标记没有统一的载荷长度,通常不能安全跳过。这些边界见 +[扩展机制](Extensibility.md),AEAD 布局见[加密](DataPages/Encryption.md)。 diff --git a/src/zh/UserGuide/latest/FileFormat/Metadata.md b/src/zh/UserGuide/latest/FileFormat/Metadata.md new file mode 100644 index 000000000..a52196594 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Metadata.md @@ -0,0 +1,295 @@ + + +# 元数据 + +TsFile 中的元数据分为两类: + +- 与数据一起保存的 Chunk Header、Page Header 和 Page Statistics。 +- 数据区之后的 Timeseries Metadata、Metadata Index 和 File Metadata。 + +Footer 元数据从 `SEPARATOR (0x02)` 之后开始。`TimeseriesMetadata` 范围与测量索引 +节点可以交错出现;索引中的 offset 和 `end_offset` 决定每个范围的实际边界。所有 +offset 都是从文件首字节开始计算的绝对偏移量。 + +查找数据时,读取器沿下列层级逐步缩小范围,从表和设备定位到可能包含目标值的 Page: + +| 层级 | 标识对象 | 主要用途 | +| --- | --- | --- | +| 文件元数据 | 表及设备索引根 | 进入正确的索引树 | +| 设备索引 | 设备 | 定位一个设备的测量索引 | +| 测量索引 | 测量名称区间 | 定位 Timeseries Metadata | +| Timeseries Metadata | 一个逻辑序列 | 选择 Chunk 偏移及序列 Statistics | +| Chunk Metadata | 一个物理 Chunk | 跳转到数据并利用 Chunk Statistics 裁剪 | +| Page Header | 一个 Page | 利用 Page Statistics 裁剪并读取载荷 | + +
+ +## 统计信息 + +Page、Chunk 和时间序列级 Statistics 使用相同的类型相关结构: + +```text +Statistics := + UVarInt count + int64 start_time + int64 end_time + ... type-specific fields +``` + +`count` 是该结构覆盖的值数。非对齐序列的值数等于点数;对齐值列的 `count` 是非 null +值数;对齐时间列的 `count` 是时间戳数。 + +类型专有字段按以下顺序紧随公共前缀: + +| 数据类型 | 类型专有字段 | +| --- | --- | +| `BOOLEAN` | `bool first`, `bool last`, `int64 true_count` | +| `INT32`, `DATE` | `int32 min`, `int32 max`, `int32 first`, `int32 last`, `int64 sum` | +| `INT64`, `TIMESTAMP` | `int64 min`, `int64 max`, `int64 first`, `int64 last`, `float64 sum` | +| `FLOAT` | `float32 min`, `float32 max`, `float32 first`, `float32 last`, `float64 sum` | +| `DOUBLE` | `float64 min`, `float64 max`, `float64 first`, `float64 last`, `float64 sum` | +| `TEXT` | `Binary32 first`, `Binary32 last` | +| `STRING` | `Binary32 first`, `Binary32 last`, `Binary32 min`, `Binary32 max` | +| `BLOB`, `OBJECT`, `VECTOR` | 无类型专有字段 | + +Statistics 自身不携带数据类型编号。解析 Statistics 时,数据类型来自 Chunk Header 或 +Timeseries Metadata。 + +`first` 和 `last` 表示值的先后顺序,不等同于最小值和最大值。对于按时间有序的序列, +它们对应覆盖区间首尾时间戳处的值。`start_time` 和 `end_time` 均包含在统计区间内。数值 +`sum` 用于聚合;其中 `INT64` 的 sum 以 `float64` 保存,数值较大时可能无法精确表示每个 +整数值。 + +统计信息用于排除不可能匹配的数据。例如,Page 的时间范围完全早于查询范围时,可以 +跳过该 Page。范围有重叠只说明“可能匹配”,仍需读取数据检查具体值。 +`BLOB`、`OBJECT`、`TEXT` 等没有最小/最大值字段的类型不能通过 Statistics 做值范围裁剪。 + +Chunk Statistics 汇总该 Chunk 的所有 Page,序列级 Statistics 再汇总该序列的所有 +Chunk。父层计数是各子层计数之和,时间范围也应覆盖各子层的时间范围。 + +
+ +## 时间序列元数据 + +每条时间序列对应一个 `TimeseriesMetadata`: + +```text +TimeseriesMetadata := + byte timeseries_type + VarString measurement_id + byte data_type + UVarInt chunk_metadata_list_size + Statistics series_statistics + byte chunk_metadata_list[chunk_metadata_list_size] +``` + +`timeseries_type` 的高位表示列角色,低 6 位表示 Chunk Metadata 的布局: + +```text +bit 7 = 1: aligned time column +bit 6 = 1: aligned value column +low 6 bits = 0: exactly one Chunk; ChunkMetadata omits Statistics +low 6 bits = 1: more than one Chunk; every ChunkMetadata contains Statistics +``` + +
+ +## Chunk 元数据 + +当 `(timeseries_type & 0x3f) == 0` 时,时间序列只有一个 Chunk: + +```text +ChunkMetadata := int64 offset_of_chunk_header +``` + +该 Chunk 的 Statistics 直接使用 `series_statistics`。 + +当 `(timeseries_type & 0x3f) != 0` 时: + +```text +ChunkMetadata := + int64 offset_of_chunk_header + Statistics chunk_statistics +``` + +`offset_of_chunk_header` 指向 Chunk marker,即 Chunk Header 的第一个字节。 + +`chunk_metadata_list_size` 保存列表的序列化字节数,不是 Chunk 数量。将它加到列表起点 +即可得到结束位置,然后逐条读取 Chunk Metadata,直到恰好到达该位置。 + +
+ +## 元数据索引 + +```text +MetadataIndexNode := + UVarInt child_count + MetadataEntry child[child_count] + int64 end_offset + byte node_type +``` + +`node_type` 决定节点层级、Entry 的编码和 Entry 指向的数据: + +| 值 | 名称 | Entry 编码 | Entry 指向 | +| ---: | --- | --- | --- | +| 0 | `INTERNAL_DEVICE` | `DeviceIndexEntry` | 子设备索引节点 | +| 1 | `LEAF_DEVICE` | `DeviceIndexEntry` | 设备的测量索引根节点 | +| 2 | `INTERNAL_MEASUREMENT` | `MeasurementIndexEntry` | 子测量索引节点 | +| 3 | `LEAF_MEASUREMENT` | `MeasurementIndexEntry` | 一段 Timeseries Metadata | + +Entry 格式为: + +```text +DeviceIndexEntry := + DeviceID first_device_id + int64 offset + +MeasurementIndexEntry := + VarString first_measurement_id + int64 offset +``` + +每个 Entry 描述半开字节区间 `[offset, next_offset)`。`next_offset` 取下一个 Entry 的 +偏移量;最后一个 Entry 则取节点的 `end_offset`。结束偏移处的字节属于下一个区间。 + +```text +entry[0].offset <= entry[1].offset <= ... <= end_offset +``` + +Entry 的 key 是其覆盖范围内的第一个 key。设备索引按 `DeviceID` 片段逐段排序,null +小于非 null;短元组在公共前缀相同时排在长元组之前。测量索引按测量名词典序排序。 + +### 索引查找 + +在索引节点中查找键 `K` 时,选择 first key 小于或等于 `K` 的最后一个 Entry。所选字节 +区间从该 Entry 的 `offset` 开始,在下一 Entry 的 offset 处结束;最后一个 Entry 则在节点 +`end_offset` 处结束。内部节点区间包含下一级索引节点,叶子区间包含上表中对应的数据对象。 + +如果没有 Entry 的键小于或等于 `K`,该节点中就不存在目标键。测量索引的一个叶子区间 +可以包含多条 Timeseries Metadata,需要在该区间中继续查找完整的测量名。 + +沿索引继续读取前,检查: + +- `child_count` 与节点内完整 Entry 数量一致。 +- Entry key 按该节点的比较规则有序。 +- Entry offset 单调不减,每个引用区间长度非负。 +- 最后一个 Entry offset 不超过 `end_offset`。 +- 子节点层级与父节点的 node type 相符。 + +first-key 表示允许实现在节点内部对 Entry 使用二分查找,但文件格式不规定具体内存搜索 +算法。 + +
+ +## 文件元数据 + +`TsFileMetadata` 位于文件尾部长度字段之前: + +```text +TsFileMetadata := + UVarInt table_index_count + TableIndexRoot table_index[table_index_count] + UVarInt table_schema_count + NamedTableSchema table_schema[table_schema_count] + int64 meta_offset + BloomFilter bloom_filter + VarInt property_count + Property property[property_count] + +TableIndexRoot := + VarString table_name + MetadataIndexNode device_index_root + +NamedTableSchema := + VarString table_name + TableSchema schema + +Property := + VarString key + VarBinary value +``` + +属性键是 UTF-8 字符串,值是可空的原始字节数组,不要求为 UTF-8。例如,值字节 +`ff fe 00 80` 序列化为 `08 ff fe 00 80`,其中 `08` 是 `VarInt(4)`。null 值为 +`01`(`VarInt(-1)`),空值为 `00`。未知属性值必须逐字节保留,仅在具体属性约定要求时 +才按文本解码。 + +`meta_offset` 指向数据区末尾的 `0x02` separator。`TsFileMetadata` 前没有 marker, +其起始位置由文件尾保存的 4 字节长度确定。 + +Map 项的物理顺序不参与文件语义。`table_index` 保存树模型和表模型的设备索引根; +`table_schema` 保存显式表模型的 schema。 + +文件尾四字节元数据长度位于 `TsFileMetadata` 之外。设文件长度为 `F`、元数据长度为 `L`, +读取器使用下式得到精确字节区间: + +```text +file_metadata_start = F - 6 - 4 - L +file_metadata_end = F - 6 - 4 # 半开区间结束位置 +``` + +读取器必须恰好消费 `L` 字节。该检查可以在接受尾部魔数之前发现 Schema 不匹配或变长字段 +损坏。 + +
+ +## 表 Schema + +```text +TableSchema := + UVarInt column_count + TableColumn column[column_count] + +TableColumn := + MeasurementSchema schema + int32 column_category + +MeasurementSchema := + String32 column_name + byte data_type + byte encoding_type + byte compression_type + int32 property_count + SchemaProperty property[property_count] + +SchemaProperty := + String32 key + String32 value +``` + +`MeasurementSchema` 使用 `String32`,而不是 Footer 其他位置常用的 `VarString`。 + +列类别编号为: + +| 值 | 类别 | +| ---: | --- | +| 0 | `TAG` | +| 1 | `FIELD` | +| 2 | `ATTRIBUTE` | +| 3 | `TIME` | + +Schema 保留列顺序,其中 TAG 列的顺序决定 DeviceID 片段的顺序。FIELD Chunk 通过 +`measurement_id` 标识测点列,其物理排列顺序不必与 Schema 中 FIELD 列的顺序相同。 + +`TAG` 列标识设备,`FIELD` 列保存测量值,`ATTRIBUTE` 列描述实体,`TIME` 标识时间列。 +列类别说明这一列的用途,`data_type` 说明它保存哪种值,两者共同决定列的含义。 diff --git a/src/zh/UserGuide/latest/FileFormat/Nulls.md b/src/zh/UserGuide/latest/FileFormat/Nulls.md new file mode 100644 index 000000000..2ccdfaf76 --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Nulls.md @@ -0,0 +1,69 @@ + + +# 空值 + +非对齐时间序列只保存实际存在的 `(timestamp, value)` 点,Page 内没有 null 标记。 + +对齐值列通过 Value Page 的位图保存 null: + +```text +AlignedValuePagePayload := + int32 position_count + byte present_bitmap[ceil(position_count / 8)] + byte encoded_non_null_values[remaining bytes] +``` + +位置 `i` 对应 `present_bitmap[i / 8]` 的位 `7 - (i mod 8)`: + +- 位为 1:该位置有值。 +- 位为 0:该位置为 null。 + +值编码流只保存位图中为 1 的值,顺序与位置顺序一致。`position_count` 等于对应 +Time Page 的时间戳数量。 + +例如,`position_count = 10` 且位图为 `10110000 01000000` 时,位置 `0`、`2`、`3` 和 +`9` 有值。值解码器因此产生 4 个值并放到这些位置,其余 6 个位置为 null。最后一个 +字节中未使用的低位不对应逻辑位置,应当忽略。 + +非空 Value Page 满足以下约束: + +```text +bitmap_bytes = ceil(position_count / 8) +decoded_value_count = popcount(present_bitmap 的 position_count 个有效位) +position_count = 对应 Time Page 的 decoded_timestamp_count +``` + +任何不相等都表示结构损坏;读取器不能根据位图尾部的填充位或多余编码值推导新位置。 + +如果整个 Value Page 都是 null,Page 只写一个值为 0 的 `uncompressed_size`: + +```text +EmptyAlignedValuePage := UVarInt(0) +``` + +空 Page 不保存 `compressed_size`、Statistics、位图或值流。其位置数量由对应的 +Time Page 确定。 + +空 Value Page 会保留它在 Page 序列中的位置,使后续 Value Page 仍能对应到正确的 +Time Page。缺少 Value Chunk 则表示这个对齐 Chunk 集中没有该测量的物理列。还有一种 +情况是 `data_size = 0` 的 Value Chunk:它有 Header,但不包含任何 Page。遍历对齐 +Chunk 集时,需要区分这三种情况。 diff --git a/src/zh/UserGuide/latest/FileFormat/README.md b/src/zh/UserGuide/latest/FileFormat/README.md new file mode 100644 index 000000000..fc584980b --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/README.md @@ -0,0 +1,142 @@ + + +# 文件格式 + +TsFile 按列保存时序数据。写入时,数据按设备分组依次追加,索引则放在文件尾部。 +读取时通常反过来:先从文件尾找到索引,定位目标时间序列,再跳转到保存数据的 Chunk。 + +本文介绍版本字节为 `0x04` 的 TsFile v4 布局,说明各结构的存储方式及相互关系。 +编码章节列出了编码编号,并展开 `PLAIN` 的字节布局;其他值编码算法的完整位布局不在 +本文范围内。 + +## 范围与术语 + +文件中的数据分为三个层级: + +- **Page** 保存一组编码后的值,是独立压缩的单位。 +- **Chunk** 保存某个物理列的所有 Page,头部记录数据类型、编码和压缩方式。 +- **Chunk Group** 保存同一设备的一组 Chunk。随着数据分批刷出,一个设备可以在文件中 + 出现多次,对应多个 Chunk Group。 + +**非对齐数据**的每个测点列各自保存时间戳和值。**对齐数据**把共享的时间戳放在 +Time Chunk 中,各测点列分别放在 Value Chunk 中,再用位图记录哪些位置有值。 +两种布局的细节见[数据页](DataPages/README.md)。 + + + +
+ +## 物理区域 + +文件以六字节魔数 `TsFile` 和一字节版本号开头。Page AEAD 文件随后还有一个可选的 +文件加密头。数据区由唯一的 `0x02` 分隔符结束,之后是索引和文件元数据。 +最后十个字节保存元数据长度和尾部魔数。 + +下表使用以下记号: + +- `F`:文件总长度。 +- `L`:`TsFileMetadata` 的序列化长度,由文件尾的四字节长度字段保存。 +- `E`:可选文件加密头的总序列化长度;不存在时为 `0`。 +- `data_start = 7 + E`:数据区的起点。 +- `meta_offset`:分隔符的偏移量,保存在 `TsFileMetadata` 中。 + +| 区域 | 包含首尾的字节区间 | 定位方式 | +| --- | --- | --- | +| 头部魔数 | `0 … 5` | 固定位置 | +| 版本 | `6` | 固定位置 | +| 可选文件加密头 | `7 … 6 + E` | 仅 Page AEAD 存在,marker 为 `0x06` | +| 数据区 | `data_start … meta_offset - 1` | 在分隔符前结束 | +| 分隔符 | `meta_offset` | 值为 `0x02`,由 `TsFileMetadata` 引用 | +| 索引元数据 | `meta_offset + 1 … F - 11 - L` | 由元数据偏移划分 | +| `TsFileMetadata` | `F - 10 - L … F - 11` | 根据文件尾长度反向定位 | +| 元数据长度 | `F - 10 … F - 7` | 相对文件尾的固定位置 | +| 尾部魔数 | `F - 6 … F - 1` | 相对文件尾的固定位置 | + +偏移量从文件的第一个字节开始计算。上表的区间包含首尾;本文其余位置使用的 `[start, end)` +表示左闭右开区间。除非另有说明,长度不包含长度字段自身。`E = 0` 时,文件加密头对应 +的区间不存在。 + +索引元数据区中的 Timeseries Metadata 和索引节点可以交错存放,读取时应跟随偏移量, +不依赖它们的物理排列顺序。`TsFileMetadata` 自身没有标记,需要通过文件尾保存的长度 +反向定位。 + +## 写入模型 + +写入器先追加各个 Chunk Group,结束文件时再写索引和文件尾。Page 大小、刷出阈值和 +索引扇出会影响结构边界,但这些边界都能从头部和偏移量中读出。时间单位和非对齐时间 +编码仍需读写双方约定,详见[配置](Configurations.md)。 + +头部和尾部魔数使用相同的字节: + +```text +54 73 46 69 6c 65 # "TsFile" +``` + +## 元数据优先读取算法 + +读取器无需扫描整个数据区,就能定位目标序列: + +1. 校验头部魔数和版本,检查偏移 `7` 处是否有[文件加密头](DataPages/Encryption.md#file-encryption-header), + 据此确定 Chunk Header 和 Page 载荷的解析方式。 +2. 校验尾部魔数,在 `F - 10` 处读取 `L`,计算 `file_metadata_pos = F - 10 - L`。 +3. 读取 `TsFileMetadata`,检查 `meta_offset` 是否指向分隔符。 +4. 可以先用完整路径查询[布隆过滤器](Bloom-Filter.md)。只有确认字符集、路径构造规则和 + 哈希行为与写入端一致时,才能据此判断路径不存在。 +5. 依次查找表/设备索引、测量索引,定位目标序列的元数据。 +6. 根据序列和 Chunk 的统计信息选择 Chunk,跳转后再用可用的 Page 统计信息缩小范围。 +7. 读取选中 Page 的载荷,按需解密,然后解压和解码。 + +这样可以避免读取无关 Page 的数据。单 Page Chunk 不保存 Page Statistics,筛选时要使用 +Chunk 或序列级的统计信息。 + +## 结构不变量 + +读取时需要检查的主要关系包括: + +- 头尾魔数和版本有效。 +- 可选文件加密头只在偏移 `7` 处出现一次,且长度和版本有效。 +- `0 <= L <= F - 10`,且 `file_metadata_pos` 位于分隔符之后。 +- 一个 Chunk 的全部 Page 恰好占据完整 Chunk Header 之后的 `data_size` 字节。 +- 每个非空 Page 的载荷恰好占据 `compressed_size` 字节,且没有越过 Chunk 边界。 +- 同一索引节点内的偏移量单调不减,且不超过 `end_offset`。 +- 对齐 Value Page 和相同序号的 Time Page 覆盖同一组位置,且 `position_count` 等于该 + Time Page 的位置数。 + +不满足这些关系即为格式错误。 + +通用的长度和偏移量检查见[二进制表示](Binary-Representation.md),Page AEAD 额外要求的 +认证检查见[加密](DataPages/Encryption.md)。标记字节只在结构边界处有含义,不能把 Page +载荷中碰巧相同的字节当成标记。 + +
+ +## 阅读顺序与相关章节 + +初次阅读可以先看[二进制表示](Binary-Representation.md)、[Chunk 与 Chunk Group](DataPages/Chunks.md) +和[数据页](DataPages/README.md),再通过[元数据](Metadata.md)了解索引查找过程。 + +- [数据类型](Types.md)与[编码](DataPages/Encodings.md) +- [空值](Nulls.md)与[压缩](DataPages/Compression.md) +- [布隆过滤器](Bloom-Filter.md) +- [加密](DataPages/Encryption.md)与[校验和](DataPages/Checksumming.md) +- [错误恢复](DataPages/Error-Recovery.md) +- [配置](Configurations.md)、[格式版本](Format-Versions.md)与[扩展机制](Extensibility.md) diff --git a/src/zh/UserGuide/latest/FileFormat/Types.md b/src/zh/UserGuide/latest/FileFormat/Types.md new file mode 100644 index 000000000..97e68997b --- /dev/null +++ b/src/zh/UserGuide/latest/FileFormat/Types.md @@ -0,0 +1,82 @@ + + +# 类型 + +Chunk Header、Timeseries Metadata 和 Table Schema 使用单字节 `data_type` 编号: + +| 值 | 名称 | 物理/逻辑含义 | +| ---: | --- | --- | +| 0 | `BOOLEAN` | 布尔值 | +| 1 | `INT32` | 有符号 32 位整数 | +| 2 | `INT64` | 有符号 64 位整数 | +| 3 | `FLOAT` | IEEE 754 binary32 | +| 4 | `DOUBLE` | IEEE 754 binary64 | +| 5 | `TEXT` | 长度前缀字节串 | +| 6 | `VECTOR` | 对齐 Time Chunk 使用的物理类型标记 | +| 7 | `UNKNOWN` | 未知类型标记 | +| 8 | `TIMESTAMP` | 有符号 64 位整数;时间单位由上层约定 | +| 9 | `DATE` | `YYYYMMDD` 十进制形式的有符号 32 位整数 | +| 10 | `BLOB` | 任意字节串 | +| 11 | `STRING` | UTF-8 字符串 | +| 12 | `OBJECT` | 对象值的字节表示 | + +例如,日期 2026-07-27 用十进制整数 `20260727` 表示,并非从 Unix 纪元起累计的天数。 + +数据类型决定 Statistics 的附加字段和值编码器的输入类型。值的字节布局见 +[编码](DataPages/Encodings.md)。 + +## 物理类型与逻辑类型 + +数据类型说明值的取值范围,编码方式决定这些值如何写成字节。`TIMESTAMP` 和 `DATE` +还为底层整数赋予了具体含义: + +- `TIMESTAMP` 与 `INT64` 具有相同的有符号 64 位取值范围,时间单位由使用该文件的协议 + 或数据集约定,TsFile v4 不保存这个单位。 +- `DATE` 与 `INT32` 使用相同的整数编码接口,但有效逻辑值采用十进制 `YYYYMMDD` 形式。 +- `TEXT` 是带长度的字节串。`STRING` 使用相同的物理二进制容器,但具有 UTF-8 字符串 + 语义;`BLOB` 是不解释的二进制数据。 +- `OBJECT` 保存对象的字节表示,如何从这些字节还原对象,由交换文件的应用约定。 + +`VECTOR` 是对齐数据共享时间列使用的物理标记,不是标量值类型,也不定义独立值流编码。 +`UNKNOWN` 是哨兵值,不能作为可解码测量 Page 的有效类型。 + +## 类型一致性 + +同一序列的类型会在多个层级重复,使读取器既能从 Footer 进入,也能顺序扫描文件。这些副本 +必须一致: + +- 非对齐 Chunk 或对齐 Value Chunk Header 使用测量的标量 `data_type`。 +- 对应 Timeseries Metadata 使用相同的 `data_type`。 +- Statistics 根据该类型解析,自身不包含独立类型字节。 +- 对齐 Time Chunk 使用 `VECTOR`,解码值为有符号 64 位时间戳。 + +改变已经分配的类型编号含义属于不兼容变更。不支持某个类型编号的读取器仍可利用 +`data_size` 跳过完整 Chunk,但无法解码该序列的值或类型相关 Statistics。 + +## 排序与比较 + +整数和浮点值按数值顺序生成最小/最大 Statistics。`STRING` 的最小/最大值按原始值字节的 +字典序比较,不包含长度前缀。`TEXT` 只保存 first 和 last,`BLOB` 与 `OBJECT` 不保存 +类型专有的值统计信息。 + +类型提升和应用层转换不编码在 TsFile 中。读取器可以对外提供转换,但这不会改变磁盘 +`data_type`,也不会改变计算 Statistics 时使用的字节和值域。