Skip to content

FORM: locate every cell through the index container #1007

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gemmeren merged 1 commit into from
Oct 22, 2026
Merged

FORM: locate every cell through the index container #1007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
313 changes: 53 additions & 260 deletions form/storage/storage_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,9 @@
#include "storage/factories.hpp"
#include "storage/istorage.hpp"

#include <algorithm>
#include <cctype>
#include <cstddef>
#include <map>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <typeinfo>
Expand Down Expand Up @@ -52,139 +50,6 @@ namespace {
}
return per_container->second;
}

bool is_structured_index_id(std::string const& id)
{
return id.size() >= 2 && id.front() == '[' && id.back() == ']';
}

std::string trim_copy(std::string const& value)
{
auto begin = value.find_first_not_of(' ');
if (begin == std::string::npos) {
return {};
}
auto end = value.find_last_not_of(' ');
return value.substr(begin, end - begin + 1);
}

std::optional<long long> parse_index_number(std::string const& value)
{
if (value.empty()) {
return std::nullopt;
}

for (char const ch : value) {
if (!std::isdigit(static_cast<unsigned char>(ch))) {
return std::nullopt;
}
}

try {
return std::stoll(value, nullptr, 10);
} catch (...) {
return std::nullopt;
}
}

std::optional<std::map<std::string, long long>> normalize_structured_index(std::string const& id)
{
if (!is_structured_index_id(id) || id == "[]") {
return std::nullopt;
}

std::string const body = id.substr(1, id.size() - 2);
std::string token;
std::map<std::string, long long> normalized;

auto commit_token = [&normalized](std::string const& raw_token) -> bool {
auto const token_trimmed = trim_copy(raw_token);
if (token_trimmed.empty()) {
return true;
}

auto const sep = token_trimmed.find_first_of(":=");
if (sep == std::string::npos) {
return false;
}

auto key = trim_copy(token_trimmed.substr(0, sep));
auto value = trim_copy(token_trimmed.substr(sep + 1));
if (key.empty() || value.empty()) {
return false;
}

for (char& ch : key) {
ch = static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
}

auto parsed = parse_index_number(value);
if (!parsed) {
return false;
}

normalized[key] = *parsed;
return true;
};

for (char const ch : body) {
if (ch == ',' || ch == ';') {
if (!commit_token(token)) {
return std::nullopt;
}
token.clear();
} else {
token.push_back(ch);
}
}

if (!commit_token(token)) {
return std::nullopt;
}

if (normalized.empty()) {
return std::nullopt;
}
return normalized;
}

bool all_components_zero(std::map<std::string, long long> const& components)
{
if (components.empty()) {
return false;
}
return std::ranges::all_of(components,
[](auto const& component) { return component.second == 0; });
}

std::optional<int> sequential_row_from_index_id(std::string const& id)
{
if (id == "[]") {
return 0;
}

if (id.size() < 2 || id.front() != '[' || id.back() != ']') {
return std::nullopt;
}

auto const body = id.substr(1, id.size() - 2);
auto const comma = body.find(',');
if (comma != std::string::npos) {
return std::nullopt;
}

auto const colon = body.find(':');
if (colon == std::string::npos) {
return std::nullopt;
}

try {
auto const number = std::stoi(body.substr(colon + 1));
return number;
} catch (...) {
return std::nullopt;
}
}
}
// Factory function implementation
namespace form::detail::experimental {
Expand All @@ -198,85 +63,13 @@ int storage_reader::get_index(token const& token,
std::string const& id,
form::experimental::config::tech_setting_config const& settings)
{
if (auto row = sequential_row_from_index_id(id)) {
return *row;
}

if (index_maps_[token.container_name()].empty()) {
auto cont_key = std::make_pair(token.file_name(), token.container_name());
auto cont = read_containers_.find(cont_key);
if (cont == read_containers_.end()) {
auto file = files_.find(token.file_name());
if (file == files_.end()) {
file =
files_
.insert({token.file_name(), create_file(token.technology(), token.file_name(), 'i')})
.first;
for (auto const& [key, value] :
get_file_table(settings, token.technology(), token.file_name())) {
file->second->set_attribute(key, value);
}
}
cont =
read_containers_
.insert({cont_key, create_read_container(token.technology(), token.container_name())})
.first;
for (auto const& [key, value] :
get_container_table(settings, token.technology(), token.container_name())) {
cont->second->set_attribute(key, value);
}
cont->second->set_file(file->second);
}
auto const& type = typeid(std::string);
int entry = 0;
void const* raw_data = nullptr;
while (cont->second->read(entry, &raw_data, type)) {
std::unique_ptr<std::string const> const data(static_cast<std::string const*>(raw_data));
index_maps_[token.container_name()].insert(std::make_pair(*data, entry));
entry++;
}

if (index_maps_[token.container_name()].empty()) {
if (!is_structured_index_id(id)) {
return 0;
}
throw std::runtime_error("Unable to read index data from container: " +
token.container_name());
}
}

auto const found = index_maps_[token.container_name()].find(id);
if (found != index_maps_[token.container_name()].end()) {
return found->second;
auto const& index = index_map(token, settings);
auto const found = index.find(id);
if (found == index.end()) {
throw std::runtime_error("Index id not found: " + id + " in container: " +
token.container_name() + " of file: " + token.file_name());
}

auto const normalized_query = normalize_structured_index(id);
if (normalized_query) {
for (auto const& [existing_id, entry] : index_maps_[token.container_name()]) {
auto const normalized_existing = normalize_structured_index(existing_id);
if (normalized_existing && *normalized_existing == *normalized_query) {
return entry;
}
}

if (all_components_zero(*normalized_query)) {
auto const empty_key = index_maps_[token.container_name()].find("");
if (empty_key != index_maps_[token.container_name()].end()) {
return 0;
}

// Compatibility fallback for backends that do not persist the first
// structured id string in a normalized round-trippable form.
return 0;
}
}

if (!is_structured_index_id(id)) {
return 0;
}

throw std::runtime_error("Index id not found: " + id +
" in container: " + token.container_name());
return found->second;
}

void storage_reader::prime(token const& token,
Expand Down Expand Up @@ -311,61 +104,61 @@ void storage_reader::prime(token const& token,
std::vector<std::string> storage_reader::list_indices(
token const& token, form::experimental::config::tech_setting_config const& settings)
{
if (index_maps_[token.container_name()].empty()) {
auto cont_key = std::make_pair(token.file_name(), token.container_name());
auto cont = read_containers_.find(cont_key);
if (cont == read_containers_.end()) {
auto file = files_.find(token.file_name());
if (file == files_.end()) {
file =
files_
.insert({token.file_name(), create_file(token.technology(), token.file_name(), 'i')})
.first;
for (auto const& [key, value] :
get_file_table(settings, token.technology(), token.file_name())) {
file->second->set_attribute(key, value);
}
}
cont =
read_containers_
.insert({cont_key, create_read_container(token.technology(), token.container_name())})
auto const& index = index_map(token, settings);
std::vector<std::string> result(index.size());
for (auto const& [id, row] : index) {
result[static_cast<std::size_t>(row)] = id;
}
return result;
}

std::map<std::string, int> const& storage_reader::index_map(
token const& token, form::experimental::config::tech_setting_config const& settings)
{
auto const cont_key = std::make_pair(token.file_name(), token.container_name());
if (auto const built = index_maps_.find(cont_key); built != index_maps_.end()) {
return built->second;
}
Comment thread
wwuoneway marked this conversation as resolved.

auto cont = read_containers_.find(cont_key);
if (cont == read_containers_.end()) {
auto file = files_.find(token.file_name());
if (file == files_.end()) {
file =
files_.insert({token.file_name(), create_file(token.technology(), token.file_name(), 'i')})
.first;
for (auto const& [key, value] :
get_container_table(settings, token.technology(), token.container_name())) {
cont->second->set_attribute(key, value);
get_file_table(settings, token.technology(), token.file_name())) {
file->second->set_attribute(key, value);
}
cont->second->set_file(file->second);
}

auto const& type = typeid(std::string);
int entry = 0;
void const* raw_data = nullptr;
while (cont->second->read(entry, &raw_data, type)) {
std::unique_ptr<std::string const> const data(static_cast<std::string const*>(raw_data));
index_maps_[token.container_name()].insert(std::make_pair(*data, entry));
entry++;
}

if (index_maps_[token.container_name()].empty()) {
throw std::runtime_error("Unable to enumerate indices from container: " +
token.container_name());
cont = read_containers_
.insert({cont_key, create_read_container(token.technology(), token.container_name())})
.first;
for (auto const& [key, value] :
get_container_table(settings, token.technology(), token.container_name())) {
cont->second->set_attribute(key, value);
}
cont->second->set_file(file->second);
}

std::vector<std::pair<int, std::string>> ordered;
ordered.reserve(index_maps_[token.container_name()].size());
for (auto const& [index_string, entry] : index_maps_[token.container_name()]) {
ordered.emplace_back(entry, index_string);
std::map<std::string, int> index;
auto const& type = typeid(std::string);
int row = 0;
void const* raw_data = nullptr;
while (cont->second->read(row, &raw_data, type)) {
std::unique_ptr<std::string const> const id(static_cast<std::string const*>(raw_data));
if (!index.emplace(*id, row).second) {
throw std::runtime_error("Index id " + *id + " appears more than once in container: " +
token.container_name() + " of file: " + token.file_name());
}
++row;
}
std::ranges::sort(ordered,
[](auto const& lhs, auto const& rhs) { return lhs.first < rhs.first; });

std::vector<std::string> result;
result.reserve(ordered.size());
for (auto const& [_, index_string] : ordered) {
result.push_back(index_string);
if (index.empty()) {
throw std::runtime_error("Unable to read index data from container: " + token.container_name() +
" of file: " + token.file_name());
}
return result;
return index_maps_.emplace(cont_key, std::move(index)).first->second;
}

void storage_reader::read_container(token const& token,
Expand Down
5 changes: 4 additions & 1 deletion form/storage/storage_reader.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,15 @@ namespace form::detail::experimental {
form::experimental::config::tech_setting_config const& settings) override;

private:
std::map<std::string, int> const& index_map(
token const& token, form::experimental::config::tech_setting_config const& settings);

std::map<std::string, std::shared_ptr<i_storage_file>> files_;
std::unordered_map<std::pair<std::string, std::string>,
std::shared_ptr<i_storage_read_container>,
pair_hash>
read_containers_;
std::map<std::string, std::map<std::string, int>> index_maps_;
std::map<std::pair<std::string, std::string>, std::map<std::string, int>> index_maps_;
};

} // namespace form::detail::experimental
Expand Down
Loading
Loading