Skip to content

Return the dataclass Metadata from the C reader #498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
oschwald wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Return the dataclass Metadata from the C reader #498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions HISTORY.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,15 @@ History
``os.pread``.
* C extension:

* Fixed segmentation faults from invalid use of ``Metadata``, ``Reader`` and
the internal iterator type.
* Fixed segmentation faults from invalid use of ``Reader`` and the internal
iterator type.
* Fixed a segmentation fault on a database with a map key that is not a
string. Such a database now raises ``InvalidDatabaseError``.
* Fixed large ``uint32`` values, which came back negative on platforms
with a 32-bit C ``long``, such as Windows.
* Fixed memory leaks and a use-after-free. A second ``__init__`` on a
``Reader`` now closes the old database, and an iterator from before it
raises ``ValueError``. A failed ``__init__`` keeps the old database.
Reinitializing a ``Metadata`` changes nothing.
* Fixed a ``RuntimeWarning`` or ``RuntimeError`` on free-threaded Python on
macOS when a ``Reader`` failed to open or was used without ``__init__``.
* Fixed a deadlock on free-threaded Python when a ``Reader`` was closed
Expand All @@ -57,8 +56,16 @@ History
This includes an error in the data of one record, but not a failure to
get the read lock. Previously, the next call returned the remaining
networks.
* Added the ``node_byte_size`` and ``search_tree_size`` properties to
``Metadata``, as the pure Python ``Metadata`` has.
* ``Reader.metadata()`` now returns the pure Python
``maxminddb.reader.Metadata`` dataclass, as the pure Python reader does.
Metadata from the two readers compares equal with ``==``, and it works
with ``repr()`` and ``dataclasses.asdict()``. It also has the
``node_byte_size`` and ``search_tree_size`` properties.
``maxminddb.extension.Metadata`` is now the same class. As with the pure
Python class, ``hash()`` raises ``TypeError``, the constructor takes
only keyword arguments and raises ``TypeError`` for a missing or unknown
argument, and assignment to an attribute raises
``dataclasses.FrozenInstanceError``. GitHub #468.

* Metadata:

Expand Down
231 changes: 34 additions & 197 deletions extension/maxminddb.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,19 +91,6 @@ typedef struct {
// Set once next() has raised StopIteration or an error.
bool done;
} ReaderIter_obj;

typedef struct {
PyObject_HEAD /* no semicolon */
PyObject *binary_format_major_version;
PyObject *binary_format_minor_version;
PyObject *build_epoch;
PyObject *database_type;
PyObject *description;
PyObject *ip_version;
PyObject *languages;
PyObject *node_count;
PyObject *record_size;
} Metadata_obj;
// clang-format on

// =============================================================================
Expand Down Expand Up @@ -139,7 +126,6 @@ static bool can_read(const char *path);
static int get_record(PyObject *self, PyObject *args, PyObject **record);
static PyObject *metadata_value(PyObject *map, const char *key);
static void set_error_from_cause(PyObject *type, const char *message);
static PyObject *Metadata_node_byte_size(PyObject *self, void *closure);
static PyObject *reader_iter_next(PyObject *self);
static PyObject *corrupt_tree_error(ReaderIter_obj *ri,
maxminddb_state *state,
Expand Down Expand Up @@ -745,18 +731,32 @@ static PyObject *Reader_metadata(PyObject *self, PyObject *UNUSED(args)) {
"Error decoding metadata.");
}
} else {
// The order of the values must match kwlist in Metadata_new.
metadata = PyObject_CallFunction(state->Metadata_Type,
"HHKOOHOIH",
binary_format_major_version,
binary_format_minor_version,
(unsigned long long)build_epoch,
database_type,
description,
ip_version,
languages,
(unsigned int)node_count,
record_size);
// Metadata takes keyword arguments only.
PyObject *kwargs =
Py_BuildValue("{s:H,s:H,s:K,s:O,s:O,s:H,s:O,s:I,s:H}",
"binary_format_major_version",
binary_format_major_version,
"binary_format_minor_version",
binary_format_minor_version,
"build_epoch",
(unsigned long long)build_epoch,
"database_type",
database_type,
"description",
description,
"ip_version",
ip_version,
"languages",
languages,
"node_count",
(unsigned int)node_count,
"record_size",
record_size);
if (kwargs != NULL) {
metadata = PyObject_VectorcallDict(
state->Metadata_Type, NULL, 0, kwargs);
Py_DECREF(kwargs);
}
}
Py_DECREF(map);
}
Expand Down Expand Up @@ -1158,74 +1158,6 @@ static void free_records(struct record *next) {
}
}

// Metadata is immutable, so tp_new sets every field and there is no tp_init.
// No object can then have a NULL field or be initialized twice.
static PyObject *
Metadata_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
PyObject *binary_format_major_version, *binary_format_minor_version,
*build_epoch, *database_type, *description, *ip_version, *languages,
*node_count, *record_size;

// Reader_metadata passes the values in this order.
static char *kwlist[] = {"binary_format_major_version",
"binary_format_minor_version",
"build_epoch",
"database_type",
"description",
"ip_version",
"languages",
"node_count",
"record_size",
NULL};

if (!PyArg_ParseTupleAndKeywords(args,
kwds,
"OOOOOOOOO:Metadata",
kwlist,
&binary_format_major_version,
&binary_format_minor_version,
&build_epoch,
&database_type,
&description,
&ip_version,
&languages,
&node_count,
&record_size)) {
return NULL;
}

Metadata_obj *obj = (Metadata_obj *)type->tp_alloc(type, 0);
if (obj == NULL) {
return NULL;
}
obj->binary_format_major_version = Py_NewRef(binary_format_major_version);
obj->binary_format_minor_version = Py_NewRef(binary_format_minor_version);
obj->build_epoch = Py_NewRef(build_epoch);
obj->database_type = Py_NewRef(database_type);
obj->description = Py_NewRef(description);
obj->ip_version = Py_NewRef(ip_version);
obj->languages = Py_NewRef(languages);
obj->node_count = Py_NewRef(node_count);
obj->record_size = Py_NewRef(record_size);
return (PyObject *)obj;
}

static void Metadata_dealloc(PyObject *self) {
Metadata_obj *obj = (Metadata_obj *)self;
Py_DECREF(obj->binary_format_major_version);
Py_DECREF(obj->binary_format_minor_version);
Py_DECREF(obj->build_epoch);
Py_DECREF(obj->database_type);
Py_DECREF(obj->description);
Py_DECREF(obj->ip_version);
Py_DECREF(obj->languages);
Py_DECREF(obj->node_count);
Py_DECREF(obj->record_size);
PyTypeObject *type = Py_TYPE(self);
PyObject_Del(self);
Py_DECREF(type);
}

static PyObject *
from_entry_data_list(maxminddb_state *state,
MMDB_entry_data_list_s **entry_data_list) {
Expand Down Expand Up @@ -1431,90 +1363,6 @@ static PyMemberDef Reader_members[] = {

static PyMethodDef ReaderIter_methods[] = {{NULL, NULL, 0, NULL}};

static PyMethodDef Metadata_methods[] = {{NULL, NULL, 0, NULL}};

static PyMemberDef Metadata_members[] = {
{"binary_format_major_version",
T_OBJECT,
offsetof(Metadata_obj, binary_format_major_version),
READONLY,
NULL},
{"binary_format_minor_version",
T_OBJECT,
offsetof(Metadata_obj, binary_format_minor_version),
READONLY,
NULL},
{"build_epoch",
T_OBJECT,
offsetof(Metadata_obj, build_epoch),
READONLY,
NULL},
{"database_type",
T_OBJECT,
offsetof(Metadata_obj, database_type),
READONLY,
NULL},
{"description",
T_OBJECT,
offsetof(Metadata_obj, description),
READONLY,
NULL},
{"ip_version",
T_OBJECT,
offsetof(Metadata_obj, ip_version),
READONLY,
NULL},
{"languages", T_OBJECT, offsetof(Metadata_obj, languages), READONLY, NULL},
{"node_count",
T_OBJECT,
offsetof(Metadata_obj, node_count),
READONLY,
NULL},
{"record_size",
T_OBJECT,
offsetof(Metadata_obj, record_size),
READONLY,
NULL},
{NULL, 0, 0, 0, NULL}};

static PyObject *Metadata_search_tree_size(PyObject *self,
void *UNUSED(closure)) {
Metadata_obj *obj = (Metadata_obj *)self;
PyObject *node_byte_size = Metadata_node_byte_size(self, NULL);
if (node_byte_size == NULL) {
return NULL;
}
PyObject *search_tree_size =
PyNumber_Multiply(obj->node_count, node_byte_size);
Py_DECREF(node_byte_size);
return search_tree_size;
}

static PyObject *Metadata_node_byte_size(PyObject *self,
void *UNUSED(closure)) {
Metadata_obj *obj = (Metadata_obj *)self;
PyObject *four = PyLong_FromLong(4);
if (four == NULL) {
return NULL;
}
PyObject *node_byte_size = PyNumber_FloorDivide(obj->record_size, four);
Py_DECREF(four);
return node_byte_size;
}

// These match the properties of the pure Python Metadata class.
static PyGetSetDef Metadata_getset[] = {{"node_byte_size",
Metadata_node_byte_size,
NULL,
"The size of a node in bytes.",
NULL},
{"search_tree_size",
Metadata_search_tree_size,
NULL,
"The size of the search tree.",
NULL},
{NULL, NULL, NULL, NULL, NULL}};

// =============================================================================
// Type specs for heap type conversion (PEP 489)
// =============================================================================
Expand All @@ -1537,23 +1385,6 @@ static PyType_Spec Reader_Type_spec = {
.slots = Reader_Type_slots,
};

static PyType_Slot Metadata_Type_slots[] = {
{Py_tp_doc, "Metadata object"},
{Py_tp_dealloc, Metadata_dealloc},
{Py_tp_new, Metadata_new},
{Py_tp_methods, Metadata_methods},
{Py_tp_members, Metadata_members},
{Py_tp_getset, Metadata_getset},
{0, NULL},
};

static PyType_Spec Metadata_Type_spec = {
.name = "maxminddb.extension.Metadata",
.basicsize = sizeof(Metadata_obj),
.flags = Py_TPFLAGS_DEFAULT,
.slots = Metadata_Type_slots,
};

static PyType_Slot ReaderIter_Type_slots[] = {
{Py_tp_doc, "Iterator for Reader object"},
{Py_tp_dealloc, ReaderIter_dealloc},
Expand Down Expand Up @@ -1614,8 +1445,14 @@ static int maxminddb_exec(PyObject *module) {
return -1;
}

state->Metadata_Type =
PyType_FromModuleAndSpec(module, &Metadata_Type_spec, NULL);
// metadata() returns the pure Python Metadata, so both readers return
// the same type.
PyObject *reader_mod = PyImport_ImportModule("maxminddb.reader");
if (reader_mod == NULL) {
return -1;
}
state->Metadata_Type = PyObject_GetAttrString(reader_mod, "Metadata");
Py_DECREF(reader_mod);
if (state->Metadata_Type == NULL) {
return -1;
}
Expand Down
Loading
Loading