Skip to content

Commit

Permalink
Merge pull request #1 from SEKOIA-IO/feat/reduce_memory_footprint
Browse files Browse the repository at this point in the history 
feat(radix): Reduce memory footprint of the C implementation
  • Loading branch information
@Darkheir
Darkheir authored Jul 10, 2020
2 parents 52d70b0 + ad1c207 commit 56b0ef9
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 39 deletions.
34 changes: 19 additions & 15 deletions README.rst
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
py-radix
========

.. image:: https://travis-ci.org/mjschultz/py-radix.svg?branch=master
:target: https://travis-ci.org/mjschultz/py-radix
Fork of the original library to reduce the memory footprint of the C implementation.

.. image:: https://coveralls.io/repos/mjschultz/py-radix/badge.png?branch=master
:target: https://coveralls.io/r/mjschultz/py-radix?branch=master
The following attributes have been converted to properties and will be computed at query time:

* network
* prefix

The `data` object is now `None` by default instead of being an empty dict. It can be used to store any type of data.

py-radix implements the radix tree data structure for the storage and
retrieval of IPv4 and IPv6 network prefixes.
Expand Down Expand Up @@ -37,6 +40,7 @@ Usage

A simple example that demonstrates most of the features: ::

import socket
import radix

# Create a new tree
Expand All @@ -45,7 +49,7 @@ A simple example that demonstrates most of the features: ::
# Adding a node returns a RadixNode object. You can create
# arbitrary members in its 'data' dict to store your data
rnode = rtree.add("10.0.0.0/8")
rnode.data["blah"] = "whatever you want"
rnode.data = {"blah": "whatever you want"}

# You can specify nodes as CIDR addresses, or networks with
# separate mask lengths. The following three invocations are
Expand All @@ -59,16 +63,16 @@ A simple example that demonstrates most of the features: ::
# functions. In this case, the radix module will assume that
# a four-byte address is an IPv4 address and a sixteen-byte
# address is an IPv6 address. For example:
binary_addr = inet_ntoa("172.18.22.0")
binary_addr = socket.inet_aton("172.18.22.0")
rnode = rtree.add(packed = binary_addr, masklen = 23)

# Exact search will only return prefixes you have entered
# You can use all of the above ways to specify the address
rnode = rtree.search_exact("10.0.0.0/8")
# Get your data back out
print rnode.data["blah"]
print(rnode.data["blah"])
# Use a packed address
addr = socket.inet_ntoa("10.0.0.0")
addr = socket.inet_aton("10.0.0.0")
rnode = rtree.search_exact(packed = addr, masklen = 8)

# Best-match search will return the longest matching prefix
Expand All @@ -85,11 +89,11 @@ A simple example that demonstrates most of the features: ::
rnodes = rtree.search_covered("10.123.0.0/16")

# There are a couple of implicit members of a RadixNode:
print rnode.network # -> "10.0.0.0"
print rnode.prefix # -> "10.0.0.0/8"
print rnode.prefixlen # -> 8
print rnode.family # -> socket.AF_INET
print rnode.packed # -> '\n\x00\x00\x00'
print(rnode.network) # -> "10.0.0.0"
print(rnode.prefix) # -> "10.0.0.0/8"
print(rnode.prefixlen) # -> 8
print(rnode.family) # -> socket.AF_INET
print(rnode.packed) # -> '\n\x00\x00\x00'

# IPv6 prefixes are fully supported in the same tree
rnode = rtree.add("2001:DB8::/3")
Expand All @@ -98,7 +102,7 @@ A simple example that demonstrates most of the features: ::
# Use the nodes() method to return all RadixNodes created
nodes = rtree.nodes()
for rnode in nodes:
print rnode.prefix
print(rnode.prefix)

# The prefixes() method will return all the prefixes (as a
# list of strings) that have been entered
Expand All @@ -111,7 +115,7 @@ A simple example that demonstrates most of the features: ::
# receive a RuntimeWarning. Changing a node's data dict
# is permitted.
for rnode in rtree:
print rnode.prefix
print(rnode.prefix)


License
Expand Down
64 changes: 40 additions & 24 deletions radix/_radix.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,6 @@ PyObject *radix_constructor;
typedef struct {
PyObject_HEAD
PyObject *user_attr; /* User-specified attributes */
PyObject *network;
PyObject *prefix;
PyObject *prefixlen;
PyObject *family;
PyObject *packed;
Expand All @@ -64,7 +62,6 @@ static RadixNodeObject *
newRadixNodeObject(radix_node_t *rn)
{
RadixNodeObject *self;
char network[256], prefix[256];

/* Sanity check */
if (rn == NULL || rn->prefix == NULL ||
Expand All @@ -77,24 +74,16 @@ newRadixNodeObject(radix_node_t *rn)

self->rn = rn;

/* Format addresses for packing into objects */
prefix_addr_ntop(rn->prefix, network, sizeof(network));
prefix_ntop(rn->prefix, prefix, sizeof(prefix));

self->user_attr = PyDict_New();
self->network = PyString_FromString(network);
self->prefix = PyString_FromString(prefix);
self->user_attr = NULL;
self->prefixlen = PyInt_FromLong(rn->prefix->bitlen);
self->family = PyInt_FromLong(rn->prefix->family);
self->packed = PyString_FromStringAndSize((char*)&rn->prefix->add,
rn->prefix->family == AF_INET ? 4 : 16);

if (self->user_attr == NULL || self->prefixlen == NULL ||
self->family == NULL || self->network == NULL ||
self->prefix == NULL) {
/* RadixNode_dealloc will clean up for us */
Py_XDECREF(self);
return (NULL);
if (self->family == NULL || self->prefixlen == NULL) {
/* RadixNode_dealloc will clean up for us */
Py_XDECREF(self);
return (NULL);
}

return self;
Expand All @@ -108,8 +97,6 @@ RadixNode_dealloc(RadixNodeObject *self)
Py_XDECREF(self->user_attr);
Py_XDECREF(self->prefixlen);
Py_XDECREF(self->family);
Py_XDECREF(self->network);
Py_XDECREF(self->prefix);
Py_XDECREF(self->packed);
PyObject_Del(self);
}
Expand Down Expand Up @@ -138,10 +125,24 @@ Radix_parent(RadixNodeObject *self, void *closure)
}
Py_RETURN_NONE;
}
static PyObject *
Radix_prefix(RadixNodeObject *self, void *closure)
{
char buf[256];
return PyString_FromString(
prefix_ntop(self->rn->prefix, buf, sizeof(buf))
);
}
static PyObject *
Radix_network(RadixNodeObject *self, void *closure)
{
char buf[256];
return PyString_FromString(
prefix_addr_ntop(self->rn->prefix, buf, sizeof(buf))
);
}
static PyMemberDef RadixNode_members[] = {
{"data", T_OBJECT, offsetof(RadixNodeObject, user_attr), READONLY},
{"network", T_OBJECT, offsetof(RadixNodeObject, network), READONLY},
{"prefix", T_OBJECT, offsetof(RadixNodeObject, prefix), READONLY},
{"data", T_OBJECT, offsetof(RadixNodeObject, user_attr), 0},
{"prefixlen", T_OBJECT, offsetof(RadixNodeObject, prefixlen), READONLY},
{"family", T_OBJECT, offsetof(RadixNodeObject, family), READONLY},
{"packed", T_OBJECT, offsetof(RadixNodeObject, packed), READONLY},
Expand All @@ -155,6 +156,18 @@ static PyGetSetDef node_getter[] = {
"parent of node", /* optional doc string */
NULL /* optional additional data for getter and setter */
},
{"prefix",
(getter) Radix_prefix, /* C function to get the attribute */
NULL, /* C function to set the attribute */
"Node prefix", /* optional doc string */
NULL /* optional additional data for getter and setter */
},
{"network",
(getter) Radix_network, /* C function to get the attribute */
NULL, /* C function to set the attribute */
"Node network", /* optional doc string */
NULL /* optional additional data for getter and setter */
},
{NULL} /* Sentinel */
};

Expand Down Expand Up @@ -524,7 +537,7 @@ add_node_to_list(radix_node_t *node, void *arg)
PyObject *ret = arg;

if (node->data != NULL)
PyList_Append(ret, ((RadixNodeObject *)node->data));
PyList_Append(ret, (PyObject *)((RadixNodeObject *)node->data));
return (0);
}

Expand Down Expand Up @@ -643,8 +656,11 @@ Radix_prefixes(RadixObject *self, PyObject *args)

RADIX_TREE_WALK(self->rt, node) {
if (node->data != NULL) {
PyList_Append(ret,
((RadixNodeObject *)node->data)->prefix);
char buf[256];
PyObject *prefix = PyString_FromString(
prefix_ntop(node->prefix, buf, sizeof(buf))
);
PyList_Append(ret, prefix);
}
} RADIX_TREE_WALK_END;

Expand Down

0 comments on commit 56b0ef9

Please sign in to comment.