diff --git a/modules/geoip/AMBuilder b/modules/geoip/AMBuilder
index 78bb0580..2e79c52e 100644
--- a/modules/geoip/AMBuilder
+++ b/modules/geoip/AMBuilder
@@ -13,6 +13,7 @@ binary.compiler.defines += [
binary.sources = [
'../../public/sdk/amxxmodule.cpp',
+ '../../third_party/libmaxminddb/data-pool.c',
'../../third_party/libmaxminddb/maxminddb.c',
'geoip_main.cpp',
'geoip_natives.cpp',
diff --git a/modules/geoip/msvc12/geoip.vcxproj b/modules/geoip/msvc12/geoip.vcxproj
index 464f19ba..e23c0fb0 100644
--- a/modules/geoip/msvc12/geoip.vcxproj
+++ b/modules/geoip/msvc12/geoip.vcxproj
@@ -102,6 +102,7 @@
+
@@ -109,6 +110,7 @@
+
@@ -124,4 +126,4 @@
-
+
\ No newline at end of file
diff --git a/modules/geoip/msvc12/geoip.vcxproj.filters b/modules/geoip/msvc12/geoip.vcxproj.filters
index ee2e53ea..6836a135 100644
--- a/modules/geoip/msvc12/geoip.vcxproj.filters
+++ b/modules/geoip/msvc12/geoip.vcxproj.filters
@@ -38,6 +38,9 @@
GeoIP2
+
+ GeoIP2
+
@@ -64,6 +67,9 @@
GeoIP2
+
+ GeoIP2
+
diff --git a/third_party/libmaxminddb/data-pool.c b/third_party/libmaxminddb/data-pool.c
new file mode 100644
index 00000000..48521b64
--- /dev/null
+++ b/third_party/libmaxminddb/data-pool.c
@@ -0,0 +1,180 @@
+#include "data-pool.h"
+#include "maxminddb.h"
+
+#include
+#include
+#include
+
+static bool can_multiply(size_t const, size_t const, size_t const);
+
+// Allocate an MMDB_data_pool_s. It initially has space for size
+// MMDB_entry_data_list_s structs.
+MMDB_data_pool_s *data_pool_new(size_t const size)
+{
+ MMDB_data_pool_s *const pool = calloc(1, sizeof(MMDB_data_pool_s));
+ if (!pool) {
+ return NULL;
+ }
+
+ if (size == 0 ||
+ !can_multiply(SIZE_MAX, size, sizeof(MMDB_entry_data_list_s))) {
+ data_pool_destroy(pool);
+ return NULL;
+ }
+ pool->size = size;
+ pool->blocks[0] = calloc(pool->size, sizeof(MMDB_entry_data_list_s));
+ if (!pool->blocks[0]) {
+ data_pool_destroy(pool);
+ return NULL;
+ }
+ pool->blocks[0]->pool = pool;
+
+ pool->sizes[0] = size;
+
+ pool->block = pool->blocks[0];
+
+ return pool;
+}
+
+// Determine if we can multiply m*n. We can do this if the result will be below
+// the given max. max will typically be SIZE_MAX.
+//
+// We want to know if we'll wrap around.
+static bool can_multiply(size_t const max, size_t const m, size_t const n)
+{
+ if (m == 0) {
+ return false;
+ }
+
+ return n <= max / m;
+}
+
+// Clean up the data pool.
+void data_pool_destroy(MMDB_data_pool_s *const pool)
+{
+ if (!pool) {
+ return;
+ }
+
+ for (size_t i = 0; i <= pool->index; i++) {
+ free(pool->blocks[i]);
+ }
+
+ free(pool);
+}
+
+// Claim a new struct from the pool. Doing this may cause the pool's size to
+// grow.
+MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const pool)
+{
+ if (!pool) {
+ return NULL;
+ }
+
+ if (pool->used < pool->size) {
+ MMDB_entry_data_list_s *const element = pool->block + pool->used;
+ pool->used++;
+ return element;
+ }
+
+ // Take it from a new block of memory.
+
+ size_t const new_index = pool->index + 1;
+ if (new_index == DATA_POOL_NUM_BLOCKS) {
+ // See the comment about not growing this on DATA_POOL_NUM_BLOCKS.
+ return NULL;
+ }
+
+ if (!can_multiply(SIZE_MAX, pool->size, 2)) {
+ return NULL;
+ }
+ size_t const new_size = pool->size * 2;
+
+ if (!can_multiply(SIZE_MAX, new_size, sizeof(MMDB_entry_data_list_s))) {
+ return NULL;
+ }
+ pool->blocks[new_index] = calloc(new_size, sizeof(MMDB_entry_data_list_s));
+ if (!pool->blocks[new_index]) {
+ return NULL;
+ }
+
+ // We don't need to set this, but it's useful for introspection in tests.
+ pool->blocks[new_index]->pool = pool;
+
+ pool->index = new_index;
+ pool->block = pool->blocks[pool->index];
+
+ pool->size = new_size;
+ pool->sizes[pool->index] = pool->size;
+
+ MMDB_entry_data_list_s *const element = pool->block;
+ pool->used = 1;
+ return element;
+}
+
+// Turn the structs in the array-like pool into a linked list.
+//
+// Before calling this function, the list isn't linked up.
+MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const pool)
+{
+ if (!pool) {
+ return NULL;
+ }
+
+ if (pool->index == 0 && pool->used == 0) {
+ return NULL;
+ }
+
+ for (size_t i = 0; i <= pool->index; i++) {
+ MMDB_entry_data_list_s *const block = pool->blocks[i];
+
+ size_t size = pool->sizes[i];
+ if (i == pool->index) {
+ size = pool->used;
+ }
+
+ for (size_t j = 0; j < size - 1; j++) {
+ MMDB_entry_data_list_s *const cur = block + j;
+ cur->next = block + j + 1;
+ }
+
+ if (i < pool->index) {
+ MMDB_entry_data_list_s *const last = block + size - 1;
+ last->next = pool->blocks[i + 1];
+ }
+ }
+
+ return pool->blocks[0];
+}
+
+#ifdef TEST_DATA_POOL
+
+#include
+#include
+
+static void test_can_multiply(void);
+
+int main(void)
+{
+ plan(NO_PLAN);
+ test_can_multiply();
+ done_testing();
+}
+
+static void test_can_multiply(void)
+{
+ {
+ ok(can_multiply(SIZE_MAX, 1, SIZE_MAX), "1*SIZE_MAX is ok");
+ }
+
+ {
+ ok(!can_multiply(SIZE_MAX, 2, SIZE_MAX), "2*SIZE_MAX is not ok");
+ }
+
+ {
+ ok(can_multiply(SIZE_MAX, 10240, sizeof(MMDB_entry_data_list_s)),
+ "1024 entry_data_list_s's are okay");
+ }
+}
+
+#endif
diff --git a/third_party/libmaxminddb/data-pool.h b/third_party/libmaxminddb/data-pool.h
new file mode 100644
index 00000000..25d09923
--- /dev/null
+++ b/third_party/libmaxminddb/data-pool.h
@@ -0,0 +1,52 @@
+#ifndef DATA_POOL_H
+#define DATA_POOL_H
+
+#include "maxminddb.h"
+
+#include
+#include
+
+// This should be large enough that we never need to grow the array of pointers
+// to blocks. 32 is enough. Even starting out of with size 1 (1 struct), the
+// 32nd element alone will provide 2**32 structs as we exponentially increase
+// the number in each block. Being confident that we do not have to grow the
+// array lets us avoid writing code to do that. That code would be risky as it
+// would rarely be hit and likely not be well tested.
+#define DATA_POOL_NUM_BLOCKS 32
+
+// A pool of memory for MMDB_entry_data_list_s structs. This is so we can
+// allocate multiple up front rather than one at a time for performance
+// reasons.
+//
+// The order you add elements to it (by calling data_pool_alloc()) ends up as
+// the order of the list.
+//
+// The memory only grows. There is no support for releasing an element you take
+// back to the pool.
+typedef struct MMDB_data_pool_s {
+ // Index of the current block we're allocating out of.
+ size_t index;
+
+ // The size of the current block, counting by structs.
+ size_t size;
+
+ // How many used in the current block, counting by structs.
+ size_t used;
+
+ // The current block we're allocating out of.
+ MMDB_entry_data_list_s *block;
+
+ // The size of each block.
+ size_t sizes[DATA_POOL_NUM_BLOCKS];
+
+ // An array of pointers to blocks of memory holding space for list
+ // elements.
+ MMDB_entry_data_list_s *blocks[DATA_POOL_NUM_BLOCKS];
+} MMDB_data_pool_s;
+
+MMDB_data_pool_s *data_pool_new(size_t const);
+void data_pool_destroy(MMDB_data_pool_s *const);
+MMDB_entry_data_list_s *data_pool_alloc(MMDB_data_pool_s *const);
+MMDB_entry_data_list_s *data_pool_to_list(MMDB_data_pool_s *const);
+
+#endif
diff --git a/third_party/libmaxminddb/maxminddb.c b/third_party/libmaxminddb/maxminddb.c
index c08b35ba..7580e1ea 100644
--- a/third_party/libmaxminddb/maxminddb.c
+++ b/third_party/libmaxminddb/maxminddb.c
@@ -1,12 +1,14 @@
#if HAVE_CONFIG_H
#include
#endif
+#include "data-pool.h"
#include "maxminddb.h"
#include "maxminddb-compat-util.h"
#include
#include
#include
#include
+#include
#include
#include
#include
@@ -21,6 +23,7 @@
#endif
#define MMDB_DATA_SECTION_SEPARATOR (16)
+#define MAXIMUM_DATA_STRUCTURE_DEPTH (512)
#ifdef MMDB_DEBUG
#define LOCAL
@@ -105,6 +108,18 @@ DEBUG_FUNC char *type_num_to_name(uint8_t num)
}
#endif
+/* None of the values we check on the lhs are bigger than uint32_t, so on
+ * platforms where SIZE_MAX is a 64-bit integer, this would be a no-op, and it
+ * makes the compiler complain if we do the check anyway. */
+#if SIZE_MAX == UINT32_MAX
+#define MAYBE_CHECK_SIZE_OVERFLOW(lhs, rhs, error) \
+ if ((lhs) > (rhs)) { \
+ return error; \
+ }
+#else
+#define MAYBE_CHECK_SIZE_OVERFLOW(...)
+#endif
+
typedef struct record_info_s {
uint16_t record_length;
uint32_t (*left_record_getter)(const uint8_t *);
@@ -116,6 +131,9 @@ typedef struct record_info_s {
/* This is 128kb */
#define METADATA_BLOCK_MAX_SIZE 131072
+// 64 leads us to allocating 4 KiB on a 64bit system.
+#define MMDB_POOL_INIT_SIZE 64
+
/* *INDENT-OFF* */
/* --prototypes automatically generated by dev-bin/regen-prototypes.pl - don't remove this comment */
LOCAL int map_file(MMDB_s *const mmdb);
@@ -123,10 +141,14 @@ LOCAL const uint8_t *find_metadata(const uint8_t *file_content,
ssize_t file_size, uint32_t *metadata_size);
LOCAL int read_metadata(MMDB_s *mmdb);
LOCAL MMDB_s make_fake_metadata_db(MMDB_s *mmdb);
-LOCAL uint16_t value_for_key_as_uint16(MMDB_entry_s *start, char *key);
-LOCAL uint32_t value_for_key_as_uint32(MMDB_entry_s *start, char *key);
-LOCAL uint64_t value_for_key_as_uint64(MMDB_entry_s *start, char *key);
-LOCAL char *value_for_key_as_string(MMDB_entry_s *start, char *key);
+LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key,
+ uint16_t *value);
+LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key,
+ uint32_t *value);
+LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key,
+ uint64_t *value);
+LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key,
+ char const **value);
LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
MMDB_entry_s *metadata_start);
LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
@@ -136,11 +158,15 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
sa_family_t address_family,
MMDB_lookup_result_s *result);
LOCAL record_info_s record_info_for_database(MMDB_s *mmdb);
-LOCAL MMDB_ipv4_start_node_s find_ipv4_start_node(MMDB_s *mmdb);
-LOCAL int populate_result(MMDB_s *mmdb, uint32_t node_count, uint32_t value,
- uint16_t netmask, MMDB_lookup_result_s *result);
+LOCAL int find_ipv4_start_node(MMDB_s *mmdb);
+LOCAL uint8_t maybe_populate_result(MMDB_s *mmdb, uint32_t record,
+ uint16_t netmask,
+ MMDB_lookup_result_s *result);
+LOCAL uint8_t record_type(MMDB_s *const mmdb, uint64_t record);
LOCAL uint32_t get_left_28_bit_record(const uint8_t *record);
LOCAL uint32_t get_right_28_bit_record(const uint8_t *record);
+LOCAL uint32_t data_section_offset_for_record(MMDB_s *const mmdb,
+ uint64_t record);
LOCAL int path_length(va_list va_path);
LOCAL int lookup_path_in_array(const char *path_elem, MMDB_s *mmdb,
MMDB_entry_data_s *entry_data);
@@ -154,8 +180,11 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
LOCAL int get_ext_type(int raw_ext_type);
LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr,
int ptr_size);
-LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
- MMDB_entry_data_list_s *const entry_data_list);
+LOCAL int get_entry_data_list(MMDB_s *mmdb,
+ uint32_t offset,
+ MMDB_entry_data_list_s *const entry_data_list,
+ MMDB_data_pool_s *const pool,
+ int depth);
LOCAL float get_ieee754_float(const uint8_t *restrict p);
LOCAL double get_ieee754_double(const uint8_t *restrict p);
LOCAL uint32_t get_uint32(const uint8_t *p);
@@ -163,7 +192,6 @@ LOCAL uint32_t get_uint24(const uint8_t *p);
LOCAL uint32_t get_uint16(const uint8_t *p);
LOCAL uint64_t get_uintX(const uint8_t *p, int length);
LOCAL int32_t get_sintX(const uint8_t *p, int length);
-LOCAL MMDB_entry_data_list_s *new_entry_data_list(void);
LOCAL void free_mmdb_struct(MMDB_s *const mmdb);
LOCAL void free_languages_metadata(MMDB_s *mmdb);
LOCAL void free_descriptions_metadata(MMDB_s *mmdb);
@@ -175,20 +203,24 @@ LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size);
/* --prototypes end - don't remove this comment-- */
/* *INDENT-ON* */
-#define CHECKED_DECODE_ONE(mmdb, offset, entry_data) \
- do { \
- int status = decode_one(mmdb, offset, entry_data); \
- if (MMDB_SUCCESS != status) { \
- return status; \
- } \
+#define CHECKED_DECODE_ONE(mmdb, offset, entry_data) \
+ do { \
+ int status = decode_one(mmdb, offset, entry_data); \
+ if (MMDB_SUCCESS != status) { \
+ DEBUG_MSGF("CHECKED_DECODE_ONE failed." \
+ " status = %d (%s)", status, MMDB_strerror(status)); \
+ return status; \
+ } \
} while (0)
-#define CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data) \
- do { \
- int status = decode_one_follow(mmdb, offset, entry_data); \
- if (MMDB_SUCCESS != status) { \
- return status; \
- } \
+#define CHECKED_DECODE_ONE_FOLLOW(mmdb, offset, entry_data) \
+ do { \
+ int status = decode_one_follow(mmdb, offset, entry_data); \
+ if (MMDB_SUCCESS != status) { \
+ DEBUG_MSGF("CHECKED_DECODE_ONE_FOLLOW failed." \
+ " status = %d (%s)", status, MMDB_strerror(status)); \
+ return status; \
+ } \
} while (0)
#define FREE_AND_SET_NULL(p) { free((void *)(p)); (p) = NULL; }
@@ -214,7 +246,7 @@ int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb)
}
mmdb->flags = flags;
- if (MMDB_SUCCESS != (status = map_file(mmdb)) ) {
+ if (MMDB_SUCCESS != (status = map_file(mmdb))) {
goto cleanup;
}
@@ -247,12 +279,38 @@ int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb)
uint32_t search_tree_size = mmdb->metadata.node_count *
mmdb->full_record_byte_size;
- mmdb->data_section = mmdb->file_content + search_tree_size;
- mmdb->data_section_size = mmdb->file_size - search_tree_size;
+ mmdb->data_section = mmdb->file_content + search_tree_size
+ + MMDB_DATA_SECTION_SEPARATOR;
+ if (search_tree_size + MMDB_DATA_SECTION_SEPARATOR >
+ (uint32_t)mmdb->file_size) {
+ status = MMDB_INVALID_METADATA_ERROR;
+ goto cleanup;
+ }
+ mmdb->data_section_size = (uint32_t)mmdb->file_size - search_tree_size -
+ MMDB_DATA_SECTION_SEPARATOR;
+
+ // Although it is likely not possible to construct a database with valid
+ // valid metadata, as parsed above, and a data_section_size less than 3,
+ // we do this check as later we assume it is at least three when doing
+ // bound checks.
+ if (mmdb->data_section_size < 3) {
+ status = MMDB_INVALID_DATA_ERROR;
+ goto cleanup;
+ }
+
mmdb->metadata_section = metadata;
mmdb->ipv4_start_node.node_value = 0;
mmdb->ipv4_start_node.netmask = 0;
+ // We do this immediately as otherwise there is a race to set
+ // ipv4_start_node.node_value and ipv4_start_node.netmask.
+ if (mmdb->metadata.ip_version == 6) {
+ status = find_ipv4_start_node(mmdb);
+ if (status != MMDB_SUCCESS) {
+ goto cleanup;
+ }
+ }
+
cleanup:
if (MMDB_SUCCESS != status) {
int saved_errno = errno;
@@ -262,16 +320,15 @@ int MMDB_open(const char *const filename, uint32_t flags, MMDB_s *const mmdb)
return status;
}
+#ifdef _WIN32
+
LOCAL int map_file(MMDB_s *const mmdb)
{
- ssize_t size;
+ DWORD size;
int status = MMDB_SUCCESS;
-#ifdef _WIN32
- HANDLE fd = INVALID_HANDLE_VALUE;
HANDLE mmh = NULL;
-
- fd = CreateFileA(mmdb->filename, GENERIC_READ, FILE_SHARE_READ, NULL,
- OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+ HANDLE fd = CreateFileA(mmdb->filename, GENERIC_READ, FILE_SHARE_READ, NULL,
+ OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (fd == INVALID_HANDLE_VALUE) {
status = MMDB_FILE_OPEN_ERROR;
goto cleanup;
@@ -282,7 +339,9 @@ LOCAL int map_file(MMDB_s *const mmdb)
goto cleanup;
}
mmh = CreateFileMappingA(fd, NULL, PAGE_READONLY, 0, size, NULL);
- if (NULL == mmh) { /* Microsoft documentation for CreateFileMapping indicates this returns NULL not INVALID_HANDLE_VALUE on error */
+ /* Microsoft documentation for CreateFileMapping indicates this returns
+ NULL not INVALID_HANDLE_VALUE on error */
+ if (NULL == mmh) {
status = MMDB_IO_ERROR;
goto cleanup;
}
@@ -292,8 +351,35 @@ LOCAL int map_file(MMDB_s *const mmdb)
status = MMDB_IO_ERROR;
goto cleanup;
}
+
+ mmdb->file_size = size;
+ mmdb->file_content = file_content;
+
+ cleanup:;
+ int saved_errno = errno;
+ if (INVALID_HANDLE_VALUE != fd) {
+ CloseHandle(fd);
+ }
+ if (NULL != mmh) {
+ CloseHandle(mmh);
+ }
+ errno = saved_errno;
+
+ return status;
+}
+
#else
- int fd = open(mmdb->filename, O_RDONLY);
+
+LOCAL int map_file(MMDB_s *const mmdb)
+{
+ ssize_t size;
+ int status = MMDB_SUCCESS;
+
+ int flags = O_RDONLY;
+#ifdef O_CLOEXEC
+ flags |= O_CLOEXEC;
+#endif
+ int fd = open(mmdb->filename, flags);
struct stat s;
if (fd < 0 || fstat(fd, &s)) {
status = MMDB_FILE_OPEN_ERROR;
@@ -301,6 +387,10 @@ LOCAL int map_file(MMDB_s *const mmdb)
}
size = s.st_size;
+ if (size < 0 || size != s.st_size) {
+ status = MMDB_OUT_OF_MEMORY_ERROR;
+ goto cleanup;
+ }
uint8_t *file_content =
(uint8_t *)mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
@@ -312,53 +402,56 @@ LOCAL int map_file(MMDB_s *const mmdb)
}
goto cleanup;
}
-#endif
mmdb->file_size = size;
mmdb->file_content = file_content;
cleanup:;
int saved_errno = errno;
-#ifdef _WIN32
- if (INVALID_HANDLE_VALUE != fd) {
- CloseHandle(fd);
- }
- if (NULL != mmh) {
- CloseHandle(mmh);
- }
-#else
if (fd >= 0) {
close(fd);
}
-#endif
errno = saved_errno;
return status;
}
+#endif
+
LOCAL const uint8_t *find_metadata(const uint8_t *file_content,
ssize_t file_size, uint32_t *metadata_size)
{
+ const ssize_t marker_len = sizeof(METADATA_MARKER) - 1;
ssize_t max_size = file_size >
METADATA_BLOCK_MAX_SIZE ? METADATA_BLOCK_MAX_SIZE :
file_size;
uint8_t *search_area = (uint8_t *)(file_content + (file_size - max_size));
+ uint8_t *start = search_area;
uint8_t *tmp;
do {
tmp = mmdb_memmem(search_area, max_size,
- METADATA_MARKER, strlen(METADATA_MARKER));
+ METADATA_MARKER, marker_len);
if (NULL != tmp) {
max_size -= tmp - search_area;
search_area = tmp;
+
+ /* Continue searching just after the marker we just read, in case
+ * there are multiple markers in the same file. This would be odd
+ * but is certainly not impossible. */
+ max_size -= marker_len;
+ search_area += marker_len;
}
- } while (NULL != tmp && tmp != search_area);
+ } while (NULL != tmp);
- const uint8_t *metadata_start = search_area + strlen(METADATA_MARKER);
- *metadata_size = file_size - (search_area - file_content);
+ if (search_area == start) {
+ return NULL;
+ }
- return metadata_start;
+ *metadata_size = (uint32_t)max_size;
+
+ return search_area;
}
LOCAL int read_metadata(MMDB_s *mmdb)
@@ -374,15 +467,22 @@ LOCAL int read_metadata(MMDB_s *mmdb)
.offset = 0
};
- mmdb->metadata.node_count =
- value_for_key_as_uint32(&metadata_start, "node_count");
+ int status =
+ value_for_key_as_uint32(&metadata_start, "node_count",
+ &mmdb->metadata.node_count);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
if (!mmdb->metadata.node_count) {
DEBUG_MSG("could not find node_count value in metadata");
return MMDB_INVALID_METADATA_ERROR;
}
- mmdb->metadata.record_size =
- value_for_key_as_uint16(&metadata_start, "record_size");
+ status = value_for_key_as_uint16(&metadata_start, "record_size",
+ &mmdb->metadata.record_size);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
if (!mmdb->metadata.record_size) {
DEBUG_MSG("could not find record_size value in metadata");
return MMDB_INVALID_METADATA_ERROR;
@@ -395,8 +495,11 @@ LOCAL int read_metadata(MMDB_s *mmdb)
return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR;
}
- mmdb->metadata.ip_version =
- value_for_key_as_uint16(&metadata_start, "ip_version");
+ status = value_for_key_as_uint16(&metadata_start, "ip_version",
+ &mmdb->metadata.ip_version);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
if (!mmdb->metadata.ip_version) {
DEBUG_MSG("could not find ip_version value in metadata");
return MMDB_INVALID_METADATA_ERROR;
@@ -407,33 +510,44 @@ LOCAL int read_metadata(MMDB_s *mmdb)
return MMDB_INVALID_METADATA_ERROR;
}
- mmdb->metadata.database_type =
- value_for_key_as_string(&metadata_start, "database_type");
- if (NULL == mmdb->metadata.database_type) {
- DEBUG_MSG("could not find database_type value in metadata");
- return MMDB_INVALID_METADATA_ERROR;
+ status = value_for_key_as_string(&metadata_start, "database_type",
+ &mmdb->metadata.database_type);
+ if (MMDB_SUCCESS != status) {
+ DEBUG_MSG("error finding database_type value in metadata");
+ return status;
}
- int status =
+ status =
populate_languages_metadata(mmdb, &metadata_db, &metadata_start);
if (MMDB_SUCCESS != status) {
DEBUG_MSG("could not populate languages from metadata");
return status;
}
- mmdb->metadata.binary_format_major_version =
- value_for_key_as_uint16(&metadata_start, "binary_format_major_version");
+ status = value_for_key_as_uint16(
+ &metadata_start, "binary_format_major_version",
+ &mmdb->metadata.binary_format_major_version);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
if (!mmdb->metadata.binary_format_major_version) {
DEBUG_MSG(
"could not find binary_format_major_version value in metadata");
return MMDB_INVALID_METADATA_ERROR;
}
- mmdb->metadata.binary_format_minor_version =
- value_for_key_as_uint16(&metadata_start, "binary_format_minor_version");
+ status = value_for_key_as_uint16(
+ &metadata_start, "binary_format_minor_version",
+ &mmdb->metadata.binary_format_minor_version);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
- mmdb->metadata.build_epoch =
- value_for_key_as_uint64(&metadata_start, "build_epoch");
+ status = value_for_key_as_uint64(&metadata_start, "build_epoch",
+ &mmdb->metadata.build_epoch);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
if (!mmdb->metadata.build_epoch) {
DEBUG_MSG("could not find build_epoch value in metadata");
return MMDB_INVALID_METADATA_ERROR;
@@ -462,36 +576,83 @@ LOCAL MMDB_s make_fake_metadata_db(MMDB_s *mmdb)
return fake_metadata_db;
}
-LOCAL uint16_t value_for_key_as_uint16(MMDB_entry_s *start, char *key)
+LOCAL int value_for_key_as_uint16(MMDB_entry_s *start, char *key,
+ uint16_t *value)
{
MMDB_entry_data_s entry_data;
const char *path[] = { key, NULL };
- MMDB_aget_value(start, &entry_data, path);
- return entry_data.uint16;
+ int status = MMDB_aget_value(start, &entry_data, path);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
+ if (MMDB_DATA_TYPE_UINT16 != entry_data.type) {
+ DEBUG_MSGF("expect uint16 for %s but received %s", key,
+ type_num_to_name(
+ entry_data.type));
+ return MMDB_INVALID_METADATA_ERROR;
+ }
+ *value = entry_data.uint16;
+ return MMDB_SUCCESS;
}
-LOCAL uint32_t value_for_key_as_uint32(MMDB_entry_s *start, char *key)
+LOCAL int value_for_key_as_uint32(MMDB_entry_s *start, char *key,
+ uint32_t *value)
{
MMDB_entry_data_s entry_data;
const char *path[] = { key, NULL };
- MMDB_aget_value(start, &entry_data, path);
- return entry_data.uint32;
+ int status = MMDB_aget_value(start, &entry_data, path);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
+ if (MMDB_DATA_TYPE_UINT32 != entry_data.type) {
+ DEBUG_MSGF("expect uint32 for %s but received %s", key,
+ type_num_to_name(
+ entry_data.type));
+ return MMDB_INVALID_METADATA_ERROR;
+ }
+ *value = entry_data.uint32;
+ return MMDB_SUCCESS;
}
-LOCAL uint64_t value_for_key_as_uint64(MMDB_entry_s *start, char *key)
+LOCAL int value_for_key_as_uint64(MMDB_entry_s *start, char *key,
+ uint64_t *value)
{
MMDB_entry_data_s entry_data;
const char *path[] = { key, NULL };
- MMDB_aget_value(start, &entry_data, path);
- return entry_data.uint64;
+ int status = MMDB_aget_value(start, &entry_data, path);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
+ if (MMDB_DATA_TYPE_UINT64 != entry_data.type) {
+ DEBUG_MSGF("expect uint64 for %s but received %s", key,
+ type_num_to_name(
+ entry_data.type));
+ return MMDB_INVALID_METADATA_ERROR;
+ }
+ *value = entry_data.uint64;
+ return MMDB_SUCCESS;
}
-LOCAL char *value_for_key_as_string(MMDB_entry_s *start, char *key)
+LOCAL int value_for_key_as_string(MMDB_entry_s *start, char *key,
+ char const **value)
{
MMDB_entry_data_s entry_data;
const char *path[] = { key, NULL };
- MMDB_aget_value(start, &entry_data, path);
- return mmdb_strndup((char *)entry_data.utf8_string, entry_data.data_size);
+ int status = MMDB_aget_value(start, &entry_data, path);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
+ if (MMDB_DATA_TYPE_UTF8_STRING != entry_data.type) {
+ DEBUG_MSGF("expect string for %s but received %s", key,
+ type_num_to_name(
+ entry_data.type));
+ return MMDB_INVALID_METADATA_ERROR;
+ }
+ *value = mmdb_strndup((char *)entry_data.utf8_string, entry_data.data_size);
+ if (NULL == *value) {
+ return MMDB_OUT_OF_MEMORY_ERROR;
+ }
+ return MMDB_SUCCESS;
}
LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
@@ -500,8 +661,10 @@ LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
MMDB_entry_data_s entry_data;
const char *path[] = { "languages", NULL };
- MMDB_aget_value(metadata_start, &entry_data, path);
-
+ int status = MMDB_aget_value(metadata_start, &entry_data, path);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
if (MMDB_DATA_TYPE_ARRAY != entry_data.type) {
return MMDB_INVALID_METADATA_ERROR;
}
@@ -512,11 +675,17 @@ LOCAL int populate_languages_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
};
MMDB_entry_data_list_s *member;
- MMDB_get_entry_data_list(&array_start, &member);
+ status = MMDB_get_entry_data_list(&array_start, &member);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
MMDB_entry_data_list_s *first_member = member;
uint32_t array_size = member->entry_data.data_size;
+ MAYBE_CHECK_SIZE_OVERFLOW(array_size, SIZE_MAX / sizeof(char *),
+ MMDB_INVALID_METADATA_ERROR);
+
mmdb->metadata.languages.count = 0;
mmdb->metadata.languages.names = malloc(array_size * sizeof(char *));
if (NULL == mmdb->metadata.languages.names) {
@@ -552,9 +721,13 @@ LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
MMDB_entry_data_s entry_data;
const char *path[] = { "description", NULL };
- MMDB_aget_value(metadata_start, &entry_data, path);
+ int status = MMDB_aget_value(metadata_start, &entry_data, path);
+ if (MMDB_SUCCESS != status) {
+ return status;
+ }
if (MMDB_DATA_TYPE_MAP != entry_data.type) {
+ DEBUG_MSGF("Unexpected entry_data type: %d", entry_data.type);
return MMDB_INVALID_METADATA_ERROR;
}
@@ -564,7 +737,13 @@ LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
};
MMDB_entry_data_list_s *member;
- MMDB_get_entry_data_list(&map_start, &member);
+ status = MMDB_get_entry_data_list(&map_start, &member);
+ if (MMDB_SUCCESS != status) {
+ DEBUG_MSGF(
+ "MMDB_get_entry_data_list failed while populating description."
+ " status = %d (%s)", status, MMDB_strerror(status));
+ return status;
+ }
MMDB_entry_data_list_s *first_member = member;
@@ -574,18 +753,22 @@ LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
mmdb->metadata.description.descriptions = NULL;
goto cleanup;
}
+ MAYBE_CHECK_SIZE_OVERFLOW(map_size, SIZE_MAX / sizeof(MMDB_description_s *),
+ MMDB_INVALID_METADATA_ERROR);
mmdb->metadata.description.descriptions =
malloc(map_size * sizeof(MMDB_description_s *));
if (NULL == mmdb->metadata.description.descriptions) {
- return MMDB_OUT_OF_MEMORY_ERROR;
+ status = MMDB_OUT_OF_MEMORY_ERROR;
+ goto cleanup;
}
for (uint32_t i = 0; i < map_size; i++) {
mmdb->metadata.description.descriptions[i] =
malloc(sizeof(MMDB_description_s));
if (NULL == mmdb->metadata.description.descriptions[i]) {
- return MMDB_OUT_OF_MEMORY_ERROR;
+ status = MMDB_OUT_OF_MEMORY_ERROR;
+ goto cleanup;
}
mmdb->metadata.description.count = i + 1;
@@ -595,7 +778,8 @@ LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
member = member->next;
if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) {
- return MMDB_INVALID_METADATA_ERROR;
+ status = MMDB_INVALID_METADATA_ERROR;
+ goto cleanup;
}
mmdb->metadata.description.descriptions[i]->language =
@@ -603,13 +787,15 @@ LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
member->entry_data.data_size);
if (NULL == mmdb->metadata.description.descriptions[i]->language) {
- return MMDB_OUT_OF_MEMORY_ERROR;
+ status = MMDB_OUT_OF_MEMORY_ERROR;
+ goto cleanup;
}
member = member->next;
if (MMDB_DATA_TYPE_UTF8_STRING != member->entry_data.type) {
- return MMDB_INVALID_METADATA_ERROR;
+ status = MMDB_INVALID_METADATA_ERROR;
+ goto cleanup;
}
mmdb->metadata.description.descriptions[i]->description =
@@ -617,14 +803,15 @@ LOCAL int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db,
member->entry_data.data_size);
if (NULL == mmdb->metadata.description.descriptions[i]->description) {
- return MMDB_OUT_OF_MEMORY_ERROR;
+ status = MMDB_OUT_OF_MEMORY_ERROR;
+ goto cleanup;
}
}
cleanup:
MMDB_free_entry_data_list(first_member);
- return MMDB_SUCCESS;
+ return status;
}
MMDB_lookup_result_s MMDB_lookup_string(MMDB_s *const mmdb,
@@ -644,20 +831,10 @@ MMDB_lookup_result_s MMDB_lookup_string(MMDB_s *const mmdb,
struct addrinfo *addresses = NULL;
*gai_error = resolve_any_address(ipstr, &addresses);
- if (*gai_error) {
- goto cleanup;
+ if (!*gai_error) {
+ result = MMDB_lookup_sockaddr(mmdb, addresses->ai_addr, mmdb_error);
}
- if (mmdb->metadata.ip_version == 4
- && addresses->ai_addr->sa_family == AF_INET6) {
-
- *mmdb_error = MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR;
- goto cleanup;
- }
-
- result = MMDB_lookup_sockaddr(mmdb, addresses->ai_addr, mmdb_error);
-
- cleanup:
if (NULL != addresses) {
freeaddrinfo(addresses);
}
@@ -668,22 +845,13 @@ MMDB_lookup_result_s MMDB_lookup_string(MMDB_s *const mmdb,
LOCAL int resolve_any_address(const char *ipstr, struct addrinfo **addresses)
{
struct addrinfo hints = {
+ .ai_family = AF_UNSPEC,
+ .ai_flags = AI_NUMERICHOST,
+ // We set ai_socktype so that we only get one result back
.ai_socktype = SOCK_STREAM
};
- int gai_status;
- if (NULL != strchr(ipstr, ':')) {
- hints.ai_flags = AI_NUMERICHOST;
-#if defined AI_V4MAPPED && !defined __FreeBSD__
- hints.ai_flags |= AI_V4MAPPED;
-#endif
- hints.ai_family = AF_INET6;
- } else {
- hints.ai_flags = AI_NUMERICHOST;
- hints.ai_family = AF_INET;
- }
-
- gai_status = getaddrinfo(ipstr, NULL, &hints, addresses);
+ int gai_status = getaddrinfo(ipstr, NULL, &hints, addresses);
if (gai_status) {
return gai_status;
}
@@ -708,6 +876,7 @@ MMDB_lookup_result_s MMDB_lookup_sockaddr(
uint8_t mapped_address[16], *address;
if (mmdb->metadata.ip_version == 4) {
if (sockaddr->sa_family == AF_INET6) {
+ *mmdb_error = MMDB_IPV6_LOOKUP_IN_IPV4_DATABASE_ERROR;
return result;
}
address = (uint8_t *)&((struct sockaddr_in *)sockaddr)->sin_addr.s_addr;
@@ -743,23 +912,34 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
DEBUG_NL;
DEBUG_MSG("Looking for address in search tree");
- uint32_t node_count = mmdb->metadata.node_count;
uint32_t value = 0;
uint16_t max_depth0 = mmdb->depth - 1;
uint16_t start_bit = max_depth0;
if (mmdb->metadata.ip_version == 6 && address_family == AF_INET) {
- MMDB_ipv4_start_node_s ipv4_start_node = find_ipv4_start_node(mmdb);
+ int mmdb_error = find_ipv4_start_node(mmdb);
+ if (MMDB_SUCCESS != mmdb_error) {
+ return mmdb_error;
+ }
DEBUG_MSGF("IPv4 start node is %u (netmask %u)",
- ipv4_start_node.node_value, ipv4_start_node.netmask);
- /* We have an IPv6 database with no IPv4 data */
- if (ipv4_start_node.node_value >= node_count) {
- return populate_result(mmdb, node_count, ipv4_start_node.node_value,
- ipv4_start_node.netmask, result);
+ mmdb->ipv4_start_node.node_value,
+ mmdb->ipv4_start_node.netmask);
+
+ uint8_t type = maybe_populate_result(mmdb,
+ mmdb->ipv4_start_node.node_value,
+ mmdb->ipv4_start_node.netmask,
+ result);
+ if (MMDB_RECORD_TYPE_INVALID == type) {
+ return MMDB_CORRUPT_SEARCH_TREE_ERROR;
}
- value = ipv4_start_node.node_value;
- start_bit -= ipv4_start_node.netmask;
+ /* We have an IPv6 database with no IPv4 data */
+ if (MMDB_RECORD_TYPE_SEARCH_NODE != type) {
+ return MMDB_SUCCESS;
+ }
+
+ value = mmdb->ipv4_start_node.node_value;
+ start_bit -= mmdb->ipv4_start_node.netmask;
}
const uint8_t *search_tree = mmdb->file_content;
@@ -774,6 +954,9 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
DEBUG_MSGF(" current node = %u", value);
record_pointer = &search_tree[value * record_info.record_length];
+ if (record_pointer + record_info.record_length > mmdb->data_section) {
+ return MMDB_CORRUPT_SEARCH_TREE_ERROR;
+ }
if (bit_is_true) {
record_pointer += record_info.right_record_offset;
value = record_info.right_record_getter(record_pointer);
@@ -781,20 +964,17 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
value = record_info.left_record_getter(record_pointer);
}
- /* Ideally we'd check to make sure that a record never points to a
- * previously seen value, but that's more complicated. For now, we can
- * at least check that we don't end up at the top of the tree again. */
- if (0 == value) {
- DEBUG_MSGF(" %s record has a value of 0",
- bit_is_true ? "right" : "left");
+ uint8_t type = maybe_populate_result(mmdb, value, (uint16_t)current_bit,
+ result);
+ if (MMDB_RECORD_TYPE_INVALID == type) {
return MMDB_CORRUPT_SEARCH_TREE_ERROR;
}
- if (value >= node_count) {
- return populate_result(mmdb, node_count, value, current_bit, result);
- } else {
- DEBUG_MSGF(" proceeding to search tree node %i", value);
+ if (MMDB_RECORD_TYPE_SEARCH_NODE != type) {
+ return MMDB_SUCCESS;
}
+
+ DEBUG_MSGF(" proceeding to search tree node %i", value);
}
DEBUG_MSG(
@@ -830,13 +1010,13 @@ LOCAL record_info_s record_info_for_database(MMDB_s *mmdb)
return record_info;
}
-LOCAL MMDB_ipv4_start_node_s find_ipv4_start_node(MMDB_s *mmdb)
+LOCAL int find_ipv4_start_node(MMDB_s *mmdb)
{
/* In a pathological case of a database with a single node search tree,
* this check will be true even after we've found the IPv4 start node, but
* that doesn't seem worth trying to fix. */
if (mmdb->ipv4_start_node.node_value != 0) {
- return mmdb->ipv4_start_node;
+ return MMDB_SUCCESS;
}
record_info_s record_info = record_info_for_database(mmdb);
@@ -844,9 +1024,12 @@ LOCAL MMDB_ipv4_start_node_s find_ipv4_start_node(MMDB_s *mmdb)
const uint8_t *search_tree = mmdb->file_content;
uint32_t node_value = 0;
const uint8_t *record_pointer;
- uint32_t netmask;
+ uint16_t netmask;
for (netmask = 0; netmask < 96; netmask++) {
record_pointer = &search_tree[node_value * record_info.record_length];
+ if (record_pointer + record_info.record_length > mmdb->data_section) {
+ return MMDB_CORRUPT_SEARCH_TREE_ERROR;
+ }
node_value = record_info.left_record_getter(record_pointer);
/* This can happen if there's no IPv4 data _or_ if there is a subnet
* with data that contains the entire IPv4 range (like ::/64) */
@@ -858,23 +1041,57 @@ LOCAL MMDB_ipv4_start_node_s find_ipv4_start_node(MMDB_s *mmdb)
mmdb->ipv4_start_node.node_value = node_value;
mmdb->ipv4_start_node.netmask = netmask;
- return mmdb->ipv4_start_node;
+ return MMDB_SUCCESS;
}
-LOCAL int populate_result(MMDB_s *mmdb, uint32_t node_count, uint32_t value,
- uint16_t netmask, MMDB_lookup_result_s *result)
+LOCAL uint8_t maybe_populate_result(MMDB_s *mmdb, uint32_t record,
+ uint16_t netmask,
+ MMDB_lookup_result_s *result)
{
- uint32_t offset = value - node_count;
- DEBUG_MSGF(" data section offset is %i (record value = %i)", offset, value);
+ uint8_t type = record_type(mmdb, record);
- if (offset > mmdb->data_section_size) {
- return MMDB_CORRUPT_SEARCH_TREE_ERROR;
+ if (MMDB_RECORD_TYPE_SEARCH_NODE == type ||
+ MMDB_RECORD_TYPE_INVALID == type) {
+ return type;
}
result->netmask = mmdb->depth - netmask;
- result->entry.offset = offset;
- result->found_entry = result->entry.offset > 0 ? true : false;
- return MMDB_SUCCESS;
+
+ result->entry.offset = data_section_offset_for_record(mmdb, record);
+
+ // type is either MMDB_RECORD_TYPE_DATA or MMDB_RECORD_TYPE_EMPTY
+ // at this point
+ result->found_entry = MMDB_RECORD_TYPE_DATA == type;
+
+ return type;
+}
+
+LOCAL uint8_t record_type(MMDB_s *const mmdb, uint64_t record)
+{
+ uint32_t node_count = mmdb->metadata.node_count;
+
+ /* Ideally we'd check to make sure that a record never points to a
+ * previously seen value, but that's more complicated. For now, we can
+ * at least check that we don't end up at the top of the tree again. */
+ if (record == 0) {
+ DEBUG_MSG("record has a value of 0");
+ return MMDB_RECORD_TYPE_INVALID;
+ }
+
+ if (record < node_count) {
+ return MMDB_RECORD_TYPE_SEARCH_NODE;
+ }
+
+ if (record == node_count) {
+ return MMDB_RECORD_TYPE_EMPTY;
+ }
+
+ if (record - node_count < mmdb->data_section_size) {
+ return MMDB_RECORD_TYPE_DATA;
+ }
+
+ DEBUG_MSG("record has a value that points outside of the database");
+ return MMDB_RECORD_TYPE_INVALID;
}
LOCAL uint32_t get_left_28_bit_record(const uint8_t *record)
@@ -908,9 +1125,31 @@ int MMDB_read_node(MMDB_s *const mmdb, uint32_t node_number,
record_pointer += record_info.right_record_offset;
node->right_record = record_info.right_record_getter(record_pointer);
+ node->left_record_type = record_type(mmdb, node->left_record);
+ node->right_record_type = record_type(mmdb, node->right_record);
+
+ // Note that offset will be invalid if the record type is not
+ // MMDB_RECORD_TYPE_DATA, but that's ok. Any use of the record entry
+ // for other data types is a programming error.
+ node->left_record_entry = (struct MMDB_entry_s) {
+ .mmdb = mmdb,
+ .offset = data_section_offset_for_record(mmdb, node->left_record),
+ };
+ node->right_record_entry = (struct MMDB_entry_s) {
+ .mmdb = mmdb,
+ .offset = data_section_offset_for_record(mmdb, node->right_record),
+ };
+
return MMDB_SUCCESS;
}
+LOCAL uint32_t data_section_offset_for_record(MMDB_s *const mmdb,
+ uint64_t record)
+{
+ return (uint32_t)record - mmdb->metadata.node_count -
+ MMDB_DATA_SECTION_SEPARATOR;
+}
+
int MMDB_get_value(MMDB_entry_s *const start,
MMDB_entry_data_s *const entry_data,
...)
@@ -930,6 +1169,9 @@ int MMDB_vget_value(MMDB_entry_s *const start,
const char *path_elem;
int i = 0;
+ MAYBE_CHECK_SIZE_OVERFLOW(length, SIZE_MAX / sizeof(const char *) - 1,
+ MMDB_INVALID_METADATA_ERROR);
+
const char **path = malloc((length + 1) * sizeof(const char *));
if (NULL == path) {
return MMDB_OUT_OF_MEMORY_ERROR;
@@ -1129,14 +1371,20 @@ LOCAL int decode_one_follow(MMDB_s *mmdb, uint32_t offset,
{
CHECKED_DECODE_ONE(mmdb, offset, entry_data);
if (entry_data->type == MMDB_DATA_TYPE_POINTER) {
+ uint32_t next = entry_data->offset_to_next;
+ CHECKED_DECODE_ONE(mmdb, entry_data->pointer, entry_data);
+ /* Pointers to pointers are illegal under the spec */
+ if (entry_data->type == MMDB_DATA_TYPE_POINTER) {
+ DEBUG_MSG("pointer points to another pointer");
+ return MMDB_INVALID_DATA_ERROR;
+ }
+
/* The pointer could point to any part of the data section but the
* next entry for this particular offset may be the one after the
* pointer, not the one after whatever the pointer points to. This
* depends on whether the pointer points to something that is a simple
* value or a compound value. For a compound value, the next one is
* the one after the pointer result, not the one after the pointer. */
- uint32_t next = entry_data->offset_to_next;
- CHECKED_DECODE_ONE(mmdb, entry_data->pointer, entry_data);
if (entry_data->type != MMDB_DATA_TYPE_MAP
&& entry_data->type != MMDB_DATA_TYPE_ARRAY) {
@@ -1164,7 +1412,12 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
{
const uint8_t *mem = mmdb->data_section;
- if (offset > mmdb->data_section_size) {
+ // We subtract rather than add as it possible that offset + 1
+ // could overflow for a corrupt database while an underflow
+ // from data_section_size - 1 should not be possible.
+ if (offset > mmdb->data_section_size - 1) {
+ DEBUG_MSGF("Offset (%d) past data section (%d)", offset,
+ mmdb->data_section_size);
return MMDB_INVALID_DATA_ERROR;
}
@@ -1181,6 +1434,13 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
DEBUG_MSGF("Type: %i (%s)", type, type_num_to_name(type));
if (type == MMDB_DATA_TYPE_EXTENDED) {
+ // Subtracting 1 to avoid possible overflow on offset + 1
+ if (offset > mmdb->data_section_size - 1) {
+ DEBUG_MSGF("Extended type offset (%d) past data section (%d)",
+ offset,
+ mmdb->data_section_size);
+ return MMDB_INVALID_DATA_ERROR;
+ }
type = get_ext_type(mem[offset++]);
DEBUG_MSGF("Extended type: %i (%s)", type, type_num_to_name(type));
}
@@ -1188,27 +1448,57 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
entry_data->type = type;
if (type == MMDB_DATA_TYPE_POINTER) {
- int psize = (ctrl >> 3) & 3;
+ uint8_t psize = ((ctrl >> 3) & 3) + 1;
DEBUG_MSGF("Pointer size: %i", psize);
+ // We check that the offset does not extend past the end of the
+ // database and that the subtraction of psize did not underflow.
+ if (offset > mmdb->data_section_size - psize ||
+ mmdb->data_section_size < psize) {
+ DEBUG_MSGF("Pointer offset (%d) past data section (%d)", offset +
+ psize,
+ mmdb->data_section_size);
+ return MMDB_INVALID_DATA_ERROR;
+ }
entry_data->pointer = get_ptr_from(ctrl, &mem[offset], psize);
DEBUG_MSGF("Pointer to: %i", entry_data->pointer);
- entry_data->data_size = psize + 1;
- entry_data->offset_to_next = offset + psize + 1;
+ entry_data->data_size = psize;
+ entry_data->offset_to_next = offset + psize;
return MMDB_SUCCESS;
}
uint32_t size = ctrl & 31;
switch (size) {
case 29:
+ // We subtract when checking offset to avoid possible overflow
+ if (offset > mmdb->data_section_size - 1) {
+ DEBUG_MSGF("String end (%d, case 29) past data section (%d)",
+ offset,
+ mmdb->data_section_size);
+ return MMDB_INVALID_DATA_ERROR;
+ }
size = 29 + mem[offset++];
break;
case 30:
+ // We subtract when checking offset to avoid possible overflow
+ if (offset > mmdb->data_section_size - 2) {
+ DEBUG_MSGF("String end (%d, case 30) past data section (%d)",
+ offset,
+ mmdb->data_section_size);
+ return MMDB_INVALID_DATA_ERROR;
+ }
size = 285 + get_uint16(&mem[offset]);
offset += 2;
break;
case 31:
+ // We subtract when checking offset to avoid possible overflow
+ if (offset > mmdb->data_section_size - 3) {
+ DEBUG_MSGF("String end (%d, case 31) past data section (%d)",
+ offset,
+ mmdb->data_section_size);
+ return MMDB_INVALID_DATA_ERROR;
+ }
size = 65821 + get_uint24(&mem[offset]);
offset += 3;
default:
@@ -1231,32 +1521,46 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
return MMDB_SUCCESS;
}
+ // Check that the data doesn't extend past the end of the memory
+ // buffer and that the calculation in doing this did not underflow.
+ if (offset > mmdb->data_section_size - size ||
+ mmdb->data_section_size < size) {
+ DEBUG_MSGF("Data end (%d) past data section (%d)", offset + size,
+ mmdb->data_section_size);
+ return MMDB_INVALID_DATA_ERROR;
+ }
+
if (type == MMDB_DATA_TYPE_UINT16) {
if (size > 2) {
+ DEBUG_MSGF("uint16 of size %d", size);
return MMDB_INVALID_DATA_ERROR;
}
entry_data->uint16 = (uint16_t)get_uintX(&mem[offset], size);
DEBUG_MSGF("uint16 value: %u", entry_data->uint16);
} else if (type == MMDB_DATA_TYPE_UINT32) {
if (size > 4) {
+ DEBUG_MSGF("uint32 of size %d", size);
return MMDB_INVALID_DATA_ERROR;
}
entry_data->uint32 = (uint32_t)get_uintX(&mem[offset], size);
DEBUG_MSGF("uint32 value: %u", entry_data->uint32);
} else if (type == MMDB_DATA_TYPE_INT32) {
if (size > 4) {
+ DEBUG_MSGF("int32 of size %d", size);
return MMDB_INVALID_DATA_ERROR;
}
entry_data->int32 = get_sintX(&mem[offset], size);
DEBUG_MSGF("int32 value: %i", entry_data->int32);
} else if (type == MMDB_DATA_TYPE_UINT64) {
if (size > 8) {
+ DEBUG_MSGF("uint64 of size %d", size);
return MMDB_INVALID_DATA_ERROR;
}
entry_data->uint64 = get_uintX(&mem[offset], size);
DEBUG_MSGF("uint64 value: %" PRIu64, entry_data->uint64);
} else if (type == MMDB_DATA_TYPE_UINT128) {
if (size > 16) {
+ DEBUG_MSGF("uint128 of size %d", size);
return MMDB_INVALID_DATA_ERROR;
}
#if MMDB_UINT128_IS_BYTE_ARRAY
@@ -1269,6 +1573,7 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
#endif
} else if (type == MMDB_DATA_TYPE_FLOAT) {
if (size != 4) {
+ DEBUG_MSGF("float of size %d", size);
return MMDB_INVALID_DATA_ERROR;
}
size = 4;
@@ -1276,6 +1581,7 @@ LOCAL int decode_one(MMDB_s *mmdb, uint32_t offset,
DEBUG_MSGF("float value: %f", entry_data->float_value);
} else if (type == MMDB_DATA_TYPE_DOUBLE) {
if (size != 8) {
+ DEBUG_MSGF("double of size %d", size);
return MMDB_INVALID_DATA_ERROR;
}
size = 8;
@@ -1313,21 +1619,21 @@ LOCAL uint32_t get_ptr_from(uint8_t ctrl, uint8_t const *const ptr,
{
uint32_t new_offset;
switch (ptr_size) {
- case 0:
- new_offset = (ctrl & 7) * 256 + ptr[0];
- break;
case 1:
- new_offset = 2048 + (ctrl & 7) * 65536 + ptr[0] * 256 + ptr[1];
+ new_offset = ( (ctrl & 7) << 8) + ptr[0];
break;
case 2:
- new_offset = 2048 + 524288 + (ctrl & 7) * 16777216 + get_uint24(ptr);
+ new_offset = 2048 + ( (ctrl & 7) << 16 ) + ( ptr[0] << 8) + ptr[1];
break;
case 3:
+ new_offset = 2048 + 524288 + ( (ctrl & 7) << 24 ) + get_uint24(ptr);
+ break;
+ case 4:
default:
new_offset = get_uint32(ptr);
break;
}
- return MMDB_DATA_SECTION_SEPARATOR + new_offset;
+ return new_offset;
}
int MMDB_get_metadata_as_entry_data_list(
@@ -1346,16 +1652,40 @@ int MMDB_get_metadata_as_entry_data_list(
int MMDB_get_entry_data_list(
MMDB_entry_s *start, MMDB_entry_data_list_s **const entry_data_list)
{
- *entry_data_list = new_entry_data_list();
- if (NULL == *entry_data_list) {
+ MMDB_data_pool_s *const pool = data_pool_new(MMDB_POOL_INIT_SIZE);
+ if (!pool) {
return MMDB_OUT_OF_MEMORY_ERROR;
}
- return get_entry_data_list(start->mmdb, start->offset, *entry_data_list);
+
+ MMDB_entry_data_list_s *const list = data_pool_alloc(pool);
+ if (!list) {
+ data_pool_destroy(pool);
+ return MMDB_OUT_OF_MEMORY_ERROR;
+ }
+
+ int const status = get_entry_data_list(start->mmdb, start->offset, list,
+ pool, 0);
+
+ *entry_data_list = data_pool_to_list(pool);
+ if (!*entry_data_list) {
+ data_pool_destroy(pool);
+ return MMDB_OUT_OF_MEMORY_ERROR;
+ }
+
+ return status;
}
-LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
- MMDB_entry_data_list_s *const entry_data_list)
+LOCAL int get_entry_data_list(MMDB_s *mmdb,
+ uint32_t offset,
+ MMDB_entry_data_list_s *const entry_data_list,
+ MMDB_data_pool_s *const pool,
+ int depth)
{
+ if (depth >= MAXIMUM_DATA_STRUCTURE_DEPTH) {
+ DEBUG_MSG("reached the maximum data structure depth");
+ return MMDB_INVALID_DATA_ERROR;
+ }
+ depth++;
CHECKED_DECODE_ONE(mmdb, offset, &entry_data_list->entry_data);
switch (entry_data_list->entry_data.type) {
@@ -1363,19 +1693,24 @@ LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
{
uint32_t next_offset = entry_data_list->entry_data.offset_to_next;
uint32_t last_offset;
- while (entry_data_list->entry_data.type ==
- MMDB_DATA_TYPE_POINTER) {
- CHECKED_DECODE_ONE(mmdb, last_offset =
- entry_data_list->entry_data.pointer,
- &entry_data_list->entry_data);
+ CHECKED_DECODE_ONE(mmdb, last_offset =
+ entry_data_list->entry_data.pointer,
+ &entry_data_list->entry_data);
+
+ /* Pointers to pointers are illegal under the spec */
+ if (entry_data_list->entry_data.type == MMDB_DATA_TYPE_POINTER) {
+ DEBUG_MSG("pointer points to another pointer");
+ return MMDB_INVALID_DATA_ERROR;
}
if (entry_data_list->entry_data.type == MMDB_DATA_TYPE_ARRAY
|| entry_data_list->entry_data.type == MMDB_DATA_TYPE_MAP) {
int status =
- get_entry_data_list(mmdb, last_offset, entry_data_list);
+ get_entry_data_list(mmdb, last_offset, entry_data_list,
+ pool, depth);
if (MMDB_SUCCESS != status) {
+ DEBUG_MSG("get_entry_data_list on pointer failed.");
return status;
}
}
@@ -1386,24 +1721,22 @@ LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
{
uint32_t array_size = entry_data_list->entry_data.data_size;
uint32_t array_offset = entry_data_list->entry_data.offset_to_next;
- MMDB_entry_data_list_s *previous = entry_data_list;
while (array_size-- > 0) {
- MMDB_entry_data_list_s *entry_data_list_to = previous->next =
- new_entry_data_list();
- if (NULL == entry_data_list_to) {
+ MMDB_entry_data_list_s *entry_data_list_to =
+ data_pool_alloc(pool);
+ if (!entry_data_list_to) {
return MMDB_OUT_OF_MEMORY_ERROR;
}
int status =
- get_entry_data_list(mmdb, array_offset, entry_data_list_to);
+ get_entry_data_list(mmdb, array_offset, entry_data_list_to,
+ pool, depth);
if (MMDB_SUCCESS != status) {
+ DEBUG_MSG("get_entry_data_list on array element failed.");
return status;
}
array_offset = entry_data_list_to->entry_data.offset_to_next;
- while (previous->next) {
- previous = previous->next;
- }
}
entry_data_list->entry_data.offset_to_next = array_offset;
@@ -1414,41 +1747,33 @@ LOCAL int get_entry_data_list(MMDB_s *mmdb, uint32_t offset,
uint32_t size = entry_data_list->entry_data.data_size;
offset = entry_data_list->entry_data.offset_to_next;
- MMDB_entry_data_list_s *previous = entry_data_list;
while (size-- > 0) {
- MMDB_entry_data_list_s *entry_data_list_to = previous->next =
- new_entry_data_list();
- if (NULL == entry_data_list_to) {
+ MMDB_entry_data_list_s *list_key = data_pool_alloc(pool);
+ if (!list_key) {
return MMDB_OUT_OF_MEMORY_ERROR;
}
int status =
- get_entry_data_list(mmdb, offset, entry_data_list_to);
+ get_entry_data_list(mmdb, offset, list_key, pool, depth);
if (MMDB_SUCCESS != status) {
+ DEBUG_MSG("get_entry_data_list on map key failed.");
return status;
}
- while (previous->next) {
- previous = previous->next;
- }
+ offset = list_key->entry_data.offset_to_next;
- offset = entry_data_list_to->entry_data.offset_to_next;
- entry_data_list_to = previous->next =
- new_entry_data_list();
-
- if (NULL == entry_data_list_to) {
+ MMDB_entry_data_list_s *list_value = data_pool_alloc(pool);
+ if (!list_value) {
return MMDB_OUT_OF_MEMORY_ERROR;
}
- status = get_entry_data_list(mmdb, offset, entry_data_list_to);
+ status = get_entry_data_list(mmdb, offset, list_value, pool,
+ depth);
if (MMDB_SUCCESS != status) {
+ DEBUG_MSG("get_entry_data_list on map element failed.");
return status;
}
-
- while (previous->next) {
- previous = previous->next;
- }
- offset = entry_data_list_to->entry_data.offset_to_next;
+ offset = list_value->entry_data.offset_to_next;
}
entry_data_list->entry_data.offset_to_next = offset;
}
@@ -1464,7 +1789,9 @@ LOCAL float get_ieee754_float(const uint8_t *restrict p)
{
volatile float f;
uint8_t *q = (void *)&f;
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+/* Windows builds don't use autoconf but we can assume they're all
+ * little-endian. */
+#if MMDB_LITTLE_ENDIAN || _WIN32
q[3] = p[0];
q[2] = p[1];
q[1] = p[2];
@@ -1479,7 +1806,7 @@ LOCAL double get_ieee754_double(const uint8_t *restrict p)
{
volatile double d;
uint8_t *q = (void *)&d;
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#if MMDB_LITTLE_ENDIAN || _WIN32
q[7] = p[0];
q[6] = p[1];
q[5] = p[2];
@@ -1525,22 +1852,12 @@ LOCAL int32_t get_sintX(const uint8_t *p, int length)
return (int32_t)get_uintX(p, length);
}
-LOCAL MMDB_entry_data_list_s *new_entry_data_list(void)
-{
- /* We need calloc here in order to ensure that the ->next pointer in the
- * struct doesn't point to some random address. */
- return calloc(1, sizeof(MMDB_entry_data_list_s));
-}
-
void MMDB_free_entry_data_list(MMDB_entry_data_list_s *const entry_data_list)
{
if (entry_data_list == NULL) {
return;
}
- if (entry_data_list->next) {
- MMDB_free_entry_data_list(entry_data_list->next);
- }
- free(entry_data_list);
+ data_pool_destroy(entry_data_list->pool);
}
void MMDB_close(MMDB_s *const mmdb)
@@ -1644,6 +1961,11 @@ LOCAL MMDB_entry_data_list_s *dump_entry_data_list(
for (entry_data_list = entry_data_list->next;
size && entry_data_list; size--) {
+ if (MMDB_DATA_TYPE_UTF8_STRING !=
+ entry_data_list->entry_data.type) {
+ *status = MMDB_INVALID_DATA_ERROR;
+ return NULL;
+ }
char *key =
mmdb_strndup(
(char *)entry_data_list->entry_data.utf8_string,
@@ -1766,6 +2088,10 @@ LOCAL MMDB_entry_data_list_s *dump_entry_data_list(
#if MMDB_UINT128_IS_BYTE_ARRAY
char *hex_string =
bytes_to_hex((uint8_t *)entry_data_list->entry_data.uint128, 16);
+ if (NULL == hex_string) {
+ *status = MMDB_OUT_OF_MEMORY_ERROR;
+ return NULL;
+ }
fprintf(stream, "0x%s \n", hex_string);
free(hex_string);
#else
@@ -1801,11 +2127,16 @@ LOCAL void print_indentation(FILE *stream, int i)
LOCAL char *bytes_to_hex(uint8_t *bytes, uint32_t size)
{
- char *hex_string = malloc((size * 2) + 1);
- char *hex_pointer = hex_string;
+ char *hex_string;
+ MAYBE_CHECK_SIZE_OVERFLOW(size, SIZE_MAX / 2 - 1, NULL);
+
+ hex_string = malloc((size * 2) + 1);
+ if (NULL == hex_string) {
+ return NULL;
+ }
for (uint32_t i = 0; i < size; i++) {
- sprintf(hex_pointer + (2 * i), "%02X", bytes[i]);
+ sprintf(hex_string + (2 * i), "%02X", bytes[i]);
}
return hex_string;
diff --git a/third_party/libmaxminddb/maxminddb.h b/third_party/libmaxminddb/maxminddb.h
index 40f15a95..1f18e13c 100644
--- a/third_party/libmaxminddb/maxminddb.h
+++ b/third_party/libmaxminddb/maxminddb.h
@@ -5,10 +5,21 @@ extern "C" {
#ifndef MAXMINDDB_H
#define MAXMINDDB_H
+/* Request POSIX.1-2008. However, we want to remain compatible with
+ * POSIX.1-2001 (since we have been historically and see no reason to drop
+ * compatibility). By requesting POSIX.1-2008, we can conditionally use
+ * features provided by that standard if the implementation provides it. We can
+ * check for what the implementation provides by checking the _POSIX_VERSION
+ * macro after including unistd.h. If a feature is in POSIX.1-2008 but not
+ * POSIX.1-2001, check that macro before using the feature (or check for the
+ * feature directly if possible). */
#ifndef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE 200112L
+#define _POSIX_C_SOURCE 200809L
#endif
+/* libmaxminddb package version from configure */
+#define PACKAGE_VERSION "1.3.2"
+
#include "maxminddb_config.h"
#include
#include
@@ -24,7 +35,7 @@ typedef ADDRESS_FAMILY sa_family_t;
#if defined(_MSC_VER)
/* MSVC doesn't define signed size_t, copy it from configure */
-#define ssize_t int
+#define ssize_t SSIZE_T
/* MSVC doesn't support restricted pointers */
#define restrict
@@ -35,9 +46,6 @@ typedef ADDRESS_FAMILY sa_family_t;
#include
#endif
-/* libmaxminddb package version from configure */
-#define PACKAGE_VERSION "1.1.0"
-
#define MMDB_DATA_TYPE_EXTENDED (0)
#define MMDB_DATA_TYPE_POINTER (1)
#define MMDB_DATA_TYPE_UTF8_STRING (2)
@@ -55,6 +63,11 @@ typedef ADDRESS_FAMILY sa_family_t;
#define MMDB_DATA_TYPE_BOOLEAN (14)
#define MMDB_DATA_TYPE_FLOAT (15)
+#define MMDB_RECORD_TYPE_SEARCH_NODE (0)
+#define MMDB_RECORD_TYPE_EMPTY (1)
+#define MMDB_RECORD_TYPE_DATA (2)
+#define MMDB_RECORD_TYPE_INVALID (3)
+
/* flags for open */
#define MMDB_MODE_MMAP (1)
#define MMDB_MODE_MASK (7)
@@ -131,6 +144,7 @@ typedef struct MMDB_entry_data_s {
typedef struct MMDB_entry_data_list_s {
MMDB_entry_data_s entry_data;
struct MMDB_entry_data_list_s *next;
+ void *pool;
} MMDB_entry_data_list_s;
typedef struct MMDB_description_s {
@@ -179,6 +193,10 @@ typedef struct MMDB_s {
typedef struct MMDB_search_node_s {
uint64_t left_record;
uint64_t right_record;
+ uint8_t left_record_type;
+ uint8_t right_record_type;
+ MMDB_entry_s left_record_entry;
+ MMDB_entry_s right_record_entry;
} MMDB_search_node_s;
/* *INDENT-OFF* */