mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-09 16:34:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 17

bpo-40609: Remove _Py_hashtable_t.key_size (GH-20060)

Browse source 
Rewrite _Py_hashtable_t type to always store the key as
a "const void *" pointer. Add an explicit "key" member to
_Py_hashtable_entry_t.

Remove _Py_hashtable_t.key_size member.

hash and compare functions drop their hash table parameter, and their
'key' parameter type becomes "const void *".
This commit is contained in:
Victor Stinner 2020-05-13 02:26:02 +02:00 • committed by GitHub
parent 9e2ca17420
commit f9b3b582b8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 120 additions and 188 deletions
Download patch file Download diff file Expand all files Collapse all files

74
Include/internal/pycore_hashtable.h
View file

@ -30,32 +30,13 @@ typedef struct {
_Py_slist_item_t _Py_slist_item; _Py_slist_item_t _Py_slist_item;
Py_uhash_t key_hash; Py_uhash_t key_hash;
void *key;
/* key (key_size bytes) and then data (data_size bytes) follows */ /* data (data_size bytes) follows */
} _Py_hashtable_entry_t; } _Py_hashtable_entry_t;
#define _Py_HASHTABLE_ENTRY_PKEY(ENTRY) \
((const void *)((char *)(ENTRY) \
+ sizeof(_Py_hashtable_entry_t)))
#define _Py_HASHTABLE_ENTRY_PDATA(TABLE, ENTRY) \ #define _Py_HASHTABLE_ENTRY_PDATA(TABLE, ENTRY) \
((const void *)((char *)(ENTRY) \ ((const void *)((char *)(ENTRY) \
+ sizeof(_Py_hashtable_entry_t) \ + sizeof(_Py_hashtable_entry_t)))
+ (TABLE)->key_size))
/* Get a key value from pkey: use memcpy() rather than a pointer dereference
to avoid memory alignment issues. */
#define _Py_HASHTABLE_READ_KEY(TABLE, PKEY, DST_KEY) \
do { \
assert(sizeof(DST_KEY) == (TABLE)->key_size); \
memcpy(&(DST_KEY), (PKEY), sizeof(DST_KEY)); \
} while (0)
#define _Py_HASHTABLE_ENTRY_READ_KEY(TABLE, ENTRY, KEY) \
do { \
assert(sizeof(KEY) == (TABLE)->key_size); \
memcpy(&(KEY), _Py_HASHTABLE_ENTRY_PKEY(ENTRY), sizeof(KEY)); \
} while (0)
#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA) \ #define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA) \
do { \ do { \
@ -78,15 +59,12 @@ typedef struct {
struct _Py_hashtable_t; struct _Py_hashtable_t;
typedef struct _Py_hashtable_t _Py_hashtable_t; typedef struct _Py_hashtable_t _Py_hashtable_t;
typedef Py_uhash_t (*_Py_hashtable_hash_func) (_Py_hashtable_t *ht, typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key);
const void *pkey); typedef int (*_Py_hashtable_compare_func) (const void *key1, const void *key2);
typedef int (*_Py_hashtable_compare_func) (_Py_hashtable_t *ht,
const void *pkey,
const _Py_hashtable_entry_t *he);
typedef _Py_hashtable_entry_t* (*_Py_hashtable_get_entry_func)(_Py_hashtable_t *ht, typedef _Py_hashtable_entry_t* (*_Py_hashtable_get_entry_func)(_Py_hashtable_t *ht,
const void *pkey); const void *key);
typedef int (*_Py_hashtable_get_func) (_Py_hashtable_t *ht, typedef int (*_Py_hashtable_get_func) (_Py_hashtable_t *ht,
const void *pkey, void *data); const void *key, void *data);
typedef struct { typedef struct {
/* allocate a memory block */ /* allocate a memory block */
@ -102,7 +80,6 @@ struct _Py_hashtable_t {
size_t num_buckets; size_t num_buckets;
size_t entries; /* Total number of entries in the table. */ size_t entries; /* Total number of entries in the table. */
_Py_slist_t *buckets; _Py_slist_t *buckets;
size_t key_size;
size_t data_size; size_t data_size;
_Py_hashtable_get_func get_func; _Py_hashtable_get_func get_func;
@ -113,24 +90,19 @@ struct _Py_hashtable_t {
}; };
/* hash a pointer (void*) */ /* hash a pointer (void*) */
PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key);
struct _Py_hashtable_t *ht,
const void *pkey);
/* comparison using memcmp() */ /* comparison using memcmp() */
PyAPI_FUNC(int) _Py_hashtable_compare_direct( PyAPI_FUNC(int) _Py_hashtable_compare_direct(
_Py_hashtable_t *ht, const void *key1,
const void *pkey, const void *key2);
const _Py_hashtable_entry_t *entry);
PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new(
size_t key_size,
size_t data_size, size_t data_size,
_Py_hashtable_hash_func hash_func, _Py_hashtable_hash_func hash_func,
_Py_hashtable_compare_func compare_func); _Py_hashtable_compare_func compare_func);
PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full(
size_t key_size,
size_t data_size, size_t data_size,
size_t init_size, size_t init_size,
_Py_hashtable_hash_func hash_func, _Py_hashtable_hash_func hash_func,
@ -165,16 +137,15 @@ PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht);
but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */ but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */
PyAPI_FUNC(int) _Py_hashtable_set( PyAPI_FUNC(int) _Py_hashtable_set(
_Py_hashtable_t *ht, _Py_hashtable_t *ht,
size_t key_size, const void *key,
const void *pkey,
size_t data_size, size_t data_size,
const void *data); const void *data);
#define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \
_Py_hashtable_set(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) _Py_hashtable_set(TABLE, (KEY), sizeof(DATA), &(DATA))
#define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ #define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \
_Py_hashtable_set(TABLE, sizeof(KEY), &(KEY), 0, NULL) _Py_hashtable_set(TABLE, (KEY), 0, NULL)
/* Get an entry. /* Get an entry.
@ -183,14 +154,13 @@ PyAPI_FUNC(int) _Py_hashtable_set(
Don't call directly this function, but use _Py_HASHTABLE_GET_ENTRY() Don't call directly this function, but use _Py_HASHTABLE_GET_ENTRY()
macro */ macro */
static inline _Py_hashtable_entry_t * static inline _Py_hashtable_entry_t *
_Py_hashtable_get_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey) _Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key)
{ {
assert(key_size == ht->key_size); return ht->get_entry_func(ht, key);
return ht->get_entry_func(ht, pkey);
} }
#define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \ #define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \
_Py_hashtable_get_entry(TABLE, sizeof(KEY), &(KEY)) _Py_hashtable_get_entry(TABLE, (const void *)(KEY))
/* Get data from an entry. Copy entry data into data and return 1 if the entry /* Get data from an entry. Copy entry data into data and return 1 if the entry
@ -198,28 +168,26 @@ _Py_hashtable_get_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey)
Don't call directly this function, but use _Py_HASHTABLE_GET() macro */ Don't call directly this function, but use _Py_HASHTABLE_GET() macro */
static inline int static inline int
_Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size, const void *pkey, _Py_hashtable_get(_Py_hashtable_t *ht, const void *key,
size_t data_size, void *data) size_t data_size, void *data)
{ {
assert(key_size == ht->key_size);
assert(data_size == ht->data_size); assert(data_size == ht->data_size);
return ht->get_func(ht, pkey, data); return ht->get_func(ht, key, data);
} }
#define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ #define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \
_Py_hashtable_get(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) _Py_hashtable_get(TABLE, (KEY), sizeof(DATA), &(DATA))
/* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */ /* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */
PyAPI_FUNC(int) _Py_hashtable_pop( PyAPI_FUNC(int) _Py_hashtable_pop(
_Py_hashtable_t *ht, _Py_hashtable_t *ht,
size_t key_size, const void *key,
const void *pkey,
size_t data_size, size_t data_size,
void *data); void *data);
#define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \ #define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \
_Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) _Py_hashtable_pop(TABLE, (KEY), sizeof(DATA), &(DATA))
#ifdef __cplusplus #ifdef __cplusplus

117
Modules/_tracemalloc.c
View file

@ -23,6 +23,9 @@ static void raw_free(void *ptr);
# define TRACE_DEBUG # define TRACE_DEBUG
#endif #endif
#define TO_PTR(key) ((const void *)(uintptr_t)key)
#define FROM_PTR(key) ((uintptr_t)key)
/* Protected by the GIL */ /* Protected by the GIL */
static struct { static struct {
PyMemAllocatorEx mem; PyMemAllocatorEx mem;
@ -203,47 +206,42 @@ set_reentrant(int reentrant)
static Py_uhash_t static Py_uhash_t
hashtable_hash_pyobject(_Py_hashtable_t *ht, const void *pkey) hashtable_hash_pyobject(const void *key)
{ {
PyObject *obj; PyObject *obj = (PyObject *)key;
_Py_HASHTABLE_READ_KEY(ht, pkey, obj);
return PyObject_Hash(obj); return PyObject_Hash(obj);
} }
static int static int
hashtable_compare_unicode(_Py_hashtable_t *ht, const void *pkey, hashtable_compare_unicode(const void *key1, const void *key2)
const _Py_hashtable_entry_t *entry)
{ {
PyObject *key1, *key2; PyObject *obj1 = (PyObject *)key1;
PyObject *obj2 = (PyObject *)key2;
_Py_HASHTABLE_READ_KEY(ht, pkey, key1); if (obj1 != NULL && obj2 != NULL) {
_Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, key2); return (PyUnicode_Compare(obj1, obj2) == 0);
}
if (key1 != NULL && key2 != NULL) else {
return (PyUnicode_Compare(key1, key2) == 0); return obj1 == obj2;
else }
return key1 == key2;
} }
static Py_uhash_t static Py_uhash_t
hashtable_hash_uint(_Py_hashtable_t *ht, const void *pkey) hashtable_hash_uint(const void *key_raw)
{ {
unsigned int key; unsigned int key = (unsigned int)FROM_PTR(key_raw);
_Py_HASHTABLE_READ_KEY(ht, pkey, key);
return (Py_uhash_t)key; return (Py_uhash_t)key;
} }
static _Py_hashtable_t * static _Py_hashtable_t *
hashtable_new(size_t key_size, size_t data_size, hashtable_new(size_t data_size,
_Py_hashtable_hash_func hash_func, _Py_hashtable_hash_func hash_func,
_Py_hashtable_compare_func compare_func) _Py_hashtable_compare_func compare_func)
{ {
_Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; _Py_hashtable_allocator_t hashtable_alloc = {malloc, free};
return _Py_hashtable_new_full(key_size, data_size, 0, return _Py_hashtable_new_full(data_size, 0,
hash_func, compare_func, hash_func, compare_func,
&hashtable_alloc); &hashtable_alloc);
} }
@ -263,39 +261,33 @@ raw_free(void *ptr)
static Py_uhash_t static Py_uhash_t
hashtable_hash_traceback(_Py_hashtable_t *ht, const void *pkey) hashtable_hash_traceback(const void *key)
{ {
traceback_t *traceback; const traceback_t *traceback = (const traceback_t *)key;
_Py_HASHTABLE_READ_KEY(ht, pkey, traceback);
return traceback->hash; return traceback->hash;
} }
static int static int
hashtable_compare_traceback(_Py_hashtable_t *ht, const void *pkey, hashtable_compare_traceback(const void *key1, const void *key2)
const _Py_hashtable_entry_t *entry)
{ {
traceback_t *traceback1, *traceback2; const traceback_t *traceback1 = (const traceback_t *)key1;
const frame_t *frame1, *frame2; const traceback_t *traceback2 = (const traceback_t *)key2;
int i;
_Py_HASHTABLE_READ_KEY(ht, pkey, traceback1); if (traceback1->nframe != traceback2->nframe) {
_Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, traceback2);
if (traceback1->nframe != traceback2->nframe)
return 0; return 0;
}
if (traceback1->total_nframe != traceback2->total_nframe) if (traceback1->total_nframe != traceback2->total_nframe) {
return 0; return 0;
}
for (i=0; i < traceback1->nframe; i++) { for (int i=0; i < traceback1->nframe; i++) {
frame1 = &traceback1->frames[i]; const frame_t *frame1 = &traceback1->frames[i];
frame2 = &traceback2->frames[i]; const frame_t *frame2 = &traceback2->frames[i];
if (frame1->lineno != frame2->lineno) if (frame1->lineno != frame2->lineno) {
return 0; return 0;
}
if (frame1->filename != frame2->filename) { if (frame1->filename != frame2->filename) {
assert(PyUnicode_Compare(frame1->filename, frame2->filename) != 0); assert(PyUnicode_Compare(frame1->filename, frame2->filename) != 0);
return 0; return 0;
@ -349,7 +341,7 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame)
_Py_hashtable_entry_t *entry; _Py_hashtable_entry_t *entry;
entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename);
if (entry != NULL) { if (entry != NULL) {
_Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_filenames, entry, filename); filename = (PyObject *)entry->key;
} }
else { else {
/* tracemalloc_filenames is responsible to keep a reference /* tracemalloc_filenames is responsible to keep a reference
@ -444,7 +436,7 @@ traceback_new(void)
/* intern the traceback */ /* intern the traceback */
entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback);
if (entry != NULL) { if (entry != NULL) {
_Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_tracebacks, entry, traceback); traceback = (traceback_t *)entry->key;
} }
else { else {
traceback_t *copy; traceback_t *copy;
@ -477,8 +469,7 @@ traceback_new(void)
static _Py_hashtable_t* static _Py_hashtable_t*
tracemalloc_create_traces_table(void) tracemalloc_create_traces_table(void)
{ {
return hashtable_new(sizeof(uintptr_t), return hashtable_new(sizeof(trace_t),
sizeof(trace_t),
_Py_hashtable_hash_ptr, _Py_hashtable_hash_ptr,
_Py_hashtable_compare_direct); _Py_hashtable_compare_direct);
} }
@ -487,8 +478,7 @@ tracemalloc_create_traces_table(void)
static _Py_hashtable_t* static _Py_hashtable_t*
tracemalloc_create_domains_table(void) tracemalloc_create_domains_table(void)
{ {
return hashtable_new(sizeof(unsigned int), return hashtable_new(sizeof(_Py_hashtable_t *),
sizeof(_Py_hashtable_t *),
hashtable_hash_uint, hashtable_hash_uint,
_Py_hashtable_compare_direct); _Py_hashtable_compare_direct);
} }
@ -522,7 +512,7 @@ tracemalloc_get_traces_table(unsigned int domain)
} }
else { else {
_Py_hashtable_t *traces = NULL; _Py_hashtable_t *traces = NULL;
(void)_Py_HASHTABLE_GET(tracemalloc_domains, domain, traces); (void)_Py_HASHTABLE_GET(tracemalloc_domains, TO_PTR(domain), traces);
return traces; return traces;
} }
} }
@ -539,7 +529,7 @@ tracemalloc_remove_trace(unsigned int domain, uintptr_t ptr)
} }
trace_t trace; trace_t trace;
if (!_Py_HASHTABLE_POP(traces, ptr, trace)) { if (!_Py_HASHTABLE_POP(traces, TO_PTR(ptr), trace)) {
return; return;
} }
assert(tracemalloc_traced_memory >= trace.size); assert(tracemalloc_traced_memory >= trace.size);
@ -568,7 +558,7 @@ tracemalloc_add_trace(unsigned int domain, uintptr_t ptr,
return -1; return -1;
} }
if (_Py_HASHTABLE_SET(tracemalloc_domains, domain, traces) < 0) { if (_Py_HASHTABLE_SET(tracemalloc_domains, TO_PTR(domain), traces) < 0) {
_Py_hashtable_destroy(traces); _Py_hashtable_destroy(traces);
return -1; return -1;
} }
@ -590,7 +580,7 @@ tracemalloc_add_trace(unsigned int domain, uintptr_t ptr,
trace.size = size; trace.size = size;
trace.traceback = traceback; trace.traceback = traceback;
int res = _Py_HASHTABLE_SET(traces, ptr, trace); int res = _Py_HASHTABLE_SET(traces, TO_PTR(ptr), trace);
if (res != 0) { if (res != 0) {
return res; return res;
} }
@ -859,9 +849,7 @@ static int
tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry,
void *user_data) void *user_data)
{ {
PyObject *filename; PyObject *filename = (PyObject *)entry->key;
_Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, filename);
Py_DECREF(filename); Py_DECREF(filename);
return 0; return 0;
} }
@ -871,9 +859,7 @@ static int
traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry,
void *user_data) void *user_data)
{ {
traceback_t *traceback; traceback_t *traceback = (traceback_t *)entry->key;
_Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, traceback);
raw_free(traceback); raw_free(traceback);
return 0; return 0;
} }
@ -936,11 +922,11 @@ tracemalloc_init(void)
} }
#endif #endif
tracemalloc_filenames = hashtable_new(sizeof(PyObject *), 0, tracemalloc_filenames = hashtable_new(0,
hashtable_hash_pyobject, hashtable_hash_pyobject,
hashtable_compare_unicode); hashtable_compare_unicode);
tracemalloc_tracebacks = hashtable_new(sizeof(traceback_t *), 0, tracemalloc_tracebacks = hashtable_new(0,
hashtable_hash_traceback, hashtable_hash_traceback,
hashtable_compare_traceback); hashtable_compare_traceback);
@ -1154,7 +1140,7 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table)
PyObject *frames, *frame; PyObject *frames, *frame;
if (intern_table != NULL) { if (intern_table != NULL) {
if (_Py_HASHTABLE_GET(intern_table, traceback, frames)) { if (_Py_HASHTABLE_GET(intern_table, (const void *)traceback, frames)) {
Py_INCREF(frames); Py_INCREF(frames);
return frames; return frames;
} }
@ -1244,13 +1230,12 @@ tracemalloc_get_traces_copy_domain(_Py_hashtable_t *domains,
{ {
get_traces_t *get_traces = user_data; get_traces_t *get_traces = user_data;
unsigned int domain; unsigned int domain = (unsigned int)FROM_PTR(entry->key);
_Py_HASHTABLE_ENTRY_READ_KEY(domains, entry, domain);
_Py_hashtable_t *traces; _Py_hashtable_t *traces;
_Py_HASHTABLE_ENTRY_READ_DATA(domains, entry, traces); _Py_HASHTABLE_ENTRY_READ_DATA(domains, entry, traces);
_Py_hashtable_t *traces2 = _Py_hashtable_copy(traces); _Py_hashtable_t *traces2 = _Py_hashtable_copy(traces);
if (_Py_HASHTABLE_SET(get_traces->domains, domain, traces2) < 0) { if (_Py_HASHTABLE_SET(get_traces->domains, TO_PTR(domain), traces2) < 0) {
_Py_hashtable_destroy(traces2); _Py_hashtable_destroy(traces2);
return -1; return -1;
} }
@ -1289,8 +1274,7 @@ tracemalloc_get_traces_domain(_Py_hashtable_t *domains,
{ {
get_traces_t *get_traces = user_data; get_traces_t *get_traces = user_data;
unsigned int domain; unsigned int domain = (unsigned int)FROM_PTR(entry->key);
_Py_HASHTABLE_ENTRY_READ_KEY(domains, entry, domain);
_Py_hashtable_t *traces; _Py_hashtable_t *traces;
_Py_HASHTABLE_ENTRY_READ_DATA(domains, entry, traces); _Py_HASHTABLE_ENTRY_READ_DATA(domains, entry, traces);
@ -1343,8 +1327,7 @@ _tracemalloc__get_traces_impl(PyObject *module)
/* the traceback hash table is used temporarily to intern traceback tuple /* the traceback hash table is used temporarily to intern traceback tuple
of (filename, lineno) tuples */ of (filename, lineno) tuples */
get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), get_traces.tracebacks = hashtable_new(sizeof(PyObject *),
sizeof(PyObject *),
_Py_hashtable_hash_ptr, _Py_hashtable_hash_ptr,
_Py_hashtable_compare_direct); _Py_hashtable_compare_direct);
if (get_traces.tracebacks == NULL) { if (get_traces.tracebacks == NULL) {
@ -1425,7 +1408,7 @@ tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr)
TABLES_LOCK(); TABLES_LOCK();
_Py_hashtable_t *traces = tracemalloc_get_traces_table(domain); _Py_hashtable_t *traces = tracemalloc_get_traces_table(domain);
if (traces) { if (traces) {
found = _Py_HASHTABLE_GET(traces, ptr, trace); found = _Py_HASHTABLE_GET(traces, TO_PTR(ptr), trace);
} }
else { else {
found = 0; found = 0;

115
Python/hashtable.c
View file

@ -59,7 +59,7 @@
#define ENTRY_NEXT(ENTRY) \ #define ENTRY_NEXT(ENTRY) \
((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY)) ((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY))
#define HASHTABLE_ITEM_SIZE(HT) \ #define HASHTABLE_ITEM_SIZE(HT) \
(sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) (sizeof(_Py_hashtable_entry_t) + (HT)->data_size)
#define ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ #define ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \
do { \ do { \
@ -105,20 +105,16 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous,
Py_uhash_t Py_uhash_t
_Py_hashtable_hash_ptr(struct _Py_hashtable_t *ht, const void *pkey) _Py_hashtable_hash_ptr(const void *key)
{ {
void *key;
_Py_HASHTABLE_READ_KEY(ht, pkey, key);
return (Py_uhash_t)_Py_HashPointerRaw(key); return (Py_uhash_t)_Py_HashPointerRaw(key);
} }
int int
_Py_hashtable_compare_direct(_Py_hashtable_t *ht, const void *pkey, _Py_hashtable_compare_direct(const void *key1, const void *key2)
const _Py_hashtable_entry_t *entry)
{ {
const void *pkey2 = _Py_HASHTABLE_ENTRY_PKEY(entry); return (key1 == key2);
return (memcmp(pkey, pkey2, ht->key_size) == 0);
} }
@ -195,16 +191,16 @@ _Py_hashtable_print_stats(_Py_hashtable_t *ht)
_Py_hashtable_entry_t * _Py_hashtable_entry_t *
_Py_hashtable_get_entry_generic(_Py_hashtable_t *ht, const void *pkey) _Py_hashtable_get_entry_generic(_Py_hashtable_t *ht, const void *key)
{ {
Py_uhash_t key_hash = ht->hash_func(ht, pkey); Py_uhash_t key_hash = ht->hash_func(key);
size_t index = key_hash & (ht->num_buckets - 1); size_t index = key_hash & (ht->num_buckets - 1);
_Py_hashtable_entry_t *entry = entry = TABLE_HEAD(ht, index); _Py_hashtable_entry_t *entry = entry = TABLE_HEAD(ht, index);
while (1) { while (1) {
if (entry == NULL) { if (entry == NULL) {
return NULL; return NULL;
} }
if (entry->key_hash == key_hash && ht->compare_func(ht, pkey, entry)) { if (entry->key_hash == key_hash && ht->compare_func(key, entry->key)) {
break; break;
} }
entry = ENTRY_NEXT(entry); entry = ENTRY_NEXT(entry);
@ -214,27 +210,26 @@ _Py_hashtable_get_entry_generic(_Py_hashtable_t *ht, const void *pkey)
static int static int
_Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, _Py_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key,
void *data, size_t data_size) void *data, size_t data_size)
{ {
Py_uhash_t key_hash;
size_t index;
_Py_hashtable_entry_t *entry, *previous;
assert(key_size == ht->key_size); Py_uhash_t key_hash = ht->hash_func(key);
size_t index = key_hash & (ht->num_buckets - 1);
key_hash = ht->hash_func(ht, pkey); _Py_hashtable_entry_t *entry = TABLE_HEAD(ht, index);
index = key_hash & (ht->num_buckets - 1); _Py_hashtable_entry_t *previous = NULL;
while (1) {
previous = NULL; if (entry == NULL) {
for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { // not found
if (entry->key_hash == key_hash && ht->compare_func(ht, pkey, entry))
break;
previous = entry;
}
if (entry == NULL)
return 0; return 0;
}
if (entry->key_hash == key_hash && ht->compare_func(key, entry->key)) {
break;
}
previous = entry;
entry = ENTRY_NEXT(entry);
}
_Py_slist_remove(&ht->buckets[index], (_Py_slist_item_t *)previous, _Py_slist_remove(&ht->buckets[index], (_Py_slist_item_t *)previous,
(_Py_slist_item_t *)entry); (_Py_slist_item_t *)entry);
@ -251,26 +246,22 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey,
int int
_Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, _Py_hashtable_set(_Py_hashtable_t *ht, const void *key,
size_t data_size, const void *data) size_t data_size, const void *data)
{ {
Py_uhash_t key_hash;
size_t index;
_Py_hashtable_entry_t *entry; _Py_hashtable_entry_t *entry;
assert(key_size == ht->key_size);
assert(data != NULL || data_size == 0); assert(data != NULL || data_size == 0);
#ifndef NDEBUG #ifndef NDEBUG
/* Don't write the assertion on a single line because it is interesting /* Don't write the assertion on a single line because it is interesting
to know the duplicated entry if the assertion failed. The entry can to know the duplicated entry if the assertion failed. The entry can
be read using a debugger. */ be read using a debugger. */
entry = ht->get_entry_func(ht, pkey); entry = ht->get_entry_func(ht, key);
assert(entry == NULL); assert(entry == NULL);
#endif #endif
key_hash = ht->hash_func(ht, pkey); Py_uhash_t key_hash = ht->hash_func(key);
index = key_hash & (ht->num_buckets - 1); size_t index = key_hash & (ht->num_buckets - 1);
entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht));
if (entry == NULL) { if (entry == NULL) {
@ -279,9 +270,10 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey,
} }
entry->key_hash = key_hash; entry->key_hash = key_hash;
memcpy((void *)_Py_HASHTABLE_ENTRY_PKEY(entry), pkey, ht->key_size); entry->key = (void *)key;
if (data) if (data) {
ENTRY_WRITE_PDATA(ht, entry, data_size, data); ENTRY_WRITE_PDATA(ht, entry, data_size, data);
}
_Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry);
ht->entries++; ht->entries++;
@ -293,10 +285,10 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey,
int int
_Py_hashtable_get_generic(_Py_hashtable_t *ht, const void *pkey, void *data) _Py_hashtable_get_generic(_Py_hashtable_t *ht, const void *key, void *data)
{ {
assert(data != NULL); assert(data != NULL);
_Py_hashtable_entry_t *entry = ht->get_entry_func(ht, pkey); _Py_hashtable_entry_t *entry = ht->get_entry_func(ht, key);
if (entry != NULL) { if (entry != NULL) {
ENTRY_READ_PDATA(ht, entry, ht->data_size, data); ENTRY_READ_PDATA(ht, entry, ht->data_size, data);
return 1; return 1;
@ -308,13 +300,12 @@ _Py_hashtable_get_generic(_Py_hashtable_t *ht, const void *pkey, void *data)
// Specialized for: // Specialized for:
// key_size == sizeof(void*)
// hash_func == _Py_hashtable_hash_ptr // hash_func == _Py_hashtable_hash_ptr
// compare_func == _Py_hashtable_compare_direct // compare_func == _Py_hashtable_compare_direct
_Py_hashtable_entry_t * _Py_hashtable_entry_t *
_Py_hashtable_get_entry_ptr(_Py_hashtable_t *ht, const void *pkey) _Py_hashtable_get_entry_ptr(_Py_hashtable_t *ht, const void *key)
{ {
Py_uhash_t key_hash = _Py_hashtable_hash_ptr(ht, pkey); Py_uhash_t key_hash = _Py_hashtable_hash_ptr(key);
size_t index = key_hash & (ht->num_buckets - 1); size_t index = key_hash & (ht->num_buckets - 1);
_Py_hashtable_entry_t *entry = entry = TABLE_HEAD(ht, index); _Py_hashtable_entry_t *entry = entry = TABLE_HEAD(ht, index);
while (1) { while (1) {
@ -322,8 +313,7 @@ _Py_hashtable_get_entry_ptr(_Py_hashtable_t *ht, const void *pkey)
return NULL; return NULL;
} }
if (entry->key_hash == key_hash) { if (entry->key_hash == key_hash) {
const void *pkey2 = _Py_HASHTABLE_ENTRY_PKEY(entry); if (entry->key == key) {
if (memcmp(pkey, pkey2, sizeof(void*)) == 0) {
break; break;
} }
} }
@ -334,14 +324,13 @@ _Py_hashtable_get_entry_ptr(_Py_hashtable_t *ht, const void *pkey)
// Specialized for: // Specialized for:
// key_size == sizeof(void*)
// hash_func == _Py_hashtable_hash_ptr // hash_func == _Py_hashtable_hash_ptr
// compare_func == _Py_hashtable_compare_direct // compare_func == _Py_hashtable_compare_direct
int int
_Py_hashtable_get_ptr(_Py_hashtable_t *ht, const void *pkey, void *data) _Py_hashtable_get_ptr(_Py_hashtable_t *ht, const void *key, void *data)
{ {
assert(data != NULL); assert(data != NULL);
_Py_hashtable_entry_t *entry = _Py_hashtable_get_entry_ptr(ht, pkey); _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry_ptr(ht, key);
if (entry != NULL) { if (entry != NULL) {
ENTRY_READ_PDATA(ht, entry, ht->data_size, data); ENTRY_READ_PDATA(ht, entry, ht->data_size, data);
return 1; return 1;
@ -353,24 +342,24 @@ _Py_hashtable_get_ptr(_Py_hashtable_t *ht, const void *pkey, void *data)
int int
_Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, _Py_hashtable_pop(_Py_hashtable_t *ht, const void *key,
size_t data_size, void *data) size_t data_size, void *data)
{ {
assert(data != NULL); assert(data != NULL);
return _Py_hashtable_pop_entry(ht, key_size, pkey, data, data_size); return _Py_hashtable_pop_entry(ht, key, data, data_size);
} }
/* Code commented since the function is not needed in Python */ /* Code commented since the function is not needed in Python */
#if 0 #if 0
void void
_Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) _Py_hashtable_delete(_Py_hashtable_t *ht, size_t const void *key)
{ {
#ifndef NDEBUG #ifndef NDEBUG
int found = _Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); int found = _Py_hashtable_pop_entry(ht, key, NULL, 0);
assert(found); assert(found);
#else #else
(void)_Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); (void)_Py_hashtable_pop_entry(ht, key, NULL, 0);
#endif #endif
} }
#endif #endif
@ -427,7 +416,7 @@ hashtable_rehash(_Py_hashtable_t *ht)
size_t entry_index; size_t entry_index;
assert(ht->hash_func(ht, _Py_HASHTABLE_ENTRY_PKEY(entry)) == entry->key_hash); assert(ht->hash_func(entry->key) == entry->key_hash);
next = ENTRY_NEXT(entry); next = ENTRY_NEXT(entry);
entry_index = entry->key_hash & (new_size - 1); entry_index = entry->key_hash & (new_size - 1);
@ -440,8 +429,7 @@ hashtable_rehash(_Py_hashtable_t *ht)
_Py_hashtable_t * _Py_hashtable_t *
_Py_hashtable_new_full(size_t key_size, size_t data_size, _Py_hashtable_new_full(size_t data_size, size_t init_size,
size_t init_size,
_Py_hashtable_hash_func hash_func, _Py_hashtable_hash_func hash_func,
_Py_hashtable_compare_func compare_func, _Py_hashtable_compare_func compare_func,
_Py_hashtable_allocator_t *allocator) _Py_hashtable_allocator_t *allocator)
@ -464,7 +452,6 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size,
ht->num_buckets = round_size(init_size); ht->num_buckets = round_size(init_size);
ht->entries = 0; ht->entries = 0;
ht->key_size = key_size;
ht->data_size = data_size; ht->data_size = data_size;
buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); buckets_size = ht->num_buckets * sizeof(ht->buckets[0]);
@ -480,8 +467,7 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size,
ht->hash_func = hash_func; ht->hash_func = hash_func;
ht->compare_func = compare_func; ht->compare_func = compare_func;
ht->alloc = alloc; ht->alloc = alloc;
if (ht->key_size == sizeof(void*) if (ht->hash_func == _Py_hashtable_hash_ptr
&& ht->hash_func == _Py_hashtable_hash_ptr
&& ht->compare_func == _Py_hashtable_compare_direct) && ht->compare_func == _Py_hashtable_compare_direct)
{ {
ht->get_func = _Py_hashtable_get_ptr; ht->get_func = _Py_hashtable_get_ptr;
@ -492,12 +478,11 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size,
_Py_hashtable_t * _Py_hashtable_t *
_Py_hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_new(size_t data_size,
_Py_hashtable_hash_func hash_func, _Py_hashtable_hash_func hash_func,
_Py_hashtable_compare_func compare_func) _Py_hashtable_compare_func compare_func)
{ {
return _Py_hashtable_new_full(key_size, data_size, return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE,
HASHTABLE_MIN_SIZE,
hash_func, compare_func, hash_func, compare_func,
NULL); NULL);
} }
@ -543,15 +528,13 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht)
_Py_hashtable_t * _Py_hashtable_t *
_Py_hashtable_copy(_Py_hashtable_t *src) _Py_hashtable_copy(_Py_hashtable_t *src)
{ {
const size_t key_size = src->key_size;
const size_t data_size = src->data_size; const size_t data_size = src->data_size;
_Py_hashtable_t *dst; _Py_hashtable_t *dst;
_Py_hashtable_entry_t *entry; _Py_hashtable_entry_t *entry;
size_t bucket; size_t bucket;
int err; int err;
dst = _Py_hashtable_new_full(key_size, data_size, dst = _Py_hashtable_new_full(data_size, src->num_buckets,
src->num_buckets,
src->hash_func, src->hash_func,
src->compare_func, src->compare_func,
&src->alloc); &src->alloc);
@ -561,9 +544,9 @@ _Py_hashtable_copy(_Py_hashtable_t *src)
for (bucket=0; bucket < src->num_buckets; bucket++) { for (bucket=0; bucket < src->num_buckets; bucket++) {
entry = TABLE_HEAD(src, bucket); entry = TABLE_HEAD(src, bucket);
for (; entry; entry = ENTRY_NEXT(entry)) { for (; entry; entry = ENTRY_NEXT(entry)) {
const void *pkey = _Py_HASHTABLE_ENTRY_PKEY(entry); const void *key = entry->key;
const void *pdata = _Py_HASHTABLE_ENTRY_PDATA(src, entry); const void *pdata = _Py_HASHTABLE_ENTRY_PDATA(src, entry);
err = _Py_hashtable_set(dst, key_size, pkey, data_size, pdata); err = _Py_hashtable_set(dst, key, data_size, pdata);
if (err) { if (err) {
_Py_hashtable_destroy(dst); _Py_hashtable_destroy(dst);
return NULL; return NULL;

6
Python/marshal.c
View file

@ -549,7 +549,7 @@ static int
w_init_refs(WFILE *wf, int version) w_init_refs(WFILE *wf, int version)
{ {
if (version >= 3) { if (version >= 3) {
wf->hashtable = _Py_hashtable_new(sizeof(PyObject *), sizeof(int), wf->hashtable = _Py_hashtable_new(sizeof(int),
_Py_hashtable_hash_ptr, _Py_hashtable_hash_ptr,
_Py_hashtable_compare_direct); _Py_hashtable_compare_direct);
if (wf->hashtable == NULL) { if (wf->hashtable == NULL) {
@ -564,9 +564,7 @@ static int
w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry,
void *Py_UNUSED(data)) void *Py_UNUSED(data))
{ {
PyObject *entry_key; PyObject *entry_key = (PyObject *)entry->key;
_Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, entry_key);
Py_XDECREF(entry_key); Py_XDECREF(entry_key);
return 0; return 0;
} }