118 #define PREFETCH(addr, write_p) __builtin_prefetch(addr, write_p)
119 #define EXPECT(expr, val) __builtin_expect(expr, val)
120 #define ATTRIBUTE_UNUSED __attribute__((unused))
122 #define PREFETCH(addr, write_p)
123 #define EXPECT(expr, val) (expr)
124 #define ATTRIBUTE_UNUSED
128 #define st_assert assert
130 #define st_assert(cond) ((void)(0 && (cond)))
142 #define type_numhash st_hashtype_num
158 st_locale_insensitive_strcasecmp_i,
165 #define ST_INIT_VAL 0xafafafafafafafaf
166 #define ST_INIT_VAL_BYTE 0xafa
173 #define malloc ruby_xmalloc
174 #define calloc ruby_xcalloc
175 #define realloc ruby_xrealloc
176 #define free ruby_xfree
179 #define EQUAL(tab,x,y) ((x) == (y) || (*(tab)->type->compare)((x),(y)) == 0)
180 #define PTR_EQUAL(tab, ptr, hash_val, key_) \
181 ((ptr)->hash == (hash_val) && EQUAL((tab), (key_), (ptr)->key))
185 #define DO_PTR_EQUAL_CHECK(tab, ptr, hash_val, key, res, rebuilt_p) \
187 unsigned int _old_rebuilds_num = (tab)->rebuilds_num; \
188 res = PTR_EQUAL(tab, ptr, hash_val, key); \
189 rebuilt_p = _old_rebuilds_num != (tab)->rebuilds_num; \
208 #if SIZEOF_ST_INDEX_T == 8
209 #define MAX_POWER2 62
227 {16, 17, 2, 0x10000},
228 {17, 18, 2, 0x20000},
229 {18, 19, 2, 0x40000},
230 {19, 20, 2, 0x80000},
231 {20, 21, 2, 0x100000},
232 {21, 22, 2, 0x200000},
233 {22, 23, 2, 0x400000},
234 {23, 24, 2, 0x800000},
235 {24, 25, 2, 0x1000000},
236 {25, 26, 2, 0x2000000},
237 {26, 27, 2, 0x4000000},
238 {27, 28, 2, 0x8000000},
239 {28, 29, 2, 0x10000000},
240 {29, 30, 2, 0x20000000},
241 {30, 31, 2, 0x40000000},
242 {31, 32, 2, 0x80000000},
243 {32, 33, 3, 0x200000000},
244 {33, 34, 3, 0x400000000},
245 {34, 35, 3, 0x800000000},
246 {35, 36, 3, 0x1000000000},
247 {36, 37, 3, 0x2000000000},
248 {37, 38, 3, 0x4000000000},
249 {38, 39, 3, 0x8000000000},
250 {39, 40, 3, 0x10000000000},
251 {40, 41, 3, 0x20000000000},
252 {41, 42, 3, 0x40000000000},
253 {42, 43, 3, 0x80000000000},
254 {43, 44, 3, 0x100000000000},
255 {44, 45, 3, 0x200000000000},
256 {45, 46, 3, 0x400000000000},
257 {46, 47, 3, 0x800000000000},
258 {47, 48, 3, 0x1000000000000},
259 {48, 49, 3, 0x2000000000000},
260 {49, 50, 3, 0x4000000000000},
261 {50, 51, 3, 0x8000000000000},
262 {51, 52, 3, 0x10000000000000},
263 {52, 53, 3, 0x20000000000000},
264 {53, 54, 3, 0x40000000000000},
265 {54, 55, 3, 0x80000000000000},
266 {55, 56, 3, 0x100000000000000},
267 {56, 57, 3, 0x200000000000000},
268 {57, 58, 3, 0x400000000000000},
269 {58, 59, 3, 0x800000000000000},
270 {59, 60, 3, 0x1000000000000000},
271 {60, 61, 3, 0x2000000000000000},
272 {61, 62, 3, 0x4000000000000000},
273 {62, 63, 3, 0x8000000000000000},
277 #define MAX_POWER2 30
296 {16, 17, 2, 0x20000},
297 {17, 18, 2, 0x40000},
298 {18, 19, 2, 0x80000},
299 {19, 20, 2, 0x100000},
300 {20, 21, 2, 0x200000},
301 {21, 22, 2, 0x400000},
302 {22, 23, 2, 0x800000},
303 {23, 24, 2, 0x1000000},
304 {24, 25, 2, 0x2000000},
305 {25, 26, 2, 0x4000000},
306 {26, 27, 2, 0x8000000},
307 {27, 28, 2, 0x10000000},
308 {28, 29, 2, 0x20000000},
309 {29, 30, 2, 0x40000000},
310 {30, 31, 2, 0x80000000},
316 #define RESERVED_HASH_VAL (~(st_hash_t) 0)
317 #define RESERVED_HASH_SUBSTITUTION_VAL ((st_hash_t) 0)
334 #define MINIMAL_POWER2 2
336 #if MINIMAL_POWER2 < 2
337 #error "MINIMAL_POWER2 should be >= 2"
342 #define MAX_POWER2_FOR_TABLES_WITHOUT_BINS 4
364 return (s == 0 ? ((
unsigned char *) bins)[
n]
365 : s == 1 ? ((
unsigned short *) bins)[
n]
366 : s == 2 ? ((
unsigned int *) bins)[
n]
375 if (s == 0) ((
unsigned char *) bins)[
n] = (
unsigned char)
v;
376 else if (s == 1) ((
unsigned short *) bins)[
n] = (
unsigned short)
v;
377 else if (s == 2) ((
unsigned int *) bins)[
n] = (
unsigned int)
v;
385 #define DELETED_BIN 1
391 #define MARK_BIN_EMPTY(tab, i) (set_bin((tab)->bins, get_size_ind(tab), i, EMPTY_BIN))
395 #define UNDEFINED_ENTRY_IND (~(st_index_t) 0)
396 #define UNDEFINED_BIN_IND (~(st_index_t) 0)
400 #define REBUILT_TABLE_ENTRY_IND (~(st_index_t) 1)
401 #define REBUILT_TABLE_BIN_IND (~(st_index_t) 1)
406 #define MARK_BIN_DELETED(tab, i) \
408 st_assert(i != UNDEFINED_BIN_IND); \
409 st_assert(! IND_EMPTY_OR_DELETED_BIN_P(tab, i)); \
410 set_bin((tab)->bins, get_size_ind(tab), i, DELETED_BIN); \
415 #define EMPTY_BIN_P(b) ((b) == EMPTY_BIN)
416 #define DELETED_BIN_P(b) ((b) == DELETED_BIN)
417 #define EMPTY_OR_DELETED_BIN_P(b) ((b) <= DELETED_BIN)
421 #define IND_EMPTY_BIN_P(tab, i) (EMPTY_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
422 #define IND_DELETED_BIN_P(tab, i) (DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
423 #define IND_EMPTY_OR_DELETED_BIN_P(tab, i) (EMPTY_OR_DELETED_BIN_P(get_bin((tab)->bins, get_size_ind(tab), i)))
427 #define MARK_ENTRY_DELETED(e_ptr) ((e_ptr)->hash = RESERVED_HASH_VAL)
428 #define DELETED_ENTRY_P(e_ptr) ((e_ptr)->hash == RESERVED_HASH_VAL)
431 static inline unsigned int
448 return get_bins_num(tab) - 1;
456 return hash_value & bins_mask(tab);
461 get_allocated_entries(
const st_table *tab)
487 initialize_bins(tab);
491 #define st_assert_notinitial(ent) \
493 st_assert(ent.hash != (st_hash_t) ST_INIT_VAL); \
494 st_assert(ent.key != ST_INIT_VAL); \
495 st_assert(ent.record != ST_INIT_VAL); \
504 for (p = get_allocated_entries(tab),
i = 0; p > 1;
i++, p>>=1)
514 st_assert_notinitial(tab->
entries[
i]);
523 for (
n = d =
i = 0;
i < get_bins_num(tab);
i++) {
535 st_assert_notinitial(tab->
entries[e]);
548 int all, total, num,
str, strcase;
553 static int init_st = 0;
560 char fname[10+
sizeof(
long)*3];
562 if (!collision.total)
return;
566 fprintf(
f,
"collision: %d / %d (%6.2f)\n", collision.all, collision.total,
567 ((
double)collision.all / (collision.total)) * 100);
568 fprintf(
f,
"num: %d, str: %d, strcase: %d\n", collision.num, collision.str, collision.strcase);
585 const char *e =
getenv(
"ST_HASH_LOG");
586 if (!e || !*e) init_st = 1;
595 n = get_power2(
size);
722 + (tab->
bins ==
NULL ? 0 : bins_size(tab))
747 else if (
type == &type_strhash) {
750 else if (
type == &type_strcasehash) {
755 #define COLLISION (collision_check ? count_collision(tab->type) : (void)0)
756 #define FOUND_BIN (collision_check ? collision.total++ : (void)0)
757 #define collision_check 0
766 #define REBUILD_THRESHOLD 4
768 #if REBUILD_THRESHOLD < 2
769 #error "REBUILD_THRESHOLD should be >= 2"
780 unsigned int size_ind;
790 if ((2 * tab->
num_entries <= get_allocated_entries(tab)
796 initialize_bins(tab);
803 new_entries = new_tab->
entries;
806 bins = new_tab->
bins;
807 size_ind = get_size_ind(new_tab);
813 if (&new_entries[ni] != curr_entry_ptr)
814 new_entries[ni] = *curr_entry_ptr;
816 bin_ind = find_table_bin_ind_direct(new_tab, curr_entry_ptr->
hash,
817 curr_entry_ptr->
key);
821 set_bin(bins, size_ind, bin_ind, ni +
ENTRY_BASE);
826 if (new_tab != tab) {
863 ind = (ind << 2) + ind + *perterb + 1;
864 return hash_bin(ind, tab);
903 #ifdef QUADRATIC_PROBE
913 ind = hash_bin(hash_value, tab);
914 #ifdef QUADRATIC_PROBE
921 bin = get_bin(tab->
bins, get_size_ind(tab), ind);
930 #ifdef QUADRATIC_PROBE
931 ind = hash_bin(ind + d, tab);
934 ind = secondary_hash(ind, tab, &peterb);
950 #ifdef QUADRATIC_PROBE
960 ind = hash_bin(hash_value, tab);
961 #ifdef QUADRATIC_PROBE
968 bin = get_bin(tab->
bins, get_size_ind(tab), ind);
977 #ifdef QUADRATIC_PROBE
978 ind = hash_bin(ind + d, tab);
981 ind = secondary_hash(ind, tab, &peterb);
995 #ifdef QUADRATIC_PROBE
1005 ind = hash_bin(hash_value, tab);
1006 #ifdef QUADRATIC_PROBE
1009 peterb = hash_value;
1013 bin = get_bin(tab->
bins, get_size_ind(tab), ind);
1017 #ifdef QUADRATIC_PROBE
1018 ind = hash_bin(ind + d, tab);
1021 ind = secondary_hash(ind, tab, &peterb);
1040 int eq_p, rebuilt_p;
1042 st_hash_t curr_hash_value = *hash_value;
1043 #ifdef QUADRATIC_PROBE
1056 ind = hash_bin(curr_hash_value, tab);
1057 #ifdef QUADRATIC_PROBE
1060 peterb = curr_hash_value;
1066 entry_index = get_bin(tab->
bins, get_size_ind(tab), ind);
1072 ind = first_deleted_bin_ind;
1079 if (
EXPECT(rebuilt_p, 0))
1085 first_deleted_bin_ind = ind;
1086 #ifdef QUADRATIC_PROBE
1087 ind = hash_bin(ind + d, tab);
1090 ind = secondary_hash(ind, tab, &peterb);
1108 bin = find_entry(tab, hash,
key);
1115 bin = find_table_entry_ind(tab, hash,
key);
1137 bin = find_entry(tab, hash,
key);
1144 bin = find_table_entry_ind(tab, hash,
key);
1158 rebuild_table_if_necessary (
st_table *tab)
1162 if (bound == get_allocated_entries(tab))
1180 hash_value = do_hash(
key, tab);
1182 rebuild_table_if_necessary(tab);
1184 bin = find_entry(tab, hash_value,
key);
1193 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1204 entry->
hash = hash_value;
1208 set_bin(tab->
bins, get_size_ind(tab), bin_ind, ind +
ENTRY_BASE);
1224 st_add_direct_with_hash(
st_table *tab,
1231 rebuild_table_if_necessary(tab);
1239 bin_ind = find_table_bin_ind_direct(tab, hash,
key);
1241 set_bin(tab->
bins, get_size_ind(tab), bin_ind, ind +
ENTRY_BASE);
1255 hash_value = do_hash(
key, tab);
1256 st_add_direct_with_hash(tab,
key, value, hash_value);
1273 hash_value = do_hash(
key, tab);
1275 rebuild_table_if_necessary (tab);
1277 bin = find_entry(tab, hash_value,
key);
1286 bin = find_table_bin_ptr_and_reserve(tab, &hash_value,
1300 entry->
hash = hash_value;
1304 set_bin(tab->
bins, get_size_ind(tab), bin_ind, ind +
ENTRY_BASE);
1326 if (new_tab ==
NULL)
1329 *new_tab = *old_tab;
1350 get_allocated_entries(old_tab));
1383 hash = do_hash(*
key, tab);
1386 bin = find_entry(tab, hash, *
key);
1390 if (value != 0) *value = 0;
1395 bin_ind = find_table_bin_ind(tab, hash, *
key);
1399 if (value != 0) *value = 0;
1407 if (value != 0) *value = entry->
record;
1410 update_range_for_deleted(tab,
bin);
1420 return st_general_delete(tab,
key, value);
1432 return st_general_delete(tab,
key, value);
1455 if (value != 0) *value = curr_entry_ptr->
record;
1459 bin = find_entry(tab, entry_hash, entry_key);
1468 bin_ind = find_table_bin_ind(tab, entry_hash, entry_key);
1474 curr_entry_ptr = &
entries[get_bin(tab->
bins, get_size_ind(tab), bin_ind)
1478 st_assert(entry_hash != curr_entry_ptr->
hash && entry_key == curr_entry_ptr->
key);
1481 update_range_for_deleted(tab,
i);
1490 if (value != 0) *value = 0;
1519 int retval, existing;
1525 bin = find_entry(tab, hash,
key);
1533 bin_ind = find_table_bin_ind(tab, hash,
key);
1548 retval = (*func)(&
key, &value,
arg, existing);
1553 st_add_direct_with_hash(tab,
key, value, hash);
1556 if (old_key !=
key) {
1567 update_range_for_deleted(tab,
bin);
1599 int error_p, packed_p = tab->
bins ==
NULL;
1609 key = curr_entry_ptr->
key;
1611 hash = curr_entry_ptr->
hash;
1616 value = curr_entry_ptr->
record;
1617 retval = (*replace)(&
key, &value,
arg,
TRUE);
1618 curr_entry_ptr->
key =
key;
1619 curr_entry_ptr->
record = value;
1627 i = find_entry(tab, hash,
key);
1633 i = find_table_entry_ind(tab, hash,
key);
1639 if (error_p && check_p) {
1641 retval = (*func)(0, 0,
arg, 1);
1667 bin = find_entry(tab, hash,
key);
1674 bin_ind = find_table_bin_ind(tab, hash,
key);
1685 update_range_for_deleted(tab,
bin);
1702 return st_general_foreach(tab, func, replace,
arg,
TRUE);
1713 const struct functor *
f = (
void *)d;
1714 return f->func(k,
v,
f->arg);
1745 if (
keys == keys_end)
1748 key = curr_entry_ptr->
key;
1753 return keys - keys_start;
1759 return st_general_keys(tab,
keys,
size);
1767 return st_general_keys(tab,
keys,
size);
1779 values_start = values;
1780 values_end = values +
size;
1784 if (values == values_end)
1788 *values++ = curr_entry_ptr->
record;
1791 return values - values_start;
1797 return st_general_values(tab, values,
size);
1805 return st_general_values(tab, values,
size);
1808 #define FNV1_32A_INIT 0x811c9dc5
1813 #define FNV_32_PRIME 0x01000193
1815 #ifndef UNALIGNED_WORD_ACCESS
1816 # if defined(__i386) || defined(__i386__) || defined(_M_IX86) || \
1817 defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || \
1818 defined(__powerpc64__) || \
1819 defined(__mc68020__)
1820 # define UNALIGNED_WORD_ACCESS 1
1823 #ifndef UNALIGNED_WORD_ACCESS
1824 # define UNALIGNED_WORD_ACCESS 0
1830 #define BIG_CONSTANT(x,y) ((st_index_t)(x)<<32|(st_index_t)(y))
1831 #define ROTL(x,n) ((x)<<(n)|(x)>>(SIZEOF_ST_INDEX_T*CHAR_BIT-(n)))
1833 #if ST_INDEX_BITS <= 32
1834 #define C1 (st_index_t)0xcc9e2d51
1835 #define C2 (st_index_t)0x1b873593
1837 #define C1 BIG_CONSTANT(0x87c37b91,0x114253d5);
1838 #define C2 BIG_CONSTANT(0x4cf5ad43,0x2745937f);
1847 #if ST_INDEX_BITS <= 32
1866 #if ST_INDEX_BITS <= 32
1880 #if ST_INDEX_BITS > 64
1899 const char *data =
ptr;
1903 #define data_at(n) (st_index_t)((unsigned char)data[(n)])
1904 #define UNALIGNED_ADD_4 UNALIGNED_ADD(2); UNALIGNED_ADD(1); UNALIGNED_ADD(0)
1905 #if SIZEOF_ST_INDEX_T > 4
1906 #define UNALIGNED_ADD_8 UNALIGNED_ADD(6); UNALIGNED_ADD(5); UNALIGNED_ADD(4); UNALIGNED_ADD(3); UNALIGNED_ADD_4
1907 #if SIZEOF_ST_INDEX_T > 8
1908 #define UNALIGNED_ADD_16 UNALIGNED_ADD(14); UNALIGNED_ADD(13); UNALIGNED_ADD(12); UNALIGNED_ADD(11); \
1909 UNALIGNED_ADD(10); UNALIGNED_ADD(9); UNALIGNED_ADD(8); UNALIGNED_ADD(7); UNALIGNED_ADD_8
1910 #define UNALIGNED_ADD_ALL UNALIGNED_ADD_16
1912 #define UNALIGNED_ADD_ALL UNALIGNED_ADD_8
1914 #define UNALIGNED_ADD_ALL UNALIGNED_ADD_4
1918 #if !UNALIGNED_WORD_ACCESS
1925 #ifdef WORDS_BIGENDIAN
1926 # define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1927 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 2)
1929 # define UNALIGNED_ADD(n) case SIZEOF_ST_INDEX_T - (n) - 1: \
1930 t |= data_at(n) << CHAR_BIT*(n)
1933 #undef UNALIGNED_ADD
1936 #ifdef WORDS_BIGENDIAN
1950 #ifdef WORDS_BIGENDIAN
1951 t = (t << sr) | (d >> sl);
1953 t = (t >> sr) | (d << sl);
1955 h = murmur_step(
h, t);
1964 #ifdef WORDS_BIGENDIAN
1965 # define UNALIGNED_ADD(n) case (n) + 1: \
1966 d |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
1968 # define UNALIGNED_ADD(n) case (n) + 1: \
1969 d |= data_at(n) << CHAR_BIT*(n)
1972 #undef UNALIGNED_ADD
1974 #ifdef WORDS_BIGENDIAN
1975 t = (t << sr) | (d >> sl);
1977 t = (t >> sr) | (d << sl);
1980 if (
len < (
size_t)align)
goto skip_tail;
1981 # define SKIP_TAIL 1
1982 h = murmur_step(
h, t);
1988 #ifdef HAVE_BUILTIN___BUILTIN_ASSUME_ALIGNED
1989 #define aligned_data __builtin_assume_aligned(data, sizeof(st_index_t))
1991 #define aligned_data data
2004 #if UNALIGNED_WORD_ACCESS && SIZEOF_ST_INDEX_T <= 8 && CHAR_BIT == 8
2006 #if SIZEOF_ST_INDEX_T > 4
2007 case 7: t |=
data_at(6) << 48;
2008 case 6: t |=
data_at(5) << 40;
2009 case 5: t |=
data_at(4) << 32;
2013 # define SKIP_TAIL 1
2015 case 3: t |=
data_at(2) << 16;
2019 #ifdef WORDS_BIGENDIAN
2020 # define UNALIGNED_ADD(n) case (n) + 1: \
2021 t |= data_at(n) << CHAR_BIT*(SIZEOF_ST_INDEX_T - (n) - 1)
2023 # define UNALIGNED_ADD(n) case (n) + 1: \
2024 t |= data_at(n) << CHAR_BIT*(n)
2027 #undef UNALIGNED_ADD
2038 return murmur_finish(
h);
2044 return murmur_step(
h,
i);
2054 #if SIZEOF_ST_INDEX_T*CHAR_BIT > 8*8
2055 h = murmur_step(
h,
i >> 8*8);
2057 h = murmur_step(
h,
i);
2064 h = murmur_finish(
h);
2068 #undef st_hash_start
2078 register const char *
string = (
const char *)
arg;
2090 if (c1 ==
'\0' || c2 ==
'\0') {
2091 if (c1 !=
'\0')
return 1;
2092 if (c2 !=
'\0')
return -1;
2095 if ((
'A' <= c1) && (c1 <=
'Z')) c1 +=
'a' -
'A';
2096 if ((
'A' <= c2) && (c2 <=
'Z')) c2 +=
'a' -
'A';
2112 for (
i = 0;
i <
n;
i++) {
2115 if (c1 ==
'\0' || c2 ==
'\0') {
2116 if (c1 !=
'\0')
return 1;
2117 if (c2 !=
'\0')
return -1;
2120 if ((
'A' <= c1) && (c1 <=
'Z')) c1 +=
'a' -
'A';
2121 if ((
'A' <= c2) && (c2 <=
'Z')) c2 +=
'a' -
'A';
2135 const char *s1 = (
char *)lhs;
2136 const char *
s2 = (
char *)rhs;
2143 const char *s1 = (
char *)lhs;
2144 const char *
s2 = (
char *)rhs;
2152 register const char *
string = (
const char *)
arg;
2159 unsigned int c = (
unsigned char)*
string++;
2160 if ((
unsigned int)(c -
'A') <= (
'Z' -
'A')) c +=
'a' -
'A';
2178 enum {s1 = 11,
s2 = 3};
2191 if (siz <= get_allocated_entries(tab))
2195 n = get_allocated_entries(tab);
2216 int eq_p, rebuilt_p;
2232 if (
EXPECT(rebuilt_p, 0))
2239 update_range_for_deleted(tab, j);
2251 int eq_p, rebuilt_p;
2254 unsigned int const size_ind = get_size_ind(tab);
2258 initialize_bins(tab);
2262 #ifdef QUADRATIC_PROBE
2271 ind = hash_bin(p->
hash, tab);
2282 if (
EXPECT(rebuilt_p, 0))
2290 update_range_for_deleted(tab,
bin);
2295 #ifdef QUADRATIC_PROBE
2296 ind = hash_bin(ind + d, tab);
2299 ind = secondary_hash(ind, tab, &peterb);
2318 rebuilt_p = st_rehash_linear(tab);
2320 rebuilt_p = st_rehash_indexed(tab);
2321 }
while (rebuilt_p);
2337 e.
hash = do_hash(k, tab);
2352 for (
i = 0;
i <
argc; ) {
2367 for (
i = 0;
i <
argc; ) {
2370 st_insert_single(tab, hash,
key, val);
2387 st_expand_table(tab,
n);
2389 st_insert_generic(tab,
argc,
argv, hash);
2391 st_insert_single(tab, hash,
argv[0],
argv[1]);
2393 st_insert_linear(tab,
argc,
argv, hash);
2395 st_insert_generic(tab,
argc,
argv, hash);