History

lavc/hashtable: create generic robin hood hash table

Adds a generic hash table with the DXV encoder as an initial use case.

Emma Worley committed 1 year ago

Commit 6fdb54d1 parent 854b8694 files · +420 −0Browse files

Unified · Split

4 files
  1. libavcodec/Makefile +2 −0
  2. libavcodec/hashtable.c +214 −0
  3. libavcodec/hashtable.h +94 −0
  4. libavcodec/tests/hashtable.c +110 −0
libavcodec/Makefile+2−0
@@ -42,6 +42,7 @@ OBJS = ac3_parser.o \ · 41 unmodified lines
42 dv_profile.o \42 dv_profile.o \
43 encode.o \43 encode.o \
44 get_buffer.o \44 get_buffer.o \
45 hashtable.o \
45 imgconvert.o \46 imgconvert.o \
46 jni.o \47 jni.o \
47 lcevcdec.o \48 lcevcdec.o \
@@ -1326,6 +1327,7 @@ TESTPROGS = avcodec \ · 1278 unmodified lines
1326 bitstream_le \1327 bitstream_le \
1327 celp_math \1328 celp_math \
1328 codec_desc \1329 codec_desc \
1330 hashtable \
1329 htmlsubtitles \1331 htmlsubtitles \
1330 jpeg2000dwt \1332 jpeg2000dwt \
1331 mathops \1333 mathops \
libavcodec/hashtable.c+214−0
@@ -0,0 +1,214 @@
1/*
2 * Generic hashtable
3 * Copyright (C) 2025 Emma Worley <emma@emma.gg>
4 *
5 * This file is part of FFmpeg.
6 *
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#include <stdint.h>
23#include <string.h>
24
25#include "libavutil/crc.h"
26#include "libavutil/error.h"
27#include "libavutil/mem.h"
28#include "hashtable.h"
29
30#define ALIGN _Alignof(size_t)
31
32struct FFHashtableContext {
33 size_t key_size;
34 size_t key_size_aligned;
35 size_t val_size;
36 size_t val_size_aligned;
37 size_t entry_size;
38 size_t max_entries;
39 size_t nb_entries;
40 const AVCRC *crc;
41 uint8_t *table;
42 uint8_t *swapbuf;
43};
44
45/*
46 * Hash table entries are comprised of a probe sequence length (PSL), key, and
47 * value. When the PSL of an entry is zero, it means it is not occupied by a
48 * key/value pair. When the PSL is non-zero, it represents the "distance" of
49 * the entry from its "home" location plus one, where the "home" location is
50 * hash(key) % max_entries.
51 */
52
53#define ENTRY_PSL_VAL(entry) (*(size_t*)(entry))
54#define ENTRY_KEY_PTR(entry) ((entry) + FFALIGN(sizeof(size_t), ALIGN))
55#define ENTRY_VAL_PTR(entry) (ENTRY_KEY_PTR(entry) + ctx->key_size_aligned)
56
57#define KEYS_EQUAL(k1, k2) (!memcmp((k1), (k2), ctx->key_size))
58
59int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries)
60{
61 struct FFHashtableContext *res = av_malloc(sizeof(struct FFHashtableContext));
62 if (!res)
63 return AVERROR(ENOMEM);
64 res->key_size = key_size;
65 res->key_size_aligned = FFALIGN(key_size, ALIGN);
66 res->val_size = val_size;
67 res->val_size_aligned = FFALIGN(val_size, ALIGN);
68 res->entry_size = FFALIGN(sizeof(size_t), ALIGN)
69 + res->key_size_aligned
70 + res->val_size_aligned;
71 res->max_entries = max_entries;
72 res->nb_entries = 0;
73 res->crc = av_crc_get_table(AV_CRC_32_IEEE);
74 if (!res->crc) {
75 ff_hashtable_freep(&res);
76 return AVERROR_BUG;
77 }
78 res->table = av_calloc(res->max_entries, res->entry_size);
79 if (!res->table) {
80 ff_hashtable_freep(&res);
81 return AVERROR(ENOMEM);
82 }
83
84 res->swapbuf = av_calloc(2, res->key_size_aligned + res->val_size_aligned);
85 if (!res->swapbuf) {
86 ff_hashtable_freep(&res);
87 return AVERROR(ENOMEM);
88 }
89 *ctx = res;
90 return 0;
91}
92
93static size_t hash_key(const struct FFHashtableContext *ctx, const void *key)
94{
95 return av_crc(ctx->crc, 0, key, ctx->key_size) % ctx->max_entries;
96}
97
98int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val)
99{
100 if (!ctx->nb_entries)
101 return 0;
102
103 size_t hash = hash_key(ctx, key);
104
105 for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
106 size_t wrapped_index = (hash + psl) % ctx->max_entries;
107 uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
108 if (ENTRY_PSL_VAL(entry) < psl)
109 // When PSL stops increasing it means there are no further entries
110 // with the same key hash.
111 return 0;
112 if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
113 memcpy(val, ENTRY_VAL_PTR(entry), ctx->val_size);
114 return 1;
115 }
116 }
117 return 0;
118}
119
120int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val)
121{
122 int swapping = 0;
123 size_t psl = 1;
124 size_t hash = hash_key(ctx, key);
125 size_t wrapped_index = hash % ctx->max_entries;
126 uint8_t *set = ctx->swapbuf;
127 uint8_t *tmp = ctx->swapbuf + ctx->key_size_aligned + ctx->val_size_aligned;
128
129 memcpy(set, key, ctx->key_size);
130 memcpy(set + ctx->key_size_aligned, val, ctx->val_size);
131
132 for (size_t i = 0; i < ctx->max_entries; i++) {
133 if (++wrapped_index == ctx->max_entries)
134 wrapped_index = 0;
135 uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
136 if (!ENTRY_PSL_VAL(entry) || (!swapping && KEYS_EQUAL(ENTRY_KEY_PTR(entry), set))) {
137 if (!ENTRY_PSL_VAL(entry))
138 ctx->nb_entries++;
139 ENTRY_PSL_VAL(entry) = psl;
140 memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size);
141 return 1;
142 }
143 if (ENTRY_PSL_VAL(entry) < psl) {
144 // When PSL stops increasing it means there are no further entries
145 // with the same key hash. We can only hope to find an unoccupied
146 // entry.
147 if (ctx->nb_entries == ctx->max_entries)
148 // The table is full so inserts are impossible.
149 return 0;
150 // Robin Hood hash tables "steal from the rich" by minimizing the
151 // PSL of the inserted entry.
152 swapping = 1;
153 // set needs to swap with entry
154 memcpy(tmp, ENTRY_KEY_PTR(entry), ctx->key_size_aligned + ctx->val_size_aligned);
155 memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size_aligned);
156 FFSWAP(uint8_t*, set, tmp);
157 FFSWAP(size_t, psl, ENTRY_PSL_VAL(entry));
158 }
159 psl++;
160 }
161 return 0;
162}
163
164int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key)
165{
166 if (!ctx->nb_entries)
167 return 0;
168
169 uint8_t *next_entry;
170 size_t hash = hash_key(ctx, key);
171 size_t wrapped_index = hash % ctx->max_entries;
172
173 for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
174 if (++wrapped_index == ctx->max_entries)
175 wrapped_index = 0;
176 uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
177 if (ENTRY_PSL_VAL(entry) < psl)
178 // When PSL stops increasing it means there are no further entries
179 // with the same key hash.
180 return 0;
181 if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
182 ENTRY_PSL_VAL(entry) = 0;
183 // Shift each following entry that will benefit from a reduced PSL.
184 for (psl++; psl <= ctx->max_entries; psl++) {
185 if (++wrapped_index == ctx->max_entries)
186 wrapped_index = 0;
187 next_entry = ctx->table + wrapped_index * ctx->entry_size;
188 if (ENTRY_PSL_VAL(next_entry) <= 1) {
189 ctx->nb_entries--;
190 return 1;
191 }
192 memcpy(entry, next_entry, ctx->entry_size);
193 ENTRY_PSL_VAL(entry)--;
194 ENTRY_PSL_VAL(next_entry) = 0;
195 entry = next_entry;
196 }
197 }
198 };
199 return 0;
200}
201
202void ff_hashtable_clear(struct FFHashtableContext *ctx)
203{
204 memset(ctx->table, 0, ctx->entry_size * ctx->max_entries);
205}
206
207void ff_hashtable_freep(struct FFHashtableContext **ctx)
208{
209 if (*ctx) {
210 av_freep(&(*ctx)->table);
211 av_freep(&(*ctx)->swapbuf);
212 }
213 av_freep(ctx);
214}
libavcodec/hashtable.h+94−0
@@ -0,0 +1,94 @@
1/*
2 * Generic hashtable
3 * Copyright (C) 2024 Emma Worley <emma@emma.gg>
4 *
5 * This file is part of FFmpeg.
6 *
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#ifndef AVCODEC_HASHTABLE_H
23#define AVCODEC_HASHTABLE_H
24
25#include <stddef.h>
26
27/* Implements a hash table using Robin Hood open addressing.
28 * See: https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf
29 *
30 * Keys are placed in the table based on their CRC value and are considered
31 * equal when they are bytewise-identical.
32 */
33
34typedef struct FFHashtableContext FFHashtableContext;
35
36/**
37 * Create a fixed-sized Robin Hood hash table.
38 *
39 * @param ctx context to allocate and initialize
40 * @param key_size size of key type in bytes
41 * @param val_size size of value type in bytes
42 * @param max_entries maximum number of key-value pairs to store
43 *
44 * @return zero on success, nonzero on error
45 */
46int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries);
47
48/**
49 * Look up a value from a hash table given a key.
50 *
51 * @param ctx hash table context
52 * @param key pointer to key data
53 * @param val destination pointer for value data
54 *
55 * @return 1 if the key is found, zero if the key is not found
56 */
57int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val);
58
59/**
60 * Store a value in a hash table given a key.
61 *
62 * @param ctx hash table context
63 * @param key pointer to key data
64 * @param val pointer for value data
65 *
66 * @return 1 if the key is written, zero if the key is not written due to the hash table reaching max capacity
67 */
68int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val);
69
70/**
71 * Delete a value from a hash table given a key.
72 *
73 * @param ctx hash table context
74 * @param key pointer to key data
75 *
76 * @return 1 if the key is deleted, zero if the key is not deleted due to not being found
77 */
78int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key);
79
80/**
81 * Delete all values from a hash table.
82 *
83 * @param ctx hash table context
84 */
85void ff_hashtable_clear(struct FFHashtableContext *ctx);
86
87/**
88 * Free a hash table.
89 *
90 * @param ctx hash table context
91 */
92void ff_hashtable_freep(struct FFHashtableContext **ctx);
93
94#endif
libavcodec/tests/hashtable.c+110−0
@@ -0,0 +1,110 @@
1/*
2 * Generic hashtable tests
3 * Copyright (C) 2024 Emma Worley <emma@emma.gg>
4 *
5 * This file is part of FFmpeg.
6 *
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#include <stdint.h>
23
24#include "libavutil/avassert.h"
25#include "libavcodec/hashtable.h"
26
27int main(void)
28{
29 struct FFHashtableContext *ctx;
30 uint8_t k;
31 uint64_t v;
32
33 // impossibly large allocation should fail gracefully
34 av_assert0(ff_hashtable_alloc(&ctx, -1, -1, -1) < 0);
35
36 // hashtable can store up to 3 uint8_t->uint64_t entries
37 av_assert0(!ff_hashtable_alloc(&ctx, sizeof(k), sizeof(v), 3));
38
39 // unsuccessful deletes return 0
40 k = 1;
41 av_assert0(!ff_hashtable_delete(ctx, &k));
42
43 // unsuccessful gets return 0
44 k = 1;
45 av_assert0(!ff_hashtable_get(ctx, &k, &v));
46
47 // successful sets returns 1
48 k = 1;
49 v = 1;
50 av_assert0(ff_hashtable_set(ctx, &k, &v));
51
52 // get should now contain 1
53 k = 1;
54 v = 0;
55 av_assert0(ff_hashtable_get(ctx, &k, &v));
56 av_assert0(v == 1);
57
58 // updating sets should return 1
59 k = 1;
60 v = 2;
61 av_assert0(ff_hashtable_set(ctx, &k, &v));
62
63 // get should now contain 2
64 k = 1;
65 v = 0;
66 av_assert0(ff_hashtable_get(ctx, &k, &v));
67 av_assert0(v == 2);
68
69 // fill the table
70 k = 2;
71 v = 2;
72 av_assert0(ff_hashtable_set(ctx, &k, &v));
73 k = 3;
74 v = 3;
75 av_assert0(ff_hashtable_set(ctx, &k, &v));
76
77 // inserting sets on a full table should return 0
78 k = 4;
79 v = 4;
80 av_assert0(!ff_hashtable_set(ctx, &k, &v));
81
82 // updating sets on a full table should return 1
83 k = 1;
84 v = 4;
85 av_assert0(ff_hashtable_set(ctx, &k, &v));
86 v = 0;
87 av_assert0(ff_hashtable_get(ctx, &k, &v));
88 av_assert0(v == 4);
89
90 // successful deletes should return 1
91 k = 1;
92 av_assert0(ff_hashtable_delete(ctx, &k));
93
94 // get should now return 0
95 av_assert0(!ff_hashtable_get(ctx, &k, &v));
96
97 // sanity check remaining keys
98 k = 2;
99 v = 0;
100 av_assert0(ff_hashtable_get(ctx, &k, &v));
101 av_assert0(v == 2);
102 k = 3;
103 v = 0;
104 av_assert0(ff_hashtable_get(ctx, &k, &v));
105 av_assert0(v == 3);
106
107 ff_hashtable_freep(&ctx);
108
109 return 0;
110}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t