History

lavc/hashtable: create generic robin hood hash table

Adds a generic hash table with the DXV encoder as an initial use case.

Emma Worley committed 1 year ago

Commit 6fdb54d1 parent 854b8694 files · +420 −0Browse files

Unified · Split

4 files
  1. libavcodec/Makefile +2 −0
  2. libavcodec/hashtable.c +214 −0
  3. libavcodec/hashtable.h +94 −0
  4. libavcodec/tests/hashtable.c +110 −0
libavcodec/Makefile+2−0
@@ -42,6 +42,7 @@ OBJS = ac3_parser.o \ · 41 unmodified lines
4242 dv_profile.o \
4343 encode.o \
4444 get_buffer.o \
45+ hashtable.o \
4546 imgconvert.o \
4647 jni.o \
4748 lcevcdec.o \
@@ -1326,6 +1327,7 @@ TESTPROGS = avcodec \ · 1278 unmodified lines
13261327 bitstream_le \
13271328 celp_math \
13281329 codec_desc \
1330+ hashtable \
13291331 htmlsubtitles \
13301332 jpeg2000dwt \
13311333 mathops \
libavcodec/hashtable.c+214−0
@@ -0,0 +1,214 @@
1+/*
2+ * Generic hashtable
3+ * Copyright (C) 2025 Emma Worley <emma@emma.gg>
4+ *
5+ * This file is part of FFmpeg.
6+ *
7+ * FFmpeg is free software; you can redistribute it and/or
8+ * modify it under the terms of the GNU Lesser General Public
9+ * License as published by the Free Software Foundation; either
10+ * version 2.1 of the License, or (at your option) any later version.
11+ *
12+ * FFmpeg is distributed in the hope that it will be useful,
13+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
14+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+ * Lesser General Public License for more details.
16+ *
17+ * You should have received a copy of the GNU Lesser General Public
18+ * License along with FFmpeg; if not, write to the Free Software
19+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20+ */
21+
22+#include <stdint.h>
23+#include <string.h>
24+
25+#include "libavutil/crc.h"
26+#include "libavutil/error.h"
27+#include "libavutil/mem.h"
28+#include "hashtable.h"
29+
30+#define ALIGN _Alignof(size_t)
31+
32+struct FFHashtableContext {
33+ size_t key_size;
34+ size_t key_size_aligned;
35+ size_t val_size;
36+ size_t val_size_aligned;
37+ size_t entry_size;
38+ size_t max_entries;
39+ size_t nb_entries;
40+ const AVCRC *crc;
41+ uint8_t *table;
42+ uint8_t *swapbuf;
43+};
44+
45+/*
46+ * Hash table entries are comprised of a probe sequence length (PSL), key, and
47+ * value. When the PSL of an entry is zero, it means it is not occupied by a
48+ * key/value pair. When the PSL is non-zero, it represents the "distance" of
49+ * the entry from its "home" location plus one, where the "home" location is
50+ * hash(key) % max_entries.
51+ */
52+
53+#define ENTRY_PSL_VAL(entry) (*(size_t*)(entry))
54+#define ENTRY_KEY_PTR(entry) ((entry) + FFALIGN(sizeof(size_t), ALIGN))
55+#define ENTRY_VAL_PTR(entry) (ENTRY_KEY_PTR(entry) + ctx->key_size_aligned)
56+
57+#define KEYS_EQUAL(k1, k2) (!memcmp((k1), (k2), ctx->key_size))
58+
59+int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries)
60+{
61+ struct FFHashtableContext *res = av_malloc(sizeof(struct FFHashtableContext));
62+ if (!res)
63+ return AVERROR(ENOMEM);
64+ res->key_size = key_size;
65+ res->key_size_aligned = FFALIGN(key_size, ALIGN);
66+ res->val_size = val_size;
67+ res->val_size_aligned = FFALIGN(val_size, ALIGN);
68+ res->entry_size = FFALIGN(sizeof(size_t), ALIGN)
69+ + res->key_size_aligned
70+ + res->val_size_aligned;
71+ res->max_entries = max_entries;
72+ res->nb_entries = 0;
73+ res->crc = av_crc_get_table(AV_CRC_32_IEEE);
74+ if (!res->crc) {
75+ ff_hashtable_freep(&res);
76+ return AVERROR_BUG;
77+ }
78+ res->table = av_calloc(res->max_entries, res->entry_size);
79+ if (!res->table) {
80+ ff_hashtable_freep(&res);
81+ return AVERROR(ENOMEM);
82+ }
83+
84+ res->swapbuf = av_calloc(2, res->key_size_aligned + res->val_size_aligned);
85+ if (!res->swapbuf) {
86+ ff_hashtable_freep(&res);
87+ return AVERROR(ENOMEM);
88+ }
89+ *ctx = res;
90+ return 0;
91+}
92+
93+static size_t hash_key(const struct FFHashtableContext *ctx, const void *key)
94+{
95+ return av_crc(ctx->crc, 0, key, ctx->key_size) % ctx->max_entries;
96+}
97+
98+int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val)
99+{
100+ if (!ctx->nb_entries)
101+ return 0;
102+
103+ size_t hash = hash_key(ctx, key);
104+
105+ for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
106+ size_t wrapped_index = (hash + psl) % ctx->max_entries;
107+ uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
108+ if (ENTRY_PSL_VAL(entry) < psl)
109+ // When PSL stops increasing it means there are no further entries
110+ // with the same key hash.
111+ return 0;
112+ if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
113+ memcpy(val, ENTRY_VAL_PTR(entry), ctx->val_size);
114+ return 1;
115+ }
116+ }
117+ return 0;
118+}
119+
120+int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val)
121+{
122+ int swapping = 0;
123+ size_t psl = 1;
124+ size_t hash = hash_key(ctx, key);
125+ size_t wrapped_index = hash % ctx->max_entries;
126+ uint8_t *set = ctx->swapbuf;
127+ uint8_t *tmp = ctx->swapbuf + ctx->key_size_aligned + ctx->val_size_aligned;
128+
129+ memcpy(set, key, ctx->key_size);
130+ memcpy(set + ctx->key_size_aligned, val, ctx->val_size);
131+
132+ for (size_t i = 0; i < ctx->max_entries; i++) {
133+ if (++wrapped_index == ctx->max_entries)
134+ wrapped_index = 0;
135+ uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
136+ if (!ENTRY_PSL_VAL(entry) || (!swapping && KEYS_EQUAL(ENTRY_KEY_PTR(entry), set))) {
137+ if (!ENTRY_PSL_VAL(entry))
138+ ctx->nb_entries++;
139+ ENTRY_PSL_VAL(entry) = psl;
140+ memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size);
141+ return 1;
142+ }
143+ if (ENTRY_PSL_VAL(entry) < psl) {
144+ // When PSL stops increasing it means there are no further entries
145+ // with the same key hash. We can only hope to find an unoccupied
146+ // entry.
147+ if (ctx->nb_entries == ctx->max_entries)
148+ // The table is full so inserts are impossible.
149+ return 0;
150+ // Robin Hood hash tables "steal from the rich" by minimizing the
151+ // PSL of the inserted entry.
152+ swapping = 1;
153+ // set needs to swap with entry
154+ memcpy(tmp, ENTRY_KEY_PTR(entry), ctx->key_size_aligned + ctx->val_size_aligned);
155+ memcpy(ENTRY_KEY_PTR(entry), set, ctx->key_size_aligned + ctx->val_size_aligned);
156+ FFSWAP(uint8_t*, set, tmp);
157+ FFSWAP(size_t, psl, ENTRY_PSL_VAL(entry));
158+ }
159+ psl++;
160+ }
161+ return 0;
162+}
163+
164+int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key)
165+{
166+ if (!ctx->nb_entries)
167+ return 0;
168+
169+ uint8_t *next_entry;
170+ size_t hash = hash_key(ctx, key);
171+ size_t wrapped_index = hash % ctx->max_entries;
172+
173+ for (size_t psl = 1; psl <= ctx->max_entries; psl++) {
174+ if (++wrapped_index == ctx->max_entries)
175+ wrapped_index = 0;
176+ uint8_t *entry = ctx->table + wrapped_index * ctx->entry_size;
177+ if (ENTRY_PSL_VAL(entry) < psl)
178+ // When PSL stops increasing it means there are no further entries
179+ // with the same key hash.
180+ return 0;
181+ if (KEYS_EQUAL(ENTRY_KEY_PTR(entry), key)) {
182+ ENTRY_PSL_VAL(entry) = 0;
183+ // Shift each following entry that will benefit from a reduced PSL.
184+ for (psl++; psl <= ctx->max_entries; psl++) {
185+ if (++wrapped_index == ctx->max_entries)
186+ wrapped_index = 0;
187+ next_entry = ctx->table + wrapped_index * ctx->entry_size;
188+ if (ENTRY_PSL_VAL(next_entry) <= 1) {
189+ ctx->nb_entries--;
190+ return 1;
191+ }
192+ memcpy(entry, next_entry, ctx->entry_size);
193+ ENTRY_PSL_VAL(entry)--;
194+ ENTRY_PSL_VAL(next_entry) = 0;
195+ entry = next_entry;
196+ }
197+ }
198+ };
199+ return 0;
200+}
201+
202+void ff_hashtable_clear(struct FFHashtableContext *ctx)
203+{
204+ memset(ctx->table, 0, ctx->entry_size * ctx->max_entries);
205+}
206+
207+void ff_hashtable_freep(struct FFHashtableContext **ctx)
208+{
209+ if (*ctx) {
210+ av_freep(&(*ctx)->table);
211+ av_freep(&(*ctx)->swapbuf);
212+ }
213+ av_freep(ctx);
214+}
libavcodec/hashtable.h+94−0
@@ -0,0 +1,94 @@
1+/*
2+ * Generic hashtable
3+ * Copyright (C) 2024 Emma Worley <emma@emma.gg>
4+ *
5+ * This file is part of FFmpeg.
6+ *
7+ * FFmpeg is free software; you can redistribute it and/or
8+ * modify it under the terms of the GNU Lesser General Public
9+ * License as published by the Free Software Foundation; either
10+ * version 2.1 of the License, or (at your option) any later version.
11+ *
12+ * FFmpeg is distributed in the hope that it will be useful,
13+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
14+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+ * Lesser General Public License for more details.
16+ *
17+ * You should have received a copy of the GNU Lesser General Public
18+ * License along with FFmpeg; if not, write to the Free Software
19+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20+ */
21+
22+#ifndef AVCODEC_HASHTABLE_H
23+#define AVCODEC_HASHTABLE_H
24+
25+#include <stddef.h>
26+
27+/* Implements a hash table using Robin Hood open addressing.
28+ * See: https://cs.uwaterloo.ca/research/tr/1986/CS-86-14.pdf
29+ *
30+ * Keys are placed in the table based on their CRC value and are considered
31+ * equal when they are bytewise-identical.
32+ */
33+
34+typedef struct FFHashtableContext FFHashtableContext;
35+
36+/**
37+ * Create a fixed-sized Robin Hood hash table.
38+ *
39+ * @param ctx context to allocate and initialize
40+ * @param key_size size of key type in bytes
41+ * @param val_size size of value type in bytes
42+ * @param max_entries maximum number of key-value pairs to store
43+ *
44+ * @return zero on success, nonzero on error
45+ */
46+int ff_hashtable_alloc(struct FFHashtableContext **ctx, size_t key_size, size_t val_size, size_t max_entries);
47+
48+/**
49+ * Look up a value from a hash table given a key.
50+ *
51+ * @param ctx hash table context
52+ * @param key pointer to key data
53+ * @param val destination pointer for value data
54+ *
55+ * @return 1 if the key is found, zero if the key is not found
56+ */
57+int ff_hashtable_get(const struct FFHashtableContext *ctx, const void *key, void *val);
58+
59+/**
60+ * Store a value in a hash table given a key.
61+ *
62+ * @param ctx hash table context
63+ * @param key pointer to key data
64+ * @param val pointer for value data
65+ *
66+ * @return 1 if the key is written, zero if the key is not written due to the hash table reaching max capacity
67+ */
68+int ff_hashtable_set(struct FFHashtableContext *ctx, const void *key, const void *val);
69+
70+/**
71+ * Delete a value from a hash table given a key.
72+ *
73+ * @param ctx hash table context
74+ * @param key pointer to key data
75+ *
76+ * @return 1 if the key is deleted, zero if the key is not deleted due to not being found
77+ */
78+int ff_hashtable_delete(struct FFHashtableContext *ctx, const void *key);
79+
80+/**
81+ * Delete all values from a hash table.
82+ *
83+ * @param ctx hash table context
84+ */
85+void ff_hashtable_clear(struct FFHashtableContext *ctx);
86+
87+/**
88+ * Free a hash table.
89+ *
90+ * @param ctx hash table context
91+ */
92+void ff_hashtable_freep(struct FFHashtableContext **ctx);
93+
94+#endif
libavcodec/tests/hashtable.c+110−0
@@ -0,0 +1,110 @@
1+/*
2+ * Generic hashtable tests
3+ * Copyright (C) 2024 Emma Worley <emma@emma.gg>
4+ *
5+ * This file is part of FFmpeg.
6+ *
7+ * FFmpeg is free software; you can redistribute it and/or
8+ * modify it under the terms of the GNU Lesser General Public
9+ * License as published by the Free Software Foundation; either
10+ * version 2.1 of the License, or (at your option) any later version.
11+ *
12+ * FFmpeg is distributed in the hope that it will be useful,
13+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
14+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15+ * Lesser General Public License for more details.
16+ *
17+ * You should have received a copy of the GNU Lesser General Public
18+ * License along with FFmpeg; if not, write to the Free Software
19+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20+ */
21+
22+#include <stdint.h>
23+
24+#include "libavutil/avassert.h"
25+#include "libavcodec/hashtable.h"
26+
27+int main(void)
28+{
29+ struct FFHashtableContext *ctx;
30+ uint8_t k;
31+ uint64_t v;
32+
33+ // impossibly large allocation should fail gracefully
34+ av_assert0(ff_hashtable_alloc(&ctx, -1, -1, -1) < 0);
35+
36+ // hashtable can store up to 3 uint8_t->uint64_t entries
37+ av_assert0(!ff_hashtable_alloc(&ctx, sizeof(k), sizeof(v), 3));
38+
39+ // unsuccessful deletes return 0
40+ k = 1;
41+ av_assert0(!ff_hashtable_delete(ctx, &k));
42+
43+ // unsuccessful gets return 0
44+ k = 1;
45+ av_assert0(!ff_hashtable_get(ctx, &k, &v));
46+
47+ // successful sets returns 1
48+ k = 1;
49+ v = 1;
50+ av_assert0(ff_hashtable_set(ctx, &k, &v));
51+
52+ // get should now contain 1
53+ k = 1;
54+ v = 0;
55+ av_assert0(ff_hashtable_get(ctx, &k, &v));
56+ av_assert0(v == 1);
57+
58+ // updating sets should return 1
59+ k = 1;
60+ v = 2;
61+ av_assert0(ff_hashtable_set(ctx, &k, &v));
62+
63+ // get should now contain 2
64+ k = 1;
65+ v = 0;
66+ av_assert0(ff_hashtable_get(ctx, &k, &v));
67+ av_assert0(v == 2);
68+
69+ // fill the table
70+ k = 2;
71+ v = 2;
72+ av_assert0(ff_hashtable_set(ctx, &k, &v));
73+ k = 3;
74+ v = 3;
75+ av_assert0(ff_hashtable_set(ctx, &k, &v));
76+
77+ // inserting sets on a full table should return 0
78+ k = 4;
79+ v = 4;
80+ av_assert0(!ff_hashtable_set(ctx, &k, &v));
81+
82+ // updating sets on a full table should return 1
83+ k = 1;
84+ v = 4;
85+ av_assert0(ff_hashtable_set(ctx, &k, &v));
86+ v = 0;
87+ av_assert0(ff_hashtable_get(ctx, &k, &v));
88+ av_assert0(v == 4);
89+
90+ // successful deletes should return 1
91+ k = 1;
92+ av_assert0(ff_hashtable_delete(ctx, &k));
93+
94+ // get should now return 0
95+ av_assert0(!ff_hashtable_get(ctx, &k, &v));
96+
97+ // sanity check remaining keys
98+ k = 2;
99+ v = 0;
100+ av_assert0(ff_hashtable_get(ctx, &k, &v));
101+ av_assert0(v == 2);
102+ k = 3;
103+ v = 0;
104+ av_assert0(ff_hashtable_get(ctx, &k, &v));
105+ av_assert0(v == 3);
106+
107+ ff_hashtable_freep(&ctx);
108+
109+ return 0;
110+}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t