It is unnecessary, as we already have the entries sorted by probability and therefore implicitly by length. All we need on top of that to build the tree is the number of entries of a given length. Doing so gives a 3.6% speedup of ff_mjpeg_encode_huffman_close() here; it also saves about 640B of .text here. The new code puts values with higher probability to the left of the tree. The old code did not and therefore the FATE checksums needed to be updated. Due to MJPEG's 0xFF unescaping file sizes as well as file checksums needed to be updated; the decoded picture hashes stayed the same. Given that codes on the left of the tree have on average fewer bits set than codes on the right, the file sizes mostly improve (all except vsynth3-mjpeg-444).
Showing the changes to libavcodec (2 of 22 files). Show all
Unified · Split
| @@ -43,14 +43,6 @@ typedef struct PackageMergerList { · 42 unmodified lines | |||
| 43 | int items[257 * 16]; ///< chain of all individual values that make up items A, B, A, B, C, A, B, C, D, C, D, D, E | 43 | int items[257 * 16]; ///< chain of all individual values that make up items A, B, A, B, C, A, B, C, D, C, D, D, E |
| 44 | } PackageMergerList; | 44 | } PackageMergerList; |
| 45 | 45 | ||
| 46 | /** | ||
| 47 | * Used to store optimal huffman encoding results | ||
| 48 | */ | ||
| 49 | typedef struct HuffTable { | ||
| 50 | int code; ///< code is the input value | ||
| 51 | int length; ///< length of the encoding | ||
| 52 | } HuffTable; | ||
| 53 | |||
| 54 | /** | 46 | /** |
| 55 | * Comparison function for two PTables by prob | 47 | * Comparison function for two PTables by prob |
| 56 | * | 48 | * |
| @@ -65,20 +57,6 @@ static int compare_by_prob(const void *a, const void *b) · 8 unmodified lines | |||
| 65 | return a_val.prob - b_val.prob; | 57 | return a_val.prob - b_val.prob; |
| 66 | } | 58 | } |
| 67 | 59 | ||
| 68 | /** | ||
| 69 | * Comparison function for two HuffTables by length | ||
| 70 | * | ||
| 71 | * @param a First HuffTable to compare | ||
| 72 | * @param b Second HuffTable to compare | ||
| 73 | * @return < 0 for less than, 0 for equals, > 0 for greater than | ||
| 74 | */ | ||
| 75 | static int compare_by_length(const void *a, const void *b) | ||
| 76 | { | ||
| 77 | HuffTable a_val = *(HuffTable *) a; | ||
| 78 | HuffTable b_val = *(HuffTable *) b; | ||
| 79 | return a_val.length - b_val.length; | ||
| 80 | } | ||
| 81 | |||
| 82 | /** | 60 | /** |
| 83 | * Computes the length of the Huffman encoding for each distinct input value. | 61 | * Computes the length of the Huffman encoding for each distinct input value. |
| 84 | * Uses package merge algorithm as follows: | 62 | * Uses package merge algorithm as follows: |
| @@ -92,15 +70,16 @@ static int compare_by_length(const void *a, const void *b) · 7 unmodified lines | |||
| 92 | * 8. the length of the huffman code for symbol s will be equal to the number of times the symbol occurs in the select elements | 70 | * 8. the length of the huffman code for symbol s will be equal to the number of times the symbol occurs in the select elements |
| 93 | * Go to guru.multimedia.cx/small-tasks-for-ffmpeg/ for more details | 71 | * Go to guru.multimedia.cx/small-tasks-for-ffmpeg/ for more details |
| 94 | * | 72 | * |
| 95 | * All probabilities should be positive integers. The output is sorted by code, | 73 | * All probabilities should be nonnegative integers. |
| 96 | * not by length. | ||
| 97 | * | 74 | * |
| 98 | * @param prob_table input array of a PTable for each distinct input value | 75 | * @param prob_table[in,out] array of a PTable for each distinct input value, |
| 99 | * @param distincts output array of a HuffTable that will be populated by this function | 76 | * will be sorted according to ascending probability |
| 100 | * @param size size of the prob_table array | 77 | * @param counts[out] the number of values of a given length |
| 101 | * @param max_length max length of an encoding | 78 | * @param size number of elements of the prob_table array |
| 79 | * @param max_length max length of a code | ||
| 102 | */ | 80 | */ |
| 103 | static void mjpegenc_huffman_compute_bits(PTable *prob_table, HuffTable *distincts, | 81 | static void mjpegenc_huffman_compute_bits(PTable *prob_table, |
| 82 | uint8_t counts[/* max_length + 1 */], | ||
| 104 | int size, int max_length) | 83 | int size, int max_length) |
| 105 | { | 84 | { |
| 106 | PackageMergerList list_a, list_b, *to = &list_a, *from = &list_b, *temp; | 85 | PackageMergerList list_a, list_b, *to = &list_a, *from = &list_b, *temp; |
| @@ -159,14 +138,9 @@ static void mjpegenc_huffman_compute_bits(PTable *prob_table, HuffTable *distinc · 52 unmodified lines | |||
| 159 | } | 138 | } |
| 160 | // we don't want to return the 256 bit count (it was just in here to prevent | 139 | // we don't want to return the 256 bit count (it was just in here to prevent |
| 161 | // all 1s encoding) | 140 | // all 1s encoding) |
| 162 | j = 0; | 141 | memset(counts, 0, sizeof(counts[0]) * (max_length + 1)); |
| 163 | for (i = 0; i < 256; i++) { | 142 | for (int i = 0; i < 256; ++i) |
| 164 | if (nbits[i] > 0) { | 143 | counts[nbits[i]]++; |
| 165 | distincts[j].code = i; | ||
| 166 | distincts[j].length = nbits[i]; | ||
| 167 | j++; | ||
| 168 | } | ||
| 169 | } | ||
| 170 | } | 144 | } |
| 171 | 145 | ||
| 172 | void ff_mjpeg_encode_huffman_init(MJpegEncHuffmanContext *s) | 146 | void ff_mjpeg_encode_huffman_init(MJpegEncHuffmanContext *s) |
| @@ -186,7 +160,6 @@ void ff_mjpeg_encode_huffman_close(MJpegEncHuffmanContext *s, uint8_t bits[17], · 13 unmodified lines | |||
| 186 | uint8_t val[], int max_nval) | 160 | uint8_t val[], int max_nval) |
| 187 | { | 161 | { |
| 188 | PTable val_counts[257]; | 162 | PTable val_counts[257]; |
| 189 | HuffTable distincts[256]; | ||
| 190 | 163 | ||
| 191 | av_assert1(max_nval <= FF_ARRAY_ELEMS(val_counts) - 1); | 164 | av_assert1(max_nval <= FF_ARRAY_ELEMS(val_counts) - 1); |
| 192 | 165 | ||
| @@ -201,12 +174,13 @@ void ff_mjpeg_encode_huffman_close(MJpegEncHuffmanContext *s, uint8_t bits[17], · 8 unmodified lines | |||
| 201 | } | 174 | } |
| 202 | val_counts[nval].value = 256; | 175 | val_counts[nval].value = 256; |
| 203 | val_counts[nval].prob = 0; | 176 | val_counts[nval].prob = 0; |
| 204 | mjpegenc_huffman_compute_bits(val_counts, distincts, nval + 1, 16); | ||
| 205 | AV_QSORT(distincts, nval, HuffTable, compare_by_length); | ||
| 206 | 177 | ||
| 207 | memset(bits, 0, sizeof(bits[0]) * 17); | 178 | mjpegenc_huffman_compute_bits(val_counts, bits, nval + 1, 16); |
| 208 | for (int i = 0; i < nval; i++) { | 179 | |
| 209 | val[i] = distincts[i].code; | 180 | // val_counts[0] is the fake element we added earlier. |
| 210 | bits[distincts[i].length]++; | 181 | av_assert1(val_counts[0].prob == 0 && val_counts[0].value == 256); |
| 211 | } | 182 | // The following loop puts the values with higher occurence first, |
| 183 | // ensuring that they get the shorter codes. | ||
| 184 | for (int i = 0; i < nval; ++i) | ||
| 185 | val[i] = val_counts[nval - i].value; | ||
| 212 | } | 186 | } |
| @@ -34,10 +34,9 @@ · 33 unmodified lines | |||
| 34 | static int check_lengths(int L, const int *probs, int nprobs, | 34 | static int check_lengths(int L, const int *probs, int nprobs, |
| 35 | int expected_length, const uint8_t expected_len_counts[/* L + 1 */]) | 35 | int expected_length, const uint8_t expected_len_counts[/* L + 1 */]) |
| 36 | { | 36 | { |
| 37 | HuffTable lengths[256]; | ||
| 38 | PTable val_counts[256]; | 37 | PTable val_counts[256]; |
| 39 | uint8_t len_counts[17] = { 0 }; | 38 | uint8_t len_counts[17]; |
| 40 | int actual_length = 0, i, j, k, prob, length; | 39 | int actual_length = 0, i; |
| 41 | int ret = 0; | 40 | int ret = 0; |
| 42 | av_assert0(nprobs <= 256); | 41 | av_assert0(nprobs <= 256); |
| 43 | av_assert0(L < FF_ARRAY_ELEMS(len_counts)); | 42 | av_assert0(L < FF_ARRAY_ELEMS(len_counts)); |
| @@ -46,20 +45,12 @@ static int check_lengths(int L, const int *probs, int nprobs, · 2 unmodified lines | |||
| 46 | val_counts[i] = (PTable){.value = i, .prob = probs[i]}; | 45 | val_counts[i] = (PTable){.value = i, .prob = probs[i]}; |
| 47 | } | 46 | } |
| 48 | 47 | ||
| 49 | mjpegenc_huffman_compute_bits(val_counts, lengths, nprobs, L); | 48 | mjpegenc_huffman_compute_bits(val_counts, len_counts, nprobs, L); |
| 50 | 49 | ||
| 51 | for (int i = 0; i < nprobs; ++i) { | ||
| 52 | if (lengths[i].length > L) { | ||
| 53 | fprintf(stderr, | ||
| 54 | "Len %d of element %d of Huffman table with %d elements exceeds max length %d\n", | ||
| 55 | lengths[i].length, i, nprobs, L); | ||
| 56 | return 1; | ||
| 57 | } | ||
| 58 | len_counts[lengths[i].length]++; | ||
| 59 | } | ||
| 60 | // Test that the lengths can be made part of a complete, prefix-free tree: | 50 | // Test that the lengths can be made part of a complete, prefix-free tree: |
| 61 | unsigned code = 0; | 51 | unsigned code = 0, count = 0; |
| 62 | for (int i = 1; i <= L; ++i) { | 52 | for (int i = 1; i <= L; ++i) { |
| 53 | count += len_counts[i]; | ||
| 63 | code <<= 1; | 54 | code <<= 1; |
| 64 | code += len_counts[i]; | 55 | code += len_counts[i]; |
| 65 | } | 56 | } |
| @@ -67,22 +58,36 @@ static int check_lengths(int L, const int *probs, int nprobs, · 1 unmodified line | |||
| 67 | fprintf(stderr, "Huffman tree overdetermined/invalid\n"); | 58 | fprintf(stderr, "Huffman tree overdetermined/invalid\n"); |
| 68 | ret = 1; | 59 | ret = 1; |
| 69 | } | 60 | } |
| 70 | 61 | if (count != nprobs) { | |
| 71 | for (i = 0; i < nprobs; i++) { | 62 | fprintf(stderr, "Total count %u does not match expected value %d\n", |
| 72 | // Find the value's prob and length | 63 | count, nprobs); |
| 73 | for (j = 0; j < nprobs; j++) | 64 | ret = 1; |
| 74 | if (val_counts[j].value == i) break; | 65 | } |
| 75 | for (k = 0; k < nprobs; k++) | 66 | // Test that the input values have been properly ordered. |
| 76 | if (lengths[k].code == i) break; | 67 | for (unsigned i = 0; i < count; ++i) { |
| 77 | if (!(j < nprobs && k < nprobs)) return 1; | 68 | if (val_counts[i].prob != probs[val_counts[i].value]) { |
| 78 | prob = val_counts[j].prob; | 69 | fprintf(stderr, "PTable not properly reordered\n"); |
| 79 | length = lengths[k].length; | 70 | ret = 1; |
| 80 | 71 | } | |
| 81 | if (prob) { | 72 | if (i && val_counts[i - 1].prob > val_counts[i].prob) { |
| 82 | actual_length += prob * length; | 73 | fprintf(stderr, "PTable not order ascendingly: [%u] = %d > [%u] = %d\n", |
| 74 | i - 1, val_counts[i - 1].prob, i, val_counts[i].prob); | ||
| 75 | ret = 1; | ||
| 76 | } | ||
| 77 | unsigned j; | ||
| 78 | for (j = 0; j < count; ++j) | ||
| 79 | if (val_counts[j].value == i) | ||
| 80 | break; | ||
| 81 | if (j >= count) { | ||
| 82 | fprintf(stderr, "Element %u missing after sorting\n", i); | ||
| 83 | ret = 1; | ||
| 83 | } | 84 | } |
| 84 | 85 | } | |
| 85 | if (length > L || length < 1) return 1; | 86 | for (int len = L, j = 0; len; --len) { |
| 87 | int prob = 0; | ||
| 88 | for (int end = j + len_counts[len]; j < end; ++j) | ||
| 89 | prob += val_counts[j].prob; | ||
| 90 | actual_length += prob * len; | ||
| 86 | } | 91 | } |
| 87 | // Check that the total length is optimal | 92 | // Check that the total length is optimal |
| 88 | if (actual_length != expected_length) ret = 1; | 93 | if (actual_length != expected_length) ret = 1; |
| @@ -141,7 +146,10 @@ static const uint8_t len_counts_sat[] = { · 52 unmodified lines | |||
| 141 | // http://guru.multimedia.cx/small-tasks-for-ffmpeg/ | 146 | // http://guru.multimedia.cx/small-tasks-for-ffmpeg/ |
| 142 | int main(int argc, char **argv) | 147 | int main(int argc, char **argv) |
| 143 | { | 148 | { |
| 144 | int i, ret = 0; | 149 | enum { |
| 150 | MAX_LEN = 3, | ||
| 151 | }; | ||
| 152 | int ret = 0; | ||
| 145 | // Probabilities of symbols 0..4 | 153 | // Probabilities of symbols 0..4 |
| 146 | PTable val_counts[] = { | 154 | PTable val_counts[] = { |
| 147 | {.value = 0, .prob = 1}, | 155 | {.value = 0, .prob = 1}, |
| @@ -151,30 +159,33 @@ int main(int argc, char **argv) · 3 unmodified lines | |||
| 151 | {.value = 4, .prob = 21}, | 159 | {.value = 4, .prob = 21}, |
| 152 | }; | 160 | }; |
| 153 | // Expected code lengths for each symbol | 161 | // Expected code lengths for each symbol |
| 154 | static const HuffTable expected[] = { | 162 | static const uint8_t expected[MAX_LEN + 1] = { |
| 155 | {.code = 0, .length = 3}, | 163 | [1] = 1, [3] = 4, |
| 156 | {.code = 1, .length = 3}, | ||
| 157 | {.code = 2, .length = 3}, | ||
| 158 | {.code = 3, .length = 3}, | ||
| 159 | {.code = 4, .length = 1}, | ||
| 160 | }; | 164 | }; |
| 161 | // Actual code lengths | 165 | // Actual code lengths |
| 162 | HuffTable distincts[5]; | 166 | uint8_t len_counts[MAX_LEN + 1]; |
| 163 | 167 | ||
| 164 | // Build optimal huffman tree using an internal function, to allow for | 168 | // Build optimal huffman tree using an internal function, to allow for |
| 165 | // smaller-than-normal test cases. This mutates val_counts by sorting. | 169 | // smaller-than-normal test cases. This mutates val_counts by sorting. |
| 166 | mjpegenc_huffman_compute_bits(val_counts, distincts, | 170 | mjpegenc_huffman_compute_bits(val_counts, len_counts, |
| 167 | FF_ARRAY_ELEMS(distincts), 3); | 171 | FF_ARRAY_ELEMS(val_counts), MAX_LEN); |
| 168 | 172 | ||
| 169 | for (i = 0; i < FF_ARRAY_ELEMS(distincts); i++) { | 173 | for (unsigned i = 1; i < FF_ARRAY_ELEMS(len_counts); i++) { |
| 170 | if (distincts[i].code != expected[i].code || | 174 | if (len_counts[i] != expected[i]) { |
| 171 | distincts[i].length != expected[i].length) { | ||
| 172 | fprintf(stderr, | 175 | fprintf(stderr, |
| 173 | "Built huffman does not equal expectations. " | 176 | "Built huffman does not equal expectations. " |
| 174 | "Expected: code %d probability %d, " | 177 | "Expected: %d codes of length %u, " |
| 175 | "Actual: code %d probability %d\n", | 178 | "Actual: %d codes of length %u\n", |
| 176 | expected[i].code, expected[i].length, | 179 | (int)expected[i], i, |
| 177 | distincts[i].code, distincts[i].length); | 180 | (int)len_counts[i], i); |
| 181 | ret = 1; | ||
| 182 | } | ||
| 183 | } | ||
| 184 | for (unsigned i = 1; i < FF_ARRAY_ELEMS(val_counts); ++i) { | ||
| 185 | if (val_counts[i - 1].prob > val_counts[i].prob) { | ||
| 186 | fprintf(stderr, "Probability table not ordered ascendingly. " | ||
| 187 | "val_counts[%u] == %d, val_counts[%u] == %d\n", | ||
| 188 | i - 1, val_counts[i - 1].prob, i, val_counts[i].prob); | ||
| 178 | ret = 1; | 189 | ret = 1; |
| 179 | } | 190 | } |
| 180 | } | 191 | } |