The current code used a pointer to an array (of arrays) that is offset relative to the start of the actually allocated buffer. Yet offsetting the pointer is only done on success, whereas the freeing code believes it to have happened even on error. So if any of the subarrays (or the subarrays' subarrays) can't be successfully allocated, one gets a bad free in free_lut(). Furthermore, said offsetting is only permissible in case the offsetted pointer points in the allocated buffer (here: in case the LUT's min_r is <= 0), as pointer arithmetic is undefined in case it exceeds the allocated object. Moreover, in case one of the subarrays couldn't be allocated, the code nevertheless tried to free the subarray's subarrays; and in case one of the subarray's subarrays could not be allocated successfully, there will be an invalid free, too, because the pointers for the subarrays' subarrays are also offset compared to the base pointer. This commit fixes all of this, by using the actually allocated pointer for freeing and by adding appropriate checks before freeing the subarrays. The former also allows to distinguish the cases in which the lut is currently only half-allocated due to an error in an earlier allocation attempt from the success case.
Unified · Split
| @@ -61,7 +61,10 @@ typedef struct IPlane { · 60 unmodified lines | |||
| 61 | } IPlane; | 61 | } IPlane; |
| 62 | 62 | ||
| 63 | typedef struct LUT { | 63 | typedef struct LUT { |
| 64 | /* arr is shifted from base_arr by FFMAX(min_r, 0). | ||
| 65 | * arr != NULL means "lut completely allocated" */ | ||
| 64 | uint8_t ***arr; | 66 | uint8_t ***arr; |
| 67 | uint8_t ***base_arr; | ||
| 65 | int min_r; | 68 | int min_r; |
| 66 | int max_r; | 69 | int max_r; |
| 67 | int I; | 70 | int I; |
| @@ -262,7 +265,8 @@ static void maxinplace16_fun(uint8_t *aa, const uint8_t *bb, int x) · 194 unmodified lines | |||
| 262 | 265 | ||
| 263 | static int alloc_lut(LUT *Ty, chord_set *SE, int type_size, int mode) | 266 | static int alloc_lut(LUT *Ty, chord_set *SE, int type_size, int mode) |
| 264 | { | 267 | { |
| 265 | const int size = Ty->max_r + 1 - Ty->min_r; | 268 | const int min = FFMAX(Ty->min_r, 0); |
| 269 | const int max = min + (Ty->max_r - Ty->min_r); | ||
| 266 | int pre_pad_x = 0; | 270 | int pre_pad_x = 0; |
| 267 | 271 | ||
| 268 | if (SE->minX < 0) | 272 | if (SE->minX < 0) |
| @@ -270,55 +274,58 @@ static int alloc_lut(LUT *Ty, chord_set *SE, int type_size, int mode) · 1 unmodified line | |||
| 270 | Ty->pre_pad_x = pre_pad_x; | 274 | Ty->pre_pad_x = pre_pad_x; |
| 271 | Ty->type_size = type_size; | 275 | Ty->type_size = type_size; |
| 272 | 276 | ||
| 273 | Ty->arr = av_calloc(size, sizeof(*Ty->arr)); | 277 | Ty->base_arr = av_calloc(max + 1, sizeof(*Ty->base_arr)); |
| 274 | if (!Ty->arr) | 278 | if (!Ty->base_arr) |
| 275 | return AVERROR(ENOMEM); | 279 | return AVERROR(ENOMEM); |
| 276 | for (int r = 0; r < Ty->max_r - Ty->min_r + 1; r++) { | 280 | for (int r = min; r <= max; r++) { |
| 277 | Ty->arr[r] = av_calloc(Ty->I, sizeof(uint8_t *)); | 281 | uint8_t **arr = Ty->base_arr[r] = av_calloc(Ty->I, sizeof(uint8_t *)); |
| 278 | if (!Ty->arr[r]) | 282 | if (!Ty->base_arr[r]) |
| 279 | return AVERROR(ENOMEM); | 283 | return AVERROR(ENOMEM); |
| 280 | for (int i = 0; i < Ty->I; i++) { | 284 | for (int i = 0; i < Ty->I; i++) { |
| 281 | Ty->arr[r][i] = av_calloc(Ty->X + pre_pad_x, type_size); | 285 | arr[i] = av_calloc(Ty->X + pre_pad_x, type_size); |
| 282 | if (!Ty->arr[r][i]) | 286 | if (!arr[i]) |
| 283 | return AVERROR(ENOMEM); | 287 | return AVERROR(ENOMEM); |
| 284 | if (mode == ERODE) | 288 | if (mode == ERODE) |
| 285 | memset(Ty->arr[r][i], UINT8_MAX, pre_pad_x * type_size); | 289 | memset(arr[i], UINT8_MAX, pre_pad_x * type_size); |
| 286 | /* Shifting the X index such that negative indices correspond to | 290 | /* Shifting the X index such that negative indices correspond to |
| 287 | * the pre-padding. | 291 | * the pre-padding. |
| 288 | */ | 292 | */ |
| 289 | Ty->arr[r][i] = &(Ty->arr[r][i][pre_pad_x * type_size]); | 293 | arr[i] = &(arr[i][pre_pad_x * type_size]); |
| 290 | } | 294 | } |
| 291 | } | 295 | } |
| 292 | 296 | ||
| 293 | Ty->arr = &(Ty->arr[0 - Ty->min_r]); | 297 | Ty->arr = &(Ty->base_arr[min - Ty->min_r]); |
| 294 | 298 | ||
| 295 | return 0; | 299 | return 0; |
| 296 | } | 300 | } |
| 297 | 301 | ||
| 298 | static void free_lut(LUT *table) | 302 | static void free_lut(LUT *table) |
| 299 | { | 303 | { |
| 300 | uint8_t ***rp; | 304 | const int min = FFMAX(table->min_r, 0); |
| 305 | const int max = min + (table->max_r - table->min_r); | ||
| 301 | 306 | ||
| 302 | if (!table->arr) | 307 | if (!table->base_arr) |
| 303 | return; | 308 | return; |
| 304 | 309 | ||
| 305 | // The R index was shifted, create a pointer to the original array | 310 | for (int r = min; r <= max; r++) { |
| 306 | rp = &(table->arr[table->min_r]); | 311 | if (!table->base_arr[r]) |
| 307 | 312 | break; | |
| 308 | for (int r = table->min_r; r <= table->max_r; r++) { | ||
| 309 | for (int i = 0; i < table->I; i++) { | 313 | for (int i = 0; i < table->I; i++) { |
| 314 | if (!table->base_arr[r][i]) | ||
| 315 | break; | ||
| 310 | // The X index was also shifted, for padding purposes. | 316 | // The X index was also shifted, for padding purposes. |
| 311 | av_free(table->arr[r][i] - table->pre_pad_x * table->type_size); | 317 | av_free(table->base_arr[r][i] - table->pre_pad_x * table->type_size); |
| 312 | } | 318 | } |
| 313 | av_freep(&table->arr[r]); | 319 | av_freep(&table->base_arr[r]); |
| 314 | } | 320 | } |
| 315 | av_freep(&rp); | 321 | av_freep(&table->base_arr); |
| 322 | table->arr = NULL; | ||
| 316 | } | 323 | } |
| 317 | 324 | ||
| 318 | static int alloc_lut_if_necessary(LUT *Ty, IPlane *f, chord_set *SE, | 325 | static int alloc_lut_if_necessary(LUT *Ty, IPlane *f, chord_set *SE, |
| 319 | int y, int num, enum MorphModes mode) | 326 | int y, int num, enum MorphModes mode) |
| 320 | { | 327 | { |
| 321 | if (Ty->I != SE->Lnum || | 328 | if (!Ty->arr || Ty->I != SE->Lnum || |
| 322 | Ty->X != f->w || | 329 | Ty->X != f->w || |
| 323 | SE->minX < 0 && -SE->minX > Ty->pre_pad_x || | 330 | SE->minX < 0 && -SE->minX > Ty->pre_pad_x || |
| 324 | Ty->min_r != SE->minY || | 331 | Ty->min_r != SE->minY || |