The RealMedia demuxer's read_header function initially initializes ret, the variable designated for the return variable to -1. Afterwards, chunks of the file are parsed in a loop until an error happens or until the actual frame data is encountered. If the first function whose return value is put into ret doesn't fail, then ret contains a value >= 0 (actually == 0) and this is what will be returned if an error is encountered afterwards. This is a regression since 35bbc1955a58ba74552c50d9161084644f00bbd3. Before that, ret had never been overwritten with a nonnegative value.
Unified · Split
| @@ -538,7 +538,7 @@ static int rm_read_header(AVFormatContext *s) · 537 unmodified lines | |||
| 538 | unsigned int data_off = 0, indx_off = 0; | 538 | unsigned int data_off = 0, indx_off = 0; |
| 539 | char buf[128], mime[128]; | 539 | char buf[128], mime[128]; |
| 540 | int flags = 0; | 540 | int flags = 0; |
| 541 | int ret = -1; | 541 | int ret; |
| 542 | unsigned size, v; | 542 | unsigned size, v; |
| 543 | int64_t codec_pos; | 543 | int64_t codec_pos; |
| 544 | 544 | ||
| @@ -554,6 +554,7 @@ static int rm_read_header(AVFormatContext *s) · 9 unmodified lines | |||
| 554 | avio_skip(pb, tag_size - 8); | 554 | avio_skip(pb, tag_size - 8); |
| 555 | 555 | ||
| 556 | for(;;) { | 556 | for(;;) { |
| 557 | ret = AVERROR_INVALIDDATA; | ||
| 557 | if (avio_feof(pb)) | 558 | if (avio_feof(pb)) |
| 558 | goto fail; | 559 | goto fail; |
| 559 | tag = avio_rl32(pb); | 560 | tag = avio_rl32(pb); |
| @@ -619,8 +620,9 @@ static int rm_read_header(AVFormatContext *s) · 59 unmodified lines | |||
| 619 | avio_seek(pb, codec_pos + size, SEEK_SET); | 620 | avio_seek(pb, codec_pos + size, SEEK_SET); |
| 620 | } else { | 621 | } else { |
| 621 | avio_skip(pb, -4); | 622 | avio_skip(pb, -4); |
| 622 | if (ff_rm_read_mdpr_codecdata(s, s->pb, st, st->priv_data, | 623 | ret = ff_rm_read_mdpr_codecdata(s, s->pb, st, st->priv_data, |
| 623 | size, mime) < 0) | 624 | size, mime); |
| 625 | if (ret < 0) | ||
| 624 | goto fail; | 626 | goto fail; |
| 625 | } | 627 | } |
| 626 | 628 | ||