82231fd74d173fb47845dd086d360bceb3e5fc17 mentioned that for the "rot" transform flip=true must be set, which "makes no sense at all". The reason this is happening is that the rotation matrix calculation is only valid for the 2D coordinate system where y axis is 90 degrees counterclockwise from x axis, but the coordinate system of chroma offset has its origin at top-left so it is the opposite, which results in the rotation going to the opposite way. Setting flip=true fixes the rotation direction, but results in a flipped y coordinate. Fix this by reversing the rotation angle for chroma offset rotation matrix calculation. This also allows removing some duplicated code.
Unified · Split
| @@ -898,7 +898,10 @@ static void pass_get_images(struct gl_video *p, struct video_image *vimg, · 897 unmodified lines | ||
| 898 | 898 | |
| 899 | 899 | if (type == PLANE_CHROMA) { |
| 900 | 900 | struct gl_transform rot; |
| 901 | − get_transform(0, 0, p->image_params.rotate, true, &rot); | |
| 901 | + // Reverse the rotation direction here because the different | |
| 902 | + // coordinate system of chroma offset results in rotation | |
| 903 | + // in the opposite direction. | |
| 904 | + get_transform(0, 0, 360 - p->image_params.rotate, t->flipped, &rot); | |
| 902 | 905 | |
| 903 | 906 | struct gl_transform tr = chroma; |
| 904 | 907 | gl_transform_vec(rot, &tr.t[0], &tr.t[1]); |
| @@ -908,15 +911,13 @@ static void pass_get_images(struct gl_video *p, struct video_image *vimg, · 3 unmodified lines | ||
| 908 | 911 | |
| 909 | 912 | // Adjust the chroma offset if the real chroma size is fractional |
| 910 | 913 | // due image sizes not aligned to chroma subsampling. |
| 911 | − struct gl_transform rot2; | |
| 912 | − get_transform(0, 0, p->image_params.rotate, t->flipped, &rot2); | |
| 913 | − if (rot2.m[0][0] < 0) | |
| 914 | + if (rot.m[0][0] < 0) | |
| 914 | 915 | tr.t[0] += dx; |
| 915 | − if (rot2.m[1][0] < 0) | |
| 916 | + if (rot.m[1][0] < 0) | |
| 916 | 917 | tr.t[0] += dy; |
| 917 | − if (rot2.m[0][1] < 0) | |
| 918 | + if (rot.m[0][1] < 0) | |
| 918 | 919 | tr.t[1] += dx; |
| 919 | − if (rot2.m[1][1] < 0) | |
| 920 | + if (rot.m[1][1] < 0) | |
| 920 | 921 | tr.t[1] += dy; |
| 921 | 922 | |
| 922 | 923 | off[n] = tr; |