/* * Android camera input device * * Copyright (C) 2017 Felix Matouschek * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include <errno.h>#include <pthread.h>#include <stdatomic.h>#include <stdbool.h>#include <stdint.h> #include <camera/NdkCameraDevice.h>#include <camera/NdkCameraManager.h>#include <media/NdkImage.h>#include <media/NdkImageReader.h> #include "libavformat/avformat.h"#include "libavformat/internal.h"#include "libavutil/avstring.h"#include "libavutil/display.h"#include "libavutil/imgutils.h"#include "libavutil/log.h"#include "libavutil/opt.h"#include "libavutil/parseutils.h"#include "libavutil/pixfmt.h"#include "libavutil/threadmessage.h"#include "libavutil/time.h" #include "version.h" /* This image format is available on all Android devices * supporting the Camera2 API */#define IMAGE_FORMAT_ANDROID AIMAGE_FORMAT_YUV_420_888 #define MAX_BUF_COUNT 2#define VIDEO_STREAM_INDEX 0#define VIDEO_TIMEBASE_ANDROID 1000000000 #define RETURN_CASE(x) case x: return AV_STRINGIFY(x);#define RETURN_DEFAULT(x) default: return AV_STRINGIFY(x); typedef struct AndroidCameraCtx {    const AVClass *class;     int requested_width;    int requested_height;    AVRational framerate;    int camera_index;    int input_queue_size;     uint8_t lens_facing;    int32_t sensor_orientation;    int width;    int height;    int32_t framerate_range[2];    int image_format;     ACameraManager *camera_mgr;    char *camera_id;    ACameraMetadata *camera_metadata;    ACameraDevice *camera_dev;    ACameraDevice_StateCallbacks camera_state_callbacks;    AImageReader *image_reader;    AImageReader_ImageListener image_listener;    ANativeWindow *image_reader_window;    ACaptureSessionOutputContainer *capture_session_output_container;    ACaptureSessionOutput *capture_session_output;    ACameraOutputTarget *camera_output_target;    ACaptureRequest *capture_request;    ACameraCaptureSession_stateCallbacks capture_session_state_callbacks;    ACameraCaptureSession *capture_session;     AVThreadMessageQueue *input_queue;    atomic_int exit;    atomic_int got_image_format;} AndroidCameraCtx; static const char *camera_status_string(camera_status_t val){    switch(val) {        RETURN_CASE(ACAMERA_OK)        RETURN_CASE(ACAMERA_ERROR_UNKNOWN)        RETURN_CASE(ACAMERA_ERROR_INVALID_PARAMETER)        RETURN_CASE(ACAMERA_ERROR_CAMERA_DISCONNECTED)        RETURN_CASE(ACAMERA_ERROR_NOT_ENOUGH_MEMORY)        RETURN_CASE(ACAMERA_ERROR_METADATA_NOT_FOUND)        RETURN_CASE(ACAMERA_ERROR_CAMERA_DEVICE)        RETURN_CASE(ACAMERA_ERROR_CAMERA_SERVICE)        RETURN_CASE(ACAMERA_ERROR_SESSION_CLOSED)        RETURN_CASE(ACAMERA_ERROR_INVALID_OPERATION)        RETURN_CASE(ACAMERA_ERROR_STREAM_CONFIGURE_FAIL)        RETURN_CASE(ACAMERA_ERROR_CAMERA_IN_USE)        RETURN_CASE(ACAMERA_ERROR_MAX_CAMERA_IN_USE)        RETURN_CASE(ACAMERA_ERROR_CAMERA_DISABLED)        RETURN_CASE(ACAMERA_ERROR_PERMISSION_DENIED)        RETURN_DEFAULT(ACAMERA_ERROR_UNKNOWN)    }} static const char *media_status_string(media_status_t val){    switch(val) {        RETURN_CASE(AMEDIA_OK)        RETURN_CASE(AMEDIA_ERROR_UNKNOWN)        RETURN_CASE(AMEDIA_ERROR_MALFORMED)        RETURN_CASE(AMEDIA_ERROR_UNSUPPORTED)        RETURN_CASE(AMEDIA_ERROR_INVALID_OBJECT)        RETURN_CASE(AMEDIA_ERROR_INVALID_PARAMETER)        RETURN_CASE(AMEDIA_ERROR_INVALID_OPERATION)        RETURN_CASE(AMEDIA_DRM_NOT_PROVISIONED)        RETURN_CASE(AMEDIA_DRM_RESOURCE_BUSY)        RETURN_CASE(AMEDIA_DRM_DEVICE_REVOKED)        RETURN_CASE(AMEDIA_DRM_SHORT_BUFFER)        RETURN_CASE(AMEDIA_DRM_SESSION_NOT_OPENED)        RETURN_CASE(AMEDIA_DRM_TAMPER_DETECTED)        RETURN_CASE(AMEDIA_DRM_VERIFY_FAILED)        RETURN_CASE(AMEDIA_DRM_NEED_KEY)        RETURN_CASE(AMEDIA_DRM_LICENSE_EXPIRED)        RETURN_CASE(AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE)        RETURN_CASE(AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED)        RETURN_CASE(AMEDIA_IMGREADER_CANNOT_LOCK_IMAGE)        RETURN_CASE(AMEDIA_IMGREADER_CANNOT_UNLOCK_IMAGE)        RETURN_CASE(AMEDIA_IMGREADER_IMAGE_NOT_LOCKED)        RETURN_DEFAULT(AMEDIA_ERROR_UNKNOWN)    }} static const char *error_state_callback_string(int val){    switch(val) {        RETURN_CASE(ERROR_CAMERA_IN_USE)        RETURN_CASE(ERROR_MAX_CAMERAS_IN_USE)        RETURN_CASE(ERROR_CAMERA_DISABLED)        RETURN_CASE(ERROR_CAMERA_DEVICE)        RETURN_CASE(ERROR_CAMERA_SERVICE)        default:            return "ERROR_CAMERA_UNKNOWN";    }} static void camera_dev_disconnected(void *context, ACameraDevice *device){    AVFormatContext *avctx = context;    AndroidCameraCtx *ctx = avctx->priv_data;    atomic_store(&ctx->exit, 1);    av_log(avctx, AV_LOG_ERROR, "Camera with id %s disconnected.\n",           ACameraDevice_getId(device));} static void camera_dev_error(void *context, ACameraDevice *device, int error){    AVFormatContext *avctx = context;    AndroidCameraCtx *ctx = avctx->priv_data;    atomic_store(&ctx->exit, 1);    av_log(avctx, AV_LOG_ERROR, "Error %s on camera with id %s.\n",           error_state_callback_string(error), ACameraDevice_getId(device));} static int open_camera(AVFormatContext *avctx){    AndroidCameraCtx *ctx = avctx->priv_data;    camera_status_t ret;    ACameraIdList *camera_ids;     ret = ACameraManager_getCameraIdList(ctx->camera_mgr, &camera_ids);    if (ret != ACAMERA_OK) {        av_log(avctx, AV_LOG_ERROR, "Failed to get camera id list, error: %s.\n",               camera_status_string(ret));        return AVERROR_EXTERNAL;    }     if (ctx->camera_index < camera_ids->numCameras) {        ctx->camera_id = av_strdup(camera_ids->cameraIds[ctx->camera_index]);        if (!ctx->camera_id) {            av_log(avctx, AV_LOG_ERROR, "Failed to allocate memory for camera_id.\n");            return AVERROR(ENOMEM);        }    } else {        av_log(avctx, AV_LOG_ERROR, "No camera with index %d available.\n",               ctx->camera_index);        return AVERROR(ENXIO);    }     ACameraManager_deleteCameraIdList(camera_ids);     ret = ACameraManager_getCameraCharacteristics(ctx->camera_mgr,            ctx->camera_id, &ctx->camera_metadata);    if (ret != ACAMERA_OK) {        av_log(avctx, AV_LOG_ERROR, "Failed to get metadata for camera with id %s, error: %s.\n",               ctx->camera_id, camera_status_string(ret));        return AVERROR_EXTERNAL;    }     ctx->camera_state_callbacks.context = avctx;    ctx->camera_state_callbacks.onDisconnected = camera_dev_disconnected;    ctx->camera_state_callbacks.onError = camera_dev_error;     ret = ACameraManager_openCamera(ctx->camera_mgr, ctx->camera_id,                                    &ctx->camera_state_callbacks, &ctx->camera_dev);    if (ret != ACAMERA_OK) {        av_log(avctx, AV_LOG_ERROR, "Failed to open camera with id %s, error: %s.\n",               ctx->camera_id, camera_status_string(ret));        return AVERROR_EXTERNAL;    }     return 0;} static void get_sensor_orientation(AVFormatContext *avctx){    AndroidCameraCtx *ctx = avctx->priv_data;    ACameraMetadata_const_entry lens_facing;    ACameraMetadata_const_entry sensor_orientation;     ACameraMetadata_getConstEntry(ctx->camera_metadata,                                  ACAMERA_LENS_FACING, &lens_facing);    ACameraMetadata_getConstEntry(ctx->camera_metadata,                                  ACAMERA_SENSOR_ORIENTATION, &sensor_orientation);     ctx->lens_facing = lens_facing.data.u8[0];    ctx->sensor_orientation = sensor_orientation.data.i32[0];} static void match_video_size(AVFormatContext *avctx){    AndroidCameraCtx *ctx = avctx->priv_data;    ACameraMetadata_const_entry available_configs;    int found = 0;     ACameraMetadata_getConstEntry(ctx->camera_metadata,                                  ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS,                                  &available_configs);     for (int i = 0; i < available_configs.count; i++) {        int32_t input = available_configs.data.i32[i * 4 + 3];        int32_t format = available_configs.data.i32[i * 4 + 0];         if (input) {            continue;        }         if (format == IMAGE_FORMAT_ANDROID) {            int32_t width = available_configs.data.i32[i * 4 + 1];            int32_t height = available_configs.data.i32[i * 4 + 2];             //Same ratio            if ((ctx->requested_width == width && ctx->requested_height == height) ||                    (ctx->requested_width == height && ctx->requested_height == width)) {                ctx->width = width;                ctx->height = height;                found = 1;                break;            }        }    }     if (!found || ctx->width == 0 || ctx->height == 0) {        ctx->width = available_configs.data.i32[1];        ctx->height = available_configs.data.i32[2];         av_log(avctx, AV_LOG_WARNING,               "Requested video_size %dx%d not available, falling back to %dx%d\n",               ctx->requested_width, ctx->requested_height, ctx->width, ctx->height);    }     return;} static void match_framerate(AVFormatContext *avctx){    AndroidCameraCtx *ctx = avctx->priv_data;    ACameraMetadata_const_entry available_framerates;    int found = 0;    int current_best_match = -1;    int requested_framerate = av_q2d(ctx->framerate);     ACameraMetadata_getConstEntry(ctx->camera_metadata,                                  ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES,                                  &available_framerates);     for (int i = 0; i < available_framerates.count; i++) {        int32_t min = available_framerates.data.i32[i * 2 + 0];        int32_t max = available_framerates.data.i32[i * 2 + 1];         if (requested_framerate == max) {            if (min == max) {                ctx->framerate_range[0] = min;                ctx->framerate_range[1] = max;                found = 1;                break;            } else if (current_best_match >= 0) {                int32_t current_best_match_min = available_framerates.data.i32[current_best_match * 2 + 0];                if (min > current_best_match_min) {                    current_best_match = i;                }            } else {                current_best_match = i;            }        }    }     if (!found) {        if (current_best_match >= 0) {            ctx->framerate_range[0] = available_framerates.data.i32[current_best_match * 2 + 0];            ctx->framerate_range[1] = available_framerates.data.i32[current_best_match * 2 + 1];         } else {            ctx->framerate_range[0] = available_framerates.data.i32[0];            ctx->framerate_range[1] = available_framerates.data.i32[1];        }         av_log(avctx, AV_LOG_WARNING,               "Requested framerate %d not available, falling back to min: %d and max: %d fps\n",               requested_framerate, ctx->framerate_range[0], ctx->framerate_range[1]);    }     return;} static int get_image_format(AVFormatContext *avctx, AImage *image){    AndroidCameraCtx *ctx = avctx->priv_data;    int32_t image_pixelstrides[2];    uint8_t *image_plane_data[2];    int plane_data_length[2];     for (int i = 0; i < 2; i++) {        AImage_getPlanePixelStride(image, i + 1, &image_pixelstrides[i]);        AImage_getPlaneData(image, i + 1, &image_plane_data[i], &plane_data_length[i]);    }     if (image_pixelstrides[0] != image_pixelstrides[1]) {        av_log(avctx, AV_LOG_ERROR,               "Pixel strides of U and V plane should have been the same.\n");        return AVERROR_EXTERNAL;    }     switch (image_pixelstrides[0]) {        case 1:            ctx->image_format = AV_PIX_FMT_YUV420P;            break;        case 2:            if (image_plane_data[0] < image_plane_data[1]) {                ctx->image_format = AV_PIX_FMT_NV12;            } else {                ctx->image_format = AV_PIX_FMT_NV21;            }            break;        default:            av_log(avctx, AV_LOG_ERROR,                   "Unknown pixel stride %d of U and V plane, cannot determine camera image format.\n",                   image_pixelstrides[0]);            return AVERROR(ENOSYS);    }     return 0;} static void image_available(void *context, AImageReader *reader){    AVFormatContext *avctx = context;    AndroidCameraCtx *ctx = avctx->priv_data;    media_status_t media_status;    int ret = 0;     AImage *image;    int64_t image_timestamp;    int32_t image_linestrides[4];    uint8_t *image_plane_data[4];    int plane_data_length[4];     AVPacket pkt;    int pkt_buffer_size = 0;     media_status = AImageReader_acquireLatestImage(reader, &image);    if (media_status != AMEDIA_OK) {        if (media_status == AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE) {            av_log(avctx, AV_LOG_WARNING,                   "An image reader frame was discarded");        } else {            av_log(avctx, AV_LOG_ERROR,                   "Failed to acquire latest image from image reader, error: %s.\n",                   media_status_string(media_status));            ret = AVERROR_EXTERNAL;        }        goto error;    }     // Silently drop frames when exit is set    if (atomic_load(&ctx->exit)) {        goto error;    }     // Determine actual image format    if (!atomic_load(&ctx->got_image_format)) {        ret = get_image_format(avctx, image);        if (ret < 0) {            av_log(avctx, AV_LOG_ERROR,                   "Could not get image format of camera.\n");            goto error;        } else {            atomic_store(&ctx->got_image_format, 1);        }    }     pkt_buffer_size = av_image_get_buffer_size(ctx->image_format, ctx->width, ctx->height, 32);    AImage_getTimestamp(image, &image_timestamp);     AImage_getPlaneRowStride(image, 0, &image_linestrides[0]);    AImage_getPlaneData(image, 0, &image_plane_data[0], &plane_data_length[0]);     switch (ctx->image_format) {        case AV_PIX_FMT_YUV420P:            AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);            AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);            AImage_getPlaneRowStride(image, 2, &image_linestrides[2]);            AImage_getPlaneData(image, 2, &image_plane_data[2], &plane_data_length[2]);            break;        case AV_PIX_FMT_NV12:            AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);            AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);            break;        case AV_PIX_FMT_NV21:            AImage_getPlaneRowStride(image, 2, &image_linestrides[1]);            AImage_getPlaneData(image, 2, &image_plane_data[1], &plane_data_length[1]);            break;        default:            av_log(avctx, AV_LOG_ERROR, "Unsupported camera image format.\n");            ret = AVERROR(ENOSYS);            goto error;    }     ret = av_new_packet(&pkt, pkt_buffer_size);    if (ret < 0) {        av_log(avctx, AV_LOG_ERROR,               "Failed to create new av packet, error: %s.\n", av_err2str(ret));        goto error;    }     pkt.stream_index = VIDEO_STREAM_INDEX;    pkt.pts = image_timestamp;    av_image_copy_to_buffer(pkt.data, pkt_buffer_size,                            (const uint8_t * const *) image_plane_data,                            image_linestrides, ctx->image_format,                            ctx->width, ctx->height, 32);     ret = av_thread_message_queue_send(ctx->input_queue, &pkt, AV_THREAD_MESSAGE_NONBLOCK); error:    if (ret < 0) {        if (ret != AVERROR(EAGAIN)) {            av_log(avctx, AV_LOG_ERROR,                   "Error while processing new image, error: %s.\n", av_err2str(ret));            av_thread_message_queue_set_err_recv(ctx->input_queue, ret);            atomic_store(&ctx->exit, 1);        } else {            av_log(avctx, AV_LOG_WARNING,                   "Input queue was full, dropping frame, consider raising the input_queue_size option (current value: %d)\n",                   ctx->input_queue_size);        }        if (pkt_buffer_size) {            av_packet_unref(&pkt);        }    }     AImage_delete(image);     return;} static int create_image_reader(AVFormatContext *avctx){    AndroidCameraCtx *ctx = avctx->priv_data;    media_status_t ret;     ret = AImageReader_new(ctx->width, ctx->height, IMAGE_FORMAT_ANDROID,                           MAX_BUF_COUNT, &ctx->image_reader);    if (ret != AMEDIA_OK) {        av_log(avctx, AV_LOG_ERROR,               "Failed to create image reader, error: %s.\n", media_status_string(ret));        return AVERROR_EXTERNAL;    } 

Lines 1–500 of 871. Load more