mirror of
https://github.com/qTox/qTox.git
synced 2024-03-22 14:00:36 +08:00
feat(video): redesign and improve VideoFrame class
This commit redesigns the VideoFrame class to be more robust, documented and performant
This commit is contained in:
initramfs
initramfs
parent
07ac196326
commit
38b1a9b63d
|
|
@ -17,294 +17,315 @@
|
|||
along with qTox. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include "videoframe.h"
|
||||
|
||||
#include <QMutexLocker>
|
||||
#include <QDebug>
|
||||
#include <vpx/vpx_image.h>
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
extern "C"{
|
||||
#include <libavutil/imgutils.h>
|
||||
#include <libswscale/swscale.h>
|
||||
}
|
||||
#include "videoframe.h"
|
||||
#include "camerasource.h"
|
||||
|
||||
/**
|
||||
@class VideoFrame
|
||||
|
||||
VideoFrame takes ownership of an AVFrame* and allows fast conversions to other formats
|
||||
Ownership of all video frame buffers is kept by the VideoFrame, even after conversion
|
||||
All references to the frame data become invalid when the VideoFrame is deleted
|
||||
We try to avoid pixel format conversions as much as possible, at the cost of some memory
|
||||
All methods are thread-safe. If provided freelistCallback will be called by the destructor,
|
||||
unless releaseFrame was called in between.
|
||||
*/
|
||||
|
||||
VideoFrame::VideoFrame(AVFrame* frame, int w, int h, int fmt, std::function<void()> freelistCallback)
|
||||
: freelistCallback{freelistCallback},
|
||||
frameOther{nullptr}, frameYUV420{nullptr}, frameRGB24{nullptr},
|
||||
width{w}, height{h}, pixFmt{fmt}
|
||||
VideoFrame::VideoFrame(AVFrame* sourceFrame, QRect dimensions, int pixFmt, std::function<void ()> destructCallback)
|
||||
: sourceDimensions(dimensions),
|
||||
sourcePixelFormat(pixFmt),
|
||||
destructCallback(destructCallback)
|
||||
{
|
||||
// Silences pointless swscale warning spam
|
||||
// See libswscale/utils.c:1153 @ 74f0bd3
|
||||
frame->color_range = AVCOL_RANGE_MPEG;
|
||||
if (pixFmt == AV_PIX_FMT_YUVJ420P)
|
||||
pixFmt = AV_PIX_FMT_YUV420P;
|
||||
else if (pixFmt == AV_PIX_FMT_YUVJ411P)
|
||||
pixFmt = AV_PIX_FMT_YUV411P;
|
||||
else if (pixFmt == AV_PIX_FMT_YUVJ422P)
|
||||
pixFmt = AV_PIX_FMT_YUV422P;
|
||||
else if (pixFmt == AV_PIX_FMT_YUVJ444P)
|
||||
pixFmt = AV_PIX_FMT_YUV444P;
|
||||
else if (pixFmt == AV_PIX_FMT_YUVJ440P)
|
||||
pixFmt = AV_PIX_FMT_YUV440P;
|
||||
else
|
||||
frame->color_range = AVCOL_RANGE_UNSPECIFIED;
|
||||
|
||||
if (pixFmt == AV_PIX_FMT_YUV420P) {
|
||||
frameYUV420 = frame;
|
||||
} else if (pixFmt == AV_PIX_FMT_RGB24) {
|
||||
frameRGB24 = frame;
|
||||
} else {
|
||||
frameOther = frame;
|
||||
}
|
||||
frameBuffer[pixFmt][dimensionsToKey(dimensions.size())] = sourceFrame;
|
||||
}
|
||||
|
||||
VideoFrame::VideoFrame(AVFrame* frame, std::function<void()> freelistCallback)
|
||||
: VideoFrame{frame, frame->width, frame->height, frame->format, freelistCallback}
|
||||
{
|
||||
}
|
||||
VideoFrame::VideoFrame(AVFrame* sourceFrame, std::function<void ()> destructCallback)
|
||||
: VideoFrame(sourceFrame, QRect {0, 0, sourceFrame->width, sourceFrame->height}, sourceFrame->format, destructCallback){}
|
||||
|
||||
VideoFrame::VideoFrame(AVFrame* frame)
|
||||
: VideoFrame{frame, frame->width, frame->height, frame->format, nullptr}
|
||||
{
|
||||
}
|
||||
VideoFrame::VideoFrame(AVFrame* sourceFrame)
|
||||
: VideoFrame(sourceFrame, QRect {0, 0, sourceFrame->width, sourceFrame->height}, sourceFrame->format, nullptr){}
|
||||
|
||||
/**
|
||||
@brief VideoFrame constructor. Disable copy.
|
||||
@note Use a shared_ptr if you need copies.
|
||||
*/
|
||||
VideoFrame::~VideoFrame()
|
||||
{
|
||||
if (freelistCallback)
|
||||
freelistCallback();
|
||||
frameLock.lockForWrite();
|
||||
|
||||
releaseFrameLockless();
|
||||
if(destructCallback && frameBuffer.size() > 0)
|
||||
{
|
||||
destructCallback();
|
||||
}
|
||||
|
||||
deleteFrameBuffer();
|
||||
|
||||
frameLock.unlock();
|
||||
}
|
||||
|
||||
/**
|
||||
@brief Converts the VideoFrame to a QImage that shares our internal video buffer.
|
||||
@param size Size of resulting image.
|
||||
@return Converted image to RGB24 color model.
|
||||
*/
|
||||
QImage VideoFrame::toQImage(QSize size)
|
||||
bool VideoFrame::isValid()
|
||||
{
|
||||
if (!convertToRGB24(size))
|
||||
return QImage();
|
||||
frameLock.lockForRead();
|
||||
bool retValue = frameBuffer.size() > 0;
|
||||
frameLock.unlock();
|
||||
|
||||
QMutexLocker locker(&biglock);
|
||||
|
||||
return QImage(*frameRGB24->data, frameRGB24->width, frameRGB24->height, *frameRGB24->linesize, QImage::Format_RGB888);
|
||||
return retValue;
|
||||
}
|
||||
|
||||
/**
|
||||
@brief Converts the VideoFrame to a vpx_image_t.
|
||||
Converts the VideoFrame to a vpx_image_t that shares our internal video buffer.
|
||||
@return Converted image to vpx_image format.
|
||||
*/
|
||||
vpx_image *VideoFrame::toVpxImage()
|
||||
const AVFrame* VideoFrame::getAVFrame(const QSize& dimensions, const int pixelFormat)
|
||||
{
|
||||
vpx_image* img = vpx_img_alloc(nullptr, VPX_IMG_FMT_I420, width, height, 0);
|
||||
frameLock.lockForRead();
|
||||
|
||||
if (!convertToYUV420())
|
||||
return img;
|
||||
|
||||
for (int i = 0; i < 3; i++)
|
||||
// We return nullptr if the VideoFrame is no longer valid
|
||||
if(frameBuffer.size() == 0)
|
||||
{
|
||||
int dstStride = img->stride[i];
|
||||
int srcStride = frameYUV420->linesize[i];
|
||||
int minStride = std::min(dstStride, srcStride);
|
||||
int size = (i == 0) ? img->d_h : img->d_h / 2;
|
||||
|
||||
for (int j = 0; j < size; j++)
|
||||
{
|
||||
uint8_t *dst = img->planes[i] + dstStride * j;
|
||||
uint8_t *src = frameYUV420->data[i] + srcStride * j;
|
||||
memcpy(dst, src, minStride);
|
||||
}
|
||||
frameLock.unlock();
|
||||
return nullptr;
|
||||
}
|
||||
return img;
|
||||
|
||||
AVFrame* ret = retrieveAVFrame(dimensions, pixelFormat);
|
||||
|
||||
if(ret)
|
||||
{
|
||||
frameLock.unlock();
|
||||
return ret;
|
||||
}
|
||||
|
||||
// VideoFrame does not contain an AVFrame to spec, generate one here
|
||||
ret = generateAVFrame(dimensions, pixelFormat);
|
||||
|
||||
/*
|
||||
* We need to "upgrade" the lock to a write lock so we can update our frameBuffer map.
|
||||
*
|
||||
* It doesn't matter if another thread obtains the write lock before we finish since it is
|
||||
* likely writing to somewhere else. Absolute worst-case scenario, we merely perform the
|
||||
* generation process twice, and discard the old result.
|
||||
*/
|
||||
frameLock.unlock();
|
||||
frameLock.lockForWrite();
|
||||
|
||||
storeAVFrame(ret, dimensions, pixelFormat);
|
||||
|
||||
frameLock.unlock();
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool VideoFrame::convertToRGB24(QSize size)
|
||||
{
|
||||
QMutexLocker locker(&biglock);
|
||||
|
||||
AVFrame* sourceFrame;
|
||||
if (frameOther)
|
||||
{
|
||||
sourceFrame = frameOther;
|
||||
}
|
||||
else if (frameYUV420)
|
||||
{
|
||||
sourceFrame = frameYUV420;
|
||||
}
|
||||
else
|
||||
{
|
||||
qWarning() << "None of the frames are valid! Did someone release us?";
|
||||
return false;
|
||||
}
|
||||
//std::cout << "converting to RGB24" << std::endl;
|
||||
|
||||
if (size.isEmpty())
|
||||
{
|
||||
size.setWidth(sourceFrame->width);
|
||||
size.setHeight(sourceFrame->height);
|
||||
}
|
||||
|
||||
if (frameRGB24)
|
||||
{
|
||||
if (frameRGB24->width == size.width() && frameRGB24->height == size.height())
|
||||
return true;
|
||||
|
||||
av_free(frameRGB24->opaque);
|
||||
av_frame_unref(frameRGB24);
|
||||
av_frame_free(&frameRGB24);
|
||||
}
|
||||
|
||||
frameRGB24=av_frame_alloc();
|
||||
if (!frameRGB24)
|
||||
{
|
||||
qCritical() << "av_frame_alloc failed";
|
||||
return false;
|
||||
}
|
||||
|
||||
int imgBufferSize = av_image_get_buffer_size(AV_PIX_FMT_RGB24, size.width(), size.height(), 1);
|
||||
uint8_t* buf = (uint8_t*)av_malloc(imgBufferSize);
|
||||
if (!buf)
|
||||
{
|
||||
qCritical() << "av_malloc failed";
|
||||
av_frame_free(&frameRGB24);
|
||||
return false;
|
||||
}
|
||||
frameRGB24->opaque = buf;
|
||||
|
||||
uint8_t** data = frameRGB24->data;
|
||||
int* linesize = frameRGB24->linesize;
|
||||
av_image_fill_arrays(data, linesize, buf, AV_PIX_FMT_RGB24, size.width(), size.height(), 1);
|
||||
frameRGB24->width = size.width();
|
||||
frameRGB24->height = size.height();
|
||||
|
||||
// Bilinear is better for shrinking, bicubic better for upscaling
|
||||
int resizeAlgo = size.width()<=width ? SWS_BILINEAR : SWS_BICUBIC;
|
||||
|
||||
SwsContext *swsCtx = sws_getContext(width, height, (AVPixelFormat)pixFmt,
|
||||
size.width(), size.height(), AV_PIX_FMT_RGB24,
|
||||
resizeAlgo, nullptr, nullptr, nullptr);
|
||||
sws_scale(swsCtx, (uint8_t const * const *)sourceFrame->data,
|
||||
sourceFrame->linesize, 0, height,
|
||||
frameRGB24->data, frameRGB24->linesize);
|
||||
sws_freeContext(swsCtx);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool VideoFrame::convertToYUV420()
|
||||
{
|
||||
QMutexLocker locker(&biglock);
|
||||
|
||||
if (frameYUV420)
|
||||
return true;
|
||||
|
||||
AVFrame* sourceFrame;
|
||||
if (frameOther)
|
||||
{
|
||||
sourceFrame = frameOther;
|
||||
}
|
||||
else if (frameRGB24)
|
||||
{
|
||||
sourceFrame = frameRGB24;
|
||||
}
|
||||
else
|
||||
{
|
||||
qCritical() << "None of the frames are valid! Did someone release us?";
|
||||
return false;
|
||||
}
|
||||
//std::cout << "converting to YUV420" << std::endl;
|
||||
|
||||
frameYUV420=av_frame_alloc();
|
||||
if (!frameYUV420)
|
||||
{
|
||||
qCritical() << "av_frame_alloc failed";
|
||||
return false;
|
||||
}
|
||||
|
||||
int imgBufferSize = av_image_get_buffer_size(AV_PIX_FMT_RGB24, width, height, 1);
|
||||
uint8_t* buf = (uint8_t*)av_malloc(imgBufferSize);
|
||||
if (!buf)
|
||||
{
|
||||
qCritical() << "av_malloc failed";
|
||||
av_frame_free(&frameYUV420);
|
||||
return false;
|
||||
}
|
||||
frameYUV420->opaque = buf;
|
||||
|
||||
uint8_t** data = frameYUV420->data;
|
||||
int* linesize = frameYUV420->linesize;
|
||||
av_image_fill_arrays(data, linesize, buf, AV_PIX_FMT_YUV420P, width, height, 1);
|
||||
|
||||
SwsContext *swsCtx = sws_getContext(width, height, (AVPixelFormat)pixFmt,
|
||||
width, height, AV_PIX_FMT_YUV420P,
|
||||
SWS_BILINEAR, nullptr, nullptr, nullptr);
|
||||
sws_scale(swsCtx, (uint8_t const * const *)sourceFrame->data,
|
||||
sourceFrame->linesize, 0, height,
|
||||
frameYUV420->data, frameYUV420->linesize);
|
||||
sws_freeContext(swsCtx);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
@brief Frees all frame memory.
|
||||
|
||||
Frees all internal buffers and frame data, removes the freelistCallback
|
||||
This makes all converted objects that shares our internal buffers invalid.
|
||||
*/
|
||||
void VideoFrame::releaseFrame()
|
||||
{
|
||||
QMutexLocker locker(&biglock);
|
||||
freelistCallback = nullptr;
|
||||
releaseFrameLockless();
|
||||
frameLock.lockForWrite();
|
||||
|
||||
deleteFrameBuffer();
|
||||
|
||||
frameLock.unlock();
|
||||
}
|
||||
|
||||
void VideoFrame::releaseFrameLockless()
|
||||
QImage VideoFrame::toQImage(QSize frameSize)
|
||||
{
|
||||
if (frameOther)
|
||||
frameLock.lockForRead();
|
||||
|
||||
// We return an empty QImage if the VideoFrame is no longer valid
|
||||
if(frameBuffer.size() == 0)
|
||||
{
|
||||
av_free(frameOther->opaque);
|
||||
av_frame_unref(frameOther);
|
||||
av_frame_free(&frameOther);
|
||||
frameLock.unlock();
|
||||
return QImage {};
|
||||
}
|
||||
if (frameYUV420)
|
||||
|
||||
if(frameSize.width() == 0 && frameSize.height() == 0)
|
||||
{
|
||||
av_free(frameYUV420->opaque);
|
||||
av_frame_unref(frameYUV420);
|
||||
av_frame_free(&frameYUV420);
|
||||
frameSize = sourceDimensions.size();
|
||||
}
|
||||
if (frameRGB24)
|
||||
|
||||
AVFrame* frame = retrieveAVFrame(frameSize, static_cast<int>(AV_PIX_FMT_RGB24));
|
||||
|
||||
if(frame)
|
||||
{
|
||||
av_free(frameRGB24->opaque);
|
||||
av_frame_unref(frameRGB24);
|
||||
av_frame_free(&frameRGB24);
|
||||
QImage ret {*(frame->data), frameSize.width(), frameSize.height(), *(frame->linesize), QImage::Format_RGB888};
|
||||
|
||||
frameLock.unlock();
|
||||
return ret;
|
||||
}
|
||||
|
||||
// VideoFrame does not contain an AVFrame to spec, generate one here
|
||||
frame = generateAVFrame(frameSize, static_cast<int>(AV_PIX_FMT_RGB24));
|
||||
|
||||
/*
|
||||
* We need to "upgrade" the lock to a write lock so we can update our frameBuffer map.
|
||||
*
|
||||
* It doesn't matter if another thread obtains the write lock before we finish since it is
|
||||
* likely writing to somewhere else. Absolute worst-case scenario, we merely perform the
|
||||
* generation process twice, and discard the old result.
|
||||
*/
|
||||
frameLock.unlock();
|
||||
frameLock.lockForWrite();
|
||||
|
||||
storeAVFrame(frame, frameSize, static_cast<int>(AV_PIX_FMT_RGB24));
|
||||
|
||||
QImage ret {*(frame->data), frameSize.width(), frameSize.height(), *(frame->linesize), QImage::Format_RGB888};
|
||||
|
||||
frameLock.unlock();
|
||||
return ret;
|
||||
}
|
||||
|
||||
vpx_image* VideoFrame::toVpxImage(QSize frameSize)
|
||||
{
|
||||
frameLock.lockForRead();
|
||||
|
||||
// We return nullptr if the VideoFrame is no longer valid
|
||||
if(frameBuffer.size() == 0)
|
||||
{
|
||||
frameLock.unlock();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if(frameSize.width() == 0 && frameSize.height() == 0)
|
||||
{
|
||||
frameSize = sourceDimensions.size();
|
||||
}
|
||||
|
||||
AVFrame* frame = retrieveAVFrame(frameSize, static_cast<int>(AV_PIX_FMT_YUV420P));
|
||||
|
||||
if(frame)
|
||||
{
|
||||
vpx_image* ret = new vpx_image {};
|
||||
memset(ret, 0, sizeof(vpx_image));
|
||||
|
||||
ret->w = ret->d_w = frameSize.width();
|
||||
ret->h = ret->d_h = frameSize.height();
|
||||
ret->fmt = VPX_IMG_FMT_I420;
|
||||
ret->planes[0] = frame->data[0];
|
||||
ret->planes[1] = frame->data[1];
|
||||
ret->planes[2] = frame->data[2];
|
||||
ret->planes[3] = nullptr;
|
||||
ret->stride[0] = frame->linesize[0];
|
||||
ret->stride[1] = frame->linesize[1];
|
||||
ret->stride[2] = frame->linesize[2];
|
||||
ret->stride[3] = frame->linesize[3];
|
||||
|
||||
frameLock.unlock();
|
||||
return ret;
|
||||
}
|
||||
|
||||
// VideoFrame does not contain an AVFrame to spec, generate one here
|
||||
frame = generateAVFrame(frameSize, static_cast<int>(AV_PIX_FMT_YUV420P));
|
||||
|
||||
/*
|
||||
* We need to "upgrade" the lock to a write lock so we can update our frameBuffer map.
|
||||
*
|
||||
* It doesn't matter if another thread obtains the write lock before we finish since it is
|
||||
* likely writing to somewhere else. Absolute worst-case scenario, we merely perform the
|
||||
* generation process twice, and discard the old result.
|
||||
*/
|
||||
frameLock.unlock();
|
||||
frameLock.lockForWrite();
|
||||
|
||||
storeAVFrame(frame, frameSize, static_cast<int>(AV_PIX_FMT_YUV420P));
|
||||
|
||||
vpx_image* ret = new vpx_image {};
|
||||
memset(ret, 0, sizeof(vpx_image));
|
||||
|
||||
ret->w = ret->d_w = frameSize.width();
|
||||
ret->h = ret->d_h = frameSize.height();
|
||||
ret->fmt = VPX_IMG_FMT_I420;
|
||||
ret->planes[0] = frame->data[0];
|
||||
ret->planes[1] = frame->data[1];
|
||||
ret->planes[2] = frame->data[2];
|
||||
ret->planes[3] = nullptr;
|
||||
ret->stride[0] = frame->linesize[0];
|
||||
ret->stride[1] = frame->linesize[1];
|
||||
ret->stride[2] = frame->linesize[2];
|
||||
ret->stride[3] = frame->linesize[3];
|
||||
|
||||
frameLock.unlock();
|
||||
return ret;
|
||||
}
|
||||
|
||||
AVFrame* VideoFrame::retrieveAVFrame(const QSize& dimensions, const int pixelFormat)
|
||||
{
|
||||
if(frameBuffer.contains(pixelFormat) && frameBuffer[pixelFormat].contains(dimensionsToKey(dimensions)))
|
||||
{
|
||||
return frameBuffer[pixelFormat][dimensionsToKey(dimensions)];
|
||||
}
|
||||
else
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@brief Return the size of the original frame
|
||||
@return The size of the original frame
|
||||
*/
|
||||
QSize VideoFrame::getSize()
|
||||
AVFrame* VideoFrame::generateAVFrame(const QSize& dimensions, const int pixelFormat)
|
||||
{
|
||||
return {width, height};
|
||||
AVFrame* ret = av_frame_alloc();
|
||||
|
||||
if(!ret){
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int bufSize = av_image_alloc(ret->data, ret->linesize,
|
||||
dimensions.width(), dimensions.height(),
|
||||
static_cast<AVPixelFormat>(pixelFormat), data_alignment);
|
||||
|
||||
if(bufSize < 0){
|
||||
av_frame_free(&ret);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Bilinear is better for shrinking, bicubic better for upscaling
|
||||
int resizeAlgo = sourceDimensions.width() > dimensions.width() ? SWS_BILINEAR : SWS_BICUBIC;
|
||||
|
||||
SwsContext* swsCtx = sws_getContext(sourceDimensions.width(), sourceDimensions.height(),
|
||||
static_cast<AVPixelFormat>(sourcePixelFormat),
|
||||
dimensions.width(), dimensions.height(),
|
||||
static_cast<AVPixelFormat>(pixelFormat),
|
||||
resizeAlgo, nullptr, nullptr, nullptr);
|
||||
|
||||
if(!swsCtx){
|
||||
av_frame_free(&ret);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
AVFrame* source = frameBuffer[sourcePixelFormat][dimensionsToKey(sourceDimensions.size())];
|
||||
|
||||
sws_scale(swsCtx, source->data, source->linesize, 0, sourceDimensions.height(), ret->data, ret->linesize);
|
||||
|
||||
// Populate AVFrame fields
|
||||
ret->width = dimensions.width();
|
||||
ret->height = dimensions.height();
|
||||
ret->format = pixelFormat;
|
||||
|
||||
sws_freeContext(swsCtx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void VideoFrame::storeAVFrame(AVFrame* frame, const QSize& dimensions, const int pixelFormat)
|
||||
{
|
||||
// We check the prescence of the frame in case of double-computation
|
||||
if(frameBuffer.contains(pixelFormat) && frameBuffer[pixelFormat].contains(dimensionsToKey(dimensions)))
|
||||
{
|
||||
AVFrame* old_ret = frameBuffer[pixelFormat][dimensionsToKey(dimensions)];
|
||||
|
||||
// Free old frame
|
||||
av_freep(&frame->data[0]);
|
||||
av_frame_unref(old_ret);
|
||||
av_frame_free(&old_ret);
|
||||
}
|
||||
|
||||
frameBuffer[pixelFormat].insert(dimensionsToKey(dimensions), frame);
|
||||
}
|
||||
|
||||
void VideoFrame::deleteFrameBuffer()
|
||||
{
|
||||
const quint64 sourceKey = dimensionsToKey(sourceDimensions.size());
|
||||
|
||||
for(auto pixFmtIter = frameBuffer.begin(); pixFmtIter != frameBuffer.end(); ++pixFmtIter)
|
||||
{
|
||||
const auto& pixFmtMap = pixFmtIter.value();
|
||||
|
||||
for(auto dimKeyIter = pixFmtMap.begin(); dimKeyIter != pixFmtMap.end(); ++dimKeyIter)
|
||||
{
|
||||
AVFrame* frame = dimKeyIter.value();
|
||||
|
||||
// We only need to free allocated buffers for derived frames and not the original source
|
||||
if(pixFmtIter.key() == sourcePixelFormat && dimKeyIter.key() == sourceKey)
|
||||
{
|
||||
av_frame_unref(frame);
|
||||
av_frame_free(&frame);
|
||||
}
|
||||
else
|
||||
{
|
||||
av_freep(&frame->data[0]);
|
||||
av_frame_unref(frame);
|
||||
av_frame_free(&frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
frameBuffer.clear();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,44 +20,203 @@
|
|||
#ifndef VIDEOFRAME_H
|
||||
#define VIDEOFRAME_H
|
||||
|
||||
#include <QMutex>
|
||||
#include <QHash>
|
||||
#include <QImage>
|
||||
#include <QReadWriteLock>
|
||||
#include <QRect>
|
||||
#include <QSize>
|
||||
|
||||
extern "C"{
|
||||
#include <libavcodec/avcodec.h>
|
||||
}
|
||||
|
||||
#include <vpx/vpx_image.h>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
|
||||
struct AVFrame;
|
||||
struct AVCodecContext;
|
||||
struct vpx_image;
|
||||
|
||||
/**
|
||||
* @brief An ownernship and management class for AVFrames.
|
||||
*
|
||||
* VideoFrame takes ownership of an AVFrame* and allows fast conversions to other formats.
|
||||
* Ownership of all video frame buffers is kept by the VideoFrame, even after conversion. All
|
||||
* references to the frame data become invalid when the VideoFrame is deleted. We try to avoid
|
||||
* pixel format conversions as much as possible, at the cost of some memory.
|
||||
*
|
||||
* Every function in this class is thread safe apart from concurrent construction and deletion of
|
||||
* the object.
|
||||
*/
|
||||
class VideoFrame
|
||||
{
|
||||
public:
|
||||
explicit VideoFrame(AVFrame* frame);
|
||||
VideoFrame(AVFrame* frame, std::function<void()> freelistCallback);
|
||||
VideoFrame(AVFrame* frame, int w, int h, int fmt, std::function<void()> freelistCallback);
|
||||
/**
|
||||
* @brief Constructs a new instance of a VideoFrame, sourced by a given AVFrame pointer.
|
||||
* @param sourceFrame the source AVFrame pointer to use, must be valid.
|
||||
* @param dimensions the dimensions of the AVFrame, obtained from the AVFrame if not given.
|
||||
* @param pixFmt the pixel format of the AVFrame, obtained from the AVFrame if not given.
|
||||
* @param destructCallback callback function to run upon destruction of the VideoFrame
|
||||
* this callback is only run when destroying a valid VideoFrame (e.g. a VideoFrame instance in
|
||||
* which releaseFrame() was called upon it will not call the callback).
|
||||
*/
|
||||
VideoFrame(AVFrame* sourceFrame, QRect dimensions, int pixFmt, std::function<void()> destructCallback);
|
||||
VideoFrame(AVFrame* sourceFrame, std::function<void()> destructCallback);
|
||||
VideoFrame(AVFrame* sourceFrame);
|
||||
|
||||
/**
|
||||
* Destructor for VideoFrame.
|
||||
*/
|
||||
~VideoFrame();
|
||||
|
||||
QSize getSize();
|
||||
// Copy/Move operations are disabled for the VideoFrame, encapsulate with a std::shared_ptr to manage.
|
||||
|
||||
VideoFrame(const VideoFrame& other) = delete;
|
||||
VideoFrame(VideoFrame&& other) = delete;
|
||||
|
||||
const VideoFrame& operator=(const VideoFrame& other) = delete;
|
||||
const VideoFrame& operator=(VideoFrame&& other) = delete;
|
||||
|
||||
/**
|
||||
* @brief Returns the validity of this VideoFrame.
|
||||
*
|
||||
* A VideoFrame is valid if it manages at least one AVFrame. A VideoFrame can be invalidated
|
||||
* by calling releaseFrame() on it.
|
||||
*
|
||||
* @return true if the VideoFrame is valid, false otherwise.
|
||||
*/
|
||||
bool isValid();
|
||||
|
||||
/**
|
||||
* @brief Retrieves an AVFrame derived from the source based on the given parameters.
|
||||
*
|
||||
* If a given frame does not exist, this function will perform appropriate conversions to
|
||||
* return a frame that fulfills the given parameters.
|
||||
*
|
||||
* @param dimensions the dimensions of the frame.
|
||||
* @param pixelFormat the desired pixel format of the frame.
|
||||
* @return a pointer to a AVFrame with the given parameters or nullptr if the VideoFrame is no
|
||||
* longer valid.
|
||||
*/
|
||||
const AVFrame* getAVFrame(const QSize& dimensions, const int pixelFormat);
|
||||
|
||||
/**
|
||||
* @brief Releases all frames managed by this VideoFrame and invalidates it.
|
||||
*/
|
||||
void releaseFrame();
|
||||
|
||||
QImage toQImage(QSize size = QSize());
|
||||
vpx_image* toVpxImage();
|
||||
/**
|
||||
* @brief Retrieves a copy of the source VideoFrame's dimensions.
|
||||
* @return QRect copy representing the source VideoFrame's dimensions.
|
||||
*/
|
||||
inline QRect getSourceDimensions()
|
||||
{
|
||||
return sourceDimensions;
|
||||
}
|
||||
|
||||
protected:
|
||||
bool convertToRGB24(QSize size = QSize());
|
||||
bool convertToYUV420();
|
||||
void releaseFrameLockless();
|
||||
/**
|
||||
* @brief Retrieves a copy of the source VideoFormat's pixel format.
|
||||
* @return integer copy represetning the source VideoFrame's pixel format.
|
||||
*/
|
||||
inline int getSourcePixelFormat()
|
||||
{
|
||||
return sourcePixelFormat;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Converts this VideoFrame to a QImage that shares this VideoFrame's buffer.
|
||||
*
|
||||
* The VideoFrame will be scaled into the RGB24 pixel format along with the given
|
||||
* dimension.
|
||||
*
|
||||
* @param frameSize the given frame size of QImage to generate. If frame size is 0, defaults to
|
||||
* source frame size.
|
||||
* @return a QImage that represents this VideoFrame, sharing it's buffers or a null image if
|
||||
* this VideoFrame is no longer valid.
|
||||
*/
|
||||
QImage toQImage(QSize frameSize = {0, 0});
|
||||
|
||||
/**
|
||||
* @brief Converts this VideoFrame to a vpx_image that shares this VideoFrame's buffer.
|
||||
*
|
||||
* Given that libvpx does not provide a way to create vpx_images that uses external buffers,
|
||||
* the vpx_image constructed by this function is done in a non-compliant way, requiring the
|
||||
* use of the C++ delete keyword to properly deallocate memory associated with this image.
|
||||
*
|
||||
* @param frameSize the given frame size of vpx_image to generate. If frame size is 0, defaults
|
||||
* to source frame size.
|
||||
* @return a vpx_image that represents this VideoFrame, sharing it's buffers or nullptr if this
|
||||
* VideoFrame is no longer valid.
|
||||
*/
|
||||
vpx_image* toVpxImage(QSize frameSize = {0, 0});
|
||||
private:
|
||||
VideoFrame(const VideoFrame& other)=delete;
|
||||
VideoFrame& operator=(const VideoFrame& other)=delete;
|
||||
/**
|
||||
* @brief A function to create a hashable key from a given QSize dimension.
|
||||
* @param dimensions the dimensions to hash.
|
||||
* @return a hashable unsigned 64-bit number representing the dimension.
|
||||
*/
|
||||
static inline quint64 dimensionsToKey(const QSize& dimensions)
|
||||
{
|
||||
return (static_cast<quint64>(dimensions.width()) << 32) | dimensions.height();
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Retrieves an AVFrame derived from the source based on the given parameters without
|
||||
* obtaining a lock.
|
||||
*
|
||||
* This function is not thread-safe and must be called from a thread-safe context.
|
||||
*
|
||||
* Note: this function differs from getAVFrame() in that it returns a nullptr if no frame was
|
||||
* found.
|
||||
*
|
||||
* @param dimensions the dimensions of the frame.
|
||||
* @param pixelFormat the desired pixel format of the frame.
|
||||
* @return a pointer to a AVFrame with the given parameters or nullptr if no such frame was
|
||||
* found.
|
||||
*/
|
||||
AVFrame* retrieveAVFrame(const QSize& dimensions, const int pixelFormat);
|
||||
|
||||
/**
|
||||
* @brief Generates an AVFrame based on the given specifications.
|
||||
*
|
||||
* This function is not thread-safe and must be called from a thread-safe context.
|
||||
*
|
||||
* @param dimensions the required dimensions for the frame.
|
||||
* @param pixelFormat the required pixel format for the frame.
|
||||
* @return an AVFrame with the given specifications.
|
||||
*/
|
||||
AVFrame* generateAVFrame(const QSize& dimensions, const int pixelFormat);
|
||||
|
||||
/**
|
||||
* @brief Stores a given AVFrame within the frameBuffer map.
|
||||
*
|
||||
* This function is not thread-safe and must be called from a thread-safe context.
|
||||
*
|
||||
* @param frame the given frame to store.
|
||||
* @param dimensions the dimensions of the frame.
|
||||
* @param pixelFormat the pixel format of the frame.
|
||||
*/
|
||||
void storeAVFrame(AVFrame* frame, const QSize& dimensions, const int pixelFormat);
|
||||
|
||||
/**
|
||||
* @brief Releases all frames within the frame buffer.
|
||||
*
|
||||
* This function is not thread-safe and must be called from a thread-safe context.
|
||||
*/
|
||||
void deleteFrameBuffer();
|
||||
private:
|
||||
std::function<void()> freelistCallback;
|
||||
QMutex biglock;
|
||||
AVFrame* frameOther, *frameYUV420, *frameRGB24;
|
||||
int width, height;
|
||||
int pixFmt;
|
||||
// Data alignment for framebuffers
|
||||
static constexpr int data_alignment = 32;
|
||||
|
||||
// Main framebuffer store
|
||||
QHash<int, QHash<quint64, AVFrame*>> frameBuffer {};
|
||||
|
||||
// Source frame
|
||||
const QRect sourceDimensions;
|
||||
const int sourcePixelFormat;
|
||||
|
||||
// Destructor callback
|
||||
const std::function<void ()> destructCallback;
|
||||
|
||||
// Concurrency
|
||||
QReadWriteLock frameLock {};
|
||||
};
|
||||
|
||||
#endif // VIDEOFRAME_H
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user