1
0
mirror of https://github.com/qTox/qTox.git synced 2024-03-22 14:00:36 +08:00

NetVideoSource, YUV shader, fixes

This commit is contained in:
krepa098 2014-10-15 14:46:01 +02:00
parent 8b8a541826
commit f2b730cdb9
18 changed files with 223 additions and 166 deletions

View File

@ -134,7 +134,9 @@ HEADERS += src/widget/form/addfriendform.h \
src/widget/maskablepixmapwidget.h \
src/videosource.h \
src/cameraworker.h \
src/widget/videosurface.h
src/widget/videosurface.h \
src/netvideosource.h \
src/videosource.h
SOURCES += \
src/widget/form/addfriendform.cpp \
@ -181,4 +183,5 @@ SOURCES += \
src/widget/tool/chatactions/actionaction.cpp \
src/widget/maskablepixmapwidget.cpp \
src/cameraworker.cpp \
src/widget/videosurface.cpp
src/widget/videosurface.cpp \
src/netvideosource.cpp

View File

@ -140,6 +140,7 @@ void CameraWorker::unsubscribe()
if(refCount <= 0)
{
cam.release();
refCount = 0;
}
}

View File

@ -29,6 +29,7 @@ class Camera;
class QTimer;
class QString;
class CString;
class VideoSource;
class Core : public QObject
{
@ -60,6 +61,7 @@ public:
QString getStatusMessage();
ToxID getSelfId();
VideoSource* getVideoSourceFromCall(int callNumber);
void increaseVideoBusyness();
void decreaseVideoBusyness();

View File

@ -231,10 +231,8 @@ void Core::playCallVideo(ToxAv*, int32_t callId, vpx_image_t* img, void *user_da
if (!calls[callId].active || !calls[callId].videoEnabled)
return;
if (videoBusyness >= 1)
qWarning() << "Core: playCallVideo: Busy, dropping current frame";
else
emit Core::getInstance()->videoFrameReceived(img);
calls[callId].videoSource.pushVPXFrame(img);
vpx_img_free(img);
}
@ -550,3 +548,9 @@ void Core::playAudioBuffer(int callId, int16_t *data, int samples, unsigned chan
qDebug() << "Core: Starting audio source of call " << callId;
}
}
VideoSource *Core::getVideoSourceFromCall(int callNumber)
{
return &calls[callNumber].videoSource;
}

View File

@ -2,6 +2,7 @@
#define COREAV_H
#include <tox/toxav.h>
#include "netvideosource.h"
#if defined(__APPLE__) && defined(__MACH__)
#include <OpenAL/al.h>
@ -24,6 +25,7 @@ public:
bool active;
bool muteMic;
ALuint alSource;
NetVideoSource videoSource;
};
#endif // COREAV_H

64
src/netvideosource.cpp Normal file
View File

@ -0,0 +1,64 @@
/*
Copyright (C) 2014 by Project Tox <https://tox.im>
This file is part of qTox, a Qt-based graphical interface for Tox.
This program is libre software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the COPYING file for more details.
*/
#include "netvideosource.h"
#include <QDebug>
#include <vpx/vpx_image.h>
NetVideoSource::NetVideoSource()
{
}
void NetVideoSource::pushFrame(VideoFrame frame)
{
emit frameAvailable(frame);
}
void NetVideoSource::pushVPXFrame(vpx_image *image)
{
int dw = image->d_w;
int dh = image->d_h;
int bpl = image->stride[VPX_PLANE_Y];
int cxbpl = image->stride[VPX_PLANE_V];
VideoFrame frame;
frame.frameData.resize(dw * dh * 3); //YUV 24bit
frame.resolution = QSize(dw, dh);
frame.format = VideoFrame::YUV;
uint8_t* yData = image->planes[VPX_PLANE_Y];
uint8_t* uData = image->planes[VPX_PLANE_V];
uint8_t* vData = image->planes[VPX_PLANE_U];
for (int x = 0; x < dw; x += 1)
{
for (int y = 0; y < dh; y += 1)
{
uint8_t Y = yData[x + y * bpl];
uint8_t U = uData[x/2 + y/2*cxbpl];
uint8_t V = vData[x/2 + y/2*cxbpl];
frame.frameData.data()[dw * 3 * y + x * 3 + 0] = Y;
frame.frameData.data()[dw * 3 * y + x * 3 + 1] = U;
frame.frameData.data()[dw * 3 * y + x * 3 + 2] = V;
}
}
pushFrame(frame);
}

36
src/netvideosource.h Normal file
View File

@ -0,0 +1,36 @@
/*
Copyright (C) 2014 by Project Tox <https://tox.im>
This file is part of qTox, a Qt-based graphical interface for Tox.
This program is libre software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the COPYING file for more details.
*/
#ifndef NETVIDEOSOURCE_H
#define NETVIDEOSOURCE_H
#include "videosource.h"
class vpx_image;
class NetVideoSource : public VideoSource
{
public:
NetVideoSource();
void pushFrame(VideoFrame frame);
void pushVPXFrame(vpx_image* image);
virtual void subscribe() {}
virtual void unsubscribe() {}
};
#endif // NETVIDEOSOURCE_H

View File

@ -9,30 +9,34 @@ struct VideoFrame
{
enum ColorFormat
{
NONE,
BGR,
YUV,
};
QByteArray data;
QByteArray frameData;
QSize resolution;
ColorFormat format;
VideoFrame() : format(NONE) {}
VideoFrame(QByteArray d, QSize r, ColorFormat f) : frameData(d), resolution(r), format(f) {}
void setNull()
{
data = QByteArray();
frameData = QByteArray();
}
bool isNull()
{
return data.isEmpty();
return frameData.isEmpty();
}
// assumes format is BGR
QRgb getPixel(int x, int y)
{
char b = data.data()[resolution.width() * 3 * y + x * 3 + 0];
char g = data.data()[resolution.width() * 3 * y + x * 3 + 1];
char r = data.data()[resolution.width() * 3 * y + x * 3 + 2];
char b = frameData.data()[resolution.width() * 3 * y + x * 3 + 0];
char g = frameData.data()[resolution.width() * 3 * y + x * 3 + 1];
char r = frameData.data()[resolution.width() * 3 * y + x * 3 + 2];
return qRgb(r, g, b);
}
@ -47,7 +51,6 @@ class VideoSource : public QObject
public:
virtual void subscribe() = 0;
virtual void unsubscribe() = 0;
virtual VideoFrame::ColorFormat getColorFormat() = 0;
signals:
void frameAvailable(const VideoFrame frame);

View File

@ -71,11 +71,6 @@ void Camera::unsubscribe()
}
}
VideoFrame::ColorFormat Camera::getColorFormat()
{
return VideoFrame::BGR;
}
vpx_image Camera::getLastVPXImage()
{
QMutexLocker lock(&mutex);

View File

@ -60,7 +60,6 @@ public:
// VideoSource interface
virtual void subscribe();
virtual void unsubscribe();
virtual VideoFrame::ColorFormat getColorFormat();
protected:
Camera();

View File

@ -56,7 +56,6 @@ ChatForm::ChatForm(Friend* chatFriend)
headTextLayout->setSpacing(0);
connect(Core::getInstance(), &Core::fileSendStarted, this, &ChatForm::startFileSend);
connect(Core::getInstance(), &Core::videoFrameReceived, netcam, &NetCamView::updateDisplay);
connect(sendButton, &QPushButton::clicked, this, &ChatForm::onSendTriggered);
connect(fileButton, &QPushButton::clicked, this, &ChatForm::onAttachClicked);
connect(callButton, &QPushButton::clicked, this, &ChatForm::onCallTriggered);
@ -256,6 +255,7 @@ void ChatForm::onAvCancel(int FriendId, int)
videoButton->style()->polish(videoButton);
connect(callButton, SIGNAL(clicked()), this, SLOT(onCallTriggered()));
connect(videoButton, SIGNAL(clicked()), this, SLOT(onVideoCallTriggered()));
netcam->setSource(nullptr);
netcam->hide();
}
@ -275,6 +275,7 @@ void ChatForm::onAvEnd(int FriendId, int)
videoButton->style()->polish(videoButton);
connect(callButton, SIGNAL(clicked()), this, SLOT(onCallTriggered()));
connect(videoButton, SIGNAL(clicked()), this, SLOT(onVideoCallTriggered()));
netcam->setSource(nullptr);
netcam->hide();
}
@ -304,7 +305,7 @@ void ChatForm::onAvRinging(int FriendId, int CallId, bool video)
}
}
void ChatForm::onAvStarting(int FriendId, int, bool video)
void ChatForm::onAvStarting(int FriendId, int callId, bool video)
{
if (FriendId != f->friendId)
return;
@ -318,6 +319,7 @@ void ChatForm::onAvStarting(int FriendId, int, bool video)
videoButton->setObjectName("red");
videoButton->style()->polish(videoButton);
connect(videoButton, SIGNAL(clicked()), this, SLOT(onHangupCallTriggered()));
netcam->setSource(Core::getInstance()->getVideoSourceFromCall(callId));
netcam->show();
}
else
@ -369,6 +371,7 @@ void ChatForm::onAvRequestTimeout(int FriendId, int)
videoButton->disconnect();
connect(callButton, SIGNAL(clicked()), this, SLOT(onCallTriggered()));
connect(videoButton, SIGNAL(clicked()), this, SLOT(onVideoCallTriggered()));
netcam->setSource(nullptr);
netcam->hide();
}
@ -390,6 +393,7 @@ void ChatForm::onAvPeerTimeout(int FriendId, int)
videoButton->disconnect();
connect(callButton, SIGNAL(clicked()), this, SLOT(onCallTriggered()));
connect(videoButton, SIGNAL(clicked()), this, SLOT(onVideoCallTriggered()));
netcam->setSource(nullptr);
netcam->hide();
}
@ -401,6 +405,7 @@ void ChatForm::onAvMediaChange(int, int, bool video)
}
else
{
netcam->setSource(nullptr);
netcam->hide();
}
}
@ -450,6 +455,7 @@ void ChatForm::onCancelCallTriggered()
videoButton->disconnect();
connect(callButton, SIGNAL(clicked()), this, SLOT(onCallTriggered()));
connect(videoButton, SIGNAL(clicked()), this, SLOT(onVideoCallTriggered()));
netcam->setSource(nullptr);
netcam->hide();
emit cancelCall(callId, f->friendId);
}

View File

@ -50,7 +50,7 @@ public slots:
void onAvCancel(int FriendId, int CallId);
void onAvEnd(int FriendId, int CallId);
void onAvRinging(int FriendId, int CallId, bool video);
void onAvStarting(int FriendId, int CallId, bool video);
void onAvStarting(int FriendId, int callId, bool video);
void onAvEnding(int FriendId, int CallId);
void onAvRequestTimeout(int FriendId, int CallId);
void onAvPeerTimeout(int FriendId, int CallId);

View File

@ -23,8 +23,6 @@ AVForm::AVForm() :
{
bodyUI = new Ui::AVSettings;
bodyUI->setupUi(this);
bodyUI->CamVideoSurface->setSource(Camera::getInstance());
}
AVForm::~AVForm()
@ -34,6 +32,8 @@ AVForm::~AVForm()
void AVForm::present()
{
bodyUI->CamVideoSurface->setSource(Camera::getInstance());
bodyUI->videoModescomboBox->clear();
QList<QSize> res = Camera::getInstance()->getSupportedResolutions();
for (QSize r : res)
@ -73,3 +73,8 @@ void AVForm::on_videoModescomboBox_currentIndexChanged(const QString &arg1)
Camera::getInstance()->setResolution(QSize(w,h));
}
void AVForm::hideEvent(QHideEvent *)
{
bodyUI->CamVideoSurface->setSource(nullptr);
}

View File

@ -45,6 +45,8 @@ private slots:
void on_HueSlider_sliderMoved(int position);
void on_videoModescomboBox_currentIndexChanged(const QString &arg1);
virtual void hideEvent(QHideEvent*);
private:
Ui::AVSettings *bodyUI;
VideoSurface* camView;

View File

@ -15,102 +15,29 @@
*/
#include "netcamview.h"
<<<<<<< HEAD:src/widget/netcamview.cpp
#include "src/core.h"
=======
#include "core.h"
#include "widget/videosurface.h"
>>>>>>> NetVideoSource, YUV shader, fixes:widget/netcamview.cpp
#include <QLabel>
#include <QHBoxLayout>
static inline void fromYCbCrToRGB(
uint8_t Y, uint8_t Cb, uint8_t Cr,
uint8_t& R, uint8_t& G, uint8_t& B)
{
int r = Y + ((1436 * (Cr - 128)) >> 10),
g = Y - ((354 * (Cb - 128) + 732 * (Cr - 128)) >> 10),
b = Y + ((1814 * (Cb - 128)) >> 10);
if(r < 0) {
r = 0;
} else if(r > 255) {
r = 255;
}
if(g < 0) {
g = 0;
} else if(g > 255) {
g = 255;
}
if(b < 0) {
b = 0;
} else if(b > 255) {
b = 255;
}
R = static_cast<uint8_t>(r);
G = static_cast<uint8_t>(g);
B = static_cast<uint8_t>(b);
}
NetCamView::NetCamView(QWidget* parent)
: QWidget(parent), displayLabel{new QLabel},
mainLayout{new QHBoxLayout()}
: QWidget(parent)
, mainLayout{new QHBoxLayout()}
{
setLayout(mainLayout);
setWindowTitle("Tox video");
setMinimumSize(320,240);
displayLabel->setAlignment(Qt::AlignCenter);
videoSurface = new VideoSurface(this);
mainLayout->addWidget(displayLabel);
mainLayout->addWidget(videoSurface);
}
void NetCamView::updateDisplay(vpx_image* frame)
void NetCamView::setSource(VideoSource *s)
{
if (!frame->w || !frame->h)
return;
Core* core = Core::getInstance();
core->increaseVideoBusyness();
img = convert(*frame);
vpx_img_free(frame);
displayLabel->setPixmap(QPixmap::fromImage(img).scaled(displayLabel->size(), Qt::KeepAspectRatio, Qt::SmoothTransformation));
core->decreaseVideoBusyness();
}
QImage NetCamView::convert(vpx_image& frame)
{
int w = frame.d_w, h = frame.d_h;
int bpl = frame.stride[VPX_PLANE_Y], cxbpl = frame.stride[VPX_PLANE_V];
QImage img(w, h, QImage::Format_RGB32);
uint8_t* yData = frame.planes[VPX_PLANE_Y];
uint8_t* uData = frame.planes[VPX_PLANE_V];
uint8_t* vData = frame.planes[VPX_PLANE_U];
for (int i = 0; i< h; i++)
{
uint32_t* scanline = (uint32_t*)img.scanLine(i);
for (int j=0; j < w; j++)
{
uint8_t Y = yData[i*bpl + j];
uint8_t U = uData[i/2*cxbpl + j/2];
uint8_t V = vData[i/2*cxbpl + j/2];
uint8_t R, G, B;
fromYCbCrToRGB(Y, U, V, R, G, B);
scanline[j] = (0xFF<<24) + (R<<16) + (G<<8) + B;
}
}
return img;
}
void NetCamView::resizeEvent(QResizeEvent *e)
{
Q_UNUSED(e)
displayLabel->setPixmap(QPixmap::fromImage(img).scaled(displayLabel->size(), Qt::KeepAspectRatio, Qt::SmoothTransformation));
videoSurface->setSource(s);
}

View File

@ -19,13 +19,10 @@
#include <QWidget>
class QCloseEvent;
class QShowEvent;
class QPainter;
class QLabel;
class QHBoxLayout;
class QImage;
class vpx_image;
class VideoSurface;
class VideoSource;
class NetCamView : public QWidget
{
@ -35,19 +32,11 @@ public:
NetCamView(QWidget *parent=0);
public slots:
void updateDisplay(vpx_image* frame);
void setSource(VideoSource* s);
private:
static QImage convert(vpx_image& frame);
protected:
void resizeEvent(QResizeEvent *e);
private:
QLabel *displayLabel;
QImage lastFrame;
QHBoxLayout* mainLayout;
QImage img;
VideoSurface* videoSurface;
};
#endif // NETCAMVIEW_H

View File

@ -27,7 +27,6 @@ VideoSurface::VideoSurface(QWidget* parent)
: QGLWidget(QGLFormat(QGL::SampleBuffers | QGL::SingleBuffer), parent)
, source(nullptr)
, pbo{nullptr, nullptr}
, bgrProgramm(nullptr)
, textureId(0)
, pboAllocSize(0)
, hasSubscribed(false)
@ -36,10 +35,10 @@ VideoSurface::VideoSurface(QWidget* parent)
setAutoBufferSwap(false);
}
VideoSurface::VideoSurface(VideoSource *Source, QWidget* parent)
VideoSurface::VideoSurface(VideoSource *source, QWidget* parent)
: VideoSurface(parent)
{
source = Source;
setSource(source);
}
VideoSurface::~VideoSurface()
@ -53,30 +52,17 @@ VideoSurface::~VideoSurface()
if (textureId != 0)
glDeleteTextures(1, &textureId);
if (source && hasSubscribed)
source->unsubscribe();
unsubscribe();
}
void VideoSurface::setSource(VideoSource *src)
{
source = src;
}
if (source == src)
return;
void VideoSurface::hideEvent(QHideEvent *ev)
{
unsubscribe();
QGLWidget::hideEvent(ev);
}
void VideoSurface::showEvent(QShowEvent *ev)
{
source = src;
subscribe();
QGLWidget::showEvent(ev);
}
QSize VideoSurface::sizeHint() const
{
return QGLWidget::sizeHint();
}
void VideoSurface::initializeGL()
@ -113,13 +99,33 @@ void VideoSurface::initializeGL()
bgrProgramm->bindAttributeLocation("vertices", 0);
bgrProgramm->link();
// shaders
yuvProgramm = new QOpenGLShaderProgram;
yuvProgramm->addShaderFromSourceCode(QOpenGLShader::Vertex,
"attribute vec4 vertices;"
"varying vec2 coords;"
"void main() {"
" gl_Position = vec4(vertices.xy,0.0,1.0);"
" coords = vertices.xy*vec2(0.5,0.5)+vec2(0.5,0.5);"
"}");
// brg frag-shader
yuvProgramm->addShaderFromSourceCode(QOpenGLShader::Fragment,
"uniform sampler2D texture0;"
"varying vec2 coords;"
"void main() {"
" vec3 yuv = texture2D(texture0,coords*vec2(1.0, -1.0)) - vec3(0,0.5,0.5);"
" vec3 rgb = mat3(1,1,1,0,-0.21482,2.12798,1.28033,-0.38059,0) * yuv;"
" gl_FragColor = vec4(rgb,1);"
"}");
yuvProgramm->bindAttributeLocation("vertices", 0);
yuvProgramm->link();
}
void VideoSurface::paintGL()
{
if (!source)
return;
mutex.lock();
VideoFrame currFrame = frame;
mutex.unlock();
@ -140,25 +146,22 @@ void VideoSurface::paintGL()
if (!currFrame.isNull())
{
QElapsedTimer timer;
timer.start();
pboIndex = (pboIndex + 1) % 2;
int nextPboIndex = (pboIndex + 1) % 2;
if (pboAllocSize != currFrame.data.size())
if (pboAllocSize != currFrame.frameData.size())
{
qDebug() << "VideoSurface: Resize pbo " << currFrame.data.size() << "bytes (before" << pboAllocSize << ")";
qDebug() << "VideoSurface: Resize pbo " << currFrame.frameData.size() << "bytes (before" << pboAllocSize << ")";
pbo[0]->bind();
pbo[0]->allocate(currFrame.data.size());
pbo[0]->allocate(currFrame.frameData.size());
pbo[0]->release();
pbo[1]->bind();
pbo[1]->allocate(currFrame.data.size());
pbo[1]->allocate(currFrame.frameData.size());
pbo[1]->release();
pboAllocSize = currFrame.data.size();
pboAllocSize = currFrame.frameData.size();
}
@ -172,7 +175,7 @@ void VideoSurface::paintGL()
pbo[nextPboIndex]->bind();
void* ptr = pbo[nextPboIndex]->map(QOpenGLBuffer::WriteOnly);
if (ptr)
memcpy(ptr, currFrame.data.data(), currFrame.data.size());
memcpy(ptr, currFrame.frameData.data(), currFrame.frameData.size());
pbo[nextPboIndex]->unmap();
pbo[nextPboIndex]->release();
@ -206,17 +209,36 @@ void VideoSurface::paintGL()
glViewport((width() - w)*0.5f, 0, w, height());
}
bgrProgramm->bind();
bgrProgramm->setAttributeArray(0, GL_FLOAT, values, 2);
bgrProgramm->enableAttributeArray(0);
QOpenGLShaderProgram* programm = nullptr;
switch (frame.format)
{
case VideoFrame::YUV:
programm = yuvProgramm;
break;
case VideoFrame::BGR:
programm = bgrProgramm;
break;
}
if (programm)
{
programm->bind();
programm->setAttributeArray(0, GL_FLOAT, values, 2);
programm->enableAttributeArray(0);
}
glBindTexture(GL_TEXTURE_2D, textureId);
//draw fullscreen quad
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glBindTexture(GL_TEXTURE_2D, 0);
bgrProgramm->disableAttributeArray(0);
bgrProgramm->release();
if (programm)
{
programm->disableAttributeArray(0);
programm->release();
}
}
void VideoSurface::subscribe()

View File

@ -19,7 +19,7 @@
#include <QGLWidget>
#include <QMutex>
#include "videosource.h"
#include "src/videosource.h"
class QOpenGLBuffer;
class QOpenGLShaderProgram;
@ -33,10 +33,7 @@ public:
VideoSurface(VideoSource* source, QWidget* parent=0);
~VideoSurface();
void setSource(VideoSource* src);
virtual void hideEvent(QHideEvent* ev);
virtual void showEvent(QShowEvent* ev);
virtual QSize sizeHint() const;
void setSource(VideoSource* src); //NULL is a valid option
// QGLWidget interface
protected:
@ -46,7 +43,6 @@ protected:
void subscribe();
void unsubscribe();
private slots:
void onNewFrameAvailable(const VideoFrame newFrame);
@ -54,6 +50,7 @@ private:
VideoSource* source;
QOpenGLBuffer* pbo[2];
QOpenGLShaderProgram* bgrProgramm;
QOpenGLShaderProgram* yuvProgramm;
GLuint textureId;
int pboAllocSize;
QSize res;