simplified VideoSource, interface, BRG->YUV conversion changes + more

src/cameraworker.cpp

@@ -24,6 +24,7 @@ CameraWorker::CameraWorker(int index)
     , camIndex(index)
     , refCount(0)
 {
+    qRegisterMetaType<VideoFrame>();
 }
 
 void CameraWorker::onStart()
@@ -147,42 +148,15 @@ void CameraWorker::doWork()
     if (!cam.isOpened())
         return;
 
-    if (queue.size() > 3)
-        return;
-
     if (!cam.read(frame))
     {
-        cam.release();
+        qDebug() << "CameraWorker: Cannot read frame";
-        qDebug() << "CameraWorker: received empty frame -> closing";
         return;
     }
 
-    mutex.lock();
+    QByteArray frameData(reinterpret_cast<char*>(frame.data), frame.total() * frame.channels());
-    queue.enqueue(frame);
-    mutex.unlock();
 
-    emit newFrameAvailable();
+    emit newFrameAvailable(VideoFrame{frameData, QSize(frame.cols, frame.rows), VideoFrame::BGR});
-}
-
-bool CameraWorker::hasFrame()
-{
-    mutex.lock();
-    bool b = !queue.empty();
-    mutex.unlock();
-
-    return b;
-}
-
-cv::Mat3b CameraWorker::dequeueFrame()
-{
-    if (queue.isEmpty())
-        return cv::Mat3b();
-
-    mutex.lock();
-    cv::Mat3b f = queue.dequeue();
-    mutex.unlock();
-
-    return f;
 }
 
 void CameraWorker::suspend()

src/cameraworker.h

@@ -25,6 +25,7 @@
 #include <QSize>
 
 #include "opencv2/opencv.hpp"
+#include "videosource.h"
 
 class QTimer;
 
@@ -34,8 +35,6 @@ class CameraWorker : public QObject
 public:
     CameraWorker(int index);
     void doWork();
-    bool hasFrame();
-    cv::Mat3b dequeueFrame();
 
     void suspend();
     void resume();
@@ -48,7 +47,7 @@ public slots:
 
 signals:
     void started();
-    void newFrameAvailable();
+    void newFrameAvailable(const VideoFrame frame);
     void resProbingFinished(QList<QSize> res);
 
 private slots:

src/videosource.h

@@ -3,24 +3,54 @@
 
 #include <QObject>
 #include <QSize>
+#include <QRgb>
+
+struct VideoFrame
+{
+    enum ColorFormat
+    {
+        BGR,
+        YUV,
+    };
+
+    QByteArray data;
+    QSize resolution;
+    ColorFormat format;
+
+    void setNull()
+    {
+        data = QByteArray();
+    }
+
+    bool isNull()
+    {
+        return data.isEmpty();
+    }
+
+    // assumes format is BGR
+    QRgb getPixel(int x, int y)
+    {
+        char b = data.data()[resolution.width() * 3 * y + x * 3 + 0];
+        char g = data.data()[resolution.width() * 3 * y + x * 3 + 1];
+        char r = data.data()[resolution.width() * 3 * y + x * 3 + 2];
+
+        return qRgb(r, g, b);
+    }
+};
+
+Q_DECLARE_METATYPE(VideoFrame)
 
 class VideoSource : public QObject
 {
     Q_OBJECT
+
 public:
-    virtual void* getData() = 0; // a pointer to a frame
-    virtual int getDataSize() = 0; // size of a frame in bytes
-
-    virtual void lock() = 0; // locks a frame so that it can't change
-    virtual void unlock() = 0;
-
-    virtual QSize resolution() = 0; // resolution of a frame
-
     virtual void subscribe() = 0;
     virtual void unsubscribe() = 0;
+    virtual VideoFrame::ColorFormat getColorFormat() = 0;
 
 signals:
-    void frameAvailable();
+    void frameAvailable(const VideoFrame frame);
 
 };
 

src/widget/camera.cpp

@@ -19,6 +19,7 @@
 #include "src/cameraworker.h"
 #include <QDebug>
 #include <QThread>
+#include <QMutexLocker>
 
 Camera* Camera::instance = nullptr;
 
@@ -70,52 +71,58 @@ void Camera::unsubscribe()
     }
 }
 
+VideoFrame::ColorFormat Camera::getColorFormat()
+{
+    return VideoFrame::BGR;
+}
+
 vpx_image Camera::getLastVPXImage()
 {
-    lock();
+    QMutexLocker lock(&mutex);
-    vpx_image img;
-    int w = currFrame.size().width, h = currFrame.size().height;
-    vpx_img_alloc(&img, VPX_IMG_FMT_I420, w, h, 1); // I420 == YUV420P, same as YV12 with U and V switched
 
-    size_t i=0, j=0;
+    vpx_image img;
-    for( int line = 0; line < h; ++line )
+
-    {
+    if (currFrame.isNull())
-        const cv::Vec3b *srcrow = currFrame[line];
-        if( !(line % 2) )
     {
+        img.w = 0;
+        img.h = 0;
+        return img;
+    }
+
+    int w = currFrame.resolution.width();
+    int h = currFrame.resolution.height();
+
+    // I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes."
+    // http://fourcc.org/yuv.php#IYUV
+    vpx_img_alloc(&img, VPX_IMG_FMT_VPXI420, w, h, 1);
+
     for (int x = 0; x < w; x += 2)
     {
-                uint8_t r = srcrow[x][2];
+        for (int y = 0; y < h; y += 2)
-                uint8_t g = srcrow[x][1];
-                uint8_t b = srcrow[x][0];
-
-                img.planes[VPX_PLANE_Y][i] = ((66*r + 129*g + 25*b) >> 8) + 16;
-                img.planes[VPX_PLANE_V][j] = ((-38*r + -74*g + 112*b) >> 8) + 128;
-                img.planes[VPX_PLANE_U][j] = ((112*r + -94*g + -18*b) >> 8) + 128;
-                i++;
-                j++;
-
-                r = srcrow[x+1][2];
-                g = srcrow[x+1][1];
-                b = srcrow[x+1][0];
-                img.planes[VPX_PLANE_Y][i] = ((66*r + 129*g + 25*b) >> 8) + 16;
-                i++;
-            }
-        }
-        else
         {
-            for( int x = 0; x < w; x += 1 )
+            QRgb p1 = currFrame.getPixel(x, y);
-            {
+            QRgb p2 = currFrame.getPixel(x + 1, y);
-                uint8_t r = srcrow[x][2];
+            QRgb p3 = currFrame.getPixel(x, y + 1);
-                uint8_t g = srcrow[x][1];
+            QRgb p4 = currFrame.getPixel(x + 1, y + 1);
-                uint8_t b = srcrow[x][0];
 
-                img.planes[VPX_PLANE_Y][i] = ((66*r + 129*g + 25*b) >> 8) + 16;
+            img.planes[VPX_PLANE_Y][x + y * w] = ((66 * qRed(p1) + 129 * qGreen(p1) + 25 * qBlue(p1)) >> 8) + 16;
-                i++;
+            img.planes[VPX_PLANE_Y][x + 1 + y * w] = ((66 * qRed(p2) + 129 * qGreen(p2) + 25 * qBlue(p2)) >> 8) + 16;
+            img.planes[VPX_PLANE_Y][x + (y + 1) * w] = ((66 * qRed(p3) + 129 * qGreen(p3) + 25 * qBlue(p3)) >> 8) + 16;
+            img.planes[VPX_PLANE_Y][x + 1 + (y + 1) * w] = ((66 * qRed(p4) + 129 * qGreen(p4) + 25 * qBlue(p4)) >> 8) + 16;
+
+            if (!(x % 2) && !(y % 2))
+            {
+                // TODO: consider p1 to p4?
+
+                int i = x / 2;
+                int j = y / 2;
+
+                img.planes[VPX_PLANE_U][i + j * w / 2] = ((112 * qRed(p1) + -94 * qGreen(p1) + -18 * qBlue(p1)) >> 8) + 128;
+                img.planes[VPX_PLANE_V][i + j * w / 2] = ((-38 * qRed(p1) + -74 * qGreen(p1) + 112 * qBlue(p1)) >> 8) + 128;
             }
         }
     }
-    unlock();
+
     return img;
 }
 
@@ -190,9 +197,13 @@ double Camera::getProp(Camera::Prop prop)
     return 0.0;
 }
 
-void Camera::onNewFrameAvailable()
+void Camera::onNewFrameAvailable(const VideoFrame frame)
 {
-    emit frameAvailable();
+    emit frameAvailable(frame);
+
+    mutex.lock();
+    currFrame = frame;
+    mutex.unlock();
 }
 
 void Camera::onResProbingFinished(QList<QSize> res)
@@ -200,34 +211,6 @@ void Camera::onResProbingFinished(QList<QSize> res)
     resolutions = res;
 }
 
-void *Camera::getData()
-{
-    return currFrame.data;
-}
-
-int Camera::getDataSize()
-{
-    return currFrame.total() * currFrame.channels();
-}
-
-void Camera::lock()
-{
-    mutex.lock();
-
-    if (worker->hasFrame())
-        currFrame = worker->dequeueFrame();
-}
-
-void Camera::unlock()
-{
-    mutex.unlock();
-}
-
-QSize Camera::resolution()
-{
-    return QSize(currFrame.cols, currFrame.rows);
-}
-
 Camera* Camera::getInstance()
 {
     if (!instance)

src/widget/camera.h

@@ -58,20 +58,16 @@ public:
     double getProp(Prop prop);
 
     // VideoSource interface
-    virtual void *getData();
-    virtual int getDataSize();
-    virtual void lock();
-    virtual void unlock();
-    virtual QSize resolution();
     virtual void subscribe();
     virtual void unsubscribe();
+    virtual VideoFrame::ColorFormat getColorFormat();
 
 protected:
     Camera();
 
 private:
     int refcount; ///< Number of users suscribed to the camera
-    cv::Mat3b currFrame;
+    VideoFrame currFrame;
     QMutex mutex;
 
     QThread* workerThread;
@@ -83,7 +79,7 @@ private:
 
 private slots:
     void onWorkerStarted();
-    void onNewFrameAvailable();
+    void onNewFrameAvailable(const VideoFrame frame);
     void onResProbingFinished(QList<QSize> res);
 
 };

src/widget/form/settings/avsettings.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>390</width>
+    <width>382</width>
-    <height>387</height>
+    <height>379</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -54,7 +54,7 @@
    <item>
     <widget class="QGroupBox" name="videoGroup">
      <property name="title">
-      <string>Video settings</string>
+      <string>Video Settings</string>
      </property>
      <layout class="QHBoxLayout" name="horizontalLayout">
       <item>
@@ -90,7 +90,7 @@
         <item row="0" column="0">
          <widget class="QLabel" name="label">
           <property name="text">
-           <string>Modes</string>
+           <string>Resolution</string>
           </property>
          </widget>
         </item>

src/widget/videosurface.cpp

@@ -21,19 +21,19 @@
 #include <QOpenGLBuffer>
 #include <QOpenGLShaderProgram>
 #include <QDebug>
+#include <QElapsedTimer>
 
 VideoSurface::VideoSurface(QWidget* parent)
-    : QGLWidget(QGLFormat(QGL::SampleBuffers), parent)
+    : QGLWidget(QGLFormat(QGL::SampleBuffers | QGL::SingleBuffer), parent)
     , source(nullptr)
-    , pbo(nullptr)
+    , pbo{nullptr, nullptr}
-    , program(nullptr)
+    , bgrProgramm(nullptr)
     , textureId(0)
     , pboAllocSize(0)
-    , uploadFrame(false)
     , hasSubscribed(false)
-    , lastWidth(0)
+    , pboIndex(0)
 {
-
+    setAutoBufferSwap(false);
 }
 
 VideoSurface::VideoSurface(VideoSource *Source, QWidget* parent)
@@ -44,8 +44,11 @@ VideoSurface::VideoSurface(VideoSource *Source, QWidget* parent)
 
 VideoSurface::~VideoSurface()
 {
-    if (pbo)
+    if (pbo[0])
-        delete pbo;
+    {
+        delete pbo[0];
+        delete pbo[1];
+    }
 
     if (textureId != 0)
         glDeleteTextures(1, &textureId);
@@ -61,25 +64,13 @@ void VideoSurface::setSource(VideoSource *src)
 
 void VideoSurface::hideEvent(QHideEvent *ev)
 {
-    if (source && hasSubscribed)
+    unsubscribe();
-    {
-        source->unsubscribe();
-        hasSubscribed = false;
-        disconnect(source, &VideoSource::frameAvailable, this, &VideoSurface::updateGL);
-    }
-
     QGLWidget::hideEvent(ev);
 }
 
 void VideoSurface::showEvent(QShowEvent *ev)
 {
-    if (source && !hasSubscribed)
+    subscribe();
-    {
-        source->subscribe();
-        hasSubscribed = true;
-        connect(source, &VideoSource::frameAvailable, this, &VideoSurface::updateGL);
-    }
-
     QGLWidget::showEvent(ev);
 }
 
@@ -89,27 +80,21 @@ QSize VideoSurface::sizeHint() const
 }
 
 void VideoSurface::initializeGL()
-{
-
-}
-
-void VideoSurface::paintGL()
-{
-    if (!source)
-        return;
-
-    if (!pbo)
 {
     qDebug() << "VideoSurface: Init";
 
     // pbo
-        pbo = new QOpenGLBuffer(QOpenGLBuffer::PixelUnpackBuffer);
+    pbo[0] = new QOpenGLBuffer(QOpenGLBuffer::PixelUnpackBuffer);
-        pbo->setUsagePattern(QOpenGLBuffer::StreamDraw);
+    pbo[0]->setUsagePattern(QOpenGLBuffer::StreamDraw);
-        pbo->create();
+    pbo[0]->create();
+
+    pbo[1] = new QOpenGLBuffer(QOpenGLBuffer::PixelUnpackBuffer);
+    pbo[1]->setUsagePattern(QOpenGLBuffer::StreamDraw);
+    pbo[1]->create();
 
     // shaders
-        program = new QOpenGLShaderProgram;
+    bgrProgramm = new QOpenGLShaderProgram;
-        program->addShaderFromSourceCode(QOpenGLShader::Vertex,
+    bgrProgramm->addShaderFromSourceCode(QOpenGLShader::Vertex,
                                      "attribute vec4 vertices;"
                                      "varying vec2 coords;"
                                      "void main() {"
@@ -118,7 +103,7 @@ void VideoSurface::paintGL()
                                      "}");
 
     // brg frag-shader
-        program->addShaderFromSourceCode(QOpenGLShader::Fragment,
+    bgrProgramm->addShaderFromSourceCode(QOpenGLShader::Fragment,
                                      "uniform sampler2D texture0;"
                                      "varying vec2 coords;"
                                      "void main() {"
@@ -126,14 +111,22 @@ void VideoSurface::paintGL()
                                      "    gl_FragColor = vec4(color.b, color.g, color.r, 1);"
                                      "}");
 
-        program->bindAttributeLocation("vertices", 0);
+    bgrProgramm->bindAttributeLocation("vertices", 0);
-        program->link();
+    bgrProgramm->link();
 }
 
-    if (res != source->resolution())
+void VideoSurface::paintGL()
 {
-        qDebug() << "VideoSurface: Change resolution from " << res << " to " << source->resolution();
+    if (!source)
-        res = source->resolution();
+        return;
+
+    mutex.lock();
+    VideoFrame currFrame = frame;
+    mutex.unlock();
+
+    if (res != currFrame.resolution)
+    {
+        res = currFrame.resolution;
 
         // a texture used to render the pbo (has the match the pixelformat of the source)
         glGenTextures(1,&textureId);
@@ -145,53 +138,58 @@ void VideoSurface::paintGL()
     }
 
 
-    if (uploadFrame)
+    if (!currFrame.isNull())
     {
-        source->lock();
+        QElapsedTimer timer;
-        void* frame = source->getData();
+        timer.start();
-        int frameBytes = source->getDataSize();
 
-        if (pboAllocSize != frameBytes && frameBytes > 0)
+        pboIndex = (pboIndex + 1) % 2;
+        int nextPboIndex = (pboIndex + 1) % 2;
+
+        if (pboAllocSize != currFrame.data.size())
         {
-            qDebug() << "VideoSurface: Resize pbo " << frameBytes << "bytes (before" << pboAllocSize << ")";
+            qDebug() << "VideoSurface: Resize pbo " << currFrame.data.size() << "bytes (before" << pboAllocSize << ")";
 
-            pbo->bind();
+            pbo[0]->bind();
-            pbo->allocate(frameBytes);
+            pbo[0]->allocate(currFrame.data.size());
-            pbo->release();
+            pbo[0]->release();
 
-            pboAllocSize = frameBytes;
+            pbo[1]->bind();
+            pbo[1]->allocate(currFrame.data.size());
+            pbo[1]->release();
+
+            pboAllocSize = currFrame.data.size();
         }
 
-        // transfer data
-        pbo->bind();
 
-        void* ptr = pbo->map(QOpenGLBuffer::WriteOnly);
+        pbo[pboIndex]->bind();
-        if (ptr && frame)
-            memcpy(ptr, frame, frameBytes);
-        pbo->unmap();
-
-        source->unlock();
-
-        //transfer pbo data to texture
         glBindTexture(GL_TEXTURE_2D, textureId);
         glTexSubImage2D(GL_TEXTURE_2D,0,0,0, res.width(), res.height(), GL_RGB, GL_UNSIGNED_BYTE, 0);
-        glBindTexture(GL_TEXTURE_2D, 0);
+        pbo[pboIndex]->unmap();
+        pbo[pboIndex]->release();
 
-        pbo->release();
+        // transfer data
+        pbo[nextPboIndex]->bind();
+        void* ptr = pbo[nextPboIndex]->map(QOpenGLBuffer::WriteOnly);
+        if (ptr)
+            memcpy(ptr, currFrame.data.data(), currFrame.data.size());
+        pbo[nextPboIndex]->unmap();
+        pbo[nextPboIndex]->release();
 
-        uploadFrame = false;
+        mutex.lock();
+        frame.setNull();
+        mutex.unlock();
     }
 
     // render pbo
-    float values[] = {
+    static float values[] = {
         -1, -1,
         1, -1,
         -1, 1,
         1, 1
     };
 
-    program->setAttributeArray(0, GL_FLOAT, values, 2);
+    // background
-
     glClearColor(0, 0, 0, 1);
     glClear(GL_COLOR_BUFFER_BIT);
 
@@ -208,23 +206,46 @@ void VideoSurface::paintGL()
         glViewport((width() - w)*0.5f, 0, w, height());
     }
 
-    program->bind();
+    bgrProgramm->bind();
-    program->enableAttributeArray(0);
+    bgrProgramm->setAttributeArray(0, GL_FLOAT, values, 2);
-
+    bgrProgramm->enableAttributeArray(0);
     glBindTexture(GL_TEXTURE_2D, textureId);
 
     //draw fullscreen quad
     glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
     glBindTexture(GL_TEXTURE_2D, 0);
-
+    bgrProgramm->disableAttributeArray(0);
-    program->disableAttributeArray(0);
+    bgrProgramm->release();
-    program->release();
 }
 
-void VideoSurface::updateGL()
+void VideoSurface::subscribe()
 {
-    uploadFrame = true;
+    if (source && !hasSubscribed)
-    QGLWidget::updateGL();
+    {
+        source->subscribe();
+        hasSubscribed = true;
+        connect(source, &VideoSource::frameAvailable, this, &VideoSurface::onNewFrameAvailable);
+    }
+}
+
+void VideoSurface::unsubscribe()
+{
+    if (source && hasSubscribed)
+    {
+        source->unsubscribe();
+        hasSubscribed = false;
+        disconnect(source, &VideoSource::frameAvailable, this, &VideoSurface::onNewFrameAvailable);
+    }
+}
+
+void VideoSurface::onNewFrameAvailable(const VideoFrame newFrame)
+{
+    mutex.lock();
+    frame = newFrame;
+    mutex.unlock();
+
+    updateGL();
 }
 
 

src/widget/videosurface.h

@@ -18,11 +18,11 @@
 #define SELFCAMVIEW_H
 
 #include <QGLWidget>
+#include <QMutex>
+#include "videosource.h"
 
 class QOpenGLBuffer;
 class QOpenGLShaderProgram;
-class QTimer;
-class VideoSource;
 
 class VideoSurface : public QGLWidget
 {
@@ -42,20 +42,26 @@ public:
 protected:
     virtual void initializeGL();
     virtual void paintGL();
-    virtual void updateGL();
 
-    void update();
+    void subscribe();
+    void unsubscribe();
+
+
+private slots:
+    void onNewFrameAvailable(const VideoFrame newFrame);
 
 private:
     VideoSource* source;
-    QOpenGLBuffer* pbo;
+    QOpenGLBuffer* pbo[2];
-    QOpenGLShaderProgram* program;
+    QOpenGLShaderProgram* bgrProgramm;
     GLuint textureId;
     int pboAllocSize;
     QSize res;
-    bool uploadFrame;
     bool hasSubscribed;
-    mutable int lastWidth;
+
+    QMutex mutex;
+    VideoFrame frame;
+    int pboIndex;
 };
 
 #endif // SELFCAMVIEW_H