camera: initial PBO support, mode probing

qtox.pro

@@ -20,7 +20,7 @@
 #    See the COPYING file for more details.
 
 
-QT       += core gui network xml
+QT       += core gui network xml opengl
 greaterThan(QT_MAJOR_VERSION, 4): QT += widgets
 
 TARGET    = qtox
@@ -64,7 +64,7 @@ win32 {
             INSTALLS += target
             LIBS += -L$$PWD/libs/lib/ -Wl,-Bstatic -ltoxcore -ltoxav -lsodium -Wl,-Bdynamic -lopus -lvpx -lopenal -lopencv_core -lopencv_highgui
         } else {
-            LIBS += -L$$PWD/libs/lib/ -ltoxcore -ltoxav -lvpx -lopenal -lopencv_core -lopencv_highgui
+            LIBS += -L$$PWD/libs/lib/ -ltoxcore -ltoxav -lvpx -lopenal -lopencv_core -lopencv_highgui -lopencv_imgproc
         }
 
         contains(JENKINS, YES) {

widget/camera.cpp

@@ -16,6 +16,7 @@
 
 #include "camera.h"
 #include "widget.h"
+#include <QDebug>
 
 using namespace cv;
 
@@ -50,21 +51,12 @@ Mat Camera::getLastFrame()
 {
     Mat frame;
     cam >> frame;
-    return frame;
-}
 
-QImage Camera::getLastImage()
-{
-    Mat3b src = getLastFrame();
-    QImage dest(src.cols, src.rows, QImage::Format_ARGB32);
-    for (int y = 0; y < src.rows; ++y)
-    {
-            const cv::Vec3b *srcrow = src[y];
-            QRgb *destrow = (QRgb*)dest.scanLine(y);
-            for (int x = 0; x < src.cols; ++x)
-                    destrow[x] = qRgba(srcrow[x][2], srcrow[x][1], srcrow[x][0], 255);
-    }
-    return dest;
+    Mat out;
+    if (!frame.empty())
+        cv::cvtColor(frame, out, CV_BGR2RGB);
+
+    return out;
 }
 
 vpx_image Camera::getLastVPXImage()
@@ -115,6 +107,69 @@ vpx_image Camera::getLastVPXImage()
     return img;
 }
 
+QList<Camera::VideoMode> Camera::getVideoModes()
+{
+    // probe resolutions
+    QList<QSize> resolutions = {
+        QSize( 160, 120), // QQVGA
+        QSize( 320, 240), // HVGA
+        QSize(1024, 768), // XGA
+        QSize( 432, 240), // WQVGA
+        QSize( 640, 360), // nHD
+    };
+
+    QList<VideoMode> modes;
+
+    for (QSize res : resolutions)
+    {
+        cam.set(CV_CAP_PROP_FRAME_WIDTH, res.width());
+        cam.set(CV_CAP_PROP_FRAME_HEIGHT, res.height());
+
+        double w = cam.get(CV_CAP_PROP_FRAME_WIDTH);
+        double h = cam.get(CV_CAP_PROP_FRAME_HEIGHT);
+
+        if (w == res.width() && h == res.height())
+        {
+            modes.append({res, 60}); // assume 60fps for now
+        }
+    }
+
+    return modes;
+}
+
+Camera::VideoMode Camera::getBestVideoMode()
+{
+    int bestScore = 0;
+    VideoMode bestMode;
+
+    for (VideoMode mode : getVideoModes())
+    {
+        int score = mode.res.width() * mode.res.height();
+
+        if (score > bestScore)
+        {
+            bestScore = score;
+            bestMode = mode;
+        }
+    }
+
+    return bestMode;
+}
+
+void Camera::setVideoMode(Camera::VideoMode mode)
+{
+    if (cam.isOpened())
+    {
+        cam.set(CV_CAP_PROP_FRAME_WIDTH, mode.res.width());
+        cam.set(CV_CAP_PROP_FRAME_HEIGHT, mode.res.height());
+    }
+}
+
+Camera::VideoMode Camera::getVideoMode()
+{
+    return VideoMode{QSize(cam.get(CV_CAP_PROP_FRAME_WIDTH), cam.get(CV_CAP_PROP_FRAME_HEIGHT)), 60};
+}
+
 Camera* Camera::getInstance()
 {
     return Widget::getInstance()->getCamera();

widget/camera.h

@@ -18,6 +18,7 @@
 #define CAMERA_H
 
 #include <QImage>
+#include <QList>
 #include "vpx/vpx_image.h"
 #include "opencv2/opencv.hpp"
 
@@ -30,14 +31,24 @@
 class Camera
 {
 public:
+    struct VideoMode {
+        QSize res;
+        double fps;
+    };
+
     Camera();
     static Camera* getInstance(); ///< Returns the global widget's Camera instance
     void suscribe(); ///< Call this once before trying to get frames
     void unsuscribe(); ///< Call this once when you don't need frames anymore
     cv::Mat getLastFrame(); ///< Get the last captured frame
-    QImage getLastImage(); ///< Convert the last frame to a QImage (can be expensive !)
     vpx_image getLastVPXImage(); ///< Convert the last frame to a vpx_image (can be expensive !)
 
+    QList<VideoMode> getVideoModes();
+    VideoMode getBestVideoMode();
+
+    void setVideoMode(VideoMode mode);
+    VideoMode getVideoMode();
+
 private:
     int refcount; ///< Number of users suscribed to the camera
     cv::VideoCapture cam; ///< OpenCV camera capture opbject

widget/form/settings/avform.cpp

@@ -26,32 +26,16 @@ AVForm::AVForm(Camera* cam) :
 
     camView = new SelfCamView(cam, this);
     bodyUI->videoGroup->layout()->addWidget(camView);
-    camView->hide(); // hide by default
 
-    connect(bodyUI->testVideoBtn, &QPushButton::clicked, this, &AVForm::onTestVideoPressed);
+    auto modes = cam->getVideoModes();
+    for (Camera::VideoMode m : modes)
+    {
+        bodyUI->videoModescomboBox->addItem(QString("%1x%2").arg(QString::number(m.res.width())
+                                                                 ,QString::number(m.res.height())));
+    }
 }
 
 AVForm::~AVForm()
 {
     delete bodyUI;
 }
-
-void AVForm::showTestVideo()
-{
-    bodyUI->testVideoBtn->setText(tr("Hide video preview","On a button"));
-    camView->show();
-}
-
-void AVForm::closeTestVideo()
-{
-    bodyUI->testVideoBtn->setText(tr("Show video preview","On a button"));
-    camView->close();
-}
-
-void AVForm::onTestVideoPressed()
-{
-    if (camView->isVisible())
-        closeTestVideo();
-    else
-        showTestVideo();
-}

widget/form/settings/avform.h

@@ -37,16 +37,12 @@ public:
     ~AVForm();
 
 private slots:
-    void onTestVideoPressed();
 
 private:
     Ui::AVSettings *bodyUI;
 
     SelfCamView* camView;
-    
-    void showTestVideo();
-    void closeTestVideo();
-       
+
 };
 
 #endif

widget/form/settings/avsettings.ui

@@ -6,43 +6,57 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>400</width>
-    <height>300</height>
+    <width>398</width>
+    <height>298</height>
    </rect>
   </property>
   <property name="windowTitle">
    <string>Form</string>
   </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
+  <layout class="QVBoxLayout" name="verticalLayout_2">
    <item>
     <widget class="QGroupBox" name="videoGroup">
      <property name="title">
       <string>Video settings</string>
      </property>
-     <layout class="QVBoxLayout" name="verticalLayout_2">
+     <layout class="QVBoxLayout" name="verticalLayout">
       <item>
-       <widget class="QPushButton" name="testVideoBtn">
-        <property name="text">
-         <string>Show video preview</string>
+       <layout class="QFormLayout" name="formLayout">
+        <item row="0" column="0">
+         <widget class="QLabel" name="label">
+          <property name="text">
+           <string>Modes</string>
+          </property>
+         </widget>
+        </item>
+        <item row="0" column="1">
+         <widget class="QComboBox" name="videoModescomboBox">
+          <property name="sizePolicy">
+           <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+            <horstretch>0</horstretch>
+            <verstretch>0</verstretch>
+           </sizepolicy>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item>
+       <spacer name="verticalSpacer">
+        <property name="orientation">
+         <enum>Qt::Vertical</enum>
         </property>
-       </widget>
+        <property name="sizeHint" stdset="0">
+         <size>
+          <width>20</width>
+          <height>212</height>
+         </size>
+        </property>
+       </spacer>
       </item>
      </layout>
     </widget>
    </item>
-   <item>
-    <spacer name="verticalSpacer">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>40</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
   </layout>
  </widget>
  <resources/>

widget/selfcamview.cpp

@@ -16,54 +16,145 @@
 
 #include "selfcamview.h"
 #include "camera.h"
-#include <QCloseEvent>
-#include <QShowEvent>
 #include <QTimer>
-#include <QLabel>
-#include <QHBoxLayout>
 #include <opencv2/opencv.hpp>
-
-using namespace cv;
+#include <QOpenGLBuffer>
+#include <QOpenGLShaderProgram>
+#include <QDebug>
 
 SelfCamView::SelfCamView(Camera* Cam, QWidget* parent)
-    : QWidget(parent), displayLabel{new QLabel},
-      mainLayout{new QHBoxLayout()}, cam(Cam), updateDisplayTimer{new QTimer}
+    : QGLWidget(QGLFormat(QGL::SampleBuffers), parent)
+    , camera(Cam)
+    , pbo(nullptr)
+    , program(nullptr)
+    , textureId(0)
+    , pboAllocSize(0)
 {
-    setLayout(mainLayout);
-    setWindowTitle(SelfCamView::tr("Tox video test","Title of the window to test the video/webcam"));
-    setMinimumSize(320,240);
+    camera->suscribe();
+    camera->setVideoMode(camera->getBestVideoMode());
 
-    updateDisplayTimer->setInterval(5);
-    updateDisplayTimer->setSingleShot(false);
-
-    displayLabel->setAlignment(Qt::AlignCenter);
-
-    mainLayout->addWidget(displayLabel);
-
-    connect(updateDisplayTimer, SIGNAL(timeout()), this, SLOT(updateDisplay()));
+    setFixedSize(camera->getBestVideoMode().res);
 }
 
-void SelfCamView::closeEvent(QCloseEvent* event)
+SelfCamView::~SelfCamView()
 {
-    cam->unsuscribe();
-    updateDisplayTimer->stop();
-    event->accept();
+    if (pbo)
+        delete pbo;
+
+    if (textureId != 0)
+        glDeleteTextures(1, &textureId);
 }
 
-void SelfCamView::showEvent(QShowEvent* event)
+void SelfCamView::initializeGL()
 {
-    cam->suscribe();
-    updateDisplayTimer->start();
-    event->accept();
+    updateTimer = new QTimer(this);
+    updateTimer->setSingleShot(false);
+    updateTimer->setInterval(1000.0 / camera->getVideoMode().fps);
+
+    connect(updateTimer, &QTimer::timeout, this, &SelfCamView::update);
+    updateTimer->start();
 }
 
-void SelfCamView::updateDisplay()
+void SelfCamView::paintGL()
 {
-    displayLabel->setPixmap(QPixmap::fromImage(cam->getLastImage()).scaled(displayLabel->size(), Qt::KeepAspectRatio, Qt::SmoothTransformation));
+    cv::Mat3b frame = camera->getLastFrame();
+    int frameBytes = frame.total() * frame.channels();
+
+    if (!pbo)
+    {
+        qDebug() << "Creating pbo, program";
+
+        // pbo
+        pbo = new QOpenGLBuffer(QOpenGLBuffer::PixelUnpackBuffer);
+        pbo->setUsagePattern(QOpenGLBuffer::StreamDraw);
+        pbo->create();
+
+        // shaders
+        program = new QOpenGLShaderProgram;
+        program->addShaderFromSourceCode(QOpenGLShader::Vertex,
+                                         "attribute vec4 vertices;"
+                                         "varying vec2 coords;"
+                                         "void main() {"
+                                         "    gl_Position = vec4(vertices.xy,0.0,1.0);"
+                                         "    coords = vertices.xy*vec2(0.5,0.5)+vec2(0.5,0.5);"
+                                         "}");
+        program->addShaderFromSourceCode(QOpenGLShader::Fragment,
+                                         "uniform sampler2D texture0;"
+                                         "varying vec2 coords;"
+                                         "void main() {"
+                                         "    gl_FragColor = texture2D(texture0,coords*vec2(1.0, -1.0));"
+                                         "}");
+
+        program->bindAttributeLocation("vertices", 0);
+        program->link();
+
+        // a texture used to render the pbo (has the match the pixelformat of the source)
+        GLuint texture[1];
+        glGenTextures(1,texture);
+        glBindTexture(GL_TEXTURE_2D,texture[0]);
+        glTexImage2D(GL_TEXTURE_2D,0, GL_RGB, frame.cols, frame.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, 0);
+        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+
+        textureId = texture[0];
+    }
+
+    if (pboAllocSize != frameBytes)
+    {
+        qDebug() << "Resize pbo " << frameBytes << "bytes (was" << pboAllocSize << ")";
+
+        pbo->bind();
+        pbo->allocate(frameBytes);
+        pbo->release();
+
+        pboAllocSize = frameBytes;
+    }
+
+    // transfer data
+    pbo->bind();
+
+    void* ptr = pbo->map(QOpenGLBuffer::WriteOnly);
+    if (ptr)
+        memcpy(ptr, frame.data, frameBytes);
+    pbo->unmap();
+
+    //transfer pbo data to texture
+    glBindTexture(GL_TEXTURE_2D, textureId);
+    glTexSubImage2D(GL_TEXTURE_2D,0,0,0, frame.cols, frame.rows, GL_RGB, GL_UNSIGNED_BYTE, 0);
+    glBindTexture(GL_TEXTURE_2D, 0);
+
+    // render pbo
+    float values[] = {
+        -1, -1,
+        1, -1,
+        -1, 1,
+        1, 1
+    };
+
+    program->setAttributeArray(0, GL_FLOAT, values, 2);
+
+
+    glClearColor(0, 0, 0, 1);
+    glClear(GL_COLOR_BUFFER_BIT);
+
+    glViewport(0, 0, width(), height());
+
+    program->bind();
+    program->enableAttributeArray(0);
+
+    glBindTexture(GL_TEXTURE_2D, textureId);
+
+    //draw fullscreen quad
+    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+    program->disableAttributeArray(0);
+    program->release();
+
+    pbo->release();
 }
 
-void SelfCamView::resizeEvent(QResizeEvent *e)
+void SelfCamView::update()
 {
-    Q_UNUSED(e)
-    updateDisplay();
+    QGLWidget::update();
 }
+
+

widget/selfcamview.h

@@ -17,39 +17,35 @@
 #ifndef SELFCAMVIEW_H
 #define SELFCAMVIEW_H
 
-#include <QWidget>
+#include <QGLWidget>
 
-class QCloseEvent;
-class QShowEvent;
-class QPainter;
 class Camera;
-class QLabel;
-class QHBoxLayout;
+class QOpenGLBuffer;
+class QOpenGLShaderProgram;
 class QTimer;
 
-class SelfCamView : public QWidget
+class SelfCamView : public QGLWidget
 {
     Q_OBJECT
 
 public:
     SelfCamView(Camera* Cam, QWidget *parent=0);
+    ~SelfCamView();
 
-private slots:
-    void updateDisplay();
-
-private:
-    void closeEvent(QCloseEvent*);
-    void showEvent(QShowEvent*);
-    void paint(QPainter *painter);
-
+    // QGLWidget interface
 protected:
-    void resizeEvent(QResizeEvent *e);
+    virtual void initializeGL();
+    virtual void paintGL();
+
+    void update();
 
 private:
-    QLabel *displayLabel;
-    QHBoxLayout* mainLayout;
-    Camera* cam;
-    QTimer* updateDisplayTimer;
+    Camera* camera;
+    QOpenGLBuffer* pbo;
+    QOpenGLShaderProgram* program;
+    QTimer* updateTimer;
+    GLuint textureId;
+    int pboAllocSize;
 };
 
 #endif // SELFCAMVIEW_H