Implement video modesetting for dshow

qtox.pro

@@ -492,4 +492,5 @@ HEADERS += \
     src/video/cameradevice.h \
     src/video/camerasource.h \
     src/video/corevideosource.h \
+    src/video/videomode.h \
     src/core/toxid.h

src/platform/camera/directshow.cpp

@@ -3,7 +3,11 @@
 #include <cstdint>
 #include <Objbase.h>
 #include <Strmif.h>
+#include <Amvideo.h>
+#include <Dvdmedia.h>
 #include <uuids.h>
+#include <cassert>
+#include <QDebug>
 
 /**
  * Most of this file is adapted from libavdevice's dshow.c,
@@ -48,9 +52,10 @@ QVector<QPair<QString,QString>> DirectShow::getDeviceList()
             goto fail;
         if (CreateBindCtx(0, &bindCtx) != S_OK)
             goto fail;
+
+        // Get an uuid for the device that we can pass to ffmpeg directly
         if (m->GetDisplayName(bindCtx, nullptr, &olestr) != S_OK)
             goto fail;
-
         devIdString = wcharToUtf8(olestr);
 
         // replace ':' with '_' since FFmpeg uses : to delimitate sources
@@ -58,6 +63,7 @@ QVector<QPair<QString,QString>> DirectShow::getDeviceList()
             if (devIdString[i] == ':')
                 devIdString[i] = '_';
 
+        // Get a human friendly name/description
         if (m->BindToStorage(nullptr, nullptr, IID_IPropertyBag, (void**)&bag) != S_OK)
             goto fail;
 
@@ -83,3 +89,144 @@ fail:
 
     return devices;
 }
+
+// Used (by getDeviceModes) to select a device
+// so we can list its properties
+static IBaseFilter* getDevFilter(QString devName)
+{
+    IBaseFilter* devFilter = nullptr;
+    devName = devName.mid(6); // Remove the "video="
+    IMoniker* m = nullptr;
+
+    ICreateDevEnum* devenum = nullptr;
+    if (CoCreateInstance(CLSID_SystemDeviceEnum, nullptr, CLSCTX_INPROC_SERVER,
+                             IID_ICreateDevEnum, (void**) &devenum) != S_OK)
+        return devFilter;
+
+    IEnumMoniker* classenum = nullptr;
+    if (devenum->CreateClassEnumerator(CLSID_VideoInputDeviceCategory,
+                                (IEnumMoniker**)&classenum, 0) != S_OK)
+        return devFilter;
+
+    while (classenum->Next(1, &m, nullptr) == S_OK)
+    {
+        LPMALLOC coMalloc = nullptr;
+        IBindCtx* bindCtx = nullptr;
+        LPOLESTR olestr = nullptr;
+        char* devIdString;
+
+        if (CoGetMalloc(1, &coMalloc) != S_OK)
+            goto fail;
+        if (CreateBindCtx(0, &bindCtx) != S_OK)
+            goto fail;
+
+        if (m->GetDisplayName(bindCtx, nullptr, &olestr) != S_OK)
+            goto fail;
+        devIdString = wcharToUtf8(olestr);
+
+        // replace ':' with '_' since FFmpeg uses : to delimitate sources
+        for (unsigned i = 0; i < strlen(devIdString); i++)
+            if (devIdString[i] == ':')
+                devIdString[i] = '_';
+
+        if (devName != devIdString)
+            goto fail;
+
+        if (m->BindToObject(0, 0, IID_IBaseFilter, (void**)&devFilter) != S_OK)
+            goto fail;
+
+fail:
+        if (olestr && coMalloc)
+            coMalloc->Free(olestr);
+        if (bindCtx)
+            bindCtx->Release();
+        delete[] devIdString;
+        m->Release();
+    }
+    classenum->Release();
+
+    if (!devFilter)
+        qWarning() << "Could't find the device "<<devName;
+
+    return devFilter;
+}
+
+QVector<VideoMode> DirectShow::getDeviceModes(QString devName)
+{
+    QVector<VideoMode> modes;
+
+    IBaseFilter* devFilter = getDevFilter(devName);
+    if (!devFilter)
+        return modes;
+
+    // The outter loop tries to find a valid output pin
+    GUID category;
+    DWORD r2;
+    IEnumPins *pins = nullptr;
+    IPin *pin;
+    if (devFilter->EnumPins(&pins) != S_OK)
+        return modes;
+    while (pins->Next(1, &pin, nullptr) == S_OK)
+    {
+        IKsPropertySet *p = nullptr;
+        PIN_INFO info;
+
+        pin->QueryPinInfo(&info);
+        info.pFilter->Release();
+        if (info.dir != PINDIR_OUTPUT)
+            goto next;
+        if (pin->QueryInterface(IID_IKsPropertySet, (void**)&p) != S_OK)
+            goto next;
+        if (p->Get(AMPROPSETID_Pin, AMPROPERTY_PIN_CATEGORY,
+                nullptr, 0, &category, sizeof(GUID), &r2) != S_OK)
+            goto next;
+        if (!IsEqualGUID(category, PIN_CATEGORY_CAPTURE))
+            goto next;
+
+        // Now we can list the video modes for the current pin
+        // Prepare for another wall of spaghetti DIRECT SHOW QUALITY code
+        {
+            IAMStreamConfig *config = nullptr;
+            VIDEO_STREAM_CONFIG_CAPS *vcaps = nullptr;
+            int size, n;
+            if (pin->QueryInterface(IID_IAMStreamConfig, (void**)&config) != S_OK)
+                goto next;
+            if (config->GetNumberOfCapabilities(&n, &size) != S_OK)
+                goto pinend;
+            assert(size == sizeof(VIDEO_STREAM_CONFIG_CAPS));
+            vcaps = new VIDEO_STREAM_CONFIG_CAPS;
+
+            for (int i=0; i<n; ++i)
+            {
+                AM_MEDIA_TYPE* type = nullptr;
+                if (config->GetStreamCaps(i, &type, (BYTE*)vcaps) != S_OK)
+                    goto nextformat;
+
+                if (!IsEqualGUID(type->formattype, FORMAT_VideoInfo)
+                    && !IsEqualGUID(type->formattype, FORMAT_VideoInfo2))
+                    goto nextformat;
+
+                VideoMode mode;
+                mode.width = vcaps->MaxOutputSize.cx;
+                mode.height = vcaps->MaxOutputSize.cy;
+                mode.FPS = 1e7 / vcaps->MinFrameInterval;
+                if (!modes.contains(mode))
+                    modes.append(std::move(mode));
+
+nextformat:
+                if (type->pbFormat)
+                    CoTaskMemFree(type->pbFormat);
+                CoTaskMemFree(type);
+            }
+pinend:
+            config->Release();
+            delete vcaps;
+        }
+next:
+        if (p)
+            p->Release();
+        pin->Release();
+    }
+
+    return modes;
+}

src/platform/camera/directshow.h

@@ -4,6 +4,7 @@
 #include <QString>
 #include <QVector>
 #include <QPair>
+#include "src/video/videomode.h"
 
 #ifndef Q_OS_WIN
 #error "This file is only meant to be compiled for Windows targets"
@@ -12,6 +13,7 @@
 namespace DirectShow
 {
     QVector<QPair<QString,QString>> getDeviceList();
+    QVector<VideoMode> getDeviceModes(QString devName);
 }
 
 #endif // DIRECTSHOW_H

src/video/cameradevice.cpp

@@ -17,10 +17,9 @@ AVInputFormat* CameraDevice::iformat{nullptr};
 CameraDevice::CameraDevice(const QString devName, AVFormatContext *context)
     : devName{devName}, context{context}, refcount{1}
 {
-
 }
 
-CameraDevice* CameraDevice::open(QString devName)
+CameraDevice* CameraDevice::open(QString devName, AVDictionary** options)
 {
     openDeviceLock.lock();
     AVFormatContext* fctx = nullptr;
@@ -28,7 +27,7 @@ CameraDevice* CameraDevice::open(QString devName)
     if (dev)
         goto out;
 
-    if (avformat_open_input(&fctx, devName.toStdString().c_str(), iformat, nullptr)<0)
+    if (avformat_open_input(&fctx, devName.toStdString().c_str(), iformat, options)<0)
         goto out;
 
     if (avformat_find_stream_info(fctx, NULL) < 0)
@@ -45,6 +44,36 @@ out:
     return dev;
 }
 
+CameraDevice* CameraDevice::open(QString devName)
+{
+    return open(devName, nullptr);
+}
+
+CameraDevice* CameraDevice::open(QString devName, VideoMode mode)
+{
+    if (!getDefaultInputFormat())
+        return nullptr;
+
+    AVDictionary* options = nullptr;
+    if (false);
+#ifdef Q_OS_WIN
+    else if (iformat->name == QString("dshow"))
+    {
+        av_dict_set(&options, "video_size", QString("%1x%2").arg(mode.width).arg(mode.height).toStdString().c_str(), 0);
+        av_dict_set(&options, "framerate", QString().setNum(mode.FPS).toStdString().c_str(), 0);
+    }
+#endif
+    else
+    {
+        qWarning() << "Video mode-setting not implemented for input "<<iformat->name;
+    }
+
+    CameraDevice* dev = open(devName, &options);
+    if (options)
+        av_dict_free(&options);
+    return dev;
+}
+
 void CameraDevice::open()
 {
     ++refcount;
@@ -130,7 +159,6 @@ QVector<QPair<QString, QString>> CameraDevice::getRawDeviceListGeneric()
 
 QVector<QPair<QString, QString>> CameraDevice::getDeviceList()
 {
-    if (!iformat)
     if (!getDefaultInputFormat())
             return {};
 
@@ -158,6 +186,19 @@ QString CameraDevice::getDefaultDeviceName()
     return devlist[0].first;
 }
 
+QVector<VideoMode> CameraDevice::getVideoModes(QString devName)
+{
+    if (false);
+#ifdef Q_OS_WIN
+    else if (iformat->name == QString("dshow"))
+        return DirectShow::getDeviceModes(devName);
+#endif
+    else
+        qWarning() << "Video mode listing not implemented for input "<<iformat->name;
+
+    return {};
+}
+
 bool CameraDevice::getDefaultInputFormat()
 {
     QMutexLocker locker(&iformatLock);

src/video/cameradevice.h

@@ -6,10 +6,12 @@
 #include <QMutex>
 #include <QVector>
 #include <atomic>
+#include "videomode.h"
 
 struct AVFormatContext;
 struct AVInputFormat;
 struct AVDeviceInfoList;
+struct AVDictionary;
 
 /// Maintains an FFmpeg context for open camera devices,
 /// takes care of sharing the context accross users
@@ -22,12 +24,21 @@ public:
     /// Opens a device, creating a new one if needed
     /// Returns a nullptr if the device couldn't be opened
     static CameraDevice* open(QString devName);
+    /// Opens a device, creating a new one if needed
+    /// If the device is alreay open in another mode, the mode
+    /// will be ignored and the existing device is used
+    /// If the mode does not exist, a new device can't be opened
+    /// Returns a nullptr if the device couldn't be opened
+    static CameraDevice* open(QString devName, VideoMode mode);
     void open(); ///< Opens the device again. Never fails
     bool close(); ///< Closes the device. Never fails. If returns true, "this" becomes invalid
 
     /// Returns a list of device names and descriptions
     /// The names are the first part of the pair and can be passed to open(QString)
-    static QVector<QPair<QString, QString> > getDeviceList();
+    static QVector<QPair<QString, QString>> getDeviceList();
+
+    /// Get the list of video modes for a device
+    static QVector<VideoMode> getVideoModes(QString devName);
 
     /// Returns the short name of the default defice
     /// This is either the device in the settings
@@ -36,6 +47,7 @@ public:
 
 private:
     CameraDevice(const QString devName, AVFormatContext *context);
+    static CameraDevice* open(QString devName, AVDictionary** options);
     static bool getDefaultInputFormat(); ///< Sets CameraDevice::iformat, returns success/failure
     static QVector<QPair<QString, QString> > getRawDeviceListGeneric(); ///< Uses avdevice_list_devices
 

src/video/camerasource.cpp

@@ -33,7 +33,12 @@ CameraSource::CameraSource()
 }
 
 CameraSource::CameraSource(const QString deviceName)
-    : deviceName{deviceName}, device{nullptr},
+    : CameraSource{deviceName, VideoMode{0,0,0}}
+{
+}
+
+CameraSource::CameraSource(const QString deviceName, VideoMode mode)
+    : deviceName{deviceName}, device{nullptr}, mode(mode),
       cctx{nullptr}, videoStreamIndex{-1},
       biglock{false}, freelistLock{false}, subscriptions{0}
 {
@@ -50,8 +55,20 @@ CameraSource::~CameraSource()
             expected = false;
     }
 
+    // Free all remaining VideoFrame
+    // Locking must be done precisely this way to avoid races
+    for (int i=0; i<freelist.size(); i++)
+    {
+        std::shared_ptr<VideoFrame> vframe = freelist[i].lock();
+        if (!vframe)
+            continue;
+        vframe->releaseFrame();
+    }
+
     if (cctx)
         avcodec_free_context(&cctx);
+    if (cctxOrig)
+        avcodec_close(cctxOrig);
 
     for (int i=subscriptions; i; --i)
         device->close();
@@ -82,6 +99,9 @@ bool CameraSource::subscribe()
 
     // We need to create a new CameraDevice
     AVCodec* codec;
+    if (mode)
+        device = CameraDevice::open(deviceName, mode);
+    else
         device = CameraDevice::open(deviceName);
     if (!device)
     {

src/video/camerasource.h

@@ -21,6 +21,7 @@
 #include <QVector>
 #include <atomic>
 #include "src/video/videosource.h"
+#include "src/video/videomode.h"
 
 class CameraDevice;
 struct AVCodecContext;
@@ -37,6 +38,7 @@ class CameraSource : public VideoSource
 public:
     CameraSource(); ///< Opens the camera device in the settings, or the system default
     CameraSource(const QString deviceName);
+    CameraSource(const QString deviceName, VideoMode mode);
     ~CameraSource();
 
     // VideoSource interface
@@ -63,6 +65,7 @@ private:
     QFuture<void> streamFuture; ///< Future of the streaming thread
     const QString deviceName; ///< Short name of the device for CameraDevice's open(QString)
     CameraDevice* device; ///< Non-owning pointer to an open CameraDevice, or nullptr
+    VideoMode mode; ///< What mode we tried to open the device in, all zeros means default mode
     AVCodecContext* cctx, *cctxOrig; ///< Codec context of the camera's selected video stream
     int videoStreamIndex; ///< A camera can have multiple streams, this is the one we're decoding
     std::atomic_bool biglock, freelistLock; ///< True when locked. Faster than mutexes for video decoding.

src/video/videomode.h

@@ -0,0 +1,25 @@
+#ifndef VIDEOMODE_H
+#define VIDEOMODE_H
+
+/// Describes a video mode supported by a device
+struct VideoMode
+{
+    short width, height; ///< Displayed video resolution (NOT frame resolution)
+    short FPS; ///< Max frames per second supported by the device at this resolution
+
+    /// All zeros means a default/unspecified mode
+    operator bool()
+    {
+        return width || height || FPS;
+    }
+
+    bool operator==(const VideoMode& other)
+    {
+        return width == other.width
+                && height == other.height
+                && FPS == other.FPS;
+    }
+};
+
+#endif // VIDEOMODE_H
+

src/widget/form/settings/avform.cpp

@@ -78,18 +78,101 @@ void AVForm::present()
 {
     getAudioOutDevices();
     getAudioInDevices();
-
     createVideoSurface();
-	
-	bodyUI->videoModescomboBox->blockSignals(true);
-	bodyUI->videoModescomboBox->addItem(tr("Initializing Camera..."));
-	bodyUI->videoModescomboBox->blockSignals(false);
+    getVideoDevices();
 }
 
 void AVForm::on_videoModescomboBox_currentIndexChanged(int index)
 {
-    QSize res = bodyUI->videoModescomboBox->itemData(index).toSize();
-    Settings::getInstance().setCamVideoRes(res);
+    if (index<0 || index>=videoModes.size())
+    {
+        qWarning() << "Invalid mode index";
+        return;
+    }
+    int devIndex = bodyUI->videoDevCombobox->currentIndex();
+    if (devIndex<0 || devIndex>=videoModes.size())
+    {
+        qWarning() << "Invalid device index";
+        return;
+    }
+    QString devName = videoDeviceList[devIndex].first;
+    VideoMode mode = videoModes[index];
+    Settings::getInstance().setCamVideoRes(QSize(mode.width, mode.height));
+    camVideoSurface->setSource(nullptr);
+    if (camera)
+        delete camera;
+    camera = new CameraSource(devName, mode);
+    camVideoSurface->setSource(camera);
+}
+
+void AVForm::updateVideoModes(int curIndex)
+{
+    if (curIndex<0 || curIndex>=videoDeviceList.size())
+    {
+        qWarning() << "Invalid index";
+        return;
+    }
+    QString devName = videoDeviceList[curIndex].first;
+    videoModes = CameraDevice::getVideoModes(devName);
+    std::sort(videoModes.begin(), videoModes.end(),
+        [](const VideoMode& a, const VideoMode& b)
+            {return a.width!=b.width ? a.width>b.width :
+                    a.height!=b.height ? a.height>b.height :
+                    a.FPS>b.FPS;});
+    bodyUI->videoModescomboBox->blockSignals(true);
+    bodyUI->videoModescomboBox->clear();
+    int prefResIndex = -1;
+    QSize prefRes = Settings::getInstance().getCamVideoRes();
+    for (int i=0; i<videoModes.size(); ++i)
+    {
+        VideoMode mode = videoModes[i];
+        if (mode.width==prefRes.width() && mode.height==prefRes.height() && prefResIndex==-1)
+            prefResIndex = i;
+        QString str = tr("%1x%2 at %3 FPS").arg(mode.width).arg(mode.height).arg(mode.FPS);
+        bodyUI->videoModescomboBox->addItem(str);
+    }
+    if (videoModes.isEmpty())
+        bodyUI->videoModescomboBox->addItem(tr("Default resolution"));
+    bodyUI->videoModescomboBox->blockSignals(false);
+    if (prefResIndex != -1)
+    {
+        bodyUI->videoModescomboBox->setCurrentIndex(prefResIndex);
+    }
+    else
+    {
+        // If the user hasn't set a preffered resolution yet,
+        // we'll pick the resolution in the middle of the list,
+        // and the best FPS for that resolution.
+        // If we picked the lowest resolution, the quality would be awful
+        // but if we picked the largest, FPS would be bad and thus quality bad too.
+        int numRes=0;
+        QSize lastSize;
+        for (int i=0; i<videoModes.size(); i++)
+        {
+            if (lastSize != QSize{videoModes[i].width, videoModes[i].height})
+            {
+                numRes++;
+                lastSize = {videoModes[i].width, videoModes[i].height};
+            }
+        }
+        int target = numRes/2;
+        numRes=0;
+        for (int i=0; i<videoModes.size(); i++)
+        {
+            if (lastSize != QSize{videoModes[i].width, videoModes[i].height})
+            {
+                numRes++;
+                lastSize = {videoModes[i].width, videoModes[i].height};
+            }
+            if (numRes==target)
+            {
+                bodyUI->videoModescomboBox->setCurrentIndex(i);
+                break;
+            }
+        }
+    }
+
+
 }
 
 void AVForm::onVideoDevChanged(int index)
@@ -104,6 +187,7 @@ void AVForm::onVideoDevChanged(int index)
         delete camera;
     QString dev = videoDeviceList[index].first;
     Settings::getInstance().setVideoDev(dev);
+    updateVideoModes(index);
     camera = new CameraSource(dev);
     camVideoSurface->setSource(camera);
 }
@@ -146,20 +230,14 @@ void AVForm::hideEvent(QHideEvent *)
     videoDeviceList.clear();
 }
 
-void AVForm::showEvent(QShowEvent *)
-{
-    createVideoSurface();
-    getVideoDevices();
-}
-
 void AVForm::getVideoDevices()
 {
     QString settingsInDev = Settings::getInstance().getVideoDev();
     int videoDevIndex = 0;
     videoDeviceList = CameraDevice::getDeviceList();
-    bodyUI->videoDevCombobox->clear();
     //prevent currentIndexChanged to be fired while adding items
     bodyUI->videoDevCombobox->blockSignals(true);
+    bodyUI->videoDevCombobox->clear();
     for (QPair<QString, QString> device : videoDeviceList)
     {
         bodyUI->videoDevCombobox->addItem(device.second);
@@ -170,6 +248,7 @@ void AVForm::getVideoDevices()
     bodyUI->videoDevCombobox->setCurrentIndex(-1);
     bodyUI->videoDevCombobox->blockSignals(false);
     bodyUI->videoDevCombobox->setCurrentIndex(videoDevIndex);
+    updateVideoModes(videoDevIndex);
 }
 
 void AVForm::getAudioInDevices()

src/widget/form/settings/avform.h

@@ -18,6 +18,7 @@
 #include <QObject>
 #include <QList>
 #include "genericsettings.h"
+#include "src/video/videomode.h"
 
 namespace Ui {
 class AVSettings;
@@ -57,16 +58,17 @@ private slots:
     void onResProbingFinished(QList<QSize> res);
 
     virtual void hideEvent(QHideEvent*);
-    virtual void showEvent(QShowEvent*);
     
 protected:
     bool eventFilter(QObject *o, QEvent *e);    
+    void updateVideoModes(int curIndex);
 
 private:
     Ui::AVSettings *bodyUI;
     VideoSurface* camVideoSurface;
     CameraSource* camera;
     QVector<QPair<QString, QString>> videoDeviceList;
+    QVector<VideoMode> videoModes;
 };
 
 #endif