StarMirror/qTox
1
0
Fork 0
mirror of https://github.com/qTox/qTox.git synced 2024-03-22 14:00:36 +08:00
Code Issues Projects Releases Wiki Activity

refactor(audio): introduce IAudioSource interface and use it

Browse Source
This commit is contained in:
sudden6 2019-05-01 21:45:18 +02:00
parent 2ccb1ec150
commit e7e35642d7
No known key found for this signature in database
GPG Key ID: 279509B499E032B9
13 changed files with 210 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -238,10 +238,13 @@ set(${PROJECT_NAME}_SOURCES
src/audio/audio.h
src/audio/backend/alsink.cpp
src/audio/backend/alsink.h
src/audio/backend/alsource.cpp
src/audio/backend/alsource.h
src/audio/backend/openal.cpp
src/audio/backend/openal.h
src/audio/iaudiosettings.h
src/audio/iaudiosink.h
src/audio/iaudiosource.h
src/chatlog/chatlinecontent.cpp
src/chatlog/chatlinecontent.h
src/chatlog/chatlinecontentproxy.cpp

74
src/audio/audio.cpp
View File

@ -24,34 +24,9 @@
#endif
#include "src/persistence/settings.h"
#include <QDebug>
#include <cassert>
/**
* @class Audio
*
* @enum Audio::Sound
* @brief Provides the different sounds for use in the getSound function.
* @see getSound
*
* @value NewMessage Returns the new message notification sound.
* @value Test Returns the test sound.
* @value IncomingCall Returns the incoming call sound.
* @value OutgoingCall Returns the outgoing call sound.
*
* @fn QString Audio::getSound(Sound s)
* @brief Function to get the path of the requested sound.
*
* @param s Name of the sound to get the path of.
* @return The path of the requested sound.
*
* @fn void Audio::frameAvailable(const int16_t *pcm, size_t sample_count, uint8_t channels,
* uint32_t sampling_rate);
*
* When there are input subscribers, we regularly emit captured audio frames with this signal
* Always connect with a blocking queued connection lambda, else the behaviour is undefined
*
* @var Audio::AUDIO_SAMPLE_RATE
* @brief The next best Opus would take is 24k
*
@ -89,37 +64,6 @@
*
* @note Default is 30dB; usually you don't need to alter this value.
*
* @fn void Audio::subscribeInput()
* @brief Subscribe to capture sound from the opened input device.
*
* If the input device is not open, it will be opened before capturing.
*
* @fn void Audio::unsubscribeInput()
* @brief Unsubscribe from capturing from an opened input device.
*
* If the input device has no more subscriptions, it will be closed.
*
* @fn void Audio::playMono16Sound(const QString& path)
* @brief Play a 44100Hz mono 16bit PCM sound from a file
*
* @param[in] path the path to the sound file
*
* @fn void Audio::playMono16Sound(const QByteArray& data)
* @brief Play a 44100Hz mono 16bit PCM sound
*
* @param[in] data 44100Hz mono 16bit PCM data in host byte order
*
* @fn void Audio::playAudioBuffer(uint sourceId, const int16_t* data, int samples,
* unsigned channels, int sampleRate)
* @brief adds a number of audio frames to the play buffer
*
* @param[in] sourceId id obtained by subscribeOutput(uint &)
* @param[in] data 16bit mono or stereo PCM data with alternating channel
* mapping for stereo (LRLR)
* @param[in] samples number of samples per channel
* @param[in] channels number of channels, currently 1 or 2 is supported
* @param[in] sampleRate sample rate in Hertz
*
* @fn bool Audio::isOutputReady() const
* @brief check if the output is ready to play audio
*
@ -135,24 +79,6 @@
*
* @return list of input devices
*
* @fn void Audio::subscribeOutput(uint& sid)
* @brief register a new output source
*
* param[out] sid contains the sourceId if source creation was successful,
* unchanged otherwise
*
* @fn void Audio::unsubscribeOutput(uint& sid)
* @brief unregisters an output source
*
* param[out] sid contains 0 if source deletion was successful,
* unchanged otherwise
*
* @fn void Audio::startLoop()
* @brief starts looping the sound played with playMono16Sound()
*
* @fn void Audio::stopLoop()
* @brief stops looping the sound played with playMono16Sound()
*
* @fn qreal Audio::inputGain() const
* @brief get the current input gain
*

13
src/audio/audio.h
View File

@ -25,6 +25,7 @@
#include <memory>
class IAudioSink;
class IAudioSource;
class Audio : public QObject
{
Q_OBJECT
@ -60,12 +61,8 @@ public:
virtual QStringList outDeviceNames() = 0;
virtual QStringList inDeviceNames() = 0;
virtual void subscribeInput() = 0;
virtual void unsubscribeInput() = 0;
virtual void stopActive() = 0;
virtual std::unique_ptr<IAudioSink> makeSink() = 0;
virtual std::unique_ptr<IAudioSource> makeSource() = 0;
protected:
// Public default audio settings
@ -75,12 +72,6 @@ protected:
static constexpr uint32_t AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL =
AUDIO_FRAME_DURATION * AUDIO_SAMPLE_RATE / 1000;
uint32_t AUDIO_FRAME_SAMPLE_COUNT_TOTAL = 0;
signals:
void frameAvailable(const int16_t* pcm, size_t sample_count, uint8_t channels,
uint32_t sampling_rate);
void volumeAvailable(float value);
void startActive(qreal msec);
};
#endif // AUDIO_H

41
src/audio/backend/alsource.cpp Normal file
View File

@ -0,0 +1,41 @@
#include "src/audio/backend/alsource.h"
#include "src/audio/backend/openal.h"
/**
* @brief Emits audio frames captured by an input device or other audio source.
*/
/**
* @brief Reserves ressources for an audio source
* @param audio Main audio object, must have longer lifetime than this object.
*/
AlSource::AlSource(OpenAL& al)
: audio(al)
, killLock(QMutex::Recursive)
{}
AlSource::~AlSource()
{
QMutexLocker{&killLock};
// unsubscribe only if not already killed
if (!killed) {
audio.destroySource(*this);
killed = true;
}
}
AlSource::operator bool() const
{
QMutexLocker{&killLock};
return !killed;
}
void AlSource::kill()
{
killLock.lock();
// this flag is only set once here, afterwards the object is considered dead
killed = true;
killLock.unlock();
emit invalidated();
}

30
src/audio/backend/alsource.h Normal file
View File

@ -0,0 +1,30 @@
#ifndef ALSOURCE_H
#define ALSOURCE_H
#include "src/audio/iaudiosource.h"
#include <QMutex>
#include <QObject>
class OpenAL;
class AlSource : public IAudioSource
{
Q_OBJECT
public:
AlSource(OpenAL& al);
AlSource(AlSource& src) = delete;
AlSource& operator=(const AlSource&) = delete;
AlSource(AlSource&& other) = delete;
AlSource& operator=(AlSource&& other) = delete;
~AlSource();
operator bool() const;
void kill();
private:
OpenAL& audio;
bool killed = false;
mutable QMutex killLock;
};
#endif // ALSOURCE_H

61
src/audio/backend/openal.cpp
View File

@ -18,8 +18,6 @@
*/
#include "openal.h"
#include "src/core/core.h"
#include "src/core/coreav.h"
#include "src/persistence/settings.h"
#include <QDebug>
@ -74,8 +72,9 @@ OpenAL::OpenAL()
voiceTimer.setSingleShot(true);
voiceTimer.moveToThread(audioThread);
connect(this, &Audio::startActive, &voiceTimer, static_cast<void (QTimer::*)(int)>(&QTimer::start));
connect(&voiceTimer, &QTimer::timeout, this, &Audio::stopActive);
connect(this, &OpenAL::startActive, &voiceTimer,
static_cast<void (QTimer::*)(int)>(&QTimer::start));
connect(&voiceTimer, &QTimer::timeout, this, &OpenAL::stopActive);
connect(&captureTimer, &QTimer::timeout, this, &OpenAL::doAudio);
captureTimer.setInterval(AUDIO_FRAME_DURATION / 2);
@ -218,8 +217,19 @@ qreal OpenAL::maxInputThreshold() const
void OpenAL::reinitInput(const QString& inDevDesc)
{
QMutexLocker locker(&audioLock);
const auto bakSources = sources;
sources.clear();
cleanupInput();
initInput(inDevDesc);
locker.unlock();
// this must happen outside `audioLock`, to avoid a deadlock when
// a slot on AlSource::invalidate tries to create a new source immedeately.
for (auto& source : bakSources) {
source->kill();
}
}
bool OpenAL::reinitOutput(const QString& outDevDesc)
@ -308,17 +318,25 @@ void OpenAL::destroySink(AlSink& sink)
*
* If the input device is not open, it will be opened before capturing.
*/
void OpenAL::subscribeInput()
std::unique_ptr<IAudioSource> OpenAL::makeSource()
{
QMutexLocker locker(&audioLock);
if (!autoInitInput()) {
qWarning("Failed to subscribe to audio input device.");
return;
return {};
}
++inSubscriptions;
qDebug() << "Subscribed to audio input device [" << inSubscriptions << "subscriptions ]";
auto const source = new AlSource(*this);
if (source == nullptr) {
return {};
}
sources.insert(source);
qDebug() << "Subscribed to audio input device [" << sources.size() << "subscriptions ]";
return std::unique_ptr<IAudioSource>{source};
}
/**
@ -326,19 +344,24 @@ void OpenAL::subscribeInput()
*
* If the input device has no more subscriptions, it will be closed.
*/
void OpenAL::unsubscribeInput()
void OpenAL::destroySource(AlSource& source)
{
QMutexLocker locker(&audioLock);
if (!inSubscriptions)
const auto s = sources.find(&source);
if (s == sources.end()) {
qWarning() << "Destroyed non-existant source";
return;
}
inSubscriptions--;
qDebug() << "Unsubscribed from audio input device [" << inSubscriptions << "subscriptions left ]";
sources.erase(s);
if (!inSubscriptions)
qDebug() << "Unsubscribed from audio input device [" << sources.size() << "subscriptions left ]";
if (sources.empty()) {
cleanupInput();
}
}
/**
* @brief Initialize audio input device, if not initialized.
@ -646,14 +669,20 @@ void OpenAL::doInput()
volume = 0;
}
emit Audio::volumeAvailable(volume);
// NOTE(sudden6): this loop probably doesn't scale too well with many sources
for (auto source : sources) {
emit source->volumeAvailable(volume);
}
if (!isActive) {
return;
}
emit Audio::frameAvailable(inputBuffer, AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL, channels,
// NOTE(sudden6): this loop probably doesn't scale too well with many sources
for (auto source : sources) {
emit source->frameAvailable(inputBuffer, AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL, channels,
AUDIO_SAMPLE_RATE);
}
}
void OpenAL::doOutput()
{
@ -670,7 +699,7 @@ void OpenAL::doAudio()
// Output section does nothing
// Input section
if (alInDev && inSubscriptions) {
if (alInDev && !sources.empty()) {
doInput();
}
}

12
src/audio/backend/openal.h
View File

@ -23,6 +23,7 @@
#include "src/audio/audio.h"
#include "src/audio/backend/alsink.h"
#include "src/audio/backend/alsource.h"
#include <memory>
#include <unordered_set>
@ -89,8 +90,8 @@ public:
std::unique_ptr<IAudioSink> makeSink();
void destroySink(AlSink& sink);
void subscribeInput();
void unsubscribeInput();
std::unique_ptr<IAudioSource> makeSource();
void destroySource(AlSource& source);
void startLoop(uint sourceId);
void stopLoop(uint sourceId);
@ -99,6 +100,8 @@ public:
void playAudioBuffer(uint sourceId, const int16_t* data, int samples, unsigned channels,
int sampleRate);
signals:
void startActive(qreal msec);
protected:
static void checkAlError() noexcept;
@ -135,7 +138,6 @@ protected:
QString outDev{};
ALCdevice* alInDev = nullptr;
quint32 inSubscriptions = 0;
QTimer captureTimer;
QTimer cleanupTimer;
@ -147,9 +149,7 @@ protected:
// Qt containers need copy operators, so use stdlib containers
std::unordered_set<AlSink*> sinks;
std::unordered_set<AlSink*> soundSinks;
// number of output sources
int outCount = 0;
std::unordered_set<AlSource*> sources;
int channels = 0;
qreal gain = 0;

2
src/audio/backend/openal2.cpp
View File

@ -18,8 +18,6 @@
*/
#include "openal2.h"
#include "src/core/core.h"
#include "src/core/coreav.h"
#include "src/persistence/settings.h"
#include <QDebug>

9
src/audio/iaudiosource.h
View File

@ -3,7 +3,14 @@
#include <QObject>
class Audio;
/**
* @fn void Audio::frameAvailable(const int16_t *pcm, size_t sample_count, uint8_t channels,
* uint32_t sampling_rate);
*
* When there are input subscribers, we regularly emit captured audio frames with this signal
* Always connect with a blocking queued connection lambda, else the behaviour is undefined
*/
class IAudioSource : public QObject
{
Q_OBJECT

65
src/core/toxcall.cpp
View File

@ -31,14 +31,14 @@
ToxCall::ToxCall(bool VideoEnabled, CoreAV& av)
: av{&av}
, videoEnabled{VideoEnabled}
, audioSource{Audio::getInstance().makeSource()}
{}
ToxCall::~ToxCall()
{
Audio& audio = Audio::getInstance();
QObject::disconnect(audioInConn);
audio.unsubscribeInput();
QObject::disconnect(audioSrcInvalid);
if (videoEnabled) {
QObject::disconnect(videoInConn);
CameraSource::getInstance().unsubscribe();
@ -105,15 +105,16 @@ ToxFriendCall::ToxFriendCall(uint32_t FriendNum, bool VideoEnabled, CoreAV& av)
, sink(Audio::getInstance().makeSink())
, friendId{FriendNum}
{
// register audio
Audio& audio = Audio::getInstance();
audio.subscribeInput();
audioInConn = QObject::connect(&Audio::getInstance(), &Audio::frameAvailable,
[&av, FriendNum](const int16_t* pcm, size_t samples,
uint8_t chans, uint32_t rate) {
av.sendCallAudio(FriendNum, pcm, samples, chans, rate);
// TODO(sudden6): move this to audio source
audioInConn =
QObject::connect(audioSource.get(), &IAudioSource::frameAvailable,
[this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
this->av->sendCallAudio(this->friendId, pcm, samples, chans, rate);
});
audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
[this]() { this->onAudioSourceInvalidated(); });
if (!audioInConn) {
qDebug() << "Audio input connection not working";
}
@ -146,6 +147,20 @@ ToxFriendCall::~ToxFriendCall()
QObject::disconnect(audioSinkInvalid);
}
void ToxFriendCall::onAudioSourceInvalidated()
{
auto newSrc = Audio::getInstance().makeSource();
audioInConn =
QObject::connect(newSrc.get(), &IAudioSource::frameAvailable,
[this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
this->av->sendCallAudio(this->friendId, pcm, samples, chans, rate);
});
audioSource = std::move(newSrc);
audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
[this]() { this->onAudioSourceInvalidated(); });
}
void ToxFriendCall::onAudioSinkInvalidated()
{
auto newSink = Audio::getInstance().makeSink();
@ -201,17 +216,18 @@ ToxGroupCall::ToxGroupCall(int GroupNum, CoreAV& av)
, groupId{GroupNum}
{
// register audio
Audio& audio = Audio::getInstance();
audio.subscribeInput();
audioInConn = QObject::connect(&Audio::getInstance(), &Audio::frameAvailable,
[&av, GroupNum](const int16_t* pcm, size_t samples,
uint8_t chans, uint32_t rate) {
av.sendGroupCallAudio(GroupNum, pcm, samples, chans, rate);
audioInConn =
QObject::connect(audioSource.get(), &IAudioSource::frameAvailable,
[this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
this->av->sendGroupCallAudio(this->groupId, pcm, samples, chans, rate);
});
if (!audioInConn) {
qDebug() << "Audio input connection not working";
}
audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
[this]() { this->onAudioSourceInvalidated(); });
}
ToxGroupCall::~ToxGroupCall()
@ -220,6 +236,23 @@ ToxGroupCall::~ToxGroupCall()
clearPeers();
}
void ToxGroupCall::onAudioSourceInvalidated()
{
auto newSrc = Audio::getInstance().makeSource();
// TODO(sudden6): move this to audio source
audioInConn =
QObject::connect(audioSource.get(), &IAudioSource::frameAvailable,
[this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) {
this->av->sendGroupCallAudio(this->groupId, pcm, samples, chans, rate);
});
audioSource = std::move(newSrc);
audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated,
[this]() { this->onAudioSourceInvalidated(); });
}
void ToxGroupCall::onAudioSinkInvalidated(ToxPk peerId)
{
removePeer(peerId);

9
src/core/toxcall.h
View File

@ -2,6 +2,7 @@
#define TOXCALL_H
#include "src/audio/iaudiosink.h"
#include "src/audio/iaudiosource.h"
#include <src/core/toxpk.h>
#include <tox/toxav.h>
@ -60,6 +61,8 @@ protected:
QMetaObject::Connection videoInConn;
bool videoEnabled{false};
bool nullVideoBitrate{false};
std::unique_ptr<IAudioSource> audioSource = nullptr;
QMetaObject::Connection audioSrcInvalid;
};
class ToxFriendCall : public ToxCall
@ -77,8 +80,7 @@ public:
TOXAV_FRIEND_CALL_STATE getState() const;
void setState(const TOXAV_FRIEND_CALL_STATE& value);
void playAudioBuffer(const int16_t* data, int samples, unsigned channels,
int sampleRate) const;
void playAudioBuffer(const int16_t* data, int samples, unsigned channels, int sampleRate) const;
protected:
std::unique_ptr<QTimer> timeoutTimer;
@ -106,7 +108,8 @@ public:
ToxGroupCall& operator=(ToxGroupCall&& other) = delete;
void removePeer(ToxPk peerId);
void playAudioBuffer(const ToxPk& peer, const int16_t* data, int samples, unsigned channels, int sampleRate);
void playAudioBuffer(const ToxPk& peer, const int16_t* data, int samples, unsigned channels,
int sampleRate);
private:
void addPeer(ToxPk peerId);

27
src/widget/form/settings/avform.cpp
View File

@ -29,6 +29,7 @@
#include "src/audio/audio.h"
#include "src/audio/iaudiosettings.h"
#include "src/audio/iaudiosource.h"
#include "src/core/core.h"
#include "src/core/coreav.h"
#include "src/video/cameradevice.h"
@ -50,7 +51,6 @@ AVForm::AVForm(Audio* audio, CoreAV* coreAV, CameraSource& camera, IAudioSetting
, coreAV{coreAV}
, audioSettings{audioSettings}
, videoSettings{videoSettings}
, subscribedToAudioIn(false)
, camVideoSurface(nullptr)
, camera(camera)
{
@ -98,7 +98,6 @@ AVForm::AVForm(Audio* audio, CoreAV* coreAV, CameraSource& camera, IAudioSetting
audioThresholdSlider->setTracking(false);
audioThresholdSlider->installEventFilter(this);
connect(audio, &Audio::volumeAvailable, this, &AVForm::setVolume);
volumeDisplay->setMaximum(totalSliderSteps);
fillAudioQualityComboBox();
@ -120,13 +119,8 @@ AVForm::~AVForm()
void AVForm::hideEvent(QHideEvent* event)
{
if (subscribedToAudioIn) {
// TODO: This should not be done in show/hide events
audio->unsubscribeInput();
subscribedToAudioIn = false;
}
audioSink.reset();
audioSrc.reset();
if (camVideoSurface) {
camVideoSurface->setSource(nullptr);
@ -144,10 +138,9 @@ void AVForm::showEvent(QShowEvent* event)
createVideoSurface();
getVideoDevices();
if (!subscribedToAudioIn) {
// TODO: This should not be done in show/hide events
audio->subscribeInput();
subscribedToAudioIn = true;
if (audioSrc == nullptr) {
audioSrc = audio->makeSource();
connect(audioSrc.get(), &IAudioSource::volumeAvailable, this, &AVForm::setVolume);
}
if (audioSink == nullptr) {
@ -540,17 +533,17 @@ void AVForm::on_inDevCombobox_currentIndexChanged(int deviceIndex)
const bool inputEnabled = deviceIndex > 0;
audioSettings->setAudioInDevEnabled(inputEnabled);
QString deviceName;
QString deviceName{};
if (inputEnabled) {
deviceName = inDevCombobox->itemText(deviceIndex);
}
const QString oldName = audioSettings->getInDev();
if (oldName != deviceName) {
audioSettings->setInDev(deviceName);
audio->reinitInput(deviceName);
subscribedToAudioIn = inputEnabled;
if (inputEnabled) {
audio->subscribeInput();
audioSrc = audio->makeSource();
connect(audioSrc.get(), &IAudioSource::volumeAvailable, this, &AVForm::setVolume);
}
microphoneSlider->setEnabled(inputEnabled);

6
src/widget/form/settings/avform.h
View File

@ -31,12 +31,13 @@
#include <memory>
class Audio;
class IAudioSettings;
class IAudioSink;
class IAudioSource;
class CameraSource;
class CoreAV;
class IAudioSettings;
class IVideoSettings;
class VideoSurface;
class IAudioSink;
class AVForm : public GenericForm, private Ui::AVForm
{
Q_OBJECT
@ -103,6 +104,7 @@ private:
bool subscribedToAudioIn;
std::unique_ptr<IAudioSink> audioSink = nullptr;
std::unique_ptr<IAudioSource> audioSrc = nullptr;
VideoSurface* camVideoSurface;
CameraSource& camera;
QVector<QPair<QString, QString>> videoDeviceList;