diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c5860a8c..726f2137d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -238,10 +238,13 @@ set(${PROJECT_NAME}_SOURCES src/audio/audio.h src/audio/backend/alsink.cpp src/audio/backend/alsink.h + src/audio/backend/alsource.cpp + src/audio/backend/alsource.h src/audio/backend/openal.cpp src/audio/backend/openal.h src/audio/iaudiosettings.h src/audio/iaudiosink.h + src/audio/iaudiosource.h src/chatlog/chatlinecontent.cpp src/chatlog/chatlinecontent.h src/chatlog/chatlinecontentproxy.cpp diff --git a/src/audio/audio.cpp b/src/audio/audio.cpp index 0c460b6f5..5b04fce0d 100644 --- a/src/audio/audio.cpp +++ b/src/audio/audio.cpp @@ -24,34 +24,9 @@ #endif #include "src/persistence/settings.h" -#include - -#include - /** * @class Audio * - * @enum Audio::Sound - * @brief Provides the different sounds for use in the getSound function. - * @see getSound - * - * @value NewMessage Returns the new message notification sound. - * @value Test Returns the test sound. - * @value IncomingCall Returns the incoming call sound. - * @value OutgoingCall Returns the outgoing call sound. - * - * @fn QString Audio::getSound(Sound s) - * @brief Function to get the path of the requested sound. - * - * @param s Name of the sound to get the path of. - * @return The path of the requested sound. - * - * @fn void Audio::frameAvailable(const int16_t *pcm, size_t sample_count, uint8_t channels, - * uint32_t sampling_rate); - * - * When there are input subscribers, we regularly emit captured audio frames with this signal - * Always connect with a blocking queued connection lambda, else the behaviour is undefined - * * @var Audio::AUDIO_SAMPLE_RATE * @brief The next best Opus would take is 24k * @@ -89,37 +64,6 @@ * * @note Default is 30dB; usually you don't need to alter this value. * - * @fn void Audio::subscribeInput() - * @brief Subscribe to capture sound from the opened input device. - * - * If the input device is not open, it will be opened before capturing. - * - * @fn void Audio::unsubscribeInput() - * @brief Unsubscribe from capturing from an opened input device. - * - * If the input device has no more subscriptions, it will be closed. - * - * @fn void Audio::playMono16Sound(const QString& path) - * @brief Play a 44100Hz mono 16bit PCM sound from a file - * - * @param[in] path the path to the sound file - * - * @fn void Audio::playMono16Sound(const QByteArray& data) - * @brief Play a 44100Hz mono 16bit PCM sound - * - * @param[in] data 44100Hz mono 16bit PCM data in host byte order - * - * @fn void Audio::playAudioBuffer(uint sourceId, const int16_t* data, int samples, - * unsigned channels, int sampleRate) - * @brief adds a number of audio frames to the play buffer - * - * @param[in] sourceId id obtained by subscribeOutput(uint &) - * @param[in] data 16bit mono or stereo PCM data with alternating channel - * mapping for stereo (LRLR) - * @param[in] samples number of samples per channel - * @param[in] channels number of channels, currently 1 or 2 is supported - * @param[in] sampleRate sample rate in Hertz - * * @fn bool Audio::isOutputReady() const * @brief check if the output is ready to play audio * @@ -135,24 +79,6 @@ * * @return list of input devices * - * @fn void Audio::subscribeOutput(uint& sid) - * @brief register a new output source - * - * param[out] sid contains the sourceId if source creation was successful, - * unchanged otherwise - * - * @fn void Audio::unsubscribeOutput(uint& sid) - * @brief unregisters an output source - * - * param[out] sid contains 0 if source deletion was successful, - * unchanged otherwise - * - * @fn void Audio::startLoop() - * @brief starts looping the sound played with playMono16Sound() - * - * @fn void Audio::stopLoop() - * @brief stops looping the sound played with playMono16Sound() - * * @fn qreal Audio::inputGain() const * @brief get the current input gain * diff --git a/src/audio/audio.h b/src/audio/audio.h index af352fced..3ad575135 100644 --- a/src/audio/audio.h +++ b/src/audio/audio.h @@ -25,6 +25,7 @@ #include class IAudioSink; +class IAudioSource; class Audio : public QObject { Q_OBJECT @@ -60,12 +61,8 @@ public: virtual QStringList outDeviceNames() = 0; virtual QStringList inDeviceNames() = 0; - virtual void subscribeInput() = 0; - virtual void unsubscribeInput() = 0; - - virtual void stopActive() = 0; - virtual std::unique_ptr makeSink() = 0; + virtual std::unique_ptr makeSource() = 0; protected: // Public default audio settings @@ -75,12 +72,6 @@ protected: static constexpr uint32_t AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL = AUDIO_FRAME_DURATION * AUDIO_SAMPLE_RATE / 1000; uint32_t AUDIO_FRAME_SAMPLE_COUNT_TOTAL = 0; - -signals: - void frameAvailable(const int16_t* pcm, size_t sample_count, uint8_t channels, - uint32_t sampling_rate); - void volumeAvailable(float value); - void startActive(qreal msec); }; #endif // AUDIO_H diff --git a/src/audio/backend/alsource.cpp b/src/audio/backend/alsource.cpp new file mode 100644 index 000000000..39dd9bed0 --- /dev/null +++ b/src/audio/backend/alsource.cpp @@ -0,0 +1,41 @@ +#include "src/audio/backend/alsource.h" +#include "src/audio/backend/openal.h" + +/** + * @brief Emits audio frames captured by an input device or other audio source. + */ + +/** + * @brief Reserves ressources for an audio source + * @param audio Main audio object, must have longer lifetime than this object. + */ +AlSource::AlSource(OpenAL& al) + : audio(al) + , killLock(QMutex::Recursive) +{} + +AlSource::~AlSource() +{ + QMutexLocker{&killLock}; + + // unsubscribe only if not already killed + if (!killed) { + audio.destroySource(*this); + killed = true; + } +} + +AlSource::operator bool() const +{ + QMutexLocker{&killLock}; + return !killed; +} + +void AlSource::kill() +{ + killLock.lock(); + // this flag is only set once here, afterwards the object is considered dead + killed = true; + killLock.unlock(); + emit invalidated(); +} diff --git a/src/audio/backend/alsource.h b/src/audio/backend/alsource.h new file mode 100644 index 000000000..bccabbf0b --- /dev/null +++ b/src/audio/backend/alsource.h @@ -0,0 +1,30 @@ +#ifndef ALSOURCE_H +#define ALSOURCE_H + +#include "src/audio/iaudiosource.h" +#include +#include + +class OpenAL; +class AlSource : public IAudioSource +{ + Q_OBJECT +public: + AlSource(OpenAL& al); + AlSource(AlSource& src) = delete; + AlSource& operator=(const AlSource&) = delete; + AlSource(AlSource&& other) = delete; + AlSource& operator=(AlSource&& other) = delete; + ~AlSource(); + + operator bool() const; + + void kill(); + +private: + OpenAL& audio; + bool killed = false; + mutable QMutex killLock; +}; + +#endif // ALSOURCE_H diff --git a/src/audio/backend/openal.cpp b/src/audio/backend/openal.cpp index bcd5a995e..d3c950390 100644 --- a/src/audio/backend/openal.cpp +++ b/src/audio/backend/openal.cpp @@ -18,8 +18,6 @@ */ #include "openal.h" -#include "src/core/core.h" -#include "src/core/coreav.h" #include "src/persistence/settings.h" #include @@ -74,8 +72,9 @@ OpenAL::OpenAL() voiceTimer.setSingleShot(true); voiceTimer.moveToThread(audioThread); - connect(this, &Audio::startActive, &voiceTimer, static_cast(&QTimer::start)); - connect(&voiceTimer, &QTimer::timeout, this, &Audio::stopActive); + connect(this, &OpenAL::startActive, &voiceTimer, + static_cast(&QTimer::start)); + connect(&voiceTimer, &QTimer::timeout, this, &OpenAL::stopActive); connect(&captureTimer, &QTimer::timeout, this, &OpenAL::doAudio); captureTimer.setInterval(AUDIO_FRAME_DURATION / 2); @@ -218,8 +217,19 @@ qreal OpenAL::maxInputThreshold() const void OpenAL::reinitInput(const QString& inDevDesc) { QMutexLocker locker(&audioLock); + + const auto bakSources = sources; + sources.clear(); + cleanupInput(); initInput(inDevDesc); + + locker.unlock(); + // this must happen outside `audioLock`, to avoid a deadlock when + // a slot on AlSource::invalidate tries to create a new source immedeately. + for (auto& source : bakSources) { + source->kill(); + } } bool OpenAL::reinitOutput(const QString& outDevDesc) @@ -308,17 +318,25 @@ void OpenAL::destroySink(AlSink& sink) * * If the input device is not open, it will be opened before capturing. */ -void OpenAL::subscribeInput() +std::unique_ptr OpenAL::makeSource() { QMutexLocker locker(&audioLock); if (!autoInitInput()) { qWarning("Failed to subscribe to audio input device."); - return; + return {}; } - ++inSubscriptions; - qDebug() << "Subscribed to audio input device [" << inSubscriptions << "subscriptions ]"; + auto const source = new AlSource(*this); + if (source == nullptr) { + return {}; + } + + sources.insert(source); + + qDebug() << "Subscribed to audio input device [" << sources.size() << "subscriptions ]"; + + return std::unique_ptr{source}; } /** @@ -326,18 +344,23 @@ void OpenAL::subscribeInput() * * If the input device has no more subscriptions, it will be closed. */ -void OpenAL::unsubscribeInput() +void OpenAL::destroySource(AlSource& source) { QMutexLocker locker(&audioLock); - if (!inSubscriptions) + const auto s = sources.find(&source); + if (s == sources.end()) { + qWarning() << "Destroyed non-existant source"; return; + } - inSubscriptions--; - qDebug() << "Unsubscribed from audio input device [" << inSubscriptions << "subscriptions left ]"; + sources.erase(s); - if (!inSubscriptions) + qDebug() << "Unsubscribed from audio input device [" << sources.size() << "subscriptions left ]"; + + if (sources.empty()) { cleanupInput(); + } } /** @@ -646,13 +669,19 @@ void OpenAL::doInput() volume = 0; } - emit Audio::volumeAvailable(volume); + // NOTE(sudden6): this loop probably doesn't scale too well with many sources + for (auto source : sources) { + emit source->volumeAvailable(volume); + } if (!isActive) { return; } - emit Audio::frameAvailable(inputBuffer, AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL, channels, - AUDIO_SAMPLE_RATE); + // NOTE(sudden6): this loop probably doesn't scale too well with many sources + for (auto source : sources) { + emit source->frameAvailable(inputBuffer, AUDIO_FRAME_SAMPLE_COUNT_PER_CHANNEL, channels, + AUDIO_SAMPLE_RATE); + } } void OpenAL::doOutput() @@ -670,7 +699,7 @@ void OpenAL::doAudio() // Output section does nothing // Input section - if (alInDev && inSubscriptions) { + if (alInDev && !sources.empty()) { doInput(); } } diff --git a/src/audio/backend/openal.h b/src/audio/backend/openal.h index f78a8b33d..6079ad4bb 100644 --- a/src/audio/backend/openal.h +++ b/src/audio/backend/openal.h @@ -23,6 +23,7 @@ #include "src/audio/audio.h" #include "src/audio/backend/alsink.h" +#include "src/audio/backend/alsource.h" #include #include @@ -89,8 +90,8 @@ public: std::unique_ptr makeSink(); void destroySink(AlSink& sink); - void subscribeInput(); - void unsubscribeInput(); + std::unique_ptr makeSource(); + void destroySource(AlSource& source); void startLoop(uint sourceId); void stopLoop(uint sourceId); @@ -99,6 +100,8 @@ public: void playAudioBuffer(uint sourceId, const int16_t* data, int samples, unsigned channels, int sampleRate); +signals: + void startActive(qreal msec); protected: static void checkAlError() noexcept; @@ -135,7 +138,6 @@ protected: QString outDev{}; ALCdevice* alInDev = nullptr; - quint32 inSubscriptions = 0; QTimer captureTimer; QTimer cleanupTimer; @@ -147,9 +149,7 @@ protected: // Qt containers need copy operators, so use stdlib containers std::unordered_set sinks; std::unordered_set soundSinks; - - // number of output sources - int outCount = 0; + std::unordered_set sources; int channels = 0; qreal gain = 0; diff --git a/src/audio/backend/openal2.cpp b/src/audio/backend/openal2.cpp index 5da941a4f..1271bc485 100644 --- a/src/audio/backend/openal2.cpp +++ b/src/audio/backend/openal2.cpp @@ -18,8 +18,6 @@ */ #include "openal2.h" -#include "src/core/core.h" -#include "src/core/coreav.h" #include "src/persistence/settings.h" #include diff --git a/src/audio/iaudiosource.h b/src/audio/iaudiosource.h index c9f1c1da2..bcebe7d15 100644 --- a/src/audio/iaudiosource.h +++ b/src/audio/iaudiosource.h @@ -3,7 +3,14 @@ #include -class Audio; +/** + * @fn void Audio::frameAvailable(const int16_t *pcm, size_t sample_count, uint8_t channels, + * uint32_t sampling_rate); + * + * When there are input subscribers, we regularly emit captured audio frames with this signal + * Always connect with a blocking queued connection lambda, else the behaviour is undefined + */ + class IAudioSource : public QObject { Q_OBJECT diff --git a/src/core/toxcall.cpp b/src/core/toxcall.cpp index ae04b27af..9944cbe2a 100644 --- a/src/core/toxcall.cpp +++ b/src/core/toxcall.cpp @@ -31,14 +31,14 @@ ToxCall::ToxCall(bool VideoEnabled, CoreAV& av) : av{&av} , videoEnabled{VideoEnabled} + , audioSource{Audio::getInstance().makeSource()} {} ToxCall::~ToxCall() { - Audio& audio = Audio::getInstance(); - QObject::disconnect(audioInConn); - audio.unsubscribeInput(); + QObject::disconnect(audioSrcInvalid); + if (videoEnabled) { QObject::disconnect(videoInConn); CameraSource::getInstance().unsubscribe(); @@ -105,14 +105,15 @@ ToxFriendCall::ToxFriendCall(uint32_t FriendNum, bool VideoEnabled, CoreAV& av) , sink(Audio::getInstance().makeSink()) , friendId{FriendNum} { - // register audio - Audio& audio = Audio::getInstance(); - audio.subscribeInput(); - audioInConn = QObject::connect(&Audio::getInstance(), &Audio::frameAvailable, - [&av, FriendNum](const int16_t* pcm, size_t samples, - uint8_t chans, uint32_t rate) { - av.sendCallAudio(FriendNum, pcm, samples, chans, rate); - }); + // TODO(sudden6): move this to audio source + audioInConn = + QObject::connect(audioSource.get(), &IAudioSource::frameAvailable, + [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) { + this->av->sendCallAudio(this->friendId, pcm, samples, chans, rate); + }); + + audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated, + [this]() { this->onAudioSourceInvalidated(); }); if (!audioInConn) { qDebug() << "Audio input connection not working"; @@ -146,6 +147,20 @@ ToxFriendCall::~ToxFriendCall() QObject::disconnect(audioSinkInvalid); } +void ToxFriendCall::onAudioSourceInvalidated() +{ + auto newSrc = Audio::getInstance().makeSource(); + audioInConn = + QObject::connect(newSrc.get(), &IAudioSource::frameAvailable, + [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) { + this->av->sendCallAudio(this->friendId, pcm, samples, chans, rate); + }); + audioSource = std::move(newSrc); + + audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated, + [this]() { this->onAudioSourceInvalidated(); }); +} + void ToxFriendCall::onAudioSinkInvalidated() { auto newSink = Audio::getInstance().makeSink(); @@ -189,7 +204,7 @@ void ToxFriendCall::setState(const TOXAV_FRIEND_CALL_STATE& value) } void ToxFriendCall::playAudioBuffer(const int16_t* data, int samples, unsigned channels, - int sampleRate) const + int sampleRate) const { if (sink) { sink->playAudioBuffer(data, samples, channels, sampleRate); @@ -201,17 +216,18 @@ ToxGroupCall::ToxGroupCall(int GroupNum, CoreAV& av) , groupId{GroupNum} { // register audio - Audio& audio = Audio::getInstance(); - audio.subscribeInput(); - audioInConn = QObject::connect(&Audio::getInstance(), &Audio::frameAvailable, - [&av, GroupNum](const int16_t* pcm, size_t samples, - uint8_t chans, uint32_t rate) { - av.sendGroupCallAudio(GroupNum, pcm, samples, chans, rate); - }); + audioInConn = + QObject::connect(audioSource.get(), &IAudioSource::frameAvailable, + [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) { + this->av->sendGroupCallAudio(this->groupId, pcm, samples, chans, rate); + }); if (!audioInConn) { qDebug() << "Audio input connection not working"; } + + audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated, + [this]() { this->onAudioSourceInvalidated(); }); } ToxGroupCall::~ToxGroupCall() @@ -220,6 +236,23 @@ ToxGroupCall::~ToxGroupCall() clearPeers(); } +void ToxGroupCall::onAudioSourceInvalidated() +{ + auto newSrc = Audio::getInstance().makeSource(); + // TODO(sudden6): move this to audio source + audioInConn = + QObject::connect(audioSource.get(), &IAudioSource::frameAvailable, + [this](const int16_t* pcm, size_t samples, uint8_t chans, uint32_t rate) { + this->av->sendGroupCallAudio(this->groupId, pcm, samples, chans, rate); + }); + + audioSource = std::move(newSrc); + + audioSrcInvalid = QObject::connect(audioSource.get(), &IAudioSource::invalidated, + [this]() { this->onAudioSourceInvalidated(); }); +} + + void ToxGroupCall::onAudioSinkInvalidated(ToxPk peerId) { removePeer(peerId); diff --git a/src/core/toxcall.h b/src/core/toxcall.h index 7c7eeac22..f404b7623 100644 --- a/src/core/toxcall.h +++ b/src/core/toxcall.h @@ -2,6 +2,7 @@ #define TOXCALL_H #include "src/audio/iaudiosink.h" +#include "src/audio/iaudiosource.h" #include #include @@ -60,6 +61,8 @@ protected: QMetaObject::Connection videoInConn; bool videoEnabled{false}; bool nullVideoBitrate{false}; + std::unique_ptr audioSource = nullptr; + QMetaObject::Connection audioSrcInvalid; }; class ToxFriendCall : public ToxCall @@ -77,8 +80,7 @@ public: TOXAV_FRIEND_CALL_STATE getState() const; void setState(const TOXAV_FRIEND_CALL_STATE& value); - void playAudioBuffer(const int16_t* data, int samples, unsigned channels, - int sampleRate) const; + void playAudioBuffer(const int16_t* data, int samples, unsigned channels, int sampleRate) const; protected: std::unique_ptr timeoutTimer; @@ -106,7 +108,8 @@ public: ToxGroupCall& operator=(ToxGroupCall&& other) = delete; void removePeer(ToxPk peerId); - void playAudioBuffer(const ToxPk& peer, const int16_t* data, int samples, unsigned channels, int sampleRate); + void playAudioBuffer(const ToxPk& peer, const int16_t* data, int samples, unsigned channels, + int sampleRate); private: void addPeer(ToxPk peerId); diff --git a/src/widget/form/settings/avform.cpp b/src/widget/form/settings/avform.cpp index 5722cfd49..db2b629dc 100644 --- a/src/widget/form/settings/avform.cpp +++ b/src/widget/form/settings/avform.cpp @@ -29,6 +29,7 @@ #include "src/audio/audio.h" #include "src/audio/iaudiosettings.h" +#include "src/audio/iaudiosource.h" #include "src/core/core.h" #include "src/core/coreav.h" #include "src/video/cameradevice.h" @@ -50,7 +51,6 @@ AVForm::AVForm(Audio* audio, CoreAV* coreAV, CameraSource& camera, IAudioSetting , coreAV{coreAV} , audioSettings{audioSettings} , videoSettings{videoSettings} - , subscribedToAudioIn(false) , camVideoSurface(nullptr) , camera(camera) { @@ -98,7 +98,6 @@ AVForm::AVForm(Audio* audio, CoreAV* coreAV, CameraSource& camera, IAudioSetting audioThresholdSlider->setTracking(false); audioThresholdSlider->installEventFilter(this); - connect(audio, &Audio::volumeAvailable, this, &AVForm::setVolume); volumeDisplay->setMaximum(totalSliderSteps); fillAudioQualityComboBox(); @@ -120,13 +119,8 @@ AVForm::~AVForm() void AVForm::hideEvent(QHideEvent* event) { - if (subscribedToAudioIn) { - // TODO: This should not be done in show/hide events - audio->unsubscribeInput(); - subscribedToAudioIn = false; - } - audioSink.reset(); + audioSrc.reset(); if (camVideoSurface) { camVideoSurface->setSource(nullptr); @@ -144,10 +138,9 @@ void AVForm::showEvent(QShowEvent* event) createVideoSurface(); getVideoDevices(); - if (!subscribedToAudioIn) { - // TODO: This should not be done in show/hide events - audio->subscribeInput(); - subscribedToAudioIn = true; + if (audioSrc == nullptr) { + audioSrc = audio->makeSource(); + connect(audioSrc.get(), &IAudioSource::volumeAvailable, this, &AVForm::setVolume); } if (audioSink == nullptr) { @@ -540,17 +533,17 @@ void AVForm::on_inDevCombobox_currentIndexChanged(int deviceIndex) const bool inputEnabled = deviceIndex > 0; audioSettings->setAudioInDevEnabled(inputEnabled); - QString deviceName; + QString deviceName{}; if (inputEnabled) { deviceName = inDevCombobox->itemText(deviceIndex); } - audioSettings->setInDev(deviceName); - - audio->reinitInput(deviceName); - subscribedToAudioIn = inputEnabled; - if (inputEnabled) { - audio->subscribeInput(); + const QString oldName = audioSettings->getInDev(); + if (oldName != deviceName) { + audioSettings->setInDev(deviceName); + audio->reinitInput(deviceName); + audioSrc = audio->makeSource(); + connect(audioSrc.get(), &IAudioSource::volumeAvailable, this, &AVForm::setVolume); } microphoneSlider->setEnabled(inputEnabled); diff --git a/src/widget/form/settings/avform.h b/src/widget/form/settings/avform.h index 118f6ab93..4b4675650 100644 --- a/src/widget/form/settings/avform.h +++ b/src/widget/form/settings/avform.h @@ -31,12 +31,13 @@ #include class Audio; +class IAudioSettings; +class IAudioSink; +class IAudioSource; class CameraSource; class CoreAV; -class IAudioSettings; class IVideoSettings; class VideoSurface; -class IAudioSink; class AVForm : public GenericForm, private Ui::AVForm { Q_OBJECT @@ -103,6 +104,7 @@ private: bool subscribedToAudioIn; std::unique_ptr audioSink = nullptr; + std::unique_ptr audioSrc = nullptr; VideoSurface* camVideoSurface; CameraSource& camera; QVector> videoDeviceList;