542 lines · 19.6 KB · C
RawDownload
////  AudioClient.h//  libraries/audio-client/src////  Created by Stephen Birarda on 1/22/13.//  Copyright 2013 High Fidelity, Inc.//  Copyright 2021 Vircadia contributors.//  Copyright 2025 Overte e.V.////  Distributed under the Apache License, Version 2.0.//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html// #ifndef hifi_AudioClient_h#define hifi_AudioClient_h #include <fstream>#include <memory>#include <vector>#include <mutex>#include <queue> #include <QFuture>#include <QtCore/QtGlobal>#include <QtCore/QByteArray>#include <QtCore/QElapsedTimer>#include <QtCore/QObject>#include <QtCore/QSharedPointer>#include <QtCore/QVector>#include <QtMultimedia/QAudio>#include <QtMultimedia/QAudioFormat>#include <QtMultimedia/QAudioInput>#include <AbstractAudioInterface.h>#include <AudioEffectOptions.h>#include <AudioStreamStats.h>#include <shared/WebRTC.h> #include <DependencyManager.h>#include <SockAddr.h>#include <NLPacket.h>#include <MixedProcessedAudioStream.h>#include <RingBufferHistory.h>#include <SettingHandle.h>#include <Sound.h>#include <StDev.h>#include <AudioHRTF.h>#include <AudioSRC.h>#include <AudioInjector.h>#include <AudioReverb.h>#include <AudioLimiter.h>#include <AudioConstants.h>#include <AudioGate.h> #include <shared/RateCounter.h> #include <plugins/CodecPlugin.h> #include "AudioIOStats.h"#include "AudioFileWav.h"#include "HifiAudioDeviceInfo.h" #if defined(WEBRTC_AUDIO)#  define WEBRTC_APM_DEBUG_DUMP 0#  include <api/audio/audio_processing.h>#endif #ifdef _WIN32#pragma warning( push )#pragma warning( disable : 4273 )#pragma warning( disable : 4305 )#endif #ifdef _WIN32#pragma warning( pop )#endif #if defined (Q_OS_ANDROID)#define VOICE_RECOGNITION "voicerecognition"#define VOICE_COMMUNICATION "voicecommunication" #define SETTING_AEC_KEY "Android/aec"#define DEFAULT_AEC_ENABLED true#endif class QAudioInput;class QAudioOutput;class QIODevice; class Transform;class NLPacket; #define DEFAULT_STARVE_DETECTION_ENABLED true#define DEFAULT_BUFFER_FRAMES 1 class AudioClient : public AbstractAudioInterface, public Dependency {    Q_OBJECT    SINGLETON_DEPENDENCY     using LocalInjectorsStream = AudioMixRingBuffer;public:    static const int MIN_BUFFER_FRAMES;    static const int MAX_BUFFER_FRAMES;     using AudioPositionGetter = std::function<glm::vec3()>;    using AudioOrientationGetter = std::function<glm::quat()>;     using Mutex = std::mutex;    using Lock = std::unique_lock<Mutex>;     class AudioOutputIODevice : public QIODevice {    public:        AudioOutputIODevice(LocalInjectorsStream& localInjectorsStream, MixedProcessedAudioStream& receivedAudioStream,                AudioClient* audio) :            _localInjectorsStream(localInjectorsStream), _receivedAudioStream(receivedAudioStream),            _audio(audio), _unfulfilledReads(0) {}         void start() { open(QIODevice::ReadOnly | QIODevice::Unbuffered); }        qint64 readData(char* data, qint64 maxSize) override;        qint64 writeData(const char* data, qint64 maxSize) override { return 0; }        int getRecentUnfulfilledReads() { int unfulfilledReads = _unfulfilledReads; _unfulfilledReads = 0; return unfulfilledReads; }    private:        LocalInjectorsStream& _localInjectorsStream;        MixedProcessedAudioStream& _receivedAudioStream;        AudioClient* _audio;        int _unfulfilledReads;    };        void startThread();    void negotiateAudioFormat();    void selectAudioFormat(const QString& selectedCodecName);     Q_INVOKABLE QString getSelectedAudioFormat() const { return _selectedCodecName; }    Q_INVOKABLE bool getNoiseGateOpen() const { return _audioGateOpen; }    Q_INVOKABLE float getSilentInboundPPS() const { return _silentInbound.rate(); }    Q_INVOKABLE float getAudioInboundPPS() const { return _audioInbound.rate(); }    Q_INVOKABLE float getSilentOutboundPPS() const { return _silentOutbound.rate(); }    Q_INVOKABLE float getAudioOutboundPPS() const { return _audioOutbound.rate(); }     const MixedProcessedAudioStream& getReceivedAudioStream() const { return _receivedAudioStream; }    MixedProcessedAudioStream& getReceivedAudioStream() { return _receivedAudioStream; }     const QAudioFormat& getOutputFormat() const { return _outputFormat; }     float getLastInputLoudness() const { return _lastInputLoudness; }     float getTimeSinceLastClip() const { return _timeSinceLastClip; }    float getAudioAverageInputLoudness() const { return _lastInputLoudness; }     const AudioIOStats& getStats() const { return _stats; }     int getOutputBufferSize() { return _outputBufferSizeFrames.get(); }     bool getOutputStarveDetectionEnabled() { return _outputStarveDetectionEnabled.get(); }    void setOutputStarveDetectionEnabled(bool enabled) { _outputStarveDetectionEnabled.set(enabled); }     bool isSimulatingJitter() { return _gate.isSimulatingJitter(); }    void setIsSimulatingJitter(bool enable) { _gate.setIsSimulatingJitter(enable); }     int getGateThreshold() { return _gate.getThreshold(); }    void setGateThreshold(int threshold) { _gate.setThreshold(threshold); }     void setPositionGetter(AudioPositionGetter positionGetter) { _positionGetter = positionGetter; }    void setOrientationGetter(AudioOrientationGetter orientationGetter) { _orientationGetter = orientationGetter; }     void setIsPlayingBackRecording(bool isPlayingBackRecording) { _isPlayingBackRecording = isPlayingBackRecording; }     Q_INVOKABLE void setAvatarBoundingBoxParameters(glm::vec3 corner, glm::vec3 scale);     bool outputLocalInjector(const AudioInjectorPointer& injector) override;     HifiAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const;    QList<HifiAudioDeviceInfo> getAudioDevices(QAudio::Mode mode) const;      void enablePeakValues(bool enable) { _enablePeakValues = enable; }    bool peakValuesAvailable() const;     static const float CALLBACK_ACCELERATOR_RATIO;     bool getNamedAudioDeviceForModeExists(QAudio::Mode mode, const QString& deviceName);     void setRecording(bool isRecording) { _isRecording = isRecording; };    bool getRecording() { return _isRecording; };     bool startRecording(const QString& filename);    void stopRecording();    void setAudioPaused(bool pause);     AudioSolo& getAudioSolo() override { return _solo; } #ifdef Q_OS_WIN    static QString getWinDeviceName(wchar_t* guid);#endif #if defined(Q_OS_ANDROID)    bool isHeadsetPluggedIn() { return _isHeadsetPluggedIn; }#endif     int getNumLocalInjectors(); public slots:    void start();    void stop();     void handleAudioEnvironmentDataPacket(QSharedPointer<ReceivedMessage> message);    void handleAudioDataPacket(QSharedPointer<ReceivedMessage> message);    void handleNoisyMutePacket(QSharedPointer<ReceivedMessage> message);    void handleMuteEnvironmentPacket(QSharedPointer<ReceivedMessage> message);    void handleSelectedAudioFormat(QSharedPointer<ReceivedMessage> message);    void handleMismatchAudioFormat(SharedNodePointer node, const QString& currentCodec, const QString& recievedCodec);     void sendDownstreamAudioStatsPacket() { _stats.publish(); }    void handleMicAudioInput();    void audioInputStateChanged(QAudio::State state);    void checkInputTimeout();    void handleDummyAudioInput();    void handleRecordedAudioInput(const QByteArray& audio);    void reset();    void audioMixerKilled();     void setMuted(bool muted, bool emitSignal = true);    bool isMuted() { return _isMuted; }     virtual bool setIsStereoInput(bool stereo) override;    virtual bool isStereoInput() override { return _isStereoInput; }     void setNoiseReduction(bool isNoiseGateEnabled, bool emitSignal = true);    bool isNoiseReductionEnabled() const { return _isNoiseGateEnabled; }        void setNoiseReductionAutomatic(bool isNoiseGateAutomatic, bool emitSignal = true);    bool isNoiseReductionAutomatic() const { return _isNoiseReductionAutomatic; }        void setNoiseReductionThreshold(float noiseReductionThreshold, bool emitSignal = true);    float noiseReductionThreshold() const { return _noiseReductionThreshold; }     void setWarnWhenMuted(bool isNoiseGateEnabled, bool emitSignal = true);    bool isWarnWhenMutedEnabled() const { return _warnWhenMuted; }     void setAcousticEchoCancellation(bool isAECEnabled, bool emitSignal = true);    bool isAcousticEchoCancellationEnabled() const { return _isAECEnabled; }     virtual bool getLocalEcho() override { return _shouldEchoLocally; }    virtual void setLocalEcho(bool localEcho) override { _shouldEchoLocally = localEcho; }    virtual void toggleLocalEcho() override { _shouldEchoLocally = !_shouldEchoLocally; }     virtual bool getServerEcho() override { return _shouldEchoToServer; }    virtual void setServerEcho(bool serverEcho) override { _shouldEchoToServer = serverEcho; }    virtual void toggleServerEcho() override { _shouldEchoToServer = !_shouldEchoToServer; }     void processReceivedSamples(const QByteArray& inputBuffer, QByteArray& outputBuffer);    void sendMuteEnvironmentPacket();     int setOutputBufferSize(int numFrames, bool persist = true);     bool shouldLoopbackInjectors() override { return _shouldEchoToServer; }     // calling with a null QAudioDevice will use the system default    bool switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo = HifiAudioDeviceInfo());    bool switchAudioDevice(QAudio::Mode mode, const QString& deviceName, bool isHmd);    void setHmdAudioName(QAudio::Mode mode, const QString& name);    // Qt opensles plugin is not able to detect when the headset is plugged in    void setHeadsetPluggedIn(bool pluggedIn);     float getInputVolume() const { return (_audioInput) ? (float)_audioInput->volume() : 0.0f; }    void setInputVolume(float volume, bool emitSignal = true);    void setReverb(bool reverb);    void setReverbOptions(const AudioEffectOptions* options);     void setLocalInjectorGain(float gain) { _localInjectorGain = gain; };    void setSystemInjectorGain(float gain) { _systemInjectorGain = gain; };    void setOutputGain(float gain) { _outputGain = gain; };     void outputNotify();    void noteAwakening();     void loadSettings();    void saveSettings(); signals:    void inputVolumeChanged(float volume);    void muteToggled(bool muted);    void noiseReductionChanged(bool noiseReductionEnabled);    void noiseReductionAutomaticChanged(bool noiseReductionAutomatic);    void noiseReductionThresholdChanged(bool noiseReductionThreshold);    void warnWhenMutedChanged(bool warnWhenMutedEnabled);    void acousticEchoCancellationChanged(bool acousticEchoCancellationEnabled);    void mutedByMixer();    void inputReceived(const QByteArray& inputSamples);    void inputLoudnessChanged(float loudness, bool isClipping);    void outputBytesToNetwork(int numBytes);    void inputBytesFromNetwork(int numBytes);    void noiseGateOpened();    void noiseGateClosed();     void changeDevice(const HifiAudioDeviceInfo& outputDeviceInfo);     void deviceChanged(QAudio::Mode mode, const HifiAudioDeviceInfo& device);    void devicesChanged(QAudio::Mode mode, const QList<HifiAudioDeviceInfo>& devices);    void peakValueListChanged(const QList<float> peakValueList);     void receivedFirstPacket();    void disconnected();     void audioFinished();     void muteEnvironmentRequested(glm::vec3 position, float radius);     void outputBufferReceived(const QByteArray _outputBuffer); protected:    AudioClient();    ~AudioClient();     virtual void customDeleter() override; private:    static const int RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES{ 100 };    // OUTPUT_CHANNEL_COUNT is audio pipeline output format, which is always 2 channel.    // _outputFormat.channelCount() is device output format, which may be 1 or multichannel.    static const int OUTPUT_CHANNEL_COUNT{ 2 };    static const int STARVE_DETECTION_THRESHOLD{ 3 };    static const int STARVE_DETECTION_PERIOD{ 10 * 1000 }; // 10 Seconds     static const AudioPositionGetter DEFAULT_POSITION_GETTER;    static const AudioOrientationGetter DEFAULT_ORIENTATION_GETTER;     friend class CheckDevicesThread;    friend class LocalInjectorsThread;     float loudnessToLevel(float loudness);     // background tasks    void checkDevices();    void checkPeakValues();     void outputFormatChanged();    void handleAudioInput(QByteArray& audioBuffer);    void prepareLocalAudioInjectors(std::unique_ptr<Lock> localAudioLock = nullptr);    bool mixLocalAudioInjectors(float* mixBuffer);    float azimuthForSource(const glm::vec3& relativePosition);    float gainForSource(float distance, float volume); #ifdef Q_OS_ANDROID    QTimer _checkInputTimer{ this };    long _inputReadsSinceLastCheck = 0l;    bool _isHeadsetPluggedIn { false };#endif     class Gate {    public:        Gate(AudioClient* audioClient);         bool isSimulatingJitter() { return _isSimulatingJitter; }        void setIsSimulatingJitter(bool enable);         int getThreshold() { return _threshold; }        void setThreshold(int threshold);         void insert(QSharedPointer<ReceivedMessage> message);     private:        void flush();         AudioClient* _audioClient;        std::queue<QSharedPointer<ReceivedMessage>> _queue;        std::mutex _mutex;         int _index{ 0 };        int _threshold{ 1 };        bool _isSimulatingJitter{ false };    };     Gate _gate{ this };     Mutex _injectorsMutex;    QAudioInput* _audioInput{ nullptr };    QTimer* _dummyAudioInput{ nullptr };    QAudioFormat _desiredInputFormat;    QAudioFormat _inputFormat;    QIODevice* _inputDevice{ nullptr };    int _numInputCallbackBytes{ 0 };    QAudioOutput* _audioOutput{ nullptr };    std::atomic<bool> _audioOutputInitialized { false };    QAudioFormat _desiredOutputFormat;    QAudioFormat _outputFormat;    int _outputFrameSize{ 0 };    int _numOutputCallbackBytes{ 0 };    QAudioOutput* _loopbackAudioOutput{ nullptr };    QIODevice* _loopbackOutputDevice{ nullptr };    AudioRingBuffer _inputRingBuffer{ 0 };    LocalInjectorsStream _localInjectorsStream{ 0 , 1 };    // In order to use _localInjectorsStream as a lock-free pipe,    // use it with a single producer/consumer, and track available samples and injectors    std::atomic<int> _localSamplesAvailable { 0 };    std::atomic<bool> _localInjectorsAvailable { false };    MixedProcessedAudioStream _receivedAudioStream{ RECEIVED_AUDIO_STREAM_CAPACITY_FRAMES };    bool _isStereoInput{ false };    std::atomic<bool> _enablePeakValues { false };     quint64 _outputStarveDetectionStartTimeMsec{ 0 };    int _outputStarveDetectionCount { 0 };     Setting::Handle<int> _outputBufferSizeFrames{"audioOutputBufferFrames", DEFAULT_BUFFER_FRAMES};    int _sessionOutputBufferSizeFrames{ _outputBufferSizeFrames.get() };    Setting::Handle<bool> _outputStarveDetectionEnabled{ "audioOutputStarveDetectionEnabled", DEFAULT_STARVE_DETECTION_ENABLED};     StDev _stdev;    QElapsedTimer _timeSinceLastReceived;    float _lastRawInputLoudness{ 0.0f };    // before mute/gate    float _lastSmoothedRawInputLoudness{ 0.0f };    float _lastInputLoudness{ 0.0f };       // after mute/gate    float _timeSinceLastClip{ -1.0f };    int _totalInputAudioSamples;     bool _isMuted{ false };    bool _shouldEchoLocally{ false };    bool _shouldEchoToServer{ false };    bool _isNoiseGateEnabled{ true };    bool _isNoiseReductionAutomatic{ true };    float _noiseReductionThreshold{ 0.1f };    bool _warnWhenMuted;    bool _isAECEnabled{ true };     bool _reverb{ false };    AudioEffectOptions _scriptReverbOptions;    AudioEffectOptions _zoneReverbOptions;    AudioEffectOptions* _reverbOptions{ &_scriptReverbOptions };    AudioReverb _sourceReverb { AudioConstants::SAMPLE_RATE };    AudioReverb _listenerReverb { AudioConstants::SAMPLE_RATE };    AudioReverb _localReverb { AudioConstants::SAMPLE_RATE };     // possible streams needed for resample    AudioSRC* _inputToNetworkResampler{ nullptr };    AudioSRC* _networkToOutputResampler{ nullptr };    AudioSRC* _localToOutputResampler{ nullptr };    AudioSRC* _loopbackResampler{ nullptr };     // for network audio (used by network audio thread)    int16_t _networkScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];     // for output audio (used by this thread)    int _outputPeriod { 0 };    float* _outputMixBuffer { NULL };    int16_t* _outputScratchBuffer { NULL };    std::atomic<float> _outputGain { 1.0f };    float _lastOutputGain { 1.0f };     // for local audio (used by audio injectors thread)    std::atomic<float> _localInjectorGain { 1.0f };    std::atomic<float> _systemInjectorGain { 1.0f };    float _localMixBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_STEREO];    int16_t _localScratchBuffer[AudioConstants::NETWORK_FRAME_SAMPLES_AMBISONIC];    float* _localOutputMixBuffer { NULL };    Mutex _localAudioMutex;    AudioLimiter _audioLimiter{ AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT };     // Adds Reverb    void configureReverb();    void updateReverbOptions();    void handleLocalEchoAndReverb(QByteArray& inputByteArray); #if defined(WEBRTC_AUDIO)    static const int WEBRTC_SAMPLE_RATE_MAX = 96000;    static const int WEBRTC_CHANNELS_MAX = 2;    static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000;     webrtc::scoped_refptr<webrtc::AudioProcessing> _apm { nullptr };     int16_t _fifoFarEnd[WEBRTC_CHANNELS_MAX * WEBRTC_FRAMES_MAX] {};    int _numFifoFarEnd = 0; // numFrames saved in fifo     void configureWebrtc();    void processWebrtcFarEnd(const int16_t* samples, int numFrames, int numChannels, int sampleRate);    void processWebrtcNearEnd(int16_t* samples, int numFrames, int numChannels, int sampleRate);#endif     bool switchInputToAudioDevice(const HifiAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest = false);    bool switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest = false);     // Callback acceleration dependent calculations    int calculateNumberOfInputCallbackBytes(const QAudioFormat& format) const;    int calculateNumberOfFrameSamples(int numBytes) const;     quint16 _outgoingAvatarAudioSequenceNumber{ 0 };     AudioOutputIODevice _audioOutputIODevice{ _localInjectorsStream, _receivedAudioStream, this };     AudioIOStats _stats{ &_receivedAudioStream };     AudioGate* _audioGate { nullptr };    bool _audioGateOpen { true };     AudioPositionGetter _positionGetter{ DEFAULT_POSITION_GETTER };    AudioOrientationGetter _orientationGetter{ DEFAULT_ORIENTATION_GETTER };     glm::vec3 avatarBoundingBoxCorner;    glm::vec3 avatarBoundingBoxScale;     HifiAudioDeviceInfo _inputDeviceInfo;    HifiAudioDeviceInfo _outputDeviceInfo; 

Lines 1–500 of 542.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t