////  Audio.cpp//  interface////  Created by Stephen Birarda on 1/22/13.//  Copyright (c) 2013 High Fidelity, Inc. All rights reserved.//#ifndef _WIN32 #include <cstring> #include <iostream>#include <pthread.h>#include <sys/stat.h> #include <AngleUtil.h>#include <NodeList.h>#include <NodeTypes.h>#include <PacketHeaders.h>#include <SharedUtil.h>#include <StdDev.h>#include <UDPSocket.h> #include "Application.h"#include "Audio.h"#include "Util.h" //  Uncomment the following definition to test audio device latency by copying output to input//#define TEST_AUDIO_LOOPBACK//#define SHOW_AUDIO_DEBUG #define VISUALIZE_ECHO_CANCELLATION static const int PHASE_DELAY_AT_90 = 20;static const float AMPLITUDE_RATIO_AT_90 = 0.5;static const int MIN_FLANGE_EFFECT_THRESHOLD = 600;static const int MAX_FLANGE_EFFECT_THRESHOLD = 1500;static const float FLANGE_BASE_RATE = 4;static const float MAX_FLANGE_SAMPLE_WEIGHT = 0.50;static const float MIN_FLANGE_INTENSITY = 0.25; static const float JITTER_BUFFER_LENGTH_MSECS = 12;static const short JITTER_BUFFER_SAMPLES = JITTER_BUFFER_LENGTH_MSECS *                                           NUM_AUDIO_CHANNELS * (SAMPLE_RATE / 1000.0); static const float AUDIO_CALLBACK_MSECS = (float)BUFFER_LENGTH_SAMPLES_PER_CHANNEL / (float)SAMPLE_RATE * 1000.0; static const int NODE_LOOPBACK_MODIFIER = 307; // Speex preprocessor and echo canceller adaptionstatic const int   AEC_N_CHANNELS_MIC = 1;                                      // Number of microphone channelsstatic const int   AEC_N_CHANNELS_PLAY = 2;                                     // Number of speaker channelsstatic const int   AEC_FILTER_LENGTH = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * 20;  // Width of the filterstatic const int   AEC_BUFFERED_FRAMES = 6;                                     // Maximum number of frames to bufferstatic const int   AEC_BUFFERED_SAMPLES_PER_CHANNEL = BUFFER_LENGTH_SAMPLES_PER_CHANNEL * AEC_BUFFERED_FRAMES;static const int   AEC_BUFFERED_SAMPLES = AEC_BUFFERED_SAMPLES_PER_CHANNEL * AEC_N_CHANNELS_PLAY;static const int   AEC_TMP_BUFFER_SIZE = (AEC_N_CHANNELS_MIC +                  // Temporary space for processing a                                          AEC_N_CHANNELS_PLAY) * BUFFER_LENGTH_SAMPLES_PER_CHANNEL;               //  single frame // Ping test configuration static const float PING_PITCH = 16.f;                                           // Ping wavelength, # samples / radianstatic const float PING_VOLUME = 32000.f;                                       // Ping peak amplitudestatic const int   PING_MIN_AMPLI = 225;                                        // Minimum amplitude static const int   PING_MAX_PERIOD_DIFFERENCE = 15;                             // Maximum # samples from expected periodstatic const int   PING_PERIOD = int(Radians::twicePi() * PING_PITCH);          // Sine period based on the given pitchstatic const int   PING_HALF_PERIOD = int(Radians::pi() * PING_PITCH);          // Distance between extremastatic const int   PING_FRAMES_TO_RECORD = AEC_BUFFERED_FRAMES;                 // Frames to record for analysisstatic const int   PING_SAMPLES_TO_ANALYZE = AEC_BUFFERED_SAMPLES_PER_CHANNEL;  // Samples to analyze (reusing AEC buffer)static const int   PING_BUFFER_OFFSET = BUFFER_LENGTH_SAMPLES_PER_CHANNEL - PING_PERIOD * 2.0f; // Signal start  inline void Audio::performIO(int16_t* inputLeft, int16_t* outputLeft, int16_t* outputRight) {     NodeList* nodeList = NodeList::getInstance();    Application* interface = Application::getInstance();    Avatar* interfaceAvatar = interface->getAvatar();     memset(outputLeft, 0, PACKET_LENGTH_BYTES_PER_CHANNEL);    memset(outputRight, 0, PACKET_LENGTH_BYTES_PER_CHANNEL);     // Add Procedural effects to input samples    addProceduralSounds(inputLeft, outputLeft, outputRight, BUFFER_LENGTH_SAMPLES_PER_CHANNEL);        if (nodeList && inputLeft) {                //  Measure the loudness of the signal from the microphone and store in audio object        float loudness = 0;        for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {            loudness += abs(inputLeft[i]);        }                loudness /= BUFFER_LENGTH_SAMPLES_PER_CHANNEL;        _lastInputLoudness = loudness;                // add input (@microphone) data to the scope        _scope->addSamples(0, inputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL);         Node* audioMixer = nodeList->soloNodeOfType(NODE_TYPE_AUDIO_MIXER);                if (audioMixer) {            audioMixer->lock();            sockaddr_in audioSocket = *(sockaddr_in*) audioMixer->getActiveSocket();            audioMixer->unlock();                        glm::vec3 headPosition = interfaceAvatar->getHeadJointPosition();            glm::quat headOrientation = interfaceAvatar->getHead().getOrientation();                        int numBytesPacketHeader = numBytesForPacketHeader((unsigned char*) &PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO);            int leadingBytes = numBytesPacketHeader + sizeof(headPosition) + sizeof(headOrientation);                        // we need the amount of bytes in the buffer + 1 for type            // + 12 for 3 floats for position + float for bearing + 1 attenuation byte            unsigned char dataPacket[MAX_PACKET_SIZE];                        PACKET_TYPE packetType = (Application::getInstance()->shouldEchoAudio())                ? PACKET_TYPE_MICROPHONE_AUDIO_WITH_ECHO                : PACKET_TYPE_MICROPHONE_AUDIO_NO_ECHO;                        unsigned char* currentPacketPtr = dataPacket + populateTypeAndVersion(dataPacket, packetType);                        // pack Source Data            uint16_t ownerID = NodeList::getInstance()->getOwnerID();            memcpy(currentPacketPtr, &ownerID, sizeof(ownerID));            currentPacketPtr += (sizeof(ownerID));            leadingBytes += (sizeof(ownerID));                        // pack Listen Mode Data            memcpy(currentPacketPtr, &_listenMode, sizeof(_listenMode));            currentPacketPtr += (sizeof(_listenMode));            leadingBytes += (sizeof(_listenMode));                if (_listenMode == AudioRingBuffer::OMNI_DIRECTIONAL_POINT) {                memcpy(currentPacketPtr, &_listenRadius, sizeof(_listenRadius));                currentPacketPtr += (sizeof(_listenRadius));                leadingBytes += (sizeof(_listenRadius));            } else if (_listenMode == AudioRingBuffer::SELECTED_SOURCES) {                int listenSourceCount = _listenSources.size();                memcpy(currentPacketPtr, &listenSourceCount, sizeof(listenSourceCount));                currentPacketPtr += (sizeof(listenSourceCount));                leadingBytes += (sizeof(listenSourceCount));                for (int i = 0; i < listenSourceCount; i++) {                    memcpy(currentPacketPtr, &_listenSources[i], sizeof(_listenSources[i]));                    currentPacketPtr += sizeof(_listenSources[i]);                    leadingBytes += sizeof(_listenSources[i]);                }            }                        // memcpy the three float positions            memcpy(currentPacketPtr, &headPosition, sizeof(headPosition));            currentPacketPtr += (sizeof(headPosition));                        // memcpy our orientation            memcpy(currentPacketPtr, &headOrientation, sizeof(headOrientation));            currentPacketPtr += sizeof(headOrientation);                        // check if we have a song to add to our audio            if (_songFileBytes > 0 && _songFileStream->tellg() != -1) {                // iterate over BUFFER_LENGTH_SAMPLES_PER_CHANNEL from the song file and add that to our audio                for (int i = 0; i < BUFFER_LENGTH_SAMPLES_PER_CHANNEL; i++) {                    int16_t songSample = 0;                                        _songFileStream->read((char*) &songSample, sizeof(songSample));                                        // attenuate the song samples since they will be loud                    const float SONG_SAMPLE_ATTENUATION = 0.25;                    songSample *= SONG_SAMPLE_ATTENUATION;                                        // add the song sample to the output and input buffersg                    inputLeft[i] = inputLeft[i] + songSample;                    outputLeft[i] = outputLeft[i] + songSample;                    outputRight[i] =  outputLeft[i] + songSample;                }            } else if (_songFileStream) {                // close the stream                _songFileStream->close();                                // delete the _songFileStream                delete _songFileStream;                _songFileStream = NULL;                                // reset the _songFileBytes back to zero                _songFileBytes = 0;                                // call Application stopMixingSong method to fix menu item                Application::getInstance()->resetSongMixMenuItem();            }                        // copy the audio data to the last BUFFER_LENGTH_BYTES bytes of the data packet            memcpy(currentPacketPtr, inputLeft, BUFFER_LENGTH_BYTES_PER_CHANNEL);                        nodeList->getNodeSocket()->send((sockaddr*) &audioSocket,                                            dataPacket,                                            BUFFER_LENGTH_BYTES_PER_CHANNEL + leadingBytes);             interface->getBandwidthMeter()->outputStream(BandwidthMeter::AUDIO).updateValue(BUFFER_LENGTH_BYTES_PER_CHANNEL                                                                                            + leadingBytes);                    }    }            AudioRingBuffer* ringBuffer = &_ringBuffer;        // if there is anything in the ring buffer, decide what to do:        if (ringBuffer->getEndOfLastWrite()) {        if (!ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() <            (PACKET_LENGTH_SAMPLES + _jitterBufferSamples * (ringBuffer->isStereo() ? 2 : 1))) {            //            //  If not enough audio has arrived to start playback, keep waiting            //#ifdef SHOW_AUDIO_DEBUG            qDebug("%i,%i,%i,%i\n",                     _packetsReceivedThisPlayback,                     ringBuffer->diffLastWriteNextOutput(),                     PACKET_LENGTH_SAMPLES,                     _jitterBufferSamples);#endif        } else if (ringBuffer->isStarted() && ringBuffer->diffLastWriteNextOutput() == 0) {            //            //  If we have started and now have run out of audio to send to the audio device,             //  this means we've starved and should restart.              //              ringBuffer->setStarted(false);                        _numStarves++;            _packetsReceivedThisPlayback = 0;            _wasStarved = 10;          //   Frames for which to render the indication that the system was starved.#ifdef SHOW_AUDIO_DEBUG            qDebug("Starved, remaining samples = %d\n",                     ringBuffer->diffLastWriteNextOutput());#endif         } else {            //            //  We are either already playing back, or we have enough audio to start playing back.            //             if (!ringBuffer->isStarted()) {                ringBuffer->setStarted(true);#ifdef SHOW_AUDIO_DEBUG                qDebug("starting playback %0.1f msecs delayed, jitter = %d, pkts recvd: %d \n",                         (usecTimestampNow() - usecTimestamp(&_firstPacketReceivedTime))/1000.0,                         _jitterBufferSamples,                         _packetsReceivedThisPlayback);#endif            }             //            // play whatever we have in the audio buffer            //            // if we haven't fired off the flange effect, check if we should            // TODO: lastMeasuredHeadYaw is now relative to body - check if this still works.                        int lastYawMeasured = fabsf(interfaceAvatar->getHeadYawRate());                        if (!_samplesLeftForFlange && lastYawMeasured > MIN_FLANGE_EFFECT_THRESHOLD) {                // we should flange for one second                if ((_lastYawMeasuredMaximum = std::max(_lastYawMeasuredMaximum, lastYawMeasured)) != lastYawMeasured) {                    _lastYawMeasuredMaximum = std::min(_lastYawMeasuredMaximum, MIN_FLANGE_EFFECT_THRESHOLD);                                        _samplesLeftForFlange = SAMPLE_RATE;                                        _flangeIntensity = MIN_FLANGE_INTENSITY +                        ((_lastYawMeasuredMaximum - MIN_FLANGE_EFFECT_THRESHOLD) /                         (float)(MAX_FLANGE_EFFECT_THRESHOLD - MIN_FLANGE_EFFECT_THRESHOLD)) *                        (1 - MIN_FLANGE_INTENSITY);                                        _flangeRate = FLANGE_BASE_RATE * _flangeIntensity;                    _flangeWeight = MAX_FLANGE_SAMPLE_WEIGHT * _flangeIntensity;                }            }                        for (int s = 0; s < PACKET_LENGTH_SAMPLES_PER_CHANNEL; s++) {                                int leftSample = ringBuffer->getNextOutput()[s];                int rightSample = ringBuffer->getNextOutput()[s + PACKET_LENGTH_SAMPLES_PER_CHANNEL];                                if (_samplesLeftForFlange > 0) {                    float exponent = (SAMPLE_RATE - _samplesLeftForFlange - (SAMPLE_RATE / _flangeRate)) /                        (SAMPLE_RATE / _flangeRate);                    int sampleFlangeDelay = (SAMPLE_RATE / (1000 * _flangeIntensity)) * powf(2, exponent);                                        if (_samplesLeftForFlange != SAMPLE_RATE || s >= (SAMPLE_RATE / 2000)) {                        // we have a delayed sample to add to this sample                                                int16_t *flangeFrame = ringBuffer->getNextOutput();                        int flangeIndex = s - sampleFlangeDelay;                                                if (flangeIndex < 0) {                            // we need to grab the flange sample from earlier in the buffer                            flangeFrame = ringBuffer->getNextOutput() != ringBuffer->getBuffer()                            ? ringBuffer->getNextOutput() - PACKET_LENGTH_SAMPLES                            : ringBuffer->getNextOutput() + RING_BUFFER_LENGTH_SAMPLES - PACKET_LENGTH_SAMPLES;                                                        flangeIndex = PACKET_LENGTH_SAMPLES_PER_CHANNEL + (s - sampleFlangeDelay);                        }                                                int16_t leftFlangeSample = flangeFrame[flangeIndex];                        int16_t rightFlangeSample = flangeFrame[flangeIndex + PACKET_LENGTH_SAMPLES_PER_CHANNEL];                                                leftSample = (1 - _flangeWeight) * leftSample + (_flangeWeight * leftFlangeSample);                        rightSample = (1 - _flangeWeight) * rightSample + (_flangeWeight * rightFlangeSample);                                                _samplesLeftForFlange--;                                                if (_samplesLeftForFlange == 0) {                            _lastYawMeasuredMaximum = 0;                        }                    }                }#ifndef TEST_AUDIO_LOOPBACK                outputLeft[s] += leftSample;                outputRight[s] += rightSample;#else                 outputLeft[s] += inputLeft[s];                outputRight[s] += inputLeft[s];#endif            }            ringBuffer->setNextOutput(ringBuffer->getNextOutput() + PACKET_LENGTH_SAMPLES);                        if (ringBuffer->getNextOutput() == ringBuffer->getBuffer() + RING_BUFFER_LENGTH_SAMPLES) {                ringBuffer->setNextOutput(ringBuffer->getBuffer());            }        }    }     eventuallySendRecvPing(inputLeft, outputLeft, outputRight);      // add output (@speakers) data just written to the scope    _scope->addSamples(1, outputLeft, BUFFER_LENGTH_SAMPLES_PER_CHANNEL);    _scope->addSamples(2, outputRight, BUFFER_LENGTH_SAMPLES_PER_CHANNEL);     gettimeofday(&_lastCallbackTime, NULL);} // inputBuffer  A pointer to an internal portaudio data buffer containing data read by portaudio.// outputBuffer A pointer to an internal portaudio data buffer to be read by the configured output device.// frames       Number of frames that portaudio requests to be read/written.// timeInfo     Portaudio time info. Currently unused.// statusFlags  Portaudio status flags. Currently unused.// userData     Pointer to supplied user data (in this case, a pointer to the parent Audio objectint Audio::audioCallback (const void* inputBuffer,                          void* outputBuffer,                          unsigned long frames,                          const PaStreamCallbackTimeInfo *timeInfo,                          PaStreamCallbackFlags statusFlags,                          void* userData) {        int16_t* inputLeft = static_cast<int16_t*const*>(inputBuffer)[0];    int16_t* outputLeft = static_cast<int16_t**>(outputBuffer)[0];    int16_t* outputRight = static_cast<int16_t**>(outputBuffer)[1];     static_cast<Audio*>(userData)->performIO(inputLeft, outputLeft, outputRight);    return paContinue;}  static void outputPortAudioError(PaError error) {    if (error != paNoError) {        qDebug("-- portaudio termination error --\n");        qDebug("PortAudio error (%d): %s\n", error, Pa_GetErrorText(error));    }} void Audio::reset() {    _packetsReceivedThisPlayback = 0;    _ringBuffer.reset();} void Audio::addListenSource(int sourceID) {    _listenSources.push_back(sourceID);} void Audio::clearListenSources() {    _listenSources.clear();} void Audio::removeListenSource(int sourceID) {    for (int i = 0; i < _listenSources.size(); i++) {        if (_listenSources[i] == sourceID) {            _listenSources.erase(_listenSources.begin() + i);            return;        }    }}  Audio::Audio(Oscilloscope* scope, int16_t initialJitterBufferSamples) :    _stream(NULL),    _ringBuffer(true),    _scope(scope),    _averagedLatency(0.0),    _measuredJitter(0),    _jitterBufferSamples(initialJitterBufferSamples),    _wasStarved(0),    _numStarves(0),    _lastInputLoudness(0),    _lastVelocity(0),    _lastAcceleration(0),    _totalPacketsReceived(0),    _firstPacketReceivedTime(),    _packetsReceivedThisPlayback(0),    _echoSamplesLeft(NULL),    _isSendingEchoPing(false),    _pingAnalysisPending(false),    _pingFramesToRecord(0),    _samplesLeftForFlange(0),    _lastYawMeasuredMaximum(0),    _flangeIntensity(0.0f),    _flangeRate(0.0f),    _flangeWeight(0.0f),    _collisionSoundMagnitude(0.0f),    _collisionSoundFrequency(0.0f),    _collisionSoundNoise(0.0f),    _collisionSoundDuration(0.0f),    _proceduralEffectSample(0),    _heartbeatMagnitude(0.0f),    _songFileStream(NULL),    _songFileBytes(0),    _listenMode(AudioRingBuffer::NORMAL),    _listenRadius(0.0f){    outputPortAudioError(Pa_Initialize());        //  NOTE:  Portaudio documentation is unclear as to whether it is safe to specify the    //         number of frames per buffer explicitly versus setting this value to zero.    //         Possible source of latency that we need to investigate further.    //     unsigned long FRAMES_PER_BUFFER = BUFFER_LENGTH_SAMPLES_PER_CHANNEL;        //  Manually initialize the portaudio stream to ask for minimum latency    PaStreamParameters inputParameters, outputParameters;        inputParameters.device = Pa_GetDefaultInputDevice();    outputParameters.device = Pa_GetDefaultOutputDevice();     if (inputParameters.device == -1 || outputParameters.device == -1) {        qDebug("Audio: Missing device.\n");        outputPortAudioError(Pa_Terminate());        return;    }     inputParameters.channelCount = 2;                    //  Stereo input    inputParameters.sampleFormat = (paInt16 | paNonInterleaved);    inputParameters.suggestedLatency = Pa_GetDeviceInfo(inputParameters.device)->defaultLowInputLatency;    inputParameters.hostApiSpecificStreamInfo = NULL;     outputParameters.channelCount = 2;                    //  Stereo output    outputParameters.sampleFormat = (paInt16 | paNonInterleaved);    outputParameters.suggestedLatency = Pa_GetDeviceInfo(outputParameters.device)->defaultLowOutputLatency;    outputParameters.hostApiSpecificStreamInfo = NULL;        outputPortAudioError(Pa_OpenStream(&_stream,                                        &inputParameters,                                       &outputParameters,                                       SAMPLE_RATE,                                       FRAMES_PER_BUFFER,                                       paNoFlag,                                       audioCallback,                                       (void*) this));            if (! _stream) {        return;    }     _echoSamplesLeft = new int16_t[AEC_BUFFERED_SAMPLES + AEC_TMP_BUFFER_SIZE];    memset(_echoSamplesLeft, 0, AEC_BUFFERED_SAMPLES * sizeof(int16_t));        // start the stream now that sources are good to go    outputPortAudioError(Pa_StartStream(_stream));        // Uncomment these lines to see the system-reported latency    //qDebug("Default low input, output latency (secs): %0.4f, %0.4f\n",    //         Pa_GetDeviceInfo(Pa_GetDefaultInputDevice())->defaultLowInputLatency,    //         Pa_GetDeviceInfo(Pa_GetDefaultOutputDevice())->defaultLowOutputLatency);        const PaStreamInfo* streamInfo = Pa_GetStreamInfo(_stream);    qDebug("Started audio with reported latency msecs In/Out: %.0f, %.0f\n", streamInfo->inputLatency * 1000.f,             streamInfo->outputLatency * 1000.f);     gettimeofday(&_lastReceiveTime, NULL);} Audio::~Audio() {        if (_stream) {        outputPortAudioError(Pa_CloseStream(_stream));        outputPortAudioError(Pa_Terminate());    }    delete[] _echoSamplesLeft;}  void Audio::importSongToMixWithMicrophone(const char* filename) {        _songFileStream = new std::ifstream(filename);        long begin = _songFileStream->tellg();    _songFileStream->seekg(0, std::ios::end);    long end = _songFileStream->tellg();        // go back to the beginning

Lines 1–500 of 907. Load more