History

added support for local audio spatialization independent of inbound audio

ZappoMan committed 12 years ago

Commit b00a67c1 parent ba361b03 files · +140 −72Browse files

Showing the changes to interface/src/AudioReflector.cpp (1 of 3 files). Show all

Unified · Split

interface/src/AudioReflector.cpp+122−66
@@ -203,7 +203,7 @@ void AudioReflector::injectAudiblePoint(const AudiblePoint& audiblePoint, · 202 unmodified lines
203 _maxAttenuation = std::max(_maxAttenuation,leftEarAttenuation);203 _maxAttenuation = std::max(_maxAttenuation,leftEarAttenuation);
204 _minAttenuation = std::min(_minAttenuation,rightEarAttenuation);204 _minAttenuation = std::min(_minAttenuation,rightEarAttenuation);
205 _minAttenuation = std::min(_minAttenuation,leftEarAttenuation);205 _minAttenuation = std::min(_minAttenuation,leftEarAttenuation);
206 206
207 // run through the samples, and attenuate them 207 // run through the samples, and attenuate them
208 for (int sample = 0; sample < totalNumberOfStereoSamples; sample++) {208 for (int sample = 0; sample < totalNumberOfStereoSamples; sample++) {
209 int16_t leftSample = originalSamplesData[sample * NUMBER_OF_CHANNELS];209 int16_t leftSample = originalSamplesData[sample * NUMBER_OF_CHANNELS];
@@ -235,25 +235,25 @@ void AudioReflector::processLocalAudio(unsigned int sampleTime, const QByteArray · 25 unmodified lines
235 if (format.channelCount() == NUM_CHANNELS_INPUT && format.sampleRate() == EXPECTED_SAMPLE_RATE) {235 if (format.channelCount() == NUM_CHANNELS_INPUT && format.sampleRate() == EXPECTED_SAMPLE_RATE) {
236 QAudioFormat outputFormat = format;236 QAudioFormat outputFormat = format;
237 outputFormat.setChannelCount(NUM_CHANNELS_OUTPUT);237 outputFormat.setChannelCount(NUM_CHANNELS_OUTPUT);
238 QByteArray stereoInputData;238 QByteArray stereoInputData(samples.size() * NUM_CHANNELS_OUTPUT, 0);
239 stereoInputData.resize(samples.size() * NUM_CHANNELS_OUTPUT);239 int numberOfSamples = (samples.size() / sizeof(int16_t));
240 int numberOfSamples = samples.size() / sizeof(int16_t);
241 int16_t* monoSamples = (int16_t*)samples.data();240 int16_t* monoSamples = (int16_t*)samples.data();
242 int16_t* stereoSamples = (int16_t*)stereoInputData.data();241 int16_t* stereoSamples = (int16_t*)stereoInputData.data();
242
243 for (int i = 0; i < numberOfSamples; i++) {243 for (int i = 0; i < numberOfSamples; i++) {
244 stereoSamples[i* NUM_CHANNELS_OUTPUT] = monoSamples[i] * _localAudioAttenuationFactor;244 stereoSamples[i* NUM_CHANNELS_OUTPUT] = monoSamples[i] * _localAudioAttenuationFactor;
245 stereoSamples[(i * NUM_CHANNELS_OUTPUT) + 1] = monoSamples[i] * _localAudioAttenuationFactor;245 stereoSamples[(i * NUM_CHANNELS_OUTPUT) + 1] = monoSamples[i] * _localAudioAttenuationFactor;
246 }246 }
247 echoAudio(sampleTime, stereoInputData, outputFormat);247 echoAudio(LOCAL_AUDIO, sampleTime, stereoInputData, outputFormat);
248 }248 }
249 }249 }
250}250}
251 251
252void AudioReflector::processInboundAudio(unsigned int sampleTime, const QByteArray& samples, const QAudioFormat& format) {252void AudioReflector::processInboundAudio(unsigned int sampleTime, const QByteArray& samples, const QAudioFormat& format) {
253 echoAudio(sampleTime, samples, format);253 echoAudio(INBOUND_AUDIO, sampleTime, samples, format);
254}254}
255 255
256void AudioReflector::echoAudio(unsigned int sampleTime, const QByteArray& samples, const QAudioFormat& format) {256void AudioReflector::echoAudio(AudioSource source, unsigned int sampleTime, const QByteArray& samples, const QAudioFormat& format) {
257 _maxDelay = 0;257 _maxDelay = 0;
258 _maxAttenuation = 0.0f;258 _maxAttenuation = 0.0f;
259 _minDelay = std::numeric_limits<int>::max();259 _minDelay = std::numeric_limits<int>::max();
@@ -265,7 +265,10 @@ void AudioReflector::echoAudio(unsigned int sampleTime, const QByteArray& sample · 5 unmodified lines
265 265
266 QMutexLocker locker(&_mutex);266 QMutexLocker locker(&_mutex);
267 267
268 foreach(const AudiblePoint& audiblePoint, _audiblePoints) {268 // depending on if we're processing local or external audio, pick the correct points vector
269 QVector<AudiblePoint>& audiblePoints = source == INBOUND_AUDIO ? _inboundAudiblePoints : _localAudiblePoints;
270
271 foreach(const AudiblePoint& audiblePoint, audiblePoints) {
269 injectAudiblePoint(audiblePoint, samples, sampleTime, format.sampleRate());272 injectAudiblePoint(audiblePoint, samples, sampleTime, format.sampleRate());
270 }273 }
271 274
@@ -294,9 +297,10 @@ void AudioReflector::drawVector(const glm::vec3& start, const glm::vec3& end, co · 22 unmodified lines
294 297
295 298
296 299
297AudioPath::AudioPath(const glm::vec3& origin, const glm::vec3& direction, 300AudioPath::AudioPath(AudioSource source, const glm::vec3& origin, const glm::vec3& direction,
298 float attenuation, float delay, float distance,bool isDiffusion, int bounceCount) :301 float attenuation, float delay, float distance,bool isDiffusion, int bounceCount) :
299 302
303 source(source),
300 isDiffusion(isDiffusion),304 isDiffusion(isDiffusion),
301 startPoint(origin),305 startPoint(origin),
302 startDirection(direction),306 startDirection(direction),
@@ -315,12 +319,15 @@ AudioPath::AudioPath(const glm::vec3& origin, const glm::vec3& direction, · 12 unmodified lines
315{319{
316}320}
317 321
318void AudioReflector::addSoundSource(const glm::vec3& origin, const glm::vec3& initialDirection, 322void AudioReflector::addAudioPath(AudioSource source, const glm::vec3& origin, const glm::vec3& initialDirection,
319 float initialAttenuation, float initialDelay, float initialDistance, bool isDiffusion) {323 float initialAttenuation, float initialDelay, float initialDistance, bool isDiffusion) {
320 324
321 AudioPath* path = new AudioPath(origin, initialDirection, initialAttenuation, initialDelay, 325 AudioPath* path = new AudioPath(source, origin, initialDirection, initialAttenuation, initialDelay,
322 initialDistance, isDiffusion, 0);326 initialDistance, isDiffusion, 0);
323 _audioPaths.push_back(path);327
328 QVector<AudioPath*>& audioPaths = source == INBOUND_AUDIO ? _inboundAudioPaths : _localAudioPaths;
329
330 audioPaths.push_back(path);
324}331}
325 332
326void AudioReflector::calculateAllReflections() {333void AudioReflector::calculateAllReflections() {
@@ -355,12 +362,15 @@ void AudioReflector::calculateAllReflections() { · 28 unmodified lines
355void AudioReflector::drawRays() {362void AudioReflector::drawRays() {
356 const glm::vec3 RED(1,0,0);363 const glm::vec3 RED(1,0,0);
357 const glm::vec3 GREEN(0,1,0);364 const glm::vec3 GREEN(0,1,0);
365 const glm::vec3 BLUE(0,0,1);
366 const glm::vec3 CYAN(0,1,1);
358 367
359 int diffusionNumber = 0;368 int diffusionNumber = 0;
360 369
361 QMutexLocker locker(&_mutex);370 QMutexLocker locker(&_mutex);
362 foreach(AudioPath* const& path, _audioPaths) {371
363 372 // draw the paths for inbound audio
373 foreach(AudioPath* const& path, _inboundAudioPaths) {
364 // if this is an original reflection, draw it in RED374 // if this is an original reflection, draw it in RED
365 if (path->isDiffusion) {375 if (path->isDiffusion) {
366 diffusionNumber++;376 diffusionNumber++;
@@ -369,6 +379,19 @@ void AudioReflector::drawRays() { · 2 unmodified lines
369 drawPath(path, RED);379 drawPath(path, RED);
370 }380 }
371 }381 }
382
383 if (Menu::getInstance()->isOptionChecked(MenuOption::AudioSpatialProcessingProcessLocalAudio)) {
384 // draw the paths for local audio
385 foreach(AudioPath* const& path, _localAudioPaths) {
386 // if this is an original reflection, draw it in RED
387 if (path->isDiffusion) {
388 diffusionNumber++;
389 drawPath(path, CYAN);
390 } else {
391 drawPath(path, BLUE);
392 }
393 }
394 }
372}395}
373 396
374void AudioReflector::drawPath(AudioPath* path, const glm::vec3& originalColor) {397void AudioReflector::drawPath(AudioPath* path, const glm::vec3& originalColor) {
@@ -383,6 +406,21 @@ void AudioReflector::drawPath(AudioPath* path, const glm::vec3& originalColor) { · 8 unmodified lines
383 }406 }
384}407}
385 408
409void AudioReflector::clearPaths() {
410 // clear our inbound audio paths
411 foreach(AudioPath* const& path, _inboundAudioPaths) {
412 delete path;
413 }
414 _inboundAudioPaths.clear();
415 _inboundAudiblePoints.clear(); // clear our inbound audible points
416
417 // clear our local audio paths
418 foreach(AudioPath* const& path, _localAudioPaths) {
419 delete path;
420 }
421 _localAudioPaths.clear();
422 _localAudiblePoints.clear(); // clear our local audible points
423}
386 424
387// Here's how this works: we have an array of AudioPaths, we loop on all of our currently calculating audio 425// Here's how this works: we have an array of AudioPaths, we loop on all of our currently calculating audio
388// paths, and calculate one ray per path. If that ray doesn't reflect, or reaches a max distance/attenuation, then it426// paths, and calculate one ray per path. If that ray doesn't reflect, or reaches a max distance/attenuation, then it
@@ -392,12 +430,7 @@ void AudioReflector::drawPath(AudioPath* path, const glm::vec3& originalColor) { · 3 unmodified lines
392// fanout number of new paths, those new paths will have an origin of the reflection point, and an initial attenuation430// fanout number of new paths, those new paths will have an origin of the reflection point, and an initial attenuation
393// of their diffusion ratio. Those new paths will be added to the active audio paths, and be analyzed for the next loop.431// of their diffusion ratio. Those new paths will be added to the active audio paths, and be analyzed for the next loop.
394void AudioReflector::analyzePaths() {432void AudioReflector::analyzePaths() {
395 // clear our _audioPaths433 clearPaths();
396 foreach(AudioPath* const& path, _audioPaths) {
397 delete path;
398 }
399 _audioPaths.clear();
400 _audiblePoints.clear(); // clear our audible points
401 434
402 // add our initial paths435 // add our initial paths
403 glm::vec3 right = glm::normalize(_orientation * IDENTITY_RIGHT);436 glm::vec3 right = glm::normalize(_orientation * IDENTITY_RIGHT);
@@ -422,36 +455,49 @@ void AudioReflector::analyzePaths() { · 18 unmodified lines
422 // NOTE: we're still calculating our initial paths based on the listeners position. But the analysis code has been455 // NOTE: we're still calculating our initial paths based on the listeners position. But the analysis code has been
423 // updated to support individual sound sources (which is how we support diffusion), we can use this new paradigm to456 // updated to support individual sound sources (which is how we support diffusion), we can use this new paradigm to
424 // add support for individual sound sources, and more directional sound sources 457 // add support for individual sound sources, and more directional sound sources
425 addSoundSource(_origin, right, initialAttenuation, preDelay);458
426 addSoundSource(_origin, front, initialAttenuation, preDelay);459 addAudioPath(INBOUND_AUDIO, _origin, front, initialAttenuation, preDelay);
427 addSoundSource(_origin, up, initialAttenuation, preDelay);460 addAudioPath(INBOUND_AUDIO, _origin, right, initialAttenuation, preDelay);
428 addSoundSource(_origin, down, initialAttenuation, preDelay);461 addAudioPath(INBOUND_AUDIO, _origin, up, initialAttenuation, preDelay);
429 addSoundSource(_origin, back, initialAttenuation, preDelay);462 addAudioPath(INBOUND_AUDIO, _origin, down, initialAttenuation, preDelay);
430 addSoundSource(_origin, left, initialAttenuation, preDelay);463 addAudioPath(INBOUND_AUDIO, _origin, back, initialAttenuation, preDelay);
431 addSoundSource(_origin, frontRightUp, initialAttenuation, preDelay);464 addAudioPath(INBOUND_AUDIO, _origin, left, initialAttenuation, preDelay);
432 addSoundSource(_origin, frontLeftUp, initialAttenuation, preDelay);465 addAudioPath(INBOUND_AUDIO, _origin, frontRightUp, initialAttenuation, preDelay);
433 addSoundSource(_origin, backRightUp, initialAttenuation, preDelay);466 addAudioPath(INBOUND_AUDIO, _origin, frontLeftUp, initialAttenuation, preDelay);
434 addSoundSource(_origin, backLeftUp, initialAttenuation, preDelay);467 addAudioPath(INBOUND_AUDIO, _origin, backRightUp, initialAttenuation, preDelay);
435 addSoundSource(_origin, frontRightDown, initialAttenuation, preDelay);468 addAudioPath(INBOUND_AUDIO, _origin, backLeftUp, initialAttenuation, preDelay);
436 addSoundSource(_origin, frontLeftDown, initialAttenuation, preDelay);469 addAudioPath(INBOUND_AUDIO, _origin, frontRightDown, initialAttenuation, preDelay);
437 addSoundSource(_origin, backRightDown, initialAttenuation, preDelay);470 addAudioPath(INBOUND_AUDIO, _origin, frontLeftDown, initialAttenuation, preDelay);
438 addSoundSource(_origin, backLeftDown, initialAttenuation, preDelay);471 addAudioPath(INBOUND_AUDIO, _origin, backRightDown, initialAttenuation, preDelay);
472 addAudioPath(INBOUND_AUDIO, _origin, backLeftDown, initialAttenuation, preDelay);
473
474 // the original paths for the local audio are directional to the front of the origin
475 addAudioPath(LOCAL_AUDIO, _origin, front, initialAttenuation, preDelay);
476 addAudioPath(LOCAL_AUDIO, _origin, frontRightUp, initialAttenuation, preDelay);
477 addAudioPath(LOCAL_AUDIO, _origin, frontLeftUp, initialAttenuation, preDelay);
478 addAudioPath(LOCAL_AUDIO, _origin, frontRightDown, initialAttenuation, preDelay);
479 addAudioPath(LOCAL_AUDIO, _origin, frontLeftDown, initialAttenuation, preDelay);
439 480
440 // loop through all our audio paths and keep analyzing them until they complete481 // loop through all our audio paths and keep analyzing them until they complete
441 int steps = 0;482 int steps = 0;
442 int acitvePaths = _audioPaths.size(); // when we start, all paths are active483 int acitvePaths = _inboundAudioPaths.size() + _localAudioPaths.size(); // when we start, all paths are active
443 while(acitvePaths > 0) {484 while(acitvePaths > 0) {
444 acitvePaths = analyzePathsSingleStep();485 acitvePaths = analyzePathsSingleStep();
445 steps++;486 steps++;
446 }487 }
447 _reflections = _audiblePoints.size();488 _reflections = _inboundAudiblePoints.size() + _localAudiblePoints.size();
448 _diffusionPathCount = countDiffusionPaths();489 _diffusionPathCount = countDiffusionPaths();
449}490}
450 491
451int AudioReflector::countDiffusionPaths() {492int AudioReflector::countDiffusionPaths() {
452 int diffusionCount = 0;493 int diffusionCount = 0;
453 494
454 foreach(AudioPath* const& path, _audioPaths) {495 foreach(AudioPath* const& path, _inboundAudioPaths) {
496 if (path->isDiffusion) {
497 diffusionCount++;
498 }
499 }
500 foreach(AudioPath* const& path, _localAudioPaths) {
455 if (path->isDiffusion) {501 if (path->isDiffusion) {
456 diffusionCount++;502 diffusionCount++;
457 }503 }
@@ -462,36 +508,44 @@ int AudioReflector::countDiffusionPaths() { · 4 unmodified lines
462int AudioReflector::analyzePathsSingleStep() {508int AudioReflector::analyzePathsSingleStep() {
463 // iterate all the active sound paths, calculate one step per active path509 // iterate all the active sound paths, calculate one step per active path
464 int activePaths = 0;510 int activePaths = 0;
465 foreach(AudioPath* const& path, _audioPaths) {511
512 QVector<AudioPath*>* pathsLists[] = { &_inboundAudioPaths, &_localAudioPaths };
513
514 for(int i = 0; i < sizeof(pathsLists) / sizeof(pathsLists[0]); i ++) {
515
516 QVector<AudioPath*>& pathList = *pathsLists[i];
517
518 foreach(AudioPath* const& path, pathList) {
466 519
467 glm::vec3 start = path->lastPoint;520 glm::vec3 start = path->lastPoint;
468 glm::vec3 direction = path->lastDirection;521 glm::vec3 direction = path->lastDirection;
469 OctreeElement* elementHit; // output from findRayIntersection522 OctreeElement* elementHit; // output from findRayIntersection
470 float distance; // output from findRayIntersection523 float distance; // output from findRayIntersection
471 BoxFace face; // output from findRayIntersection524 BoxFace face; // output from findRayIntersection
472 525
473 if (!path->finalized) {526 if (!path->finalized) {
474 activePaths++;527 activePaths++;
475 528
476 if (path->bounceCount > ABSOLUTE_MAXIMUM_BOUNCE_COUNT) {529 if (path->bounceCount > ABSOLUTE_MAXIMUM_BOUNCE_COUNT) {
477 path->finalized = true;530 path->finalized = true;
478 } else if (_voxels->findRayIntersection(start, direction, elementHit, distance, face)) {531 } else if (_voxels->findRayIntersection(start, direction, elementHit, distance, face)) {
479 // TODO: we need to decide how we want to handle locking on the ray intersection, if we force lock,532 // TODO: we need to decide how we want to handle locking on the ray intersection, if we force lock,
480 // we get an accurate picture, but it could prevent rendering of the voxels. If we trylock (default), 533 // we get an accurate picture, but it could prevent rendering of the voxels. If we trylock (default),
481 // we might not get ray intersections where they may exist, but we can't really detect that case...534 // we might not get ray intersections where they may exist, but we can't really detect that case...
482 // add last parameter of Octree::Lock to force locking535 // add last parameter of Octree::Lock to force locking
483 handlePathPoint(path, distance, elementHit, face);536 handlePathPoint(path, distance, elementHit, face);
484 537
485 } else {
486 // If we didn't intersect, but this was a diffusion ray, then we will go ahead and cast a short ray out
487 // from our last known point, in the last known direction, and leave that sound source hanging there
488 if (path->isDiffusion) {
489 const float MINIMUM_RANDOM_DISTANCE = 0.25f;
490 const float MAXIMUM_RANDOM_DISTANCE = 0.5f;
491 float distance = randFloatInRange(MINIMUM_RANDOM_DISTANCE, MAXIMUM_RANDOM_DISTANCE);
492 handlePathPoint(path, distance, NULL, UNKNOWN_FACE);
493 } else {538 } else {
494 path->finalized = true; // if it doesn't intersect, then it is finished539 // If we didn't intersect, but this was a diffusion ray, then we will go ahead and cast a short ray out
540 // from our last known point, in the last known direction, and leave that sound source hanging there
541 if (path->isDiffusion) {
542 const float MINIMUM_RANDOM_DISTANCE = 0.25f;
543 const float MAXIMUM_RANDOM_DISTANCE = 0.5f;
544 float distance = randFloatInRange(MINIMUM_RANDOM_DISTANCE, MAXIMUM_RANDOM_DISTANCE);
545 handlePathPoint(path, distance, NULL, UNKNOWN_FACE);
546 } else {
547 path->finalized = true; // if it doesn't intersect, then it is finished
548 }
495 }549 }
496 }550 }
497 }551 }
@@ -571,8 +625,8 @@ void AudioReflector::handlePathPoint(AudioPath* path, float distance, OctreeElem · 73 unmodified lines
571 625
572 diffusion = glm::normalize(diffusion);626 diffusion = glm::normalize(diffusion);
573 627
574 // add sound sources for these diffusions628 // add new audio path for these diffusions, the new path's source is the same as the original source
575 addSoundSource(end, diffusion, partialDiffusionAttenuation, currentDelay, pathDistance, true);629 addAudioPath(path->source, end, diffusion, partialDiffusionAttenuation, currentDelay, pathDistance, true);
576 }630 }
577 } else {631 } else {
578 const bool wantDebugging = false;632 const bool wantDebugging = false;
@@ -603,7 +657,9 @@ void AudioReflector::handlePathPoint(AudioPath* path, float distance, OctreeElem · 24 unmodified lines
603 // audio so that it can be adjusted to ear position657 // audio so that it can be adjusted to ear position
604 AudiblePoint point = {end, currentDelay, (reflectiveAttenuation + totalDiffusionAttenuation), pathDistance};658 AudiblePoint point = {end, currentDelay, (reflectiveAttenuation + totalDiffusionAttenuation), pathDistance};
605 659
606 _audiblePoints.push_back(point);660 QVector<AudiblePoint>& audiblePoints = path->source == INBOUND_AUDIO ? _inboundAudiblePoints : _localAudiblePoints;
661
662 audiblePoints.push_back(point);
607 663
608 // add this location to the path points, so we can visualize it664 // add this location to the path points, so we can visualize it
609 path->reflections.push_back(end);665 path->reflections.push_back(end);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t