1 /*
2 * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 #import "AVFAudioProcessor.h"
27 #import "AVFMediaPlayer.h"
28
29 #import <AVFoundation/AVFoundation.h>
30 #import <MediaToolbox/MediaToolbox.h>
31
32 #import <CoreFoundation/CoreFoundation.h>
33
34 #import <pthread.h>
35 #import <objc/message.h>
36
37 static void InitAudioTap(MTAudioProcessingTapRef tapRef, void *clientInfo, void **tapStorageOut);
38 static void FinalizeAudioTap(MTAudioProcessingTapRef tapRef);
39 static void PrepareAudioTap(MTAudioProcessingTapRef tapRef,
40 CMItemCount maxFrames,
41 const AudioStreamBasicDescription *processingFormat);
42 static void UnprepareAudioTap(MTAudioProcessingTapRef tapRef);
43 static void ProcessAudioTap(MTAudioProcessingTapRef tapRef, CMItemCount numberFrames,
44 MTAudioProcessingTapFlags flags,
45 AudioBufferList *bufferListInOut,
46 CMItemCount *numberFramesOut,
47 MTAudioProcessingTapFlags *flagsOut);
48 static OSStatus AVFTapRenderCallback(void *inRefCon,
49 AudioUnitRenderActionFlags *ioActionFlags,
50 const AudioTimeStamp *inTimeStamp,
51 UInt32 inBusNumber,
52 UInt32 inNumberFrames,
53 AudioBufferList *ioData);
54
55 @implementation AVFAudioProcessor
56
57 - (id) init {
58 if ((self = [super init]) != nil) {
59 _soundLevelUnit = AVFSoundLevelUnitPtr(new AVFSoundLevelUnit());
60 _audioSpectrum = AVFAudioSpectrumUnitPtr(new AVFAudioSpectrumUnit());
61 _audioEqualizer = AVFAudioEqualizerPtr(new AVFAudioEqualizer());
62
63 _volume = 1.0f;
64 _balance = 0.0f;
65 _audioDelay = 0LL;
66 }
67 return self;
68 }
69
70 -(void) dealloc {
71 _soundLevelUnit = nullptr;
72 _audioSpectrum = nullptr;
73 _audioEqualizer = nullptr;
74 }
75
76 -(void) setAudioTrack : (AVAssetTrack *) track {
77 if (track != _audioTrack) {
78 // reset the audio mixer if it's already been created
79 // this theoretically should never happen...
80 _mixer = nil;
81 }
82 _audioTrack = track;
83 }
84
85 -(AVAudioMix*) mixer {
86 if (!self.audioTrack) {
87 return nil;
88 }
89
90 if (!_mixer) {
91 AVMutableAudioMix *mixer = [AVMutableAudioMix audioMix];
92 if (mixer) {
93 AVMutableAudioMixInputParameters *audioMixInputParameters =
94 [AVMutableAudioMixInputParameters audioMixInputParametersWithTrack : self.audioTrack];
95 if (audioMixInputParameters &&
96 [audioMixInputParameters respondsToSelector : @selector(setAudioTapProcessor :)]) {
97 MTAudioProcessingTapCallbacks callbacks;
98
99 callbacks.version = kMTAudioProcessingTapCallbacksVersion_0;
100 callbacks.clientInfo = (__bridge void *) self;
101 callbacks.init = InitAudioTap;
102 callbacks.finalize = FinalizeAudioTap;
103 callbacks.prepare = PrepareAudioTap;
104 callbacks.unprepare = UnprepareAudioTap;
105 callbacks.process = ProcessAudioTap;
106
107 MTAudioProcessingTapRef audioProcessingTap;
108 if (noErr == MTAudioProcessingTapCreate(kCFAllocatorDefault, &callbacks,
109 kMTAudioProcessingTapCreationFlag_PreEffects,
110 &audioProcessingTap)) {
111 [audioMixInputParameters setAudioTapProcessor:audioProcessingTap];
112
113 CFRelease(audioProcessingTap); // owned by the mixer now
114 mixer.inputParameters = @[audioMixInputParameters];
115
116 _mixer = mixer;
117 }
118 }
119 }
120 }
121 return _mixer;
122 }
123
124 -(void) setVolume : (float) volume {
125 _volume = volume;
126 if (_soundLevelUnit != nullptr) {
127 _soundLevelUnit->setVolume(volume);
128 }
129 }
130
131 -(void) setBalance : (float) balance {
132 _balance = balance;
133 if (_soundLevelUnit != nullptr) {
134 _soundLevelUnit->setBalance(balance);
135 }
136 }
137
138 @end
139
140 AVFTapContext::AVFTapContext(AVFSoundLevelUnitPtr slu, AVFAudioSpectrumUnitPtr spectrum,
141 AVFAudioEqualizerPtr eq) : audioSLU(slu),
142 audioSpectrum(spectrum),
143 audioEQ(eq),
144 // Some reasonable defaults
145 mSampleRate(48000),
146 mChannels(2) {
147 }
148
149 AVFTapContext::~AVFTapContext() {
150 // AudioUnits have already been deallocated by now
151 // shared_ptrs get freed automatically
152 }
153
154 void InitAudioTap(MTAudioProcessingTapRef tapRef, void *clientInfo, void **tapStorageOut) {
155 // retain the AU kernels so they don't get freed while we're running
156 AVFAudioProcessor *processor = (__bridge AVFAudioProcessor *) clientInfo;
157 if (processor) {
158 AVFTapContext *context = new AVFTapContext(processor.soundLevelUnit,
159 processor.audioSpectrum,
160 processor.audioEqualizer);
161 *tapStorageOut = context;
162 }
163 }
164
165 void FinalizeAudioTap(MTAudioProcessingTapRef tapRef) {
166 AVFTapContext *context = (AVFTapContext*) MTAudioProcessingTapGetStorage(tapRef);
167 if (context) {
168 delete context;
169 }
170 }
171
172 static OSStatus SetupAudioUnit(AudioUnit unit,
173 const AudioStreamBasicDescription *processingFormat,
174 UInt32 maxFrames) {
175 OSStatus status = noErr;
176 if (noErr == status) {
177 status = AudioUnitSetProperty(unit,
178 kAudioUnitProperty_StreamFormat,
179 kAudioUnitScope_Input, 0,
180 processingFormat, sizeof(AudioStreamBasicDescription));
181 }
182 if (noErr == status) {
183 status = AudioUnitSetProperty(unit,
184 kAudioUnitProperty_StreamFormat,
185 kAudioUnitScope_Output, 0,
186 processingFormat, sizeof(AudioStreamBasicDescription));
187 }
188 if (noErr == status) {
189 status = AudioUnitSetProperty(unit,
190 kAudioUnitProperty_MaximumFramesPerSlice,
191 kAudioUnitScope_Global, 0,
192 &maxFrames, sizeof(UInt32));
193 }
194 if (noErr == status) {
195 status = AudioUnitInitialize(unit);
196 }
197 return status;
198 }
199
200 void PrepareAudioTap(MTAudioProcessingTapRef tapRef,
201 CMItemCount maxFrames,
202 const AudioStreamBasicDescription *processingFormat) {
203 AVFTapContext *context = (AVFTapContext*) MTAudioProcessingTapGetStorage(tapRef);
204
205 // Validate the audio format before we enable the processor
206 // Failures here should rarely, if ever, happen so leave the NSLogs in for
207 // easier diagnosis in the field
208 if (processingFormat->mFormatID != kAudioFormatLinearPCM) {
209 NSLog(@"AVFAudioProcessor needs linear PCM");
210 return;
211 }
212
213 // Use the convenient kAudioFormatFlagsNativeFloatPacked to check if we can
214 // process the incoming audio
215 if ((processingFormat->mFormatFlags & kAudioFormatFlagsNativeFloatPacked)
216 != kAudioFormatFlagsNativeFloatPacked) {
217 NSLog(@"AVFAudioProcessor needs native endian packed float samples!!");
218 return;
219 }
220
221 context->mSampleRate = processingFormat->mSampleRate;
222 context->mChannels = processingFormat->mChannelsPerFrame;
223 context->mMaxFrames = maxFrames;
224
225 // Configure audio equalizer
226 if (context->audioEQ != nullptr) {
227 context->audioEQ.get()->SetSampleRate(context->mSampleRate);
228 context->audioEQ.get()->SetChannels(context->mChannels);
229 context->audioEQ.get()->ResetBandParameters();
230 }
231
232 // Configure spectrum
233 if (context->audioSpectrum != nullptr) {
234 context->audioSpectrum.get()->SetSampleRate(context->mSampleRate);
235 context->audioSpectrum.get()->SetChannels(context->mChannels);
236 context->audioSpectrum.get()->SetMaxFrames(context->mMaxFrames);
237 }
238
239 if (context->audioSLU != nullptr) {
240 context->audioSLU.get()->SetChannels(context->mChannels);
241 }
242 }
243
244 void UnprepareAudioTap(MTAudioProcessingTapRef tapRef) {
245 // We do not need it anymore
246 }
247
248 void ProcessAudioTap(MTAudioProcessingTapRef tapRef,
249 CMItemCount numberFrames,
250 uint32_t flags,
251 AudioBufferList *bufferListInOut,
252 CMItemCount *numberFramesOut,
253 uint32_t *flagsOut) {
254 AVFTapContext *context = (AVFTapContext*) MTAudioProcessingTapGetStorage(tapRef);
255 OSStatus status = MTAudioProcessingTapGetSourceAudio(tapRef, numberFrames, bufferListInOut,
256 flagsOut, NULL, numberFramesOut);
257 if (status != noErr) {
258 NSLog(@"MTAudioProcessingTapGetSourceAudio failed: %d", status);
259 return;
260 }
261
262 if (context->audioEQ != nullptr) {
263 if (!context->audioEQ.get()->ProcessBufferLists(*bufferListInOut, numberFrames)) {
264 NSLog(@"audioEQ ProcessBufferLists() failed");
265 return;
266 }
267 }
268
269 if (context->audioSpectrum != nullptr) {
270 if (!context->audioSpectrum.get()->ProcessBufferLists(*bufferListInOut, numberFrames)) {
271 NSLog(@"audioSpectrum ProcessBufferLists() failed");
272 return;
273 }
274 }
275
276 if (context->audioSLU != nullptr) {
277 if (!context->audioSLU.get()->ProcessBufferLists(*bufferListInOut, numberFrames)) {
278 NSLog(@"audioSLU ProcessBufferLists() failed");
279 return;
280 }
281 }
282
283 if (context->audioSpectrum != nullptr) {
284 context->audioSpectrum.get()->SetFirstBufferDelivered(true);
285 }
286 }