1 /*
2 * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 #include "AVFAudioSpectrumUnit.h"
27
28 #include <iostream>
29 #include <Accelerate/Accelerate.h>
30
31 AVFAudioSpectrumUnit::AVFAudioSpectrumUnit() : mSpectrumCallbackProc(NULL),
32 mSpectrumCallbackContext(NULL),
33 mEnabled(true),
34 mBandCount(128),
35 mBands(NULL),
36 mUpdateInterval(kDefaultAudioSpectrumUpdateInterval),
37 mThreshold(kDefaultAudioSpectrumThreshold),
38 mMixBufferFrameCapacity(0),
39 mSampleRate(0),
40 mChannels(0),
41 mMaxFrames(0),
42 mSamplesPerInterval(0),
43 mRebuildCrunch(true),
44 mFirstBufferDelivered(false),
45 mSpectrumElement(NULL),
46 mSpectrum(NULL) {
47 mMixBuffer.mNumberBuffers = 1;
48 mMixBuffer.mBuffers[0].mData = NULL;
49
50 pthread_mutex_init(&mBandLock, NULL);
51
52 gst_init_check(NULL, NULL, NULL);
53 }
54
55 AVFAudioSpectrumUnit::~AVFAudioSpectrumUnit() {
56 if (mMixBuffer.mBuffers[0].mData) {
57 free(mMixBuffer.mBuffers[0].mData);
58 mMixBuffer.mBuffers[0].mData = NULL;
59 }
60
61 ReleaseSpectralProcessor();
62 }
63
64 bool AVFAudioSpectrumUnit::ProcessBufferLists(const AudioBufferList& inBuffer,
65 UInt32 inFramesToProcess) {
66 if (!mEnabled) {
67 return true;
68 }
69
70 // (Re)allocate mix buffer if needed
71 if (!mMixBuffer.mBuffers[0].mData || mMixBufferFrameCapacity < inFramesToProcess) {
72 // allocate buffer list (only need to do this once)
73 if (mMixBuffer.mBuffers[0].mData) {
74 free(mMixBuffer.mBuffers[0].mData);
75 mMixBuffer.mBuffers[0].mData = NULL;
76 }
77
78 mMixBufferFrameCapacity = mMaxFrames;
79
80 mMixBuffer.mBuffers[0].mNumberChannels = 1;
81 mMixBuffer.mBuffers[0].mData = calloc(mMixBufferFrameCapacity, sizeof (Float32));
82 mMixBuffer.mBuffers[0].mDataByteSize = 0; // size of actual contained data, not size of buffer
83 }
84
85 if (mRebuildCrunch) {
86 SetupSpectralProcessor();
87 }
88
89 if (mSpectrum != NULL) {
90 // Mix the audio into one channel since JavaFX only supports single channel spectrum
91 // Just use an arithmetic average, nothing fancy here
92 float *buffer = (float*) mMixBuffer.mBuffers[0].mData;
93 vDSP_vclr(buffer, 1, mMixBufferFrameCapacity);
94 for (int ii = 0; ii < inBuffer.mNumberBuffers; ii++) {
95 vDSP_vadd((float*) inBuffer.mBuffers[ii].mData, 1,
96 buffer, 1,
97 buffer, 1, inFramesToProcess);
98 }
99 float divisor = (float) inBuffer.mNumberBuffers;
100 vDSP_vsdiv(buffer, 1,
101 &divisor,
102 buffer, 1, inFramesToProcess);
103 mMixBuffer.mBuffers[0].mDataByteSize = inFramesToProcess * sizeof (Float32);
104
105 // Just reuse already allocated memory from mMixBuffer and do not free it
106 // in GStreamer
107 GstBuffer *gstBuffer =
108 gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, // Allow only reading
109 mMixBuffer.mBuffers[0].mData,
110 mMixBuffer.mBuffers[0].mDataByteSize,
111 0,
112 mMixBuffer.mBuffers[0].mDataByteSize,
113 NULL,
114 NULL); // No need to free memory
115 if (gstBuffer == NULL) {
116 return false;
117 }
118
119 GstFlowReturn result = gst_spectrum_transform_ip_api((GstBaseTransform *) mSpectrum, gstBuffer);
120 if (result != GST_FLOW_OK) {
121 return false;
122 }
123 gst_buffer_unref(gstBuffer);
124 }
125
126 return true;
127 }
128
129 bool AVFAudioSpectrumUnit::IsEnabled() {
130 return mEnabled;
131 }
132
133 void AVFAudioSpectrumUnit::SetEnabled(bool isEnabled) {
134 mEnabled = isEnabled;
135 mRebuildCrunch = true;
136 }
137
138 void AVFAudioSpectrumUnit::SetBands(int bands, CBandsHolder* holder) {
139 lockBands();
140 if (mBands) {
141 CBandsHolder::ReleaseRef(mBands);
142 mBands = NULL;
143 }
144 mBandCount = 0;
145 if (holder) {
146 mBands = CBandsHolder::AddRef(holder);
147 if (mBands) {
148 mBandCount = bands;
149 }
150 }
151 mRebuildCrunch = true;
152 unlockBands();
153 }
154
155 size_t AVFAudioSpectrumUnit::GetBands() {
156 return mBandCount;
157 }
158
159 double AVFAudioSpectrumUnit::GetInterval() {
160 return mUpdateInterval;
161 }
162
163 void AVFAudioSpectrumUnit::SetInterval(double interval) {
164 if (mUpdateInterval != interval) {
165 mUpdateInterval = interval;
166 mRebuildCrunch = true;
167 }
168 }
169
170 int AVFAudioSpectrumUnit::GetThreshold() {
171 return (int) mThreshold;
172 }
173
174 void AVFAudioSpectrumUnit::SetThreshold(int threshold) {
175 if (mThreshold != (Float32) threshold) {
176 mThreshold = (Float32) threshold;
177 mRebuildCrunch = true;
178 }
179 }
180
181 void AVFAudioSpectrumUnit::UpdateBands(int size, const float* magnitudes, const float* phases) {
182 // lock now otherwise the bands could change while we're processing
183 lockBands();
184 if (!mBands || size <= 0 || !mEnabled) {
185 unlockBands();
186 return;
187 }
188
189 // Update band data
190 mBands->UpdateBands(size, magnitudes, magnitudes);
191
192 // Call our listener to dispatch the spectrum event
193 if (mSpectrumCallbackProc) {
194 double duration = (double) mSamplesPerInterval / (double) 44100;
195 double timestamp = mFirstBufferDelivered ? -1.0 : 0.0;
196 mSpectrumCallbackProc(mSpectrumCallbackContext, duration, timestamp);
197 }
198
199 unlockBands();
200 }
201
202 void AVFAudioSpectrumUnit::SetSampleRate(UInt32 rate) {
203 mSampleRate = rate;
204 }
205
206 void AVFAudioSpectrumUnit::SetChannels(UInt32 count) {
207 mChannels = count;
208 }
209
210 void AVFAudioSpectrumUnit::SetMaxFrames(UInt32 maxFrames) {
211 mMaxFrames = maxFrames;
212 }
213
214 void AVFAudioSpectrumUnit::SetSpectrumCallbackProc(AVFSpectrumUnitCallbackProc proc, void *context) {
215 mSpectrumCallbackProc = proc;
216 mSpectrumCallbackContext = context;
217 }
218
219 void AVFAudioSpectrumUnit::SetFirstBufferDelivered(bool isFirstBufferDelivered) {
220 mFirstBufferDelivered = isFirstBufferDelivered;
221 }
222
223 static gboolean PostMessageCallback(GstElement * element, GstMessage * message) {
224 if (message == NULL) {
225 return FALSE;
226 }
227
228 GstSpectrum *pSpectrum = GST_SPECTRUM(element);
229 if (pSpectrum == NULL || pSpectrum->user_data == NULL) {
230 return FALSE;
231 }
232
233 AVFAudioSpectrumUnit *pSpectrumUnit = (AVFAudioSpectrumUnit*)pSpectrum->user_data;
234
235 const GstStructure *pStr = gst_message_get_structure(message);
236 if (gst_structure_has_name(pStr, "spectrum")) {
237 GstClockTime timestamp, duration;
238
239 if (!gst_structure_get_clock_time(pStr, "timestamp", ×tamp))
240 timestamp = GST_CLOCK_TIME_NONE;
241
242 if (!gst_structure_get_clock_time(pStr, "duration", &duration))
243 duration = GST_CLOCK_TIME_NONE;
244
245 size_t bandsNum = pSpectrumUnit->GetBands();
246
247 if (bandsNum > 0) {
248 float *magnitudes = new float[bandsNum];
249 float *phases = new float[bandsNum];
250
251 const GValue *magnitudes_value = gst_structure_get_value(pStr, "magnitude");
252 const GValue *phases_value = gst_structure_get_value(pStr, "phase");
253 for (int i = 0; i < bandsNum; i++) {
254 magnitudes[i] = g_value_get_float(gst_value_list_get_value(magnitudes_value, i));
255 phases[i] = g_value_get_float(gst_value_list_get_value(phases_value, i));
256 }
257 pSpectrumUnit->UpdateBands((int) bandsNum, magnitudes, phases);
258
259 delete [] magnitudes;
260 delete [] phases;
261 }
262 }
263
264 gst_message_unref(message);
265
266 return TRUE;
267 }
268
269 void AVFAudioSpectrumUnit::SetupSpectralProcessor() {
270 ReleaseSpectralProcessor();
271
272 lockBands();
273
274 mSpectrumElement = gst_element_factory_make("spectrum", NULL);
275 mSpectrum = GST_SPECTRUM(mSpectrumElement);
276 mSpectrum->user_data = (void*)this;
277
278 // Set our own callback for post message
279 GstElementClass *klass;
280 klass = GST_ELEMENT_GET_CLASS(mSpectrumElement);
281 klass->post_message = PostMessageCallback;
282
283 // Configure spectrum element
284 // Do send magnitude and phase information, off by default
285 g_object_set(mSpectrumElement, "post-messages", TRUE,
286 "message-magnitude", TRUE,
287 "message-phase", TRUE, NULL);
288
289 g_object_set(mSpectrumElement, "bands", mBandCount, NULL);
290
291 mSamplesPerInterval = (UInt32)(mSampleRate * mUpdateInterval);
292 guint64 value = (guint64) (mUpdateInterval * GST_SECOND);
293 g_object_set(mSpectrumElement, "interval", value, NULL);
294
295 g_object_set(mSpectrumElement, "threshold", (int) mThreshold, NULL);
296
297 // Since we do not run spectrum element in pipeline and it will not get configured
298 // correctly, we need to set required information directly.
299 GST_AUDIO_FILTER_RATE(mSpectrum) = mSampleRate;
300 GST_AUDIO_FILTER_CHANNELS(mSpectrum) = 1; // Always 1 channel
301
302 // gst_spectrum_setup()
303 GstAudioInfo *info = gst_audio_info_new();
304 gst_audio_info_init(info);
305 gst_audio_info_set_format(info, GST_AUDIO_FORMAT_F32, mSampleRate, 1, NULL);
306 // bps = 4 bytes - 32-bit float, bpf = 4 bytes - 32-bit float mono
307 gst_spectrum_setup_api((GstAudioFilter*) mSpectrum, info, 4, 4);
308 gst_audio_info_free(info);
309
310 // Set element to playing state
311 gst_element_set_state(mSpectrumElement, GST_STATE_PLAYING);
312
313 mRebuildCrunch = false;
314 unlockBands();
315 }
316
317 void AVFAudioSpectrumUnit::ReleaseSpectralProcessor() {
318 lockBands();
319
320 if (mSpectrumElement) {
321 gst_element_set_state(mSpectrumElement, GST_STATE_NULL);
322 gst_object_unref(GST_OBJECT(mSpectrumElement));
323 mSpectrumElement = NULL;
324 mSpectrum = NULL;
325 }
326
327 unlockBands();
328 }