1 /*
2 * Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 #include "AVFAudioSpectrumUnit.h"
27
28 #include <iostream>
29 #include <Accelerate/Accelerate.h>
30
31 AVFAudioSpectrumUnit::AVFAudioSpectrumUnit() : mSpectrumCallbackProc(NULL),
32 mSpectrumCallbackContext(NULL),
33 mEnabled(true),
34 mBandCount(128),
35 mBands(NULL),
36 mUpdateInterval(kDefaultAudioSpectrumUpdateInterval),
37 mThreshold(kDefaultAudioSpectrumThreshold),
38 mMixBufferFrameCapacity(0),
39 mSampleRate(0),
40 mChannels(0),
41 mMaxFrames(0),
42 mSamplesPerInterval(0),
43 mRebuildCrunch(true),
44 mSpectrumElement(NULL),
45 mSpectrum(NULL) {
46 mMixBuffer.mNumberBuffers = 1;
47 mMixBuffer.mBuffers[0].mData = NULL;
48
49 pthread_mutex_init(&mBandLock, NULL);
50
51 gst_init_check(NULL, NULL, NULL);
52 }
53
54 AVFAudioSpectrumUnit::~AVFAudioSpectrumUnit() {
55 if (mMixBuffer.mBuffers[0].mData) {
56 free(mMixBuffer.mBuffers[0].mData);
57 mMixBuffer.mBuffers[0].mData = NULL;
58 }
59
60 ReleaseSpectralProcessor();
61 }
62
63 bool AVFAudioSpectrumUnit::ProcessBufferLists(const AudioBufferList& inBuffer,
64 UInt32 inFramesToProcess) {
65 if (!mEnabled) {
66 return true;
67 }
68
69 // (Re)allocate mix buffer if needed
70 if (!mMixBuffer.mBuffers[0].mData || mMixBufferFrameCapacity < inFramesToProcess) {
71 // allocate buffer list (only need to do this once)
72 if (mMixBuffer.mBuffers[0].mData) {
73 free(mMixBuffer.mBuffers[0].mData);
74 mMixBuffer.mBuffers[0].mData = NULL;
75 }
76
77 mMixBufferFrameCapacity = mMaxFrames;
78
79 mMixBuffer.mBuffers[0].mNumberChannels = 1;
80 mMixBuffer.mBuffers[0].mData = calloc(mMixBufferFrameCapacity, sizeof (Float32));
81 mMixBuffer.mBuffers[0].mDataByteSize = 0; // size of actual contained data, not size of buffer
82 }
83
84 if (mRebuildCrunch) {
85 SetupSpectralProcessor();
86 }
87
88 if (mSpectrum != NULL) {
89 // Mix the audio into one channel since JavaFX only supports single channel spectrum
90 // Just use an arithmetic average, nothing fancy here
91 float *buffer = (float*) mMixBuffer.mBuffers[0].mData;
92 vDSP_vclr(buffer, 1, mMixBufferFrameCapacity);
93 for (int ii = 0; ii < inBuffer.mNumberBuffers; ii++) {
94 vDSP_vadd((float*) inBuffer.mBuffers[ii].mData, 1,
95 buffer, 1,
96 buffer, 1, inFramesToProcess);
97 }
98 float divisor = (float) inBuffer.mNumberBuffers;
99 vDSP_vsdiv(buffer, 1,
100 &divisor,
101 buffer, 1, inFramesToProcess);
102 mMixBuffer.mBuffers[0].mDataByteSize = inFramesToProcess * sizeof (Float32);
103
104 // Just reuse already allocated memory from mMixBuffer and do not free it
105 // in GStreamer
106 GstBuffer *gstBuffer =
107 gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, // Allow only reading
108 mMixBuffer.mBuffers[0].mData,
109 mMixBuffer.mBuffers[0].mDataByteSize,
110 0,
111 mMixBuffer.mBuffers[0].mDataByteSize,
112 NULL,
113 NULL); // No need to free memory
114 if (gstBuffer == NULL) {
115 return false;
116 }
117
118 GstFlowReturn result = gst_spectrum_transform_ip_api((GstBaseTransform *) mSpectrum, gstBuffer);
119 if (result != GST_FLOW_OK) {
120 return false;
121 }
122 gst_buffer_unref(gstBuffer);
123 }
124
125 return true;
126 }
127
128 bool AVFAudioSpectrumUnit::IsEnabled() {
129 return mEnabled;
130 }
131
132 void AVFAudioSpectrumUnit::SetEnabled(bool isEnabled) {
133 mEnabled = isEnabled;
134 mRebuildCrunch = true;
135 }
136
137 void AVFAudioSpectrumUnit::SetBands(int bands, CBandsHolder* holder) {
138 lockBands();
139 if (mBands) {
140 CBandsHolder::ReleaseRef(mBands);
141 mBands = NULL;
142 }
143 mBandCount = 0;
144 if (holder) {
145 mBands = CBandsHolder::AddRef(holder);
146 if (mBands) {
147 mBandCount = bands;
148 }
149 }
150 mRebuildCrunch = true;
151 unlockBands();
152 }
153
154 size_t AVFAudioSpectrumUnit::GetBands() {
155 return mBandCount;
156 }
157
158 double AVFAudioSpectrumUnit::GetInterval() {
159 return mUpdateInterval;
160 }
161
162 void AVFAudioSpectrumUnit::SetInterval(double interval) {
163 if (mUpdateInterval != interval) {
164 mUpdateInterval = interval;
165 mRebuildCrunch = true;
166 }
167 }
168
169 int AVFAudioSpectrumUnit::GetThreshold() {
170 return (int) mThreshold;
171 }
172
173 void AVFAudioSpectrumUnit::SetThreshold(int threshold) {
174 if (mThreshold != (Float32) threshold) {
175 mThreshold = (Float32) threshold;
176 mRebuildCrunch = true;
177 }
178 }
179
180 void AVFAudioSpectrumUnit::UpdateBands(int size, const float* magnitudes, const float* phases) {
181 // lock now otherwise the bands could change while we're processing
182 lockBands();
183 if (!mBands || size <= 0 || !mEnabled) {
184 unlockBands();
185 return;
186 }
187
188 // Update band data
189 mBands->UpdateBands(size, magnitudes, magnitudes);
190
191 // Call our listener to dispatch the spectrum event
192 if (mSpectrumCallbackProc) {
193 double duration = (double) mSamplesPerInterval / (double) 44100;
194 mSpectrumCallbackProc(mSpectrumCallbackContext, duration);
195 }
196
197 unlockBands();
198 }
199
200 void AVFAudioSpectrumUnit::SetSampleRate(UInt32 rate) {
201 mSampleRate = rate;
202 }
203
204 void AVFAudioSpectrumUnit::SetChannels(UInt32 count) {
205 mChannels = count;
206 }
207
208 void AVFAudioSpectrumUnit::SetMaxFrames(UInt32 maxFrames) {
209 mMaxFrames = maxFrames;
210 }
211
212 void AVFAudioSpectrumUnit::SetSpectrumCallbackProc(AVFSpectrumUnitCallbackProc proc, void *context) {
213 mSpectrumCallbackProc = proc;
214 mSpectrumCallbackContext = context;
215 }
216
217 static gboolean PostMessageCallback(GstElement * element, GstMessage * message) {
218 if (message == NULL) {
219 return FALSE;
220 }
221
222 GstSpectrum *pSpectrum = GST_SPECTRUM(element);
223 if (pSpectrum == NULL || pSpectrum->user_data == NULL) {
224 return FALSE;
225 }
226
227 AVFAudioSpectrumUnit *pSpectrumUnit = (AVFAudioSpectrumUnit*)pSpectrum->user_data;
228
229 const GstStructure *pStr = gst_message_get_structure(message);
230 if (gst_structure_has_name(pStr, "spectrum")) {
231 GstClockTime timestamp, duration;
232
233 if (!gst_structure_get_clock_time(pStr, "timestamp", ×tamp))
234 timestamp = GST_CLOCK_TIME_NONE;
235
236 if (!gst_structure_get_clock_time(pStr, "duration", &duration))
237 duration = GST_CLOCK_TIME_NONE;
238
239 size_t bandsNum = pSpectrumUnit->GetBands();
240
241 if (bandsNum > 0) {
242 float *magnitudes = new float[bandsNum];
243 float *phases = new float[bandsNum];
244
245 const GValue *magnitudes_value = gst_structure_get_value(pStr, "magnitude");
246 const GValue *phases_value = gst_structure_get_value(pStr, "phase");
247 for (int i = 0; i < bandsNum; i++) {
248 magnitudes[i] = g_value_get_float(gst_value_list_get_value(magnitudes_value, i));
249 phases[i] = g_value_get_float(gst_value_list_get_value(phases_value, i));
250 }
251 pSpectrumUnit->UpdateBands((int) bandsNum, magnitudes, phases);
252
253 delete [] magnitudes;
254 delete [] phases;
255 }
256 }
257
258 gst_message_unref(message);
259
260 return TRUE;
261 }
262
263 void AVFAudioSpectrumUnit::SetupSpectralProcessor() {
264 ReleaseSpectralProcessor();
265
266 lockBands();
267
268 mSpectrumElement = gst_element_factory_make("spectrum", NULL);
269 mSpectrum = GST_SPECTRUM(mSpectrumElement);
270 mSpectrum->user_data = (void*)this;
271
272 // Set our own callback for post message
273 GstElementClass *klass;
274 klass = GST_ELEMENT_GET_CLASS(mSpectrumElement);
275 klass->post_message = PostMessageCallback;
276
277 // Configure spectrum element
278 // Do send magnitude and phase information, off by default
279 g_object_set(mSpectrumElement, "post-messages", TRUE,
280 "message-magnitude", TRUE,
281 "message-phase", TRUE, NULL);
282
283 g_object_set(mSpectrumElement, "bands", mBandCount, NULL);
284
285 mSamplesPerInterval = (UInt32)(mSampleRate * mUpdateInterval);
286 guint64 value = (guint64) (mUpdateInterval * GST_SECOND);
287 g_object_set(mSpectrumElement, "interval", value, NULL);
288
289 g_object_set(mSpectrumElement, "threshold", (int) mThreshold, NULL);
290
291 // Since we do not run spectrum element in pipeline and it will not get configured
292 // correctly, we need to set required information directly.
293 GST_AUDIO_FILTER_RATE(mSpectrum) = mSampleRate;
294 GST_AUDIO_FILTER_CHANNELS(mSpectrum) = 1; // Always 1 channel
295
296 // gst_spectrum_setup()
297 GstAudioInfo *info = gst_audio_info_new();
298 gst_audio_info_init(info);
299 gst_audio_info_set_format(info, GST_AUDIO_FORMAT_F32, mSampleRate, 1, NULL);
300 // bps = 4 bytes - 32-bit float, bpf = 4 bytes - 32-bit float mono
301 gst_spectrum_setup_api((GstAudioFilter*) mSpectrum, info, 4, 4);
302 gst_audio_info_free(info);
303
304 // Set element to playing state
305 gst_element_set_state(mSpectrumElement, GST_STATE_PLAYING);
306
307 mRebuildCrunch = false;
308 unlockBands();
309 }
310
311 void AVFAudioSpectrumUnit::ReleaseSpectralProcessor() {
312 lockBands();
313
314 if (mSpectrumElement) {
315 gst_element_set_state(mSpectrumElement, GST_STATE_NULL);
316 gst_object_unref(GST_OBJECT(mSpectrumElement));
317 mSpectrumElement = NULL;
318 mSpectrum = NULL;
319 }
320
321 unlockBands();
322 }