From 413f597654684d3e78485bb375bcf8969d960be9 Mon Sep 17 00:00:00 2001 From: Lyubomir Marinov Date: Fri, 18 Jun 2010 00:23:57 +0000 Subject: [PATCH] In response to a report by Werner Dittmann on the dev mailing list in the thread 'Wideband PortAudio', attempts to open the PortAudio CaptureDevice and Renderer at supported sample rates. The notifications/PortAudioClipImpl and GatherEntropy still need a fix. The JNI binaries of PortAudio need rebuilding in order to implement free(void *) for PortAudio but the application should work even without it. (In the absence of PortAudio free(void *), falling back to FFmpeg's free routine will be attempted.) --- ...icator_impl_neomedia_portaudio_PortAudio.c | 7 + ...icator_impl_neomedia_portaudio_PortAudio.h | 17 +- .../impl/neomedia/MediaUtils.java | 22 +-- .../impl/neomedia/device/PortAudioAuto.java | 100 +++++++++++- .../protocol/portaudio/PortAudioStream.java | 17 +- .../renderer/audio/PortAudioRenderer.java | 147 +++++++++++++++--- .../impl/neomedia/portaudio/PortAudio.java | 45 ++++++ 7 files changed, 313 insertions(+), 42 deletions(-) diff --git a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c index 2d050ca2a..ea97907c7 100644 --- a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c +++ b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.c @@ -54,6 +54,13 @@ static const char *AUDIO_QUALITY_IMPROVEMENT_STRING_ID = "portaudio"; #define LATENCY_HIGH net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_LATENCY_HIGH #define LATENCY_LOW net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_LATENCY_LOW +JNIEXPORT void JNICALL +Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_free + (JNIEnv *jniEnv, jclass clazz, jlong ptr) +{ + free((void *) ptr); +} + JNIEXPORT void JNICALL Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_Pa_1AbortStream (JNIEnv *env, jclass clazz, jlong stream) diff --git a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.h b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.h index 4e316de3d..cf2b4d9bd 100644 --- a/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.h +++ b/src/native/portaudio/net_java_sip_communicator_impl_neomedia_portaudio_PortAudio.h @@ -1,10 +1,3 @@ -/* - * SIP Communicator, the OpenSource Java VoIP and Instant Messaging client. - * - * Distributable under LGPL license. - * See terms of license at gnu.org. - */ - /* DO NOT EDIT THIS FILE - it is machine generated */ #include /* Header for class net_java_sip_communicator_impl_neomedia_portaudio_PortAudio */ @@ -18,6 +11,16 @@ extern "C" { #define net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_LATENCY_HIGH -1.0 #undef net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_LATENCY_LOW #define net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_LATENCY_LOW -2.0 +#undef net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_LATENCY_UNSPECIFIED +#define net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_LATENCY_UNSPECIFIED 0.0 +/* + * Class: net_java_sip_communicator_impl_neomedia_portaudio_PortAudio + * Method: free + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_net_java_sip_communicator_impl_neomedia_portaudio_PortAudio_free + (JNIEnv *, jclass, jlong); + /* * Class: net_java_sip_communicator_impl_neomedia_portaudio_PortAudio * Method: Pa_AbortStream diff --git a/src/net/java/sip/communicator/impl/neomedia/MediaUtils.java b/src/net/java/sip/communicator/impl/neomedia/MediaUtils.java index fb6bc4d53..12f50eea1 100644 --- a/src/net/java/sip/communicator/impl/neomedia/MediaUtils.java +++ b/src/net/java/sip/communicator/impl/neomedia/MediaUtils.java @@ -33,6 +33,11 @@ public class MediaUtils */ public static final MediaFormat[] EMPTY_MEDIA_FORMATS = new MediaFormat[0]; + /** + * The maximum sample rate for audio that we advertise. + */ + public static final double MAX_AUDIO_SAMPLE_RATE = 32000; + /** * The Map of JMF-specific encodings to well-known encodings as * defined in RFC 3551. @@ -158,20 +163,15 @@ public class MediaUtils = new HashMap(); h264FormatParams.put("packetization-mode", "1"); - /* disable PLI since we use periodic intra-refresh feature - * of ffmpeg/x264 + /* + * Disable PLI since we use periodic intra-refresh feature of + * FFmpeg/x264. */ //h264AdvancedAttributes.put("rtcp-fb", "nack pli"); - ScreenDevice screen = NeomediaActivator.getMediaServiceImpl(). - getDefaultScreenDevice(); - - java.awt.Dimension res = null; - - if(screen != null) - { - res = screen.getSize(); - } + ScreenDevice screen + = NeomediaActivator.getMediaServiceImpl().getDefaultScreenDevice(); + java.awt.Dimension res = (screen == null) ? null : screen.getSize(); h264AdvancedAttributes.put("imageattr", createImageAttr(null, res)); diff --git a/src/net/java/sip/communicator/impl/neomedia/device/PortAudioAuto.java b/src/net/java/sip/communicator/impl/neomedia/device/PortAudioAuto.java index 8d71f4c24..3dfa2b2e5 100644 --- a/src/net/java/sip/communicator/impl/neomedia/device/PortAudioAuto.java +++ b/src/net/java/sip/communicator/impl/neomedia/device/PortAudioAuto.java @@ -23,6 +23,7 @@ * input channels. * * @author Damian Minkov + * @author Lubomir Marinov */ public class PortAudioAuto { @@ -54,7 +55,7 @@ public class PortAudioAuto public static CaptureDeviceInfo defaultCaptureDevice = null; /** - * Is portaudio supported on current platform. + * Is PortAudio supported on current platform. */ private static boolean supported = false; @@ -82,6 +83,9 @@ public class PortAudioAuto Vector playbackDevVector = new Vector(); + int channels = 1; + int sampleSizeInBits = 16; + long sampleFormat = PortAudio.getPaSampleFormat(sampleSizeInBits); for (int deviceIndex = 0; deviceIndex < deviceCount; deviceIndex++) { @@ -104,9 +108,15 @@ public class PortAudioAuto { new AudioFormat( AudioFormat.LINEAR, - PortAudio.DEFAULT_SAMPLE_RATE, - 16 /* sampleSizeInBits */, - 1 /* channels */, + (maxInputChannels > 0) + ? getSupportedSampleRate( + true, + deviceIndex, + channels, + sampleFormat) + : PortAudio.DEFAULT_SAMPLE_RATE, + sampleSizeInBits, + channels, AudioFormat.LITTLE_ENDIAN, AudioFormat.SIGNED, Format.NOT_SPECIFIED /* frameSizeInBits */, @@ -181,6 +191,88 @@ public class PortAudioAuto supported = true; } + /** + * Gets a sample rate supported by a PortAudio device with a specific device + * index with which it is to be registered with JMF. + * + * @param true if the supported sample rate is to be retrieved for + * the PortAudio device with the specified device index as an input device + * or false for an output device + * @param deviceIndex the device index of the PortAudio device for which a + * supported sample rate is to be retrieved + * @param channelCount + * @param sampleFormat + * @return a sample rate supported by the PortAudio device with the + * specified device index with which it is to be registered with JMF + */ + public static double getSupportedSampleRate( + boolean input, + int deviceIndex, + int channelCount, + long sampleFormat) + { + long deviceInfo = PortAudio.Pa_GetDeviceInfo(deviceIndex); + double supportedSampleRate; + + if (deviceInfo != 0) + { + double defaultSampleRate + = PortAudio.PaDeviceInfo_getDefaultSampleRate(deviceInfo); + + if (defaultSampleRate >= MediaUtils.MAX_AUDIO_SAMPLE_RATE) + supportedSampleRate = defaultSampleRate; + else + { + long streamParameters + = PortAudio.PaStreamParameters_new( + deviceIndex, + channelCount, + sampleFormat, + PortAudio.LATENCY_UNSPECIFIED); + + if (streamParameters == 0) + supportedSampleRate = defaultSampleRate; + else + { + try + { + long inputParameters; + long outputParameters; + + if (input) + { + inputParameters = streamParameters; + outputParameters = 0; + } + else + { + inputParameters = 0; + outputParameters = streamParameters; + } + + boolean formatIsSupported + = PortAudio.Pa_IsFormatSupported( + inputParameters, + outputParameters, + PortAudio.DEFAULT_SAMPLE_RATE); + + supportedSampleRate + = formatIsSupported + ? PortAudio.DEFAULT_SAMPLE_RATE + : defaultSampleRate; + } + finally + { + PortAudio.PaStreamParameters_free(streamParameters); + } + } + } + } + else + supportedSampleRate = PortAudio.DEFAULT_SAMPLE_RATE; + return supportedSampleRate; + } + /** * Whether portaudio is supported. * @return the supported diff --git a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/portaudio/PortAudioStream.java b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/portaudio/PortAudioStream.java index 65e3f0338..ac3c1c627 100644 --- a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/portaudio/PortAudioStream.java +++ b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/portaudio/PortAudioStream.java @@ -56,6 +56,8 @@ public class PortAudioStream */ private int framesPerBuffer; + private long inputParameters = 0; + private int sequenceNumber = 0; /** @@ -212,6 +214,11 @@ synchronized void setDeviceIndex(int deviceIndex) throw ioex; } stream = 0; + if (inputParameters != 0) + { + PortAudio.PaStreamParameters_free(inputParameters); + inputParameters = 0; + } /* * Make sure this AbstractPullBufferStream asks its DataSource @@ -238,7 +245,7 @@ synchronized void setDeviceIndex(int deviceIndex) try { - long inputParameters + inputParameters = PortAudio.PaStreamParameters_new( this.deviceIndex, channels, @@ -262,6 +269,14 @@ synchronized void setDeviceIndex(int deviceIndex) ioex.initCause(paex); throw ioex; } + finally + { + if ((stream == 0) && (inputParameters != 0)) + { + PortAudio.PaStreamParameters_free(inputParameters); + inputParameters = 0; + } + } if (stream == 0) throw new IOException("Pa_OpenStream"); diff --git a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/audio/PortAudioRenderer.java b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/audio/PortAudioRenderer.java index 7145ee645..4f3a78a2b 100644 --- a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/audio/PortAudioRenderer.java +++ b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/audio/PortAudioRenderer.java @@ -6,6 +6,8 @@ */ package net.java.sip.communicator.impl.neomedia.jmfext.media.renderer.audio; +import java.util.*; + import javax.media.*; import javax.media.format.*; @@ -31,6 +33,9 @@ public class PortAudioRenderer private static final Logger logger = Logger.getLogger(PortAudioRenderer.class); + private static final Format[] EMPTY_SUPPORTED_INPUT_FORMATS + = new Format[0]; + /** * The human-readable name of the PortAudioRenderer JMF plug-in. */ @@ -117,6 +122,8 @@ public class PortAudioRenderer */ private MediaLocator locator; + private long outputParameters = 0; + /** * The indicator which determines whether this Renderer is started. */ @@ -133,6 +140,8 @@ public class PortAudioRenderer */ private boolean streamIsBusy = false; + private Format[] supportedInputFormats; + /** * Initializes a new PortAudioRenderer instance. */ @@ -163,6 +172,11 @@ public synchronized void close() logger.error("Failed to close PortAudio stream.", paex); } } + if ((stream == 0) && (outputParameters != 0)) + { + PortAudio.PaStreamParameters_free(outputParameters); + outputParameters = 0; + } } } @@ -197,24 +211,110 @@ public String getName() */ public Format[] getSupportedInputFormats() { - /* - * We want to try to match the PortAudio CaptureDevice when rendering - * for the output device of the PortAudio audio system in order to make - * echo cancellation and denoise possible (if enabled, of course). - * Otherwise i.e. when rendering for the notification device of the - * PortAudio audio system, we will not try to make echo cancellation and - * denoise possible here. - */ -// if (getLocator() == defaultLocator) -// { -// for (Format supportedInputFormat : SUPPORTED_INPUT_FORMATS) -// { -// if (((AudioFormat) supportedInputFormat).getSampleRate() -// == PortAudio.DEFAULT_SAMPLE_RATE) -// return new Format[] { supportedInputFormat }; -// } -// } - return SUPPORTED_INPUT_FORMATS.clone(); + if (supportedInputFormats == null) + { + MediaLocator locator = getLocator(); + int deviceIndex; + + if ((locator == null) + || ((deviceIndex = DataSource.getDeviceIndex(locator)) + == PortAudio.paNoDevice)) + supportedInputFormats = SUPPORTED_INPUT_FORMATS; + else + { + long deviceInfo = PortAudio.Pa_GetDeviceInfo(deviceIndex); + + if (deviceInfo == 0) + supportedInputFormats = SUPPORTED_INPUT_FORMATS; + else + { + int minOutputChannels = 1; + /* + * The maximum output channels may be a lot and checking all + * of them will take a lot of time. Besides, we currently + * support at most 2. + */ + int maxOutputChannels + = Math.min( + PortAudio.PaDeviceInfo_getMaxOutputChannels( + deviceInfo), + 2); + List supportedInputFormats + = new ArrayList(SUPPORTED_INPUT_FORMATS.length); + + for (Format supportedInputFormat : SUPPORTED_INPUT_FORMATS) + { + getSupportedInputFormats( + supportedInputFormat, + deviceIndex, + minOutputChannels, + maxOutputChannels, + supportedInputFormats); + } + this.supportedInputFormats + = supportedInputFormats.isEmpty() + ? EMPTY_SUPPORTED_INPUT_FORMATS + : supportedInputFormats.toArray( + EMPTY_SUPPORTED_INPUT_FORMATS); + } + } + } + return + (supportedInputFormats.length == 0) + ? EMPTY_SUPPORTED_INPUT_FORMATS + : supportedInputFormats.clone(); + } + + private void getSupportedInputFormats( + Format format, + int deviceIndex, + int minOutputChannels, int maxOutputChannels, + List supportedInputFormats) + { + AudioFormat audioFormat = (AudioFormat) format; + int sampleSizeInBits = audioFormat.getSampleSizeInBits(); + long sampleFormat = PortAudio.getPaSampleFormat(sampleSizeInBits); + double sampleRate = audioFormat.getSampleRate(); + + for (int channels = minOutputChannels; + channels <= maxOutputChannels; + channels++) + { + long outputParameters + = PortAudio.PaStreamParameters_new( + deviceIndex, + channels, + sampleFormat, + PortAudio.LATENCY_UNSPECIFIED); + + if (outputParameters != 0) + { + try + { + if (PortAudio.Pa_IsFormatSupported( + 0, + outputParameters, + sampleRate)) + { + supportedInputFormats.add( + new AudioFormat( + audioFormat.getEncoding(), + sampleRate, + sampleSizeInBits, + channels, + audioFormat.getEndian(), + audioFormat.getSigned(), + Format.NOT_SPECIFIED /* frameSizeInBits */, + Format.NOT_SPECIFIED /* frameRate */, + audioFormat.getDataType())); + } + } + finally + { + PortAudio.PaStreamParameters_free(outputParameters); + } + } + } } /** @@ -243,7 +343,7 @@ public synchronized void open() try { - long outputParameters + outputParameters = PortAudio.PaStreamParameters_new( deviceIndex, channels, @@ -264,6 +364,14 @@ public synchronized void open() { throw new ResourceUnavailableException(paex.getMessage()); } + finally + { + if ((stream == 0) && (outputParameters != 0)) + { + PortAudio.PaStreamParameters_free(outputParameters); + outputParameters = 0; + } + } if (stream == 0) throw new ResourceUnavailableException("Pa_OpenStream"); @@ -461,6 +569,7 @@ else if (this.locator.equals(locator)) return; this.locator = locator; + supportedInputFormats = null; } /** diff --git a/src/net/java/sip/communicator/impl/neomedia/portaudio/PortAudio.java b/src/net/java/sip/communicator/impl/neomedia/portaudio/PortAudio.java index 3a120a8df..9743ee485 100644 --- a/src/net/java/sip/communicator/impl/neomedia/portaudio/PortAudio.java +++ b/src/net/java/sip/communicator/impl/neomedia/portaudio/PortAudio.java @@ -10,6 +10,7 @@ import java.lang.reflect.*; import java.nio.charset.*; +import net.java.sip.communicator.impl.neomedia.codec.video.*; import net.java.sip.communicator.util.*; /** @@ -158,6 +159,8 @@ public final class PortAudio public static final long STREAM_FLAGS_PRIME_OUTPUT_BUFFERS_USING_STREAM_CALLBACK = 0x00000008; + private static native void free(long ptr); + /** * Gets the native PaSampleFormat with a specific size in bits. * @@ -564,6 +567,48 @@ public static native int PaHostApiInfo_getDefaultOutputDevice( */ public static native int PaHostApiInfo_getType(long hostApiInfo); + public static void PaStreamParameters_free(long streamParameters) + { + try + { + free(streamParameters); + return; + } + catch (UnsatisfiedLinkError ulerr) + { + /* + * Ignore it because we'll try to fallback to another + * implementation. + */ + logger.warn( + "The JNI library jportaudio is out-of-date and needs to be" + + " recompiled.", + ulerr); + } + /* + * FFmpeg on Windows seems to be compiled with --enable-memalign-hack so + * av_free(void *) isn't the same as free(void *) and thus is not a + * suitable/safe fallback. + */ + if (OSUtils.IS_LINUX || OSUtils.IS_MAC) + { + try + { + FFmpeg.av_free(streamParameters); + } + catch (NoClassDefFoundError ncdferr) + { + // We'll warn bellow, don't fail. + } + catch (UnsatisfiedLinkError ulerr) + { + // We'll warn bellow, don't fail. + } + return; + } + logger.warn("Leaking a PaStreamParameters instance."); + } + /** * Creates parameters used for opening streams. * @param deviceIndex the device.