From 3569936fced1c1feff71c4ecc7cd6506f4caae60 Mon Sep 17 00:00:00 2001 From: Lyubomir Marinov Date: Thu, 29 Apr 2010 07:46:00 +0000 Subject: [PATCH] Represents work in progress on the subject of creating native VideoRenderer. Sheds light on the initial design to be shared between platforms. An implementation on Mac OS X is in the works but is not part of this commit. --- src/native/jawtrenderer/JAWTRenderer.h | 18 + ...jmfext_media_renderer_video_JAWTRenderer.c | 84 ++++ ...jmfext_media_renderer_video_JAWTRenderer.h | 45 ++ .../neomedia/codec/EncodingConfiguration.java | 52 ++- .../neomedia/device/DeviceConfiguration.java | 100 ++++ .../media/renderer/video/JAWTRenderer.java | 438 ++++++++++++++++++ 6 files changed, 718 insertions(+), 19 deletions(-) create mode 100644 src/native/jawtrenderer/JAWTRenderer.h create mode 100644 src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.c create mode 100644 src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.h create mode 100644 src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/video/JAWTRenderer.java diff --git a/src/native/jawtrenderer/JAWTRenderer.h b/src/native/jawtrenderer/JAWTRenderer.h new file mode 100644 index 000000000..0a732590b --- /dev/null +++ b/src/native/jawtrenderer/JAWTRenderer.h @@ -0,0 +1,18 @@ +#ifndef _JAWTRENDERER_H_ +#define _JAWTRENDERER_H_ + +#include +#include + +void JAWTRenderer_close + (JNIEnv *jniEnv, jclass clazz, jlong handle, jobject component); +jlong JAWTRenderer_open(JNIEnv *jniEnv, jclass clazz, jobject component); +void JAWTRenderer_paint + (JAWT_DrawingSurfaceInfo *dsi, jclass clazz, jlong handle, jobject g); +jboolean JAWTRenderer_process + (JNIEnv *jniEnv, jclass clazz, + jlong handle, jobject component, + jint *data, jint length, + jint width, jint height); + +#endif /* _JAWTRENDERER_H_ */ diff --git a/src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.c b/src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.c new file mode 100644 index 000000000..abc4e3543 --- /dev/null +++ b/src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.c @@ -0,0 +1,84 @@ +#include "net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.h" +#include "JAWTRenderer.h" + +JNIEXPORT void JNICALL +Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_close + (JNIEnv *jniEnv, jclass clazz, jlong handle, jobject component) +{ + JAWTRenderer_close(jniEnv, clazz, handle, component); +} + +JNIEXPORT jlong JNICALL +Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_open + (JNIEnv *jniEnv, jclass clazz, jobject component) +{ + return JAWTRenderer_open(jniEnv, clazz, component); +} + +JNIEXPORT void JNICALL +Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_paint + (JNIEnv *jniEnv, jclass clazz, jlong handle, jobject component, jobject g) + +{ + JAWT awt; + + awt.version = JAWT_VERSION_1_3; + if (JAWT_GetAWT(jniEnv, &awt) != JNI_FALSE) + { + JAWT_DrawingSurface *ds; + + ds = awt.GetDrawingSurface(jniEnv, component); + if (ds) + { + jint dsLock; + + dsLock = ds->Lock(ds); + if (0 == (dsLock & JAWT_LOCK_ERROR)) + { + JAWT_DrawingSurfaceInfo *dsi; + + dsi = ds->GetDrawingSurfaceInfo(ds); + if (dsi && dsi->platformInfo) + { + /* + * The function arguments jniEnv and component are now + * available as the fields env and target, respectively, of + * the JAWT_DrawingSurface which is itself the value of the + * field ds of the JAWT_DrawingSurfaceInfo. + */ + JAWTRenderer_paint(dsi, clazz, handle, g); + ds->FreeDrawingSurfaceInfo(dsi); + } + ds->Unlock(ds); + } + awt.FreeDrawingSurface(ds); + } + } +} + +JNIEXPORT jboolean JNICALL +Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_process + (JNIEnv *jniEnv, jclass clazz, + jlong handle, jobject component, + jintArray data, jint offset, jint length, + jint width, jint height) +{ + jint *dataPtr; + jboolean processed; + + dataPtr = (*jniEnv)->GetPrimitiveArrayCritical(jniEnv, data, NULL); + if (dataPtr) + { + processed + = JAWTRenderer_process + (jniEnv, clazz, + handle, component, + dataPtr + offset, length, + width, height); + (*jniEnv) + ->ReleasePrimitiveArrayCritical(jniEnv, data, dataPtr, JNI_ABORT); + } + else + processed = JNI_FALSE; + return processed; +} diff --git a/src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.h b/src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.h new file mode 100644 index 000000000..8c70c479b --- /dev/null +++ b/src/native/jawtrenderer/net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer.h @@ -0,0 +1,45 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer */ + +#ifndef _Included_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer +#define _Included_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer + * Method: close + * Signature: (JLjava/awt/Component;)V + */ +JNIEXPORT void JNICALL Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_close + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer + * Method: open + * Signature: (Ljava/awt/Component;)J + */ +JNIEXPORT jlong JNICALL Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_open + (JNIEnv *, jclass, jobject); + +/* + * Class: net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer + * Method: paint + * Signature: (JLjava/awt/Component;Ljava/awt/Graphics;)V + */ +JNIEXPORT void JNICALL Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_paint + (JNIEnv *, jclass, jlong, jobject, jobject); + +/* + * Class: net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer + * Method: process + * Signature: (JLjava/awt/Component;[IIIII)Z + */ +JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_jmfext_media_renderer_video_JAWTRenderer_process + (JNIEnv *, jclass, jlong, jobject, jintArray, jint, jint, jint, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/net/java/sip/communicator/impl/neomedia/codec/EncodingConfiguration.java b/src/net/java/sip/communicator/impl/neomedia/codec/EncodingConfiguration.java index ea5eddf89..cb9d9a513 100644 --- a/src/net/java/sip/communicator/impl/neomedia/codec/EncodingConfiguration.java +++ b/src/net/java/sip/communicator/impl/neomedia/codec/EncodingConfiguration.java @@ -78,11 +78,15 @@ public class EncodingConfiguration }; /** - * Custom Packages provided by Sip-Communicator + * The package prefixes of the additional JMF DataSources (e.g. low + * latency PortAudio and ALSA CaptureDevices). */ - private static final String[] CUSTOM_PACKAGES = new String[] - { // datasource for low latency PortAudio or ALSA input - "net.java.sip.communicator.impl.neomedia.jmfext", "net.sf.fmj" }; + private static final String[] CUSTOM_PACKAGES + = new String[] + { + "net.java.sip.communicator.impl.neomedia.jmfext", + "net.sf.fmj" + }; /** * The Comparator which sorts the sets according to the settings in @@ -381,15 +385,19 @@ public void registerCustomCodecs() exception = ex; } if (registered) + { logger.debug( "Codec " - + className - + " is successfully registered"); + + className + + " is successfully registered"); + } else + { logger.debug( "Codec " - + className - + " is NOT succsefully registered", exception); + + className + + " is NOT succsefully registered", exception); + } } } @@ -423,24 +431,30 @@ public void registerCustomCodecs() public void registerCustomPackages() { @SuppressWarnings("unchecked") - Vector currentPackagePrefix - = PackageManager.getProtocolPrefixList(); + Vector packages = PackageManager.getProtocolPrefixList(); + boolean loggerIsDebugEnabled = logger.isDebugEnabled(); - for (String className : CUSTOM_PACKAGES) + for (String customPackage : CUSTOM_PACKAGES) { - // linear search in a loop, but it doesn't have to scale since the - // list is always short - if (!currentPackagePrefix.contains(className)) + /* + * Linear search in a loop but it doesn't have to scale since the + * list is always short. + */ + if (!packages.contains(customPackage)) { - currentPackagePrefix.add(className); - logger.debug("Adding package : " + className); + packages.add(customPackage); + if (loggerIsDebugEnabled) + logger.debug("Adding package : " + customPackage); } } - PackageManager.setProtocolPrefixList(currentPackagePrefix); + PackageManager.setProtocolPrefixList(packages); PackageManager.commitProtocolPrefixList(); - logger.debug("Registering new protocol prefix list : " - + currentPackagePrefix); + if (loggerIsDebugEnabled) + { + logger.debug( + "Registering new protocol prefix list: " + packages); + } } public MediaFormat[] getAvailableEncodings(MediaType type) diff --git a/src/net/java/sip/communicator/impl/neomedia/device/DeviceConfiguration.java b/src/net/java/sip/communicator/impl/neomedia/device/DeviceConfiguration.java index d824d37f8..823dc10fb 100644 --- a/src/net/java/sip/communicator/impl/neomedia/device/DeviceConfiguration.java +++ b/src/net/java/sip/communicator/impl/neomedia/device/DeviceConfiguration.java @@ -6,6 +6,7 @@ */ package net.java.sip.communicator.impl.neomedia.device; +import java.io.*; import java.util.*; import javax.media.*; @@ -134,6 +135,16 @@ public class DeviceConfiguration static final String PROP_AUDIO_LATENCY = "net.java.sip.communicator.impl.neomedia.latency"; + /** + * The list of class names of custom Renderer implementations to be + * registered with JMF. + */ + private static final String[] CUSTOM_RENDERERS + = new String[] + { + "net.java.sip.communicator.impl.neomedia.jmfext.media.renderer.video.JAWTRenderer" + }; + /** * Used when no capture device is selected. */ @@ -191,6 +202,8 @@ public void initialize() { logger.error("Failed to initialize media.", ex); } + + registerCustomRenderers(); } /** @@ -958,4 +971,91 @@ public boolean isDenoiseEnabled() return false; } } + + /** + * Registers the custom Renderer implementations defined by class + * name in {@link #CUSTOM_RENDERERS} with JMF. + */ + private void registerCustomRenderers() + { + @SuppressWarnings("unchecked") + Vector renderers + = PlugInManager.getPlugInList(null, null, PlugInManager.RENDERER); + boolean commit = false; + + for (String customRenderer : CUSTOM_RENDERERS) + { + if ((renderers == null) || !renderers.contains(customRenderer)) + { + try + { + Renderer customRendererInstance + = (Renderer) + Class.forName(customRenderer).newInstance(); + + PlugInManager.addPlugIn( + customRenderer, + customRendererInstance.getSupportedInputFormats(), + null, + PlugInManager.RENDERER); + commit = true; + } + catch (Throwable t) + { + logger.error( + "Failed to register custom Renderer " + + customRenderer + + " with JMF."); + } + } + } + + /* + * Just in case, bubble our JMF contributions at the top so that they + * are considered preferred. + */ + int pluginType = PlugInManager.RENDERER; + @SuppressWarnings("unchecked") + Vector plugins + = PlugInManager.getPlugInList(null, null, pluginType); + + if (plugins != null) + { + int pluginCount = plugins.size(); + int pluginBeginIndex = 0; + String preferred = "net.java.sip.communicator.impl.neomedia."; + + for (int pluginIndex = pluginCount - 1; + pluginIndex >= pluginBeginIndex;) + { + String plugin = plugins.get(pluginIndex); + + if (plugin.startsWith(preferred)) + { + plugins.remove(pluginIndex); + plugins.add(0, plugin); + pluginBeginIndex++; + commit = true; + } + else + pluginIndex--; + } + PlugInManager.setPlugInList(plugins, pluginType); + if (logger.isTraceEnabled()) + logger.trace("Reordered plug-in list:" + plugins); + } + + if (commit) + { + try + { + PlugInManager.commit(); + } + catch (IOException ioex) + { + logger.warn( + "Failed to commit changes to the JMF plug-in list."); + } + } + } } diff --git a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/video/JAWTRenderer.java b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/video/JAWTRenderer.java new file mode 100644 index 000000000..d68559c8a --- /dev/null +++ b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/renderer/video/JAWTRenderer.java @@ -0,0 +1,438 @@ +/* + * SIP Communicator, the OpenSource Java VoIP and Instant Messaging client. + * + * Distributable under LGPL license. + * See terms of license at gnu.org. + */ +package net.java.sip.communicator.impl.neomedia.jmfext.media.renderer.video; + +import java.awt.*; + +import javax.media.*; +import javax.media.format.*; +import javax.media.renderer.*; + +import net.java.sip.communicator.impl.neomedia.control.*; + +/** + * Implements a VideoRenderer which uses JAWT to perform native + * painting in an AWT Canvas. + * + * @author Lubomir Marinov + */ +public class JAWTRenderer + extends ControlsAdapter + implements VideoRenderer +{ + + /** + * The human-readable PlugIn name of the JAWTRenderer + * instances. + */ + private static final String PLUGIN_NAME = "JAWT Renderer"; + + private static final Format[] SUPPORTED_INPUT_FORMATS + = new Format[] + { + new RGBFormat( + null, + Format.NOT_SPECIFIED, + Format.intArray, + Format.NOT_SPECIFIED, + 32, + 0x00FF0000, 0x0000FF00, 0x000000FF) + }; + + static + { + System.loadLibrary("jawtrenderer"); + } + + /** + * The AWT Component into which this VideoRenderer draws. + */ + private Component component; + + /** + * The handle of the native counterpart of this JAWTRenderer. + */ + long handle = 0; + + /** + * The VideoFormat of the input processed by this + * Renderer. + */ + private VideoFormat inputFormat; + + /** + * The last known height of the input processed by this + * JAWTRenderer. + */ + private int inputHeight = 0; + + /** + * The last known width of the input processed by this + * JAWTRenderer. + */ + private int inputWidth = 0; + + /** + * Initializes a new JAWTRenderer instance. + */ + public JAWTRenderer() + { + } + + /** + * Closes this PlugIn and releases the resources it has retained + * during its execution. No more data will be accepted by this + * PlugIn afterwards. A closed PlugIn can be reinstated by + * calling open again. + */ + public synchronized void close() + { + if (handle != 0) + { + close(handle, getComponent()); + handle = 0; + } + } + + /** + * Closes the native counterpart of a JAWTRenderer specified by its + * handle as returned by {@link #open(Component)} and rendering into a + * specific AWT Component. Releases the resources which the + * specified native counterpart has retained during its execution and its + * handle is considered to be invalid afterwards. + * + * @param handle the handle to the native counterpart of a + * JAWTRenderer as returned by {@link #open(Component)} which is to + * be closed + * @param component the AWT Component into which the + * JAWTRenderer and its native counterpart are drawing. The + * platform-specific info of component is not guranteed to be + * valid. + */ + private static native void close(long handle, Component component); + + /** + * Gets the region in the component of this VideoRenderer where the + * video is rendered. + * + * @return the region in the component of this VideoRenderer where + * the video is rendered; null if the entire component is used + */ + public Rectangle getBounds() + { + // TODO Auto-generated method stub + return null; + } + + /** + * Gets the AWT Component into which this VideoRenderer + * draws. + * + * @return the AWT Component into which this VideoRenderer + * draws + */ + public synchronized Component getComponent() + { + if (component == null) + { + component = new Canvas() + { + @Override + public void paint(Graphics g) + { + synchronized (JAWTRenderer.this) + { + if (handle != 0) + JAWTRenderer.this.paint(handle, this, g); + } + } + }; + } + return component; + } + + /** + * Gets the human-readable name of this PlugIn. + * + * @return the human-readable name of this PlugIn + */ + public String getName() + { + return PLUGIN_NAME; + } + + /** + * Gets the list of input Formats supported by this + * Renderer. + * + * @return an array of Format elements which represent the input + * Formats supported by this Renderer + */ + public Format[] getSupportedInputFormats() + { + return SUPPORTED_INPUT_FORMATS.clone(); + } + + /** + * Opens this PlugIn and acquires the resources that it needs to + * operate. The input format of this Renderer has to be set before + * open is called. Buffers should not be passed into this + * PlugIn without first calling open. + */ + public synchronized void open() + throws ResourceUnavailableException + { + if (handle == 0) + { + handle = open(getComponent()); + if (handle == 0) + { + throw + new ResourceUnavailableException( + "Failed to open the native counterpart of" + + " JAWTRenderer"); + } + } + } + + /** + * Opens a handle to a native counterpart of a JAWTRenderer which + * is to draw into a specific AWT Component. + * + * @param component the AWT Component into which a + * JAWTRenderer and the native counterpart to be opened are to + * draw. The platform-specific info of component is not guaranteed + * to be valid. + * @return a handle to a native counterpart of a JAWTRenderer which + * is to draw into the specified AWT Component + */ + private static native long open(Component component) + throws ResourceUnavailableException; + + /** + * Paints a specific Component which is the AWT Component + * of a JAWTRenderer specified by the handle to its native + * counterpart. + * + * @param handle the handle to the native counterpart of a + * JAWTRenderer which is to draw into the specified AWT + * Component + * @param component the AWT Component into which the + * JAWTRenderer and its native counterpart specified by + * handle are to draw. The platform-specific info of + * component is guaranteed to be valid only during the execution of + * paint. + * @param g the Graphics context into which the drawing is to be + * performed + */ + private static native void paint( + long handle, Component component, Graphics g); + + /** + * Processes the data provided in a specific Buffer and renders it + * to the output device represented by this Renderer. + * + * @param buffer a Buffer containing the data to be processed and + * rendered + * @return BUFFER_PROCESSED_OK if the processing is successful; + * otherwise, the other possible return codes defined in the PlugIn + * interface + */ + public synchronized int process(Buffer buffer) + { + if (buffer.isDiscard()) + return BUFFER_PROCESSED_OK; + + int bufferLength = buffer.getLength(); + + if (bufferLength == 0) + return BUFFER_PROCESSED_OK; + + Format bufferFormat = buffer.getFormat(); + + if ((bufferFormat != null) + && (bufferFormat != this.inputFormat) + && !bufferFormat.equals(this.inputFormat)) + { + if (setInputFormat(bufferFormat) == null) + return BUFFER_PROCESSED_FAILED; + } + + if (handle == 0) + return BUFFER_PROCESSED_FAILED; + else + { + Dimension size = null; + + if (bufferFormat != null) + size = ((VideoFormat) bufferFormat).getSize(); + if (size == null) + { + size = this.inputFormat.getSize(); + if (size == null) + return BUFFER_PROCESSED_FAILED; + } + + Component component = getComponent(); + boolean processed + = process( + handle, + component, + (int[]) buffer.getData(), buffer.getOffset(), bufferLength, + size.width, size.height); + + if (processed) + { + component.repaint(); + return BUFFER_PROCESSED_OK; + } + else + return BUFFER_PROCESSED_FAILED; + } + } + + /** + * Processes the data provided in a specific int array with a + * specific offset and length and renders it to the output device + * represented by a JAWTRenderer specified by the handle to it + * native counterpart. + * + * @param handle the handle to the native counterpart of a + * JAWTRenderer to process the specified data and render it + * @param component the AWT component into which the specified + * JAWTRenderer and its native counterpart draw + * @param data an int array which contains the data to be processed + * and rendered + * @param offset the index in data at which the data to be + * processed and rendered starts + * @param length the number of elements in data starting at + * offset which represent the data to be processed and rendered + * @param width the width of the video frame in data + * @param height the height of the video frame in data + */ + private static native boolean process( + long handle, + Component component, + int[] data, int offset, int length, + int width, int height); + + /** + * Resets the state of this PlugIn. + */ + public void reset() + { + // TODO Auto-generated method stub + } + + /** + * Sets the region in the component of this VideoRenderer where the + * video is to be rendered. + * + * @param bounds the region in the component of this VideoRenderer + * where the video is to be rendered; null if the entire component + * is to be used + */ + public void setBounds(Rectangle bounds) + { + // TODO Auto-generated method stub + } + + /** + * Sets the AWT Component into which this VideoRenderer is + * to draw. JAWTRenderer cannot draw into any other AWT + * Component but its own so it always returns false. + * + * @param component the AWT Component into which this + * VideoRenderer is to draw + * @return true if this VideoRenderer accepted the + * specified component as the AWT Component into which it + * is to draw; false, otherwise + */ + public boolean setComponent(Component component) + { + // We cannot draw into any other AWT Component but our own. + return false; + } + + /** + * Sets the Format of the input to be processed by this + * Renderer. + * + * @param format the Format to be set as the Format of the + * input to be processed by this Renderer + * @return the Format of the input to be processed by this + * Renderer if the specified format is supported or + * null if the specified format is not supported by this + * Renderer. Typically, it is the supported input Format + * which most closely matches the specified Format. + */ + public Format setInputFormat(Format format) + { + Format matchingFormat = null; + + for (Format supportedInputFormat : getSupportedInputFormats()) + { + if (supportedInputFormat.matches(format)) + { + matchingFormat = supportedInputFormat.intersects(format); + break; + } + } + if (matchingFormat == null) + return null; + + inputFormat = (VideoFormat) format; + + /* + * Know the width and height of the input because we'll be depicting it + * and we may want, for example, to report it as the preferred size of + * our AWT Component. + */ + Dimension inputSize = inputFormat.getSize(); + + if (inputSize != null) + { + inputWidth = inputSize.width; + inputHeight = inputSize.height; + } + + /* + * Reflect the width and height of the input onto the preferredSize of + * our AWT Component (if necessary). + */ + if ((inputWidth > 0) && (inputHeight > 0)) + { + Component component = getComponent(); + Dimension preferredSize = component.getPreferredSize(); + + if ((preferredSize == null) + || (preferredSize.width < 1) + || (preferredSize.height < 1)) + { + component.setPreferredSize( + new Dimension(inputWidth, inputHeight)); + } + } + + return inputFormat; + } + + /** + * Starts the rendering process. Begins rendering any data available in the + * internal buffers of this Renderer. + */ + public void start() + { + } + + /** + * Stops the rendering process. + */ + public void stop() + { + } +}