diff --git a/lib/native/freebsd-64/libscreencapture.so b/lib/native/freebsd-64/libscreencapture.so index d259e4d95..4ee183a46 100755 Binary files a/lib/native/freebsd-64/libscreencapture.so and b/lib/native/freebsd-64/libscreencapture.so differ diff --git a/lib/native/freebsd/libscreencapture.so b/lib/native/freebsd/libscreencapture.so index d2fc68024..98dd3f251 100755 Binary files a/lib/native/freebsd/libscreencapture.so and b/lib/native/freebsd/libscreencapture.so differ diff --git a/lib/native/linux-64/libscreencapture.so b/lib/native/linux-64/libscreencapture.so index b08a251eb..98bc46aa3 100755 Binary files a/lib/native/linux-64/libscreencapture.so and b/lib/native/linux-64/libscreencapture.so differ diff --git a/lib/native/linux/libscreencapture.so b/lib/native/linux/libscreencapture.so index 257956708..c0caaffaa 100755 Binary files a/lib/native/linux/libscreencapture.so and b/lib/native/linux/libscreencapture.so differ diff --git a/lib/native/mac/libscreencapture.jnilib b/lib/native/mac/libscreencapture.jnilib index b18b0b803..eef6a1d3e 100755 Binary files a/lib/native/mac/libscreencapture.jnilib and b/lib/native/mac/libscreencapture.jnilib differ diff --git a/lib/native/windows-64/screencapture.dll b/lib/native/windows-64/screencapture.dll index 768a72464..12d895772 100755 Binary files a/lib/native/windows-64/screencapture.dll and b/lib/native/windows-64/screencapture.dll differ diff --git a/lib/native/windows/screencapture.dll b/lib/native/windows/screencapture.dll index 302fb7e7f..b0d4cf907 100644 Binary files a/lib/native/windows/screencapture.dll and b/lib/native/windows/screencapture.dll differ diff --git a/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.c b/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.c index 23d828e9f..723f2786d 100644 --- a/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.c +++ b/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.c @@ -462,7 +462,7 @@ static int x11_grab_screen(const char* x11display, jbyte* data, int32_t x, int32 /** * \brief JNI native method to grab desktop screen and retrieve ARGB pixels. * \param env JVM environment - * \param obj NativeScreenCapture Java class + * \param clazz NativeScreenCapture Java class * \param x x position to start capture * \param y y position to start capture * \param width capture width @@ -470,13 +470,13 @@ static int x11_grab_screen(const char* x11display, jbyte* data, int32_t x, int32 * \param output output buffer, screen bytes will be stored in * \return true if success, false otherwise */ -JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture_grabScreen - (JNIEnv* env, jclass obj, jint x, jint y, jint width, jint height, jbyteArray output) +JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture_grabScreen__IIII_3B + (JNIEnv* env, jclass clazz, jint x, jint y, jint width, jint height, jbyteArray output) { jint size = width * height * 4; jbyte* data = NULL; - obj = obj; /* not used */ + clazz = clazz; /* not used */ if(!output || (*env)->GetArrayLength(env, output) < size) { @@ -506,3 +506,43 @@ JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_imgstrea return JNI_TRUE; } +/** + * \brief JNI native method to grab desktop screen and retrieve ARGB pixels. + * \param env JVM environment + * \param clazz NativeScreenCapture Java class + * \param x x position to start capture + * \param y y position to start capture + * \param width capture width + * \param height capture height + * \param output native output buffer + * \return true if success, false otherwise + */ +JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture_grabScreen__IIIIJI + (JNIEnv* env, jclass clazz, jint x, jint y, jint width, jint height, jlong output, jint outputLength) +{ + jint size = width * height * 4; + jbyte* data = (jbyte*)output; + + /* not used */ + clazz = clazz; + env = env; + + if(!data || outputLength < size) + { + return JNI_FALSE; + } + +#if defined (_WIN32) || defined(_WIN64) + if(windows_grab_screen(data, x, y, width, height) == -1) +#elif defined(__APPLE__) + if(quartz_grab_screen(data, x, y, width, height) == -1) +#else /* Unix */ + if(x11_grab_screen(NULL, data, x, y, width, height) == -1) +#endif + { + return JNI_FALSE; + } + + return JNI_TRUE; +} + diff --git a/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.h b/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.h index ed9d8b1ea..d1d6581de 100644 --- a/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.h +++ b/src/native/screencapture/net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture.h @@ -12,9 +12,17 @@ extern "C" { * Method: grabScreen * Signature: (IIII[B)Z */ -JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture_grabScreen +JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture_grabScreen__IIII_3B (JNIEnv *, jclass, jint, jint, jint, jint, jbyteArray); +/* + * Class: net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture + * Method: grabScreen + * Signature: (IIIIJI)Z + */ +JNIEXPORT jboolean JNICALL Java_net_java_sip_communicator_impl_neomedia_imgstreaming_NativeScreenCapture_grabScreen__IIIIJI + (JNIEnv *, jclass, jint, jint, jint, jint, jlong, jint); + #ifdef __cplusplus } #endif diff --git a/src/net/java/sip/communicator/impl/neomedia/codec/video/ByteBuffer.java b/src/net/java/sip/communicator/impl/neomedia/codec/video/ByteBuffer.java new file mode 100644 index 000000000..d5aaa7352 --- /dev/null +++ b/src/net/java/sip/communicator/impl/neomedia/codec/video/ByteBuffer.java @@ -0,0 +1,116 @@ +/* + * SIP Communicator, the OpenSource Java VoIP and Instant Messaging client. + * + * Distributable under LGPL license. + * See terms of license at gnu.org. + */ +package net.java.sip.communicator.impl.neomedia.codec.video; + +/** + * Represents a buffer of native memory with a specific size/capacity which + * either contains a specific number of bytes of valid data or is free for + * consumption. + * + * @author Lubomir Marinov + */ +public class ByteBuffer +{ + + /** + * The maximum number of bytes which can be written into the native + * memory represented by this instance. + */ + public final int capacity; + + /** + * The indicator which determines whether this instance is free to be + * written bytes into. + */ + private boolean free; + + /** + * The number of bytes of valid data that the native memory represented + * by this instance contains. + */ + private int length; + + /** + * The pointer to the native memory represented by this instance. + */ + public final long ptr; + + /** + * Initializes a new ByteBuffer instance with a specific + * capacity. + * + * @param capacity the maximum number of bytes which can be written into + * the native memory represented by the new instance + */ + public ByteBuffer(int capacity) + { + this.capacity = capacity; + this.ptr = FFmpeg.av_malloc(this.capacity); + + this.free = true; + this.length = 0; + + if (this.ptr == 0) + { + throw + new OutOfMemoryError( + getClass().getSimpleName() + + " with capacity " + + this.capacity); + } + } + + /** + * Gets the number of bytes of valid data that the native memory + * represented by this instance contains. + * + * @return the number of bytes of valid data that the native memory + * represented by this instance contains + */ + public int getLength() + { + return length; + } + + /** + * Determines whether this instance is free to be written bytes into. + * + * @return true if this instance is free to be written bytes + * into or false is the native memory represented by this + * instance is already is use + */ + public boolean isFree() + { + return free; + } + + /** + * Sets the indicator which determines whether this instance is free to + * be written bytes into. + * + * @param free true if this instance is to be made available + * for writing bytes into; otherwise, false + */ + public void setFree(boolean free) + { + this.free = free; + if (this.free) + setLength(0); + } + + /** + * Sets the number of bytes of valid data that the native memory + * represented by this instance contains. + * + * @param length the number of bytes of valid data that the native + * memory represented by this instance contains + */ + public void setLength(int length) + { + this.length = length; + } +} \ No newline at end of file diff --git a/src/net/java/sip/communicator/impl/neomedia/codec/video/SwScaler.java b/src/net/java/sip/communicator/impl/neomedia/codec/video/SwScaler.java index 2f784964b..2d481908b 100644 --- a/src/net/java/sip/communicator/impl/neomedia/codec/video/SwScaler.java +++ b/src/net/java/sip/communicator/impl/neomedia/codec/video/SwScaler.java @@ -58,6 +58,9 @@ public class SwScaler new RGBFormat(null, -1, Format.shortArray, -1.0f, 24, -1, -1, -1), }; + /** + * Libswscale context pointer. + */ private long swsContext = 0; /** diff --git a/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteract.java b/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteract.java index e45f53d40..b271c9758 100644 --- a/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteract.java +++ b/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteract.java @@ -11,7 +11,7 @@ /** * Interface to interact with the desktop such as taking * screen capture or to generate mouse/keyboard event. - * + * * @author Sebastien Vincent */ public interface DesktopInteract @@ -23,7 +23,8 @@ public interface DesktopInteract * and not BufferedImage. It is done in order to limit * slow operation such as converting ARGB images (uint32_t) to bytes * especially for big big screen. For example a 1920x1200 desktop consumes - * 9 MB of memory for grabbing and another 9 MB array for convertion operation. + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. * * @param output output buffer to store bytes in. * Be sure that output length is sufficient @@ -31,6 +32,23 @@ public interface DesktopInteract */ public boolean captureScreen(byte output[]); + /** + * Capture the full desktop screen using native grabber. + * + * Contrary to other captureScreen method, it only returns raw bytes + * and not BufferedImage. It is done in order to limit + * slow operation such as converting ARGB images (uint32_t) to bytes + * especially for big big screen. For example a 1920x1200 desktop consumes + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. + * + * @param buffer native output buffer to store bytes in. + * Be sure that output length is sufficient + * @param bufferLength length of native buffer + * @return true if success, false if JNI error or output length too short + */ + public boolean captureScreen(long buffer, int bufferLength); + /** * Capture a part of the desktop screen using native grabber. * @@ -38,7 +56,8 @@ public interface DesktopInteract * and not BufferedImage. It is done in order to limit * slow operation such as converting ARGB images (uint32_t) to bytes * especially for big big screen. For example a 1920x1200 desktop consumes - * 9 MB of memory for grabbing and another 9 MB array for convertion operation. + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. * * @param x x position to start capture * @param y y position to start capture @@ -48,7 +67,30 @@ public interface DesktopInteract * Be sure that output length is sufficient * @return true if success, false if JNI error or output length too short */ - public boolean captureScreen(int x, int y, int width, int height, byte output[]); + public boolean captureScreen(int x, int y, int width, int height, + byte output[]); + + /** + * Capture a part of the desktop screen using native grabber. + * + * Contrary to other captureScreen method, it only returns raw bytes + * and not BufferedImage. It is done in order to limit + * slow operation such as converting ARGB images (uint32_t) to bytes + * especially for big big screen. For example a 1920x1200 desktop consumes + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. + * + * @param x x position to start capture + * @param y y position to start capture + * @param width capture width + * @param height capture height + * @param buffer native output buffer to store bytes in. + * Be sure that output length is sufficient + * @param bufferLength length of native buffer + * @return true if success, false if JNI error or output length too short + */ + public boolean captureScreen(int x, int y, int width, int height, + long buffer, int bufferLength); /** * Capture the full desktop screen. diff --git a/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteractImpl.java b/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteractImpl.java index 704c1b870..d64ec5f4a 100644 --- a/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteractImpl.java +++ b/src/net/java/sip/communicator/impl/neomedia/imgstreaming/DesktopInteractImpl.java @@ -12,19 +12,20 @@ import net.java.sip.communicator.util.*; /** - * This singleton class provide screen capture and key/mouse + * This singleton class provide screen capture and key/mouse * events generation by wrapping partial or all java.awt.Robot * methods to interact with desktop. * * @see java.awt.Robot * @author Sebastien Vincent */ -public class DesktopInteractImpl implements DesktopInteract +public class DesktopInteractImpl implements DesktopInteract { /** * The Logger. */ - private static final Logger logger = Logger.getLogger(DesktopInteractImpl.class); + private static final Logger logger = + Logger.getLogger(DesktopInteractImpl.class); /** * Screen capture robot. @@ -33,11 +34,12 @@ public class DesktopInteractImpl implements DesktopInteract /** * Constructor. - * - * @throws AWTException if platform configuration does not allow low-level input control + * + * @throws AWTException if platform configuration does not allow low-level + * input control * @throws SecurityException if Robot creation is not permitted */ - public DesktopInteractImpl() throws AWTException, SecurityException + public DesktopInteractImpl() throws AWTException, SecurityException { robot = new Robot(); } @@ -49,7 +51,8 @@ public DesktopInteractImpl() throws AWTException, SecurityException * and not BufferedImage. It is done in order to limit * slow operation such as converting ARGB images (uint32_t) to bytes * especially for big big screen. For example a 1920x1200 desktop consumes - * 9 MB of memory for grabbing and another 9 MB array for convertion operation. + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. * * @param output output buffer to store bytes in. * Be sure that output length is sufficient @@ -59,7 +62,29 @@ public boolean captureScreen(byte output[]) { Dimension dim = Toolkit.getDefaultToolkit().getScreenSize(); - return captureScreen(0, 0, (int)dim.getWidth(), (int)dim.getHeight(), output); + return captureScreen(0, 0, dim.width, dim.height, + output); + } + + /** + * Capture the full desktop screen using native grabber. + * + * Contrary to other captureScreen method, it only returns raw bytes + * and not BufferedImage. It is done in order to limit + * slow operation such as converting ARGB images (uint32_t) to bytes + * especially for big big screen. For example a 1920x1200 desktop consumes + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. + * + * @param buffer native output buffer to store bytes in. + * Be sure that output length is sufficient + * @param bufferLength length of native buffer + * @return true if success, false if JNI error or output length too short + */ + public boolean captureScreen(long buffer, int bufferLength) + { + Dimension dim = Toolkit.getDefaultToolkit().getScreenSize(); + return captureScreen(0, 0, dim.width, dim.height, buffer, bufferLength); } /** @@ -67,9 +92,10 @@ public boolean captureScreen(byte output[]) * * Contrary to other captureScreen method, it only returns raw bytes * and not BufferedImage. It is done in order to limit - * slow operation such as converting ARGB images (uint32_t) to bytes + * slow operation such as converting ARGB images (uint32_t) to bytes * especially for big big screen. For example a 1920x1200 desktop consumes - * 9 MB of memory for grabbing and another 9 MB array for convertion operation. + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. * * @param x x position to start capture * @param y y position to start capture @@ -79,7 +105,8 @@ public boolean captureScreen(byte output[]) * Be sure that output length is sufficient * @return true if success, false if JNI error or output length too short */ - public boolean captureScreen(int x, int y, int width, int height, byte output[]) + public boolean captureScreen(int x, int y, int width, int height, + byte output[]) { if(OSUtils.IS_LINUX || OSUtils.IS_FREEBSD || OSUtils.IS_WINDOWS || OSUtils.IS_MAC) @@ -91,6 +118,38 @@ public boolean captureScreen(int x, int y, int width, int height, byte output[]) return false; } + /** + * Capture a part of the desktop screen using native grabber. + * + * Contrary to other captureScreen method, it only returns raw bytes + * and not BufferedImage. It is done in order to limit + * slow operation such as converting ARGB images (uint32_t) to bytes + * especially for big big screen. For example a 1920x1200 desktop consumes + * 9 MB of memory for grabbing and another 9 MB array for conversion + * operation. + * + * @param x x position to start capture + * @param y y position to start capture + * @param width capture width + * @param height capture height + * @param buffer native output buffer to store bytes in. + * Be sure that output length is sufficient + * @param bufferLength length of native buffer + * @return true if success, false if JNI error or output length too short + */ + public boolean captureScreen(int x, int y, int width, int height, + long buffer, int bufferLength) + { + if(OSUtils.IS_LINUX || OSUtils.IS_FREEBSD || OSUtils.IS_WINDOWS + || OSUtils.IS_MAC) + { + return NativeScreenCapture.grabScreen( + x, y, width, height, buffer, bufferLength); + } + + return false; + } + /** * Capture the full desktop screen using java.awt.Robot. * @@ -100,7 +159,7 @@ public BufferedImage captureScreen() { Dimension dim = Toolkit.getDefaultToolkit().getScreenSize(); - return captureScreen(0, 0, (int)dim.getWidth(), (int)dim.getHeight()); + return captureScreen(0, 0, dim.width, dim.height); } /** @@ -117,7 +176,7 @@ public BufferedImage captureScreen(int x, int y, int width, int height) { BufferedImage img = null; Rectangle rect = null; - + if(robot == null) { /* Robot has not been created so abort */ @@ -128,7 +187,7 @@ public BufferedImage captureScreen(int x, int y, int width, int height) rect = new Rectangle(x, y, width, height); img = robot.createScreenCapture(rect); logger.info("End capture: " + System.nanoTime()); - + return img; } diff --git a/src/net/java/sip/communicator/impl/neomedia/imgstreaming/NativeScreenCapture.java b/src/net/java/sip/communicator/impl/neomedia/imgstreaming/NativeScreenCapture.java index 82568e26b..895600cf1 100644 --- a/src/net/java/sip/communicator/impl/neomedia/imgstreaming/NativeScreenCapture.java +++ b/src/net/java/sip/communicator/impl/neomedia/imgstreaming/NativeScreenCapture.java @@ -6,12 +6,10 @@ */ package net.java.sip.communicator.impl.neomedia.imgstreaming; -import java.awt.image.*; - /** * This class uses native code to capture desktop screen. - * - * It should work for Windows, Mac OS X and X11-based Unix such as Linux + * + * It should work for Windows, Mac OS X and X11-based Unix such as Linux * and FreeBSD. * * @author Sebastien Vincent @@ -25,7 +23,7 @@ public class NativeScreenCapture /** * Grab desktop screen and get raw bytes. - * + * * @param x x position to start capture * @param y y position to start capture * @param width capture width @@ -33,5 +31,20 @@ public class NativeScreenCapture * @param output output buffer to store screen bytes * @return true if grab success, false otherwise */ - public static native boolean grabScreen(int x, int y, int width, int height, byte output[]); + public static native boolean grabScreen(int x, int y, int width, int height, + byte output[]); + + /** + * Grab desktop screen and get raw bytes. + * + * @param x x position to start capture + * @param y y position to start capture + * @param width capture width + * @param height capture height + * @param output native output buffer to store screen bytes + * @param outputLength native output length + * @return true if grab success, false otherwise + */ + public static native boolean grabScreen(int x, int y, int width, int height, + long output, int outputLength); } diff --git a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/DataSource.java b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/DataSource.java index cbdece5d9..2de47a571 100644 --- a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/DataSource.java +++ b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/DataSource.java @@ -12,10 +12,11 @@ import javax.media.control.*; import javax.media.format.*; +import net.java.sip.communicator.impl.neomedia.codec.video.*; import net.java.sip.communicator.impl.neomedia.jmfext.media.protocol.*; /** - * DataSource for our image streaming (which is used for + * DataSource for our image streaming (which is used for * Desktop streaming). * * @author Sebastien Vincent @@ -32,7 +33,11 @@ public class DataSource private static final Format[] formats = new Format[] { - new RGBFormat( + new AVFrameFormat( + Toolkit.getDefaultToolkit().getScreenSize(), + Format.NOT_SPECIFIED, + FFmpeg.PIX_FMT_ARGB), + new RGBFormat( Toolkit.getDefaultToolkit().getScreenSize(), // size Format.NOT_SPECIFIED, // maxDataLength Format.byteArray, // dataType diff --git a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/ImageStream.java b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/ImageStream.java index 29281d118..88c215f09 100644 --- a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/ImageStream.java +++ b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/imgstreaming/ImageStream.java @@ -14,14 +14,13 @@ import javax.media.control.*; import javax.media.format.*; +import net.java.sip.communicator.impl.neomedia.codec.video.*; import net.java.sip.communicator.impl.neomedia.imgstreaming.*; import net.java.sip.communicator.impl.neomedia.jmfext.media.protocol.*; import net.java.sip.communicator.util.*; /** * The stream used by JMF for our image streaming. - * - * This class launches a thread to handle desktop capture interactions. * * @author Sebastien Vincent * @author Lubomir Marinov @@ -45,6 +44,11 @@ public class ImageStream */ private DesktopInteract desktopInteract = null; + /** + * Native buffer pointer. + */ + ByteBuffer data = null; + /** * Initializes a new ImageStream instance which is to have a * specific FormatControl @@ -67,41 +71,80 @@ public void read(Buffer buffer) throws IOException { //System.out.println(System.currentTimeMillis()); - byte data[] = (byte[])buffer.getData(); - int dataLength = (data != null) ? data.length : 0; long begin = System.currentTimeMillis(); /* maximum time allowed for a capture to respect frame rate */ - long maxTime = 1000 / 10; + long maxTime = 1000 / 10; int wait = 0; - if((data != null) || (dataLength != 0)) + if(buffer.getFormat() instanceof AVFrameFormat) { - byte buf[] = readScreen(data); + /* native transfert: we keep data in native memory rather + * than Java Heap until we reach SwScaler + */ + Object dataAv = buffer.getData(); + AVFrame bufferFrame = null; + long bufferFramePtr = 0; - if(buf != data) + if (dataAv instanceof AVFrame) { - /* readScreen returns us a different buffer than JMF ones, - * it means that JMF's initial buffer was too short. - */ - //System.out.println("use our own buffer"); - buffer.setData(buf); + bufferFrame = (AVFrame)dataAv; + bufferFramePtr = bufferFrame.getPtr(); + } + else + { + bufferFrame = new FinalizableAVFrame(); + bufferFramePtr = bufferFrame.getPtr(); } - buffer.setOffset(0); - buffer.setLength(buf.length); - buffer.setFormat(getFormat()); - buffer.setHeader(null); - buffer.setTimeStamp(System.nanoTime()); - buffer.setSequenceNumber(seqNo); - buffer.setFlags(Buffer.FLAG_SYSTEM_TIME | Buffer.FLAG_LIVE_DATA); - seqNo++; + AVFrameFormat bufferFrameFormat = (AVFrameFormat)buffer.getFormat(); + Dimension bufferFrameSize = bufferFrameFormat.getSize(); + + if(readScreenNative(bufferFrameSize)) + { + FFmpeg.avpicture_fill( + bufferFramePtr, + data.ptr, + bufferFrameFormat.getPixFmt(), + bufferFrameSize.width, bufferFrameSize.height); + } + + buffer.setData(bufferFrame); } - + else + { + byte dataByte[] = (byte[])buffer.getData(); + int dataLength = (dataByte != null) ? dataByte.length : 0; + + if((dataByte != null) || (dataLength != 0)) + { + byte buf[] = readScreen(dataByte); + + if(buf != dataByte) + { + /* readScreen returns us a different buffer than JMF ones, + * it means that JMF's initial buffer was too short. + */ + //System.out.println("use our own buffer"); + buffer.setData(buf); + } + + buffer.setOffset(0); + buffer.setLength(buf.length); + } + } + + buffer.setFormat(buffer.getFormat()); + buffer.setHeader(null); + buffer.setTimeStamp(System.nanoTime()); + buffer.setSequenceNumber(seqNo); + buffer.setFlags(Buffer.FLAG_SYSTEM_TIME | Buffer.FLAG_LIVE_DATA); + seqNo++; + wait = (int)(maxTime - (System.currentTimeMillis() - begin)); try { - /* sleep to respect as much as possible the + /* sleep to respect as much as possible the * frame rate */ if(wait > 0) @@ -110,7 +153,7 @@ public void read(Buffer buffer) } else { - /* yield a little bit to not use all the + /* yield a little bit to not use all the * CPU */ Thread.yield(); @@ -151,14 +194,54 @@ public void start() public void stop() { logger.info("Stop stream"); + + /* native pointer is freed in FinalizableAVFrame */ + data = null; + } + + /** + * Read screen and store result in native buffer. + * + * @param dim dimension of the video + * @return true if success, false otherwise + */ + private boolean readScreenNative(Dimension dim) + { + int size = dim.width * dim.height * 4; + + /* pad the buffer */ + size += FFmpeg.FF_INPUT_BUFFER_PADDING_SIZE; + + /* allocate native array */ + if(data == null) + { + data = new ByteBuffer(size); + data.setLength(size); + } + + /* reallocate native array if capacity is not enough */ + if(data.getLength() < size) + { + data.setFree(true); + FFmpeg.av_free(data.ptr); + data = new ByteBuffer(size); + data.setLength(size); + } + + /* get desktop screen via native grabber */ + if(desktopInteract.captureScreen(data.ptr, data.getLength())) + { + return true; + } + return false; } /** * Read screen. - * + * * @param output output buffer for screen bytes - * @return raw bytes, it could be equal to output or not. Take care in the caller - * to check if output is the returned value. + * @return raw bytes, it could be equal to output or not. Take care in the + * caller to check if output is the returned value. */ public byte[] readScreen(byte output[]) { @@ -194,8 +277,8 @@ public byte[] readScreen(byte output[]) * Note that it is very memory consuming since memory are allocated * to capture screen (via Robot) and then for converting to raw bytes * - * Normally not of our supported platform (Windows (x86, x64), - * Linux (x86, x86-64), Mac OS X (i386, x86-64, ppc) and + * Normally not of our supported platform (Windows (x86, x64), + * Linux (x86, x86-64), Mac OS X (i386, x86-64, ppc) and * FreeBSD (x86, x86-64) should go here. */ screen = desktopInteract.captureScreen(); @@ -203,7 +286,7 @@ public byte[] readScreen(byte output[]) if(screen != null) { /* convert to ARGB BufferedImage */ - scaledScreen + scaledScreen = ImageStreamingUtils .getScaledImage( screen, @@ -219,4 +302,61 @@ public byte[] readScreen(byte output[]) scaledScreen = null; return data; } + + /** + * Represents an AVFrame used by this instance to provide captured + * media data in native format without representing the very frame data in + * the Java heap. Since this instance cannot know when the AVFrame + * instances are really safe for deallocation, FinalizableAVFrame + * relies on the Java finalization mechanism to reclaim the represented + * native memory. + */ + public class FinalizableAVFrame + extends AVFrame + { + + /** + * The indicator which determines whether the native memory represented + * by this instance has already been freed/deallocated. + */ + private boolean freed = false; + + /** + * Initializes a new FinalizableAVFrame instance which is to + * allocate a new native FFmpeg AVFrame and represent it. + */ + public FinalizableAVFrame() + { + super(FFmpeg.avcodec_alloc_frame()); + } + + /** + * Deallocates the native memory represented by this instance. + * + * @see Object#finalize() + */ + @Override + protected void finalize() + throws Throwable + { + try + { + if (!freed) + { + long ptr = getPtr(); + long bufferPtr = FFmpeg.avpicture_get_data0(ptr); + + if(bufferPtr != 0) + FFmpeg.av_free(bufferPtr); + + FFmpeg.av_free(ptr); + freed = true; + } + } + finally + { + super.finalize(); + } + } + } } diff --git a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/quicktime/QuickTimeStream.java b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/quicktime/QuickTimeStream.java index 89da1d641..781f996eb 100644 --- a/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/quicktime/QuickTimeStream.java +++ b/src/net/java/sip/communicator/impl/neomedia/jmfext/media/protocol/quicktime/QuickTimeStream.java @@ -694,7 +694,7 @@ private void runInTransferDataThread() nextData = null; } } - + synchronized (dataSyncRoot) { if (data == null) @@ -879,119 +879,12 @@ public void stop() } } - /** - * Represents a buffer of native memory with a specific size/capacity which - * either contains a specific number of bytes of valid data or is free for - * consumption. - */ - private static class ByteBuffer - { - - /** - * The maximum number of bytes which can be written into the native - * memory represented by this instance. - */ - public final int capacity; - - /** - * The indicator which determines whether this instance is free to be - * written bytes into. - */ - private boolean free; - - /** - * The number of bytes of valid data that the native memory represented - * by this instance contains. - */ - private int length; - - /** - * The pointer to the native memory represented by this instance. - */ - public final long ptr; - - /** - * Initializes a new ByteBuffer instance with a specific - * capacity. - * - * @param capacity the maximum number of bytes which can be written into - * the native memory represented by the new instance - */ - public ByteBuffer(int capacity) - { - this.capacity = capacity; - this.ptr = FFmpeg.av_malloc(this.capacity); - - this.free = true; - this.length = 0; - - if (this.ptr == 0) - { - throw - new OutOfMemoryError( - getClass().getSimpleName() - + " with capacity " - + this.capacity); - } - } - - /** - * Gets the number of bytes of valid data that the native memory - * represented by this instance contains. - * - * @return the number of bytes of valid data that the native memory - * represented by this instance contains - */ - public int getLength() - { - return length; - } - - /** - * Determines whether this instance is free to be written bytes into. - * - * @return true if this instance is free to be written bytes - * into or false is the native memory represented by this - * instance is already is use - */ - public boolean isFree() - { - return free; - } - - /** - * Sets the indicator which determines whether this instance is free to - * be written bytes into. - * - * @param free true if this instance is to be made available - * for writing bytes into; otherwise, false - */ - public void setFree(boolean free) - { - this.free = free; - if (this.free) - setLength(0); - } - - /** - * Sets the number of bytes of valid data that the native memory - * represented by this instance contains. - * - * @param length the number of bytes of valid data that the native - * memory represented by this instance contains - */ - public void setLength(int length) - { - this.length = length; - } - } - /** * Represents an AVFrame used by this instance to provide captured * media data in native format without representing the very frame data in * the Java heap. Since this instance cannot know when the AVFrame * instances are really safe for deallocation, FinalizableAVFrame - * relies on the Java finialization mechanism to reclaim the represented + * relies on the Java finalization mechanism to reclaim the represented * native memory. */ private class FinalizableAVFrame