Responds to INFO requests for key frames.

src/net/java/sip/communicator/impl/neomedia/codec/video/h264/JNIEncoder.java

@@ -16,6 +16,7 @@
 import net.java.sip.communicator.impl.neomedia.codec.*;
 import net.java.sip.communicator.impl.neomedia.format.*;
 import net.java.sip.communicator.service.configuration.*;
+import net.java.sip.communicator.service.neomedia.control.*;
 import net.java.sip.communicator.service.neomedia.event.*;
 import net.java.sip.communicator.util.*;
 import net.sf.fmj.media.*;
@@ -138,9 +139,18 @@ public class JNIEncoder
     private int framesSinceLastIFrame = IFRAME_INTERVAL + 1;
 
     /**
-     * Last keyframe request time.
+     * The <tt>KeyFrameControl</tt> used by this <tt>JNIEncoder</tt> to
+     * control its key frame-related logic.
      */
-    private long lastKeyframeRequestTime = System.currentTimeMillis();
+    private KeyFrameControl keyFrameControl;
+
+    private KeyFrameControl.KeyFrameRequestee keyFrameRequestee;
+
+    /**
+     * The time in milliseconds of the last request for a key frame from the
+     * remote peer to this local peer.
+     */
+    private long lastKeyFrameRequestTime = System.currentTimeMillis();
 
     /**
      * The packetization mode to be used for the H.264 RTP payload output by
@@ -210,6 +220,13 @@ public synchronized void close()
             rawFrameBuffer = 0;
 
             encFrameBuffer = null;
+
+            if (keyFrameRequestee != null)
+            {
+                if (keyFrameControl != null)
+                    keyFrameControl.removeKeyFrameRequestee(keyFrameRequestee);
+                keyFrameRequestee = null;
+            }
         }
     }
 
@@ -231,13 +248,7 @@ public void feedbackReceived(RTCPFeedbackEvent event)
             {
                 case RTCPFeedbackEvent.FMT_PLI:
                 case RTCPFeedbackEvent.FMT_FIR:
-                    if (System.currentTimeMillis()
-                            > (lastKeyframeRequestTime + PLI_INTERVAL))
-                    {
-                        lastKeyframeRequestTime = System.currentTimeMillis();
-                        // Disable PLI.
-                        //forceKeyFrame = true;
-                    }
+                    keyFrameRequest();
                     break;
                 default:
                     break;
@@ -302,6 +313,24 @@ public Format[] getSupportedOutputFormats(Format in)
         return getMatchingOutputFormats(in);
     }
 
+    /**
+     * Notifies this <tt>JNIEncoder</tt> that the remote peer has requested a
+     * key frame from this local peer.
+     *
+     * @return <tt>true</tt> if this <tt>JNIEncoder</tt> has honored the request
+     * for a key frame; otherwise, <tt>false</tt>
+     */
+    private boolean keyFrameRequest()
+    {
+        if (System.currentTimeMillis()
+                > (lastKeyFrameRequestTime + PLI_INTERVAL))
+        {
+            lastKeyFrameRequestTime = System.currentTimeMillis();
+            forceKeyFrame = true;
+        }
+        return true;
+    }
+
     /**
      * Opens this <tt>Codec</tt>.
      */
@@ -439,6 +468,23 @@ public synchronized void open()
 
         encFrameBuffer = new byte[encFrameLen];
 
+        /*
+         * Implement the ability to have the remote peer request key frames from
+         * this local peer.
+         */
+        if (keyFrameRequestee == null)
+        {
+            keyFrameRequestee = new KeyFrameControl.KeyFrameRequestee()
+            {
+                public boolean keyFrameRequest()
+                {
+                    return JNIEncoder.this.keyFrameRequest();
+                }
+            };
+        }
+        if (keyFrameControl != null)
+            keyFrameControl.addKeyFrameRequestee(-1, keyFrameRequestee);
+
         opened = true;
         super.open();
     }
@@ -585,6 +631,29 @@ public Format setInputFormat(Format in)
         return inputFormat;
     }
 
+    /**
+     * Sets the <tt>KeyFrameControl</tt> to be used by this
+     * <tt>JNIEncoder</tt> as a means of control over its key frame-related
+     * logic.
+     *
+     * @param keyFrameControl the <tt>KeyFrameControl</tt> to be used by this
+     * <tt>JNIEncoder</tt> as a means of control over its key frame-related
+     * logic
+     */
+    public void setKeyFrameControl(KeyFrameControl keyFrameControl)
+    {
+        if (this.keyFrameControl != keyFrameControl)
+        {
+            if ((this.keyFrameControl != null) && (keyFrameRequestee != null))
+                this.keyFrameControl.removeKeyFrameRequestee(keyFrameRequestee);
+
+            this.keyFrameControl = keyFrameControl;
+
+            if ((this.keyFrameControl != null) && (keyFrameRequestee != null))
+                this.keyFrameControl.addKeyFrameRequestee(-1, keyFrameRequestee);
+        }
+    }
+
     /**
      * Sets the <tt>Format</tt> in which this <tt>Codec</tt> is to output media
      * data.

src/net/java/sip/communicator/impl/neomedia/device/VideoMediaDeviceSession.java

@@ -1369,6 +1369,8 @@ protected Format setProcessorFormat(
                     logger.error("Error cannot get RTCP input stream", ioe);
                 }
             }
+            if (keyFrameControl != null)
+                encoder.setKeyFrameControl(keyFrameControl);
 
             codecCount++;
         }

src/net/java/sip/communicator/impl/protocol/sip/CallPeerSipImpl.java

@@ -338,24 +338,89 @@ public URL getCallInfoURL()
         return getMediaHandler().getCallInfoURL();
     }
 
-    /**
-     * Processes the status code of a <tt>Response<tt> to a
-     * <tt>picture_fast_update</tt> SIP INFO <tt>Request</tt> sent by the local
-     * peer to this remote peer.
-     *
-     * @param statusCode the status code of the <tt>Response</tt> to be
-     * processed
-     */
-    void processPictureFastUpdateResponseStatusCode(int statusCode)
+    void processPictureFastUpdate(
+            ClientTransaction clientTransaction,
+            Response response)
     {
         /*
          * Disable the sending of picture_fast_update because it seems to be
          * unsupported by this remote peer.
          */
-        if ((statusCode != 200) && sendPictureFastUpdate)
+        if ((response.getStatusCode() != 200) && sendPictureFastUpdate)
             sendPictureFastUpdate = false;
     }
 
+    boolean processPictureFastUpdate(
+            ServerTransaction serverTransaction,
+            Request request)
+        throws OperationFailedException
+    {
+        System.err.println("processPictureFastUpdate Request");
+        CallPeerMediaHandlerSipImpl mediaHandler = getMediaHandler();
+        boolean requested
+            = (mediaHandler == null)
+                ? false
+                : mediaHandler.processKeyFrameRequest();
+
+        Response response;
+
+        try
+        {
+            response
+                = getProtocolProvider().getMessageFactory().createResponse(
+                        Response.OK,
+                        request);
+        }
+        catch (ParseException pe)
+        {
+            throw new OperationFailedException(
+                    "Failed to create OK Response.",
+                    OperationFailedException.INTERNAL_ERROR,
+                    pe);
+        }
+
+        if (!requested)
+        {
+            ContentType ct
+                = new ContentType(
+                        "application",
+                        PICTURE_FAST_UPDATE_CONTENT_SUB_TYPE);
+            String content
+                = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\r\n"
+                    + "<media_control>\r\n"
+                    + "<general_error>\r\n"
+                    + "Failed to process picture_fast_update request.\r\n"
+                    + "</general_error>\r\n"
+                    + "</media_control>";
+
+            try
+            {
+                response.setContent(content, ct);
+            }
+            catch (ParseException pe)
+            {
+                throw new OperationFailedException(
+                        "Failed to set content of OK Response.",
+                        OperationFailedException.INTERNAL_ERROR,
+                        pe);
+            }
+        }
+
+        try
+        {
+            serverTransaction.sendResponse(response);
+        }
+        catch (Exception e)
+        {
+            throw new OperationFailedException(
+                    "Failed to send OK Response.",
+                    OperationFailedException.INTERNAL_ERROR,
+                    e);
+        }
+
+        return true;
+    }
+
     /**
      * Reinitializes the media session of the <tt>CallPeer</tt> that this
      * INVITE request is destined to.

src/net/java/sip/communicator/impl/protocol/sip/OperationSetVideoTelephonySipImpl.java

@@ -6,6 +6,7 @@
  */
 package net.java.sip.communicator.impl.protocol.sip;
 
+import java.io.*;
 import java.text.*;
 
 import javax.sip.*;
@@ -239,6 +240,106 @@ public QualityControl getQualityControl(CallPeer peer)
     private class PictureFastUpdateMethodProcessor
         extends MethodProcessorAdapter
     {
+        /**
+         * Determines whether a specific <tt>Request</tt> is a
+         * <tt>picture_fast_update</tt> one.
+         *
+         * @param request the <tt>Request</tt> to check
+         * @return <tt>true</tt> if the specified <tt>request</tt> is a
+         * <tt>picture_fast_update</tt> one; otherwise, <tt>false</tt>
+         */
+        private boolean isPictureFastUpdate(Request request)
+        {
+            ContentTypeHeader contentTypeHeader
+                = (ContentTypeHeader) request.getHeader(ContentTypeHeader.NAME);
+            if (contentTypeHeader == null)
+                return false;
+            if (!"application".equalsIgnoreCase(
+                    contentTypeHeader.getContentType()))
+                return false;
+            if (!CallPeerSipImpl.PICTURE_FAST_UPDATE_CONTENT_SUB_TYPE
+                    .equalsIgnoreCase(
+                            contentTypeHeader.getContentSubType()))
+                return false;
+
+            Object content = request.getContent();
+            if (content == null)
+                return false;
+
+            String xml;
+            if (content instanceof String)
+                xml = content.toString();
+            else if (content instanceof byte[])
+            {
+                byte[] bytes = (byte[]) content;
+                try
+                {
+                    xml = new String(bytes, "UTF-8");
+                }
+                catch (UnsupportedEncodingException uee)
+                {
+                    xml = new String(bytes);
+                }
+            }
+            else
+                return false;
+            if (xml == null)
+                return false;
+            return xml.contains("picture_fast_update");
+        }
+
+        /**
+         * Delivers <tt>picture_fast_update</tt> <tt>Request</tt>s to the
+         * targeted <tt>CallPeerSipImpl</tt>.
+         *
+         * {@inheritDoc}
+         */
+        @Override
+        public boolean processRequest(RequestEvent requestEvent)
+        {
+            if (requestEvent == null)
+                return false;
+
+            Request request = requestEvent.getRequest();
+            if (request == null)
+                return false;
+            if (!isPictureFastUpdate(request))
+                return false;
+
+            ServerTransaction serverTransaction;
+            try
+            {
+                serverTransaction
+                    = SipStackSharing.getOrCreateServerTransaction(
+                            requestEvent);
+            }
+            catch (Exception e)
+            {
+                e.printStackTrace(System.err);
+                return false;
+            }
+            if (serverTransaction == null)
+                return false;
+
+            CallPeerSipImpl callPeer
+                = basicTelephony.getActiveCallsRepository().findCallPeer(
+                    serverTransaction.getDialog());
+            if (callPeer == null)
+                return false;
+
+            try
+            {
+                return
+                    callPeer.processPictureFastUpdate(
+                            serverTransaction,
+                            request);
+            }
+            catch (OperationFailedException ofe)
+            {
+                return false;
+            }
+        }
+
         /**
          * Delivers <tt>Response</tt>s to <tt>picture_fast_update</tt>
          * <tt>Request</tt>s to the originating <tt>CallPeerSipImpl</tt>.
@@ -252,46 +353,28 @@ public boolean processResponse(ResponseEvent responseEvent)
                 return false;
 
             Response response = responseEvent.getResponse();
-
             if (response == null)
                 return false;
 
             ClientTransaction clientTransaction
                 = responseEvent.getClientTransaction();
-
             if (clientTransaction == null)
                 return false;
 
             Request request = clientTransaction.getRequest();
-
             if (request == null)
                 return false;
-
-            ContentTypeHeader contentTypeHeader
-                = (ContentTypeHeader) request.getHeader(ContentTypeHeader.NAME);
-
-            if (contentTypeHeader == null)
-                return false;
-            if (!"application".equalsIgnoreCase(
-                    contentTypeHeader.getContentType()))
-                return false;
-            if (!CallPeerSipImpl.PICTURE_FAST_UPDATE_CONTENT_SUB_TYPE
-                    .equalsIgnoreCase(
-                            contentTypeHeader.getContentSubType()))
+            if (!isPictureFastUpdate(request))
                 return false;
 
             CallPeerSipImpl callPeer
                 = basicTelephony.getActiveCallsRepository().findCallPeer(
                         clientTransaction.getDialog());
-
             if (callPeer == null)
                 return false;
-            else
-            {
-                callPeer.processPictureFastUpdateResponseStatusCode(
-                        response.getStatusCode());
-                return true;
-            }
+
+            callPeer.processPictureFastUpdate(clientTransaction, response);
+            return true;
         }
     }
 }

src/net/java/sip/communicator/service/neomedia/control/KeyFrameControl.java

@@ -16,6 +16,22 @@
  */
 public interface KeyFrameControl
 {
+    /**
+     * Adds a <tt>KeyFrameRequestee</tt> to be made available through this
+     * <tt>KeyFrameControl</tt>.
+     *
+     * @param index the zero-based index at which <tt>keyFrameRequestee</tt> is
+     * to be added to the list of <tt>KeyFrameRequestee</tt>s made available or
+     * <tt>-1</tt> to have this <tt>KeyFrameControl</tt> choose at which index
+     * it is to be added in accord with its internal logic
+     * through this <tt>KeyFrameControl</tt>
+     * @param keyFrameRequestee the <tt>KeyFrameRequestee</tt> to be added to
+     * this <tt>KeyFrameControl</tt> so that it is made available through it
+     */
+    public void addKeyFrameRequestee(
+            int index,
+            KeyFrameRequestee keyFrameRequestee);
+
     /**
      * Adds a <tt>KeyFrameRequester</tt> to be made available through this
      * <tt>KeyFrameControl</tt>.
@@ -32,6 +48,15 @@ public void addKeyFrameRequester(
             int index,
             KeyFrameRequester keyFrameRequester);
 
+    /**
+     * Gets the <tt>KeyFrameRequestee</tt>s made available through this
+     * <tt>KeyFrameControl</tt>.
+     *
+     * @return an unmodifiable list of <tt>KeyFrameRequestee</tt>s made
+     * available through this <tt>KeyFrameControl</tt>
+     */
+    public List<KeyFrameRequestee> getKeyFrameRequestees();
+
     /**
      * Gets the <tt>KeyFrameRequester</tt>s made available through this
      * <tt>KeyFrameControl</tt>.
@@ -41,6 +66,28 @@ public void addKeyFrameRequester(
      */
     public List<KeyFrameRequester> getKeyFrameRequesters();
 
+    /**
+     * Notifies this <tt>KeyFrameControl</tt> that the remote peer of the
+     * associated <tt>VideoMediaStream</tt> has requested a key frame from the
+     * local peer.
+     *
+     * @return <tt>true</tt> if the local peer has honored the request from the
+     * remote peer for a key frame; otherwise, <tt>false</tt>
+     */
+    public boolean keyFrameRequest();
+
+    /**
+     * Removes a <tt>KeyFrameRequestee</tt> to no longer be made available
+     * through this <tt>KeyFrameControl</tt>.
+     *
+     * @param keyFrameRequestee the <tt>KeyFrameRequestee</tt> to be removed
+     * from this <tt>KeyFrameControl</tt> so that it is no longer made available
+     * through it
+     * @return <tt>true</tt> if <tt>keyFrameRequestee</tt> was found in this
+     * <tt>KeyFrameControl</tt>; otherwise, <tt>false</tt>
+     */
+    public boolean removeKeyFrameRequestee(KeyFrameRequestee keyFrameRequestee);
+
     /**
      * Removes a <tt>KeyFrameRequester</tt> to no longer be made available
      * through this <tt>KeyFrameControl</tt>.
@@ -53,6 +100,25 @@ public void addKeyFrameRequester(
      */
     public boolean removeKeyFrameRequester(KeyFrameRequester keyFrameRequester);
 
+    /**
+     * Represents a way for the remote peer of a <tt>VideoMediaStream</tt> to
+     * request a key frame from its local peer.
+     *
+     * @author Lyubomir Marinov
+     */
+    public interface KeyFrameRequestee
+    {
+        /**
+         * Notifies this <tt>KeyFrameRequestee</tt> that the remote peer of the
+         * associated <tt>VideoMediaStream</tt> requests a key frame from the
+         * local peer.
+         *
+         * @return <tt>true</tt> if this <tt>KeyFrameRequestee</tt> has honored
+         * the request for a key frame; otherwise, <tt>false</tt>
+         */
+        public boolean keyFrameRequest();
+    }
+
     /**
      * Represents a way for a <tt>VideoMediaStream</tt> to request a key frame
      * from its remote peer.

src/net/java/sip/communicator/service/neomedia/control/KeyFrameControlAdapter.java

@@ -16,6 +16,13 @@
 public class KeyFrameControlAdapter
     implements KeyFrameControl
 {
+    /**
+     * The <tt>KeyFrameRequestee</tt>s made available by this
+     * <tt>KeyFrameControl</tt>.
+     */
+    private List<KeyFrameRequestee> keyFrameRequestees
+        = new ArrayList<KeyFrameRequestee>(0);
+
     /**
      * The <tt>KeyFrameRequester</tt>s made available by this
      * <tt>KeyFrameControl</tt>.
@@ -23,12 +30,62 @@ public class KeyFrameControlAdapter
     private List<KeyFrameRequester> keyFrameRequesters
         = new ArrayList<KeyFrameRequester>(0);
 
+    /**
+     * An unmodifiable view of {@link #keyFrameRequestees} appropriate to be
+     * returned by {@link #getKeyFrameRequestees()}.
+     */
+    private List<KeyFrameRequestee> unmodifiableKeyFrameRequestees;
+
     /**
      * An unmodifiable view of {@link #keyFrameRequesters} appropriate to be
      * returned by {@link #getKeyFrameRequesters()}.
      */
     private List<KeyFrameRequester> unmodifiableKeyFrameRequesters;
 
+    /**
+     * Implements
+     * {@link KeyFrameControl#addKeyFrameRequestee(int, KeyFrameRequestee)}.
+     *
+     * {@inheritDoc}
+     */
+    public void addKeyFrameRequestee(
+            int index,
+            KeyFrameRequestee keyFrameRequestee)
+    {
+        if (keyFrameRequestee == null)
+            throw new NullPointerException("keyFrameRequestee");
+        synchronized (this)
+        {
+            if (!keyFrameRequestees.contains(keyFrameRequestee))
+            {
+                List<KeyFrameRequestee> newKeyFrameRequestees
+                    = new ArrayList<KeyFrameRequestee>(
+                            keyFrameRequestees.size() + 1);
+
+                newKeyFrameRequestees.addAll(keyFrameRequestees);
+                /*
+                 * If this KeyFrameControl is to determine the index at which
+                 * keyFrameRequestee is to be added according to its own
+                 * internal logic, then it will prefer KeyFrameRequestee
+                 * implementations from outside of neomedia rather than from its
+                 * inside.
+                 */
+                if (-1 == index)
+                {
+                    if (keyFrameRequestee.getClass().getName().contains(
+                            ".neomedia."))
+                        index = newKeyFrameRequestees.size();
+                    else
+                        index = 0;
+                }
+                newKeyFrameRequestees.add(index, keyFrameRequestee);
+
+                keyFrameRequestees = newKeyFrameRequestees;
+                unmodifiableKeyFrameRequestees = null;
+            }
+        }
+    }
+
     /**
      * Implements
      * {@link KeyFrameControl#addKeyFrameRequester(int, KeyFrameRequester)}.
@@ -73,6 +130,24 @@ public void addKeyFrameRequester(
         }
     }
 
+    /**
+     * Implements {@link KeyFrameControl#getKeyFrameRequestees()}.
+     *
+     * {@inheritDoc}
+     */
+    public List<KeyFrameRequestee> getKeyFrameRequestees()
+    {
+        synchronized (this)
+        {
+            if (unmodifiableKeyFrameRequestees == null)
+            {
+                unmodifiableKeyFrameRequestees
+                    = Collections.unmodifiableList(keyFrameRequestees);
+            }
+            return unmodifiableKeyFrameRequestees;
+        }
+    }
+
     /**
      * Implements {@link KeyFrameControl#getKeyFrameRequesters()}.
      *
@@ -91,6 +166,60 @@ public List<KeyFrameRequester> getKeyFrameRequesters()
         }
     }
 
+    /**
+     * Implements {@link KeyFrameControl#keyFrameRequest()}.
+     *
+     * {@inheritDoc}
+     */
+    public boolean keyFrameRequest()
+    {
+        for (KeyFrameRequestee keyFrameRequestee : getKeyFrameRequestees())
+        {
+            try
+            {
+                if (keyFrameRequestee.keyFrameRequest())
+                    return true;
+            }
+            catch (Exception e)
+            {
+                /*
+                 * A KeyFrameRequestee has malfunctioned, do not let it
+                 * interfere with the others.
+                 */
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Implements
+     * {@link KeyFrameControl#removeKeyFrameRequestee(KeyFrameRequestee)}.
+     *
+     * {@inheritDoc}
+     */
+    public boolean removeKeyFrameRequestee(KeyFrameRequestee keyFrameRequestee)
+    {
+        synchronized (this)
+        {
+            int index = keyFrameRequestees.indexOf(keyFrameRequestee);
+
+            if (-1 != index)
+            {
+                List<KeyFrameRequestee> newKeyFrameRequestees
+                    = new ArrayList<KeyFrameRequestee>(keyFrameRequestees);
+
+                newKeyFrameRequestees.remove(index);
+
+                keyFrameRequestees = newKeyFrameRequestees;
+                unmodifiableKeyFrameRequestees = null;
+
+                return true;
+            }
+            else
+                return false;
+        }
+    }
+
     /**
      * Implements
      * {@link KeyFrameControl#removeKeyFrameRequester(KeyFrameRequester)}.

src/net/java/sip/communicator/service/protocol/media/CallPeerMediaHandler.java

@@ -1312,6 +1312,23 @@ protected void sendHolePunchPacket(MediaStreamTarget target)
         getTransportManager().sendHolePunchPacket(target, MediaType.VIDEO);
     }
 
+    /**
+     * Processes a request for a (video) key frame from the remote peer to the
+     * local peer.
+     *
+     * @return <tt>true</tt> if the request for a (video) key frame has been
+     * honored by the local peer; otherwise, <tt>false</tt>
+     */
+    public boolean processKeyFrameRequest()
+    {
+        KeyFrameControl keyFrameControl = this.keyFrameControl;
+
+        return
+            (keyFrameControl == null)
+                ? null
+                : keyFrameControl.keyFrameRequest();
+    }
+
     /**
      * Registers all audio level listeners currently known to this media handler
      * with the specified <tt>audioStream</tt>.