- AVPF Picture Loss Indication support;

- rtcp-fb nack pli support in SDP;
- Add updated ffmpeg libraries;
- Add missing javadoc.
cusax-fix
Sebastien Vincent 16 years ago
parent fd6b9abba9
commit 6166430740

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

@ -24,6 +24,9 @@ make
2. ffmpeg-r20783
First patch ffmpeg with ffmpeg-libx264-keyframe.diff:
$ patch -p0 < ffmpeg-libx264-keyframe.diff
- Windows
32-bit

@ -0,0 +1,13 @@
Index: libavcodec/libx264.c
===================================================================
--- libavcodec/libx264.c (révision 20783)
+++ libavcodec/libx264.c (copie de travail)
@@ -99,7 +99,7 @@
}
x4->pic.i_pts = frame->pts;
- x4->pic.i_type = X264_TYPE_AUTO;
+ x4->pic.i_type = frame->key_frame ? X264_TYPE_IDR : X264_TYPE_AUTO;
}
if (x264_encoder_encode(x4->enc, &nal, &nnal, frame? &x4->pic: NULL, &pic_out) < 0)

@ -686,8 +686,6 @@ public MetaContact createMetaContact( ProtocolProviderService provider,
* creation of the first protocol specific child contact in the respective
* group.
*
* @param parentGroup the <tt>MetaContactGroup</tt> that should be the
* parent of the newly created group.
* @param parent
* the meta contact group inside which the new child group must
* be created.

@ -168,7 +168,7 @@ public X509TrustManager getTrustManager(String toHost, int toPort)
*
* @param toHost the host we are connecting.
* @param toPort the port used when connecting.
* @return
* @return SSL context object
*/
public SSLContext getSSLContext(String toHost, int toPort)
throws IOException

@ -112,7 +112,7 @@ else if (MediaDeviceSession
* RTP and RTCP traffic. The instance is a <tt>TransformConnector</tt> in
* order to also enable packet transformations.
*/
private final RTPTransformConnector rtpConnector;
protected final RTPTransformConnector rtpConnector;
/**
* The one and only <tt>MediaStreamTarget</tt> this instance has added as a
@ -1607,7 +1607,7 @@ public void update(SessionEvent event)
* @param ssrc the SSRC identifier that this stream will be using in
* outgoing RTP packets from now on.
*/
private void setLocalSourceID(long ssrc)
protected void setLocalSourceID(long ssrc)
{
Long oldValue = this.localSourceID;
@ -1623,7 +1623,7 @@ private void setLocalSourceID(long ssrc)
* @param ssrc the SSRC identifier that this stream will be using in
* outgoing RTP packets from now on.
*/
private void setRemoteSourceID(long ssrc)
protected void setRemoteSourceID(long ssrc)
{
Long oldValue = this.remoteSourceID;
this.remoteSourceID = ssrc;

@ -79,6 +79,7 @@ public class MediaUtils
MediaType.AUDIO,
AudioFormat.G723_RTP,
g723FormatParams,
null,
8000);
addMediaFormats(
@ -153,13 +154,19 @@ public class MediaUtils
Map<String, String> h264FormatParams
= new HashMap<String, String>();
Map<String, String> h264AdvancedParams
= new HashMap<String, String>();
h264FormatParams.put("packetization-mode", "1");
h264AdvancedParams.put("rtcp-fb", "nack pli");
addMediaFormats(
MediaFormat.RTP_PAYLOAD_TYPE_UNKNOWN,
"H264",
MediaType.VIDEO,
Constants.H264_RTP,
h264FormatParams);
h264FormatParams,
h264AdvancedParams);
}
/**
@ -192,6 +199,7 @@ private static void addMediaFormats(
mediaType,
jmfEncoding,
null,
null,
clockRates);
}
@ -220,6 +228,7 @@ private static void addMediaFormats(
MediaType mediaType,
String jmfEncoding,
Map<String, String> formatParameters,
Map<String, String> advancedParameters,
double... clockRates)
{
int clockRateCount = clockRates.length;
@ -250,7 +259,8 @@ private static void addMediaFormats(
.createInstance(
format,
clockRate,
formatParameters);
formatParameters,
advancedParameters);
if (mediaFormat != null)
mediaFormats.add(mediaFormat);
@ -281,7 +291,8 @@ private static void addMediaFormats(
{
MediaFormat mediaFormat
= MediaFormatImpl
.createInstance(format, clockRate, formatParameters);
.createInstance(format, clockRate, formatParameters,
advancedParameters);
if (mediaFormat != null)
mediaFormats.add(mediaFormat);

@ -0,0 +1,122 @@
/*
* SIP Communicator, the OpenSource Java VoIP and Instant Messaging client.
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package net.java.sip.communicator.impl.neomedia;
import java.io.*;
import java.net.*;
import java.util.*;
import net.java.sip.communicator.service.neomedia.event.*;
/**
* @author Bing SU (nova.su@gmail.com)
* @author Lubomir Marinov
* @author Sebastien Vincent
*/
public class RTCPConnectorInputStream extends RTPConnectorInputStream
{
/**
* List of feedback listeners;
*/
private List<RTCPFeedbackListener> listeners =
new ArrayList<RTCPFeedbackListener>();
/**
* Initializes a new <tt>RTCPConnectorInputStream</tt> which is to receive
* packet data from a specific UDP socket.
*
* @param socket the UDP socket the new instance is to receive data from
*/
public RTCPConnectorInputStream(DatagramSocket socket)
{
super(socket);
}
/**
* Add an <tt>RTCPFeedbackListener</tt>.
*/
public void addRTCPFeedbackListener(RTCPFeedbackListener listener)
{
if(!listeners.contains(listener))
{
listeners.add(listener);
}
}
/**
* Removve an <tt>RTCPFeedbackListener</tt>.
*/
public void removeRTCPFeedbackListener(RTCPFeedbackListener listener)
{
if(listeners.contains(listener))
{
listeners.remove(listener);
}
}
/**
* Copies the content of the most recently received packet into
* <tt>inBuffer</tt>.
*
* @param inBuffer the <tt>byte[]</tt> that we'd like to copy the content
* of the packet to.
* @param offset the position where we are supposed to start writing in
* <tt>inBuffer</tt>.
* @param length the number of <tt>byte</tt>s available for writing in
* <tt>inBuffer</tt>.
*
* @return the number of bytes read
*
* @throws IOException if <tt>length</tt> is less than the size of the
* packet.
*/
public int read(byte[] inBuffer, int offset, int length)
throws IOException
{
if (ioError)
return -1;
int pktLength = pkt.getLength();
if (length < pktLength)
throw
new IOException("Input buffer not big enough for " + pktLength);
/* check if RTCP feedback message */
/* Feedback message size is minimum 12 bytes:
* Version/Padding/Feedback message type: 1 byte
* Payload type: 1 byte
* Length: 2 bytes
* SSRC of packet sender: 4 bytes
* SSRC of media source: 4 bytes
*/
if(pktLength >= 12)
{
byte data[] = pkt.getBuffer();
int fmt = 0;
int pt = 0;
/* get FMT field (last 5 bits of first byte) */
fmt = (data[0] & 0x1F);
pt |= (data[1] & 0xFF);
RTCPFeedbackEvent evt = new RTCPFeedbackEvent(this, fmt, pt);
/* notify feedback listeners */
for(RTCPFeedbackListener l : listeners)
{
l.feedbackReceived(evt);
}
}
System.arraycopy(
pkt.getBuffer(), pkt.getOffset(), inBuffer, offset, pktLength);
return pktLength;
}
}

@ -0,0 +1,84 @@
/*
* SIP Communicator, the OpenSource Java VoIP and Instant Messaging client.
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package net.java.sip.communicator.impl.neomedia;
import java.util.*;
import javax.media.rtp.*;
public class RTCPFeedbackPacket
{
/**
* Feedback message type.
*/
private int fmt = 0;
/**
* Payload type.
*/
private int payloadType = 0;
/**
* SSRC of packet sender.
*/
private long senderSSRC = 0;
/**
* SSRC of media source.
*/
private long sourceSSRC = 0;
/**
* Constructor.
*
* @param type feedback message type
* @param payloadType payload type
* @param sender sender SSRC
* @param src source SSRC
*/
public RTCPFeedbackPacket(int type, int payloadType, long sender, long src)
{
this.fmt = type;
this.payloadType = payloadType;
this.senderSSRC = sender;
this.sourceSSRC = src;
}
/**
* Write packet to output stream.
*
* @param out <tt>OutputDataStream</tt>
*/
public void writeTo(OutputDataStream out)
{
byte data[] = new byte[12];
byte vpfmt = (byte)((2 << 7) | (0 << 6) | (byte)fmt);
data[0] = vpfmt;
data[1] = (byte)payloadType;
/* length (in 32-bit words minus one) */
data[2] = 0;
data[3] = 2; /* common packet is 12 bytes so (12/4) - 1 */
/* sender SSRC */
data[4] = (byte)(senderSSRC >> 24);
data[5] = (byte)((senderSSRC >> 16) & 0xFF);
data[6] = (byte)((senderSSRC >> 8) & 0xFF);
data[7] = (byte)(senderSSRC & 0xFF);
/* source SSRC */
data[8] = (byte)(sourceSSRC >> 24);
data[9] = (byte)((sourceSSRC >> 16) & 0xFF);
data[10] = (byte)((sourceSSRC >> 8) & 0xFF);
data[11] = (byte)(sourceSSRC & 0xFF);
/* effective write */
out.write(data, 0, 12);
}
}

@ -129,7 +129,7 @@ public void close()
protected RTPConnectorInputStream createControlInputStream()
throws IOException
{
return new RTPConnectorInputStream(getControlSocket());
return new RTCPConnectorInputStream(getControlSocket());
}
/**

@ -46,13 +46,13 @@ public class RTPConnectorInputStream
/**
* Caught an IO exception during read from socket
*/
private boolean ioError = false;
protected boolean ioError = false;
/**
* The packet data to be read out of this instance through its
* {@link #read(byte[], int, int)} method.
*/
private RawPacket pkt;
protected RawPacket pkt;
/**
* UDP socket used to receive data.

@ -54,6 +54,11 @@ public class VideoMediaStreamImpl
*/
private Dimension outputSize;
/**
* Use or not PLI.
*/
private boolean usePLI = false;
/**
* Selects the <tt>VideoFormat</tt> from the list of supported formats of a
* specific video <tt>DataSource</tt> which has a size as close as possible
@ -511,7 +516,20 @@ public void setDevice(MediaDevice device)
{
super.setDevice(device);
((VideoMediaDeviceSession)deviceSession).setOutputSize(outputSize);
VideoMediaDeviceSession vmds = (VideoMediaDeviceSession)deviceSession;
vmds.setOutputSize(outputSize);
vmds.setConnector(rtpConnector);
vmds.setRtcpFeedbackPLI(usePLI);
}
/**
* Use or not RTCP feedback Picture Loss Indication.
*
* @param use use or not PLI
*/
public void setRtcpFeedbackPLI(boolean use)
{
usePLI = use;
}
/**
@ -523,4 +541,29 @@ public void setOutputSize(Dimension size)
{
outputSize = size;
}
/**
* Set local SSRC.
*
* @param ssrc source ID
*/
@Override
protected void setLocalSourceID(long ssrc)
{
super.setLocalSourceID(ssrc);
((VideoMediaDeviceSession)deviceSession).setLocalSSRC(ssrc);
}
/**
* Set remote SSRC.
*
* @param ssrc remote SSRC
*/
@Override
protected void setRemoteSourceID(long ssrc)
{
super.setRemoteSourceID(ssrc);
((VideoMediaDeviceSession)deviceSession).setRemoteSSRC(ssrc);
}
}

@ -15,7 +15,7 @@
* Implements a <tt>VideoFormat</tt> for a <tt>Buffer</tt> carrying
* <tt>AVFrame</tt> as its <tt>data</tt>. While the <tt>AVFrameFormat</tt> class
* is not strictly necessary and <tt>VideoFormat</tt> could have be directly
* used, it is conceived as an appripriate way to avoid possible matching with
* used, it is conceived as an appropriate way to avoid possible matching with
* other <tt>VideoFormat</tt>s and a very obvious one.
*
* @author Lubomir Marinov
@ -168,7 +168,7 @@ public Format intersects(Format format)
/**
* Determines whether a specific format matches this instance i.e. whether
* their attributes match according to the definition of "match" given by
* {@line Format#matches(Format)}.
* {@link Format#matches(Format)}.
*
* @param format the <tt>Format</tt> to compare to this instance
* @return <tt>true</tt> if the specified <tt>format</tt> matches this one;

@ -44,11 +44,11 @@ public class SwScaler
*/
private Format[] supportedOutputFormats = new Format[]
{
new YUVFormat(null, -1, Format.byteArray, -1.0f, YUVFormat.YUV_420,
new YUVFormat(null, -1, Format.byteArray, -1.0f, YUVFormat.YUV_420,
-1, -1, 0, -1, -1),
new YUVFormat(null, -1, Format.intArray, -1.0f, YUVFormat.YUV_420,
new YUVFormat(null, -1, Format.intArray, -1.0f, YUVFormat.YUV_420,
-1, -1, 0, -1, -1),
new YUVFormat(null, -1, Format.shortArray, -1.0f, YUVFormat.YUV_420,
new YUVFormat(null, -1, Format.shortArray, -1.0f, YUVFormat.YUV_420,
-1, -1, 0, -1, -1),
new RGBFormat(null, -1, Format.byteArray, -1.0f, 32, -1, -1, -1),
new RGBFormat(null, -1, Format.intArray, -1.0f, 32, -1, -1, -1),
@ -87,6 +87,9 @@ public SwScaler()
addControl(frameProcessingControl);
}
/**
* Close codec.
*/
@Override
public void close()
{
@ -157,7 +160,7 @@ public static int getNativeRGBFormat(RGBFormat rgb)
}
else
fmt = FFmpeg.PIX_FMT_RGB24;
return fmt;
}
@ -190,7 +193,7 @@ public Format[] getSupportedOutputFormats(Format input)
if(input == null)
return supportedOutputFormats;
/* if size is set for element 0 (YUVFormat), it is also set
/* if size is set for element 0 (YUVFormat), it is also set
* for element 1 (RGBFormat) and so on...
*/
Dimension size = ((VideoFormat) supportedOutputFormats[0]).getSize();
@ -207,7 +210,7 @@ public Format[] getSupportedOutputFormats(Format input)
return
new Format[]
{
{
new YUVFormat(size, -1, Format.byteArray, frameRate,
YUVFormat.YUV_420, -1, -1, 0, -1, -1),
new YUVFormat(size, -1, Format.intArray, frameRate,
@ -238,7 +241,7 @@ public Format[] getSupportedOutputFormats(Format input)
* processed
*/
@Override
public int process(Buffer input, Buffer output)
public int process(Buffer input, Buffer output)
{
if (!checkInputBuffer(input))
return BUFFER_PROCESSED_FAILED;
@ -392,7 +395,7 @@ else if (Format.shortArray.equals(outputDataType))
outFlags |= Buffer.FLAG_SYSTEM_TIME;
output.setFlags(outFlags);
return BUFFER_PROCESSED_OK;
return BUFFER_PROCESSED_OK;
}
/**

@ -11,6 +11,7 @@
import javax.media.*;
import javax.media.format.*;
import net.java.sip.communicator.impl.neomedia.*;
import net.java.sip.communicator.impl.neomedia.codec.*;
import net.java.sip.communicator.impl.neomedia.codec.video.*;
import net.java.sip.communicator.util.*;
@ -45,6 +46,11 @@ public class DePacketizer
*/
private static final byte[] NAL_PREFIX = { 0, 0, 1 };
/**
* Interval between a PLI request and its reemission.
*/
private static final long PLI_INTERVAL = 250;
/**
* The indicator which determines whether this <tt>DePacketizer</tt> has
* successfully processed an RTP packet with payload representing a
@ -71,6 +77,36 @@ public class DePacketizer
private final int outputPaddingSize
= FFmpeg.FF_INPUT_BUFFER_PADDING_SIZE;
/**
* RTCP output stream to send RTCP feedback message
* back to sender in case of packet loss for example.
*/
private RTPConnectorOutputStream rtcpOutputStream = null;
/**
* Local SSRC.
*/
private long localSSRC = -1;
/**
* Remote SSRC.
*/
private long remoteSSRC = -1;
/**
* Last timestamp of keyframe request.
*
* This is used to retransmit keyframe request in case previous get lost
* and codec doesn't receive keyframe.
*/
private long lastKeyframeRequestTimeStamp = -1;
/**
* Use or not RTCP PLI message when depacketizer miss
* packets.
*/
private boolean usePLI = false;
/**
* Initializes a new <tt>DePacketizer</tt> instance which is to depacketize
* H.264 RTP packets into NAL units.
@ -117,10 +153,19 @@ private int dePacketizeFUA(
boolean start_bit = (fu_header & 0x80) != 0;
boolean end_bit = (fu_header & 0x40) != 0;
/* key frame (IDR) type = 5 */
boolean keyframe = ((fu_header & 0x1F) == 5);
int outOffset = outBuffer.getOffset();
int newOutLength = inLength;
int octet;
/* waiting for a keyframe ? */
if(lastKeyframeRequestTimeStamp != -1 && keyframe)
{
/* keyframe received */
lastKeyframeRequestTimeStamp = -1;
}
if (start_bit)
{
/*
@ -249,7 +294,17 @@ protected int doProcess(Buffer inBuffer, Buffer outBuffer)
+ " and continuing with sequenceNumber "
+ sequenceNumber);
/* send an PLI request to inform sender that we miss
* part of video
* Do not send another one here if we are waiting for a keyframe.
*/
if(usePLI && lastKeyframeRequestTimeStamp == -1)
{
sendRTCPFeedbackPLI();
}
ret = reset(outBuffer);
if ((ret & OUTPUT_BUFFER_NOT_FILLED) == 0)
return ret;
}
@ -318,6 +373,15 @@ else if (nal_unit_type == 28) // FU-A Fragmentation unit (FU)
if ((inBuffer.getFlags() & Buffer.FLAG_RTP_MARKER) != 0)
outBuffer.setFlags(outBuffer.getFlags() | Buffer.FLAG_RTP_MARKER);
/* we do not receive keyframe so retransmit request */
if(usePLI && lastKeyframeRequestTimeStamp != -1 &&
System.currentTimeMillis() > (lastKeyframeRequestTimeStamp +
PLI_INTERVAL))
{
sendRTCPFeedbackPLI();
}
return ret;
}
@ -377,4 +441,47 @@ private int reset(Buffer outBuffer)
outBuffer.setLength(0);
return OUTPUT_BUFFER_NOT_FILLED;
}
/**
* Send RTCP Feedback PLI message.
*/
private void sendRTCPFeedbackPLI()
{
RTCPFeedbackPacket packet = new RTCPFeedbackPacket(1, 206,
localSSRC, remoteSSRC);
lastKeyframeRequestTimeStamp = System.currentTimeMillis();
packet.writeTo(rtcpOutputStream);
}
/**
* Set <tt>OutputDataStream</tt>.
*
* @param rtcpOutputStream RTCP <tt>OutputDataStream</tt>
*/
public void setConnector(RTPConnectorOutputStream rtcpOutputStream)
{
this.rtcpOutputStream = rtcpOutputStream;
}
/**
* Set local and remote SSRC. It will be used
* to send RTCP messages.
*
* @param localSSRC local SSRC
* @param remoteSSRC remote SSRC
*/
public void setSSRC(long localSSRC, long remoteSSRC)
{
this.localSSRC = localSSRC;
this.remoteSSRC = remoteSSRC;
}
/**
* Use or not RTCP feedback PLI.
*/
public void setRtcpFeedbackPLI(boolean use)
{
usePLI = use;
}
}

@ -11,6 +11,8 @@
import javax.media.*;
import javax.media.format.*;
import net.java.sip.communicator.service.neomedia.event.*;
import net.java.sip.communicator.impl.neomedia.codec.*;
import net.java.sip.communicator.impl.neomedia.codec.video.*;
import net.sf.fmj.media.*;
@ -24,36 +26,79 @@
*/
public class JNIEncoder
extends AbstractCodec
implements RTCPFeedbackListener
{
/**
* Default output formats.
*/
private static final Format[] DEFAULT_OUTPUT_FORMATS
= { new VideoFormat(Constants.H264) };
// key frame every 300 frames
/**
* Key frame every 300 frames.
*/
private static final int IFRAME_INTERVAL = 300;
/**
* Padding size.
*/
private static final int INPUT_BUFFER_PADDING_SIZE = 8;
/**
* Name of the code.
*/
private static final String PLUGIN_NAME = "H.264 Encoder";
// the frame rate we will use
/**
* Minimum interval between two PLI request processing (in milliseconds).
*/
private static final long PLI_INTERVAL = 1000;
/**
* The frame rate we will use
*/
private static final int TARGET_FRAME_RATE = 15;
// The codec we will use
/**
* The codec we will use.
*/
private long avcontext;
// the encoded data is stored in avpicture
/**
* The encoded data is stored in avpicture.
*/
private long avframe;
// we use this buffer to supply data to encoder
/**
* We use this buffer to supply data to encoder.
*/
private byte[] encFrameBuffer;
// the supplied data length
/**
* The supplied data length.
*/
private int encFrameLen;
/**
* Next interval for an automatic keyframe.
*/
private int framesSinceLastIFrame = IFRAME_INTERVAL + 1;
/**
* The raw frame buffer.
*/
private long rawFrameBuffer;
/**
* Force encoder to send a key frame.
*/
private boolean forceKeyFrame = false;
/**
* Last keyframe request time.
*/
private long lastKeyframeRequestTime = System.currentTimeMillis();
/**
* Initializes a new <tt>JNIEncoder</tt> instance.
*/
@ -76,6 +121,9 @@ public JNIEncoder()
outputFormat = null;
}
/**
* Close the <tt>Codec</tt>.
*/
@Override
public synchronized void close()
{
@ -97,6 +145,12 @@ public synchronized void close()
}
}
/**
* Get the matching output formats for a specific format.
*
* @param in input format
* @return array for formats matching input format
*/
private Format[] getMatchingOutputFormats(Format in)
{
VideoFormat videoIn = (VideoFormat) in;
@ -113,6 +167,11 @@ private Format[] getMatchingOutputFormats(Format in)
};
}
/**
* Get codec name.
*
* @return codec name
*/
@Override
public String getName()
{
@ -121,6 +180,8 @@ public String getName()
/**
* Return the list of formats supported at the output.
*
* @return array of formats supported at output
*/
public Format[] getSupportedOutputFormats(Format in)
{
@ -136,6 +197,9 @@ public Format[] getSupportedOutputFormats(Format in)
return getMatchingOutputFormats(in);
}
/**
* Inits <tt>Codec</tt> instance.
*/
@Override
public synchronized void open()
throws ResourceUnavailableException
@ -236,6 +300,14 @@ public synchronized void open()
super.open();
}
/**
* Processes (encode) a buffer.
*
* @param inBuffer input buffer
* @param outBuffer output buffer
* @return <tt>BUFFER_PROCESSED_OK</tt> if buffer has been successfully
* processed
*/
public synchronized int process(Buffer inBuffer, Buffer outBuffer)
{
if (isEOM(inBuffer))
@ -269,10 +341,11 @@ public synchronized int process(Buffer inBuffer, Buffer outBuffer)
(byte[]) inBuffer.getData(), inBuffer.getOffset(),
encFrameLen);
if (framesSinceLastIFrame >= IFRAME_INTERVAL)
if (framesSinceLastIFrame >= IFRAME_INTERVAL || forceKeyFrame)
{
FFmpeg.avframe_set_key_frame(avframe, true);
framesSinceLastIFrame = 0;
forceKeyFrame = false;
}
else
{
@ -329,6 +402,12 @@ public synchronized int process(Buffer inBuffer, Buffer outBuffer)
return BUFFER_PROCESSED_OK;
}
/**
* Sets the input format.
*
* @param in format to set
* @return format
*/
@Override
public Format setInputFormat(Format in)
{
@ -368,6 +447,16 @@ public Format setInputFormat(Format in)
return inputFormat;
}
/**
* Sets the <tt>Format</tt> in which this <tt>Codec</tt> is to output media
* data.
*
* @param out the <tt>Format</tt> in which this <tt>Codec</tt> is to
* output media data
* @return the <tt>Format</tt> in which this <tt>Codec</tt> is currently
* configured to output media data or <tt>null</tt> if <tt>format</tt> was
* found to be incompatible with this <tt>Codec</tt>
*/
@Override
public Format setOutputFormat(Format out)
{
@ -405,4 +494,34 @@ public Format setOutputFormat(Format out)
// Return the selected outputFormat
return outputFormat;
}
/**
* Event fired when RTCP feedback message is received.
*
* @param event <tt>RTCPFeedbackEvent</tt>
*/
public void feedbackReceived(RTCPFeedbackEvent event)
{
/* if RTCP message is a Picture Loss Indication (PLI)
* or a Full Intra-frame Request (FIR) the encoder will
* force the next frame to be a keyframe
*/
if(event.getPayloadType() == RTCPFeedbackEvent.PT_PS)
{
switch(event.getFeedbackMessageType())
{
case RTCPFeedbackEvent.FMT_PLI:
case RTCPFeedbackEvent.FMT_FIR:
if(System.currentTimeMillis() > lastKeyframeRequestTime
+ PLI_INTERVAL)
{
lastKeyframeRequestTime = System.currentTimeMillis();
forceKeyFrame = true;
}
break;
default:
break;
}
}
}
}

@ -81,7 +81,7 @@ public class MediaDeviceSession
* {@link #setFormat(MediaFormat)} and to be set as the output format of
* {@link #processor}.
*/
private Format format;
protected Format format;
/**
* The indicator which determines whether this <tt>MediaDeviceSession</tt>
@ -265,7 +265,8 @@ else if (sourceFormat.matches(new Format(VideoFormat.H263_RTP)))
}
else
{
// We don't know this particular format. We'll just leave it alone then.
// We don't know this particular format. We'll just leave it alone
//then.
return sourceFormat;
}
@ -1277,7 +1278,7 @@ protected void setProcessorFormat(Processor processor, Format format)
case VIDEO:
if (supportedFormats[0] instanceof VideoFormat)
{
supportedFormat
supportedFormat
= findFirstMatchingFormat(supportedFormats, format);
/*

@ -16,7 +16,9 @@
import javax.swing.*;
import net.java.sip.communicator.impl.neomedia.*;
import net.java.sip.communicator.impl.neomedia.transform.*;
import net.java.sip.communicator.impl.neomedia.codec.video.*;
import net.java.sip.communicator.impl.neomedia.codec.video.h264.*;
import net.java.sip.communicator.impl.neomedia.imgstreaming.*;
import net.java.sip.communicator.service.neomedia.*;
import net.java.sip.communicator.service.neomedia.event.*;
@ -53,6 +55,11 @@ public class VideoMediaDeviceSession
*/
private Player localPlayer = null;
/**
* Use or not RTCP feedback Picture Loss Indication.
*/
private boolean usePLI = false;
/**
* Output size of the stream.
*
@ -63,6 +70,21 @@ public class VideoMediaDeviceSession
*/
private Dimension outputSize;
/**
* The <tt>RTPConnector</tt>.
*/
private RTPConnectorImpl rtpConnector = null;
/**
* Local SSRC.
*/
private long localSSRC = -1;
/**
* Remote SSRC.
*/
private long remoteSSRC = -1;
/**
* The <tt>SwScaler</tt> inserted into the codec chain of the
* <tt>Player</tt> rendering the media received from the remote peer and
@ -94,7 +116,6 @@ public class VideoMediaDeviceSession
*
* @param device the video <tt>MediaDevice</tt> the use of which by a
* <tt>MediaStream</tt> is to be represented by the new instance
* @param outputSize output size of the video
*/
public VideoMediaDeviceSession(AbstractMediaDevice device)
{
@ -281,7 +302,7 @@ protected void fireVideoEvent(VideoEvent videoEvent)
}
/**
* Get the local <tt>Player</tt> if it exists,
* Get the local <tt>Player</tt> if it exists,
* create it otherwise
* @return local <tt>Player</tt>
*/
@ -358,7 +379,7 @@ private void controllerUpdateForCreateLocalVisualComponent(
// No listener interested in our event so free resources.
if(localPlayer == player)
localPlayer = null;
player.stop();
player.deallocate();
player.close();
@ -450,7 +471,7 @@ public void paint(Graphics g)
if (imgWidth > width)
{
scale = true;
scaleFactor = width / (float) imgWidth;
scaleFactor = width / (float) imgWidth;
}
if (imgHeight > height)
{
@ -507,9 +528,9 @@ public void disposeLocalVisualComponent()
}
/**
* Releases the resources allocated by a specific local <tt>Player</tt> in the
* course of its execution and prepares it to be garbage collected. If the
* specified <tt>Player</tt> is rendering video, notifies the
* Releases the resources allocated by a specific local <tt>Player</tt> in
* the course of its execution and prepares it to be garbage collected. If
* the specified <tt>Player</tt> is rendering video, notifies the
* <tt>VideoListener</tt>s of this instance that its visual
* <tt>Component</tt> is to no longer be used by firing a
* {@link VideoEvent#VIDEO_REMOVED} <tt>VideoEvent</tt>.
@ -524,7 +545,7 @@ protected void disposeLocalPlayer(Player player)
* its Player#getVisualComponent() (if any) should be released.
*/
Component visualComponent = getVisualComponent(player);
if(localPlayer == player)
localPlayer = null;
@ -670,7 +691,37 @@ public int process(Buffer input, Buffer output)
return result;
}
};
trackControl.setCodecChain(new Codec[] { playerScaler });
/* for H264 codec, we will use RTCP feedback
* for example to advertise sender that we miss
* a frame
*/
if(format.getEncoding().equals("h264/rtp") && usePLI)
{
DePacketizer depack = new DePacketizer();
depack.setRtcpFeedbackPLI(usePLI);
try
{
depack.setConnector(rtpConnector.
getControlOutputStream());
}
catch(Exception e)
{
logger.error("Error cannot get RTCP output stream",
e);
}
depack.setSSRC(localSSRC, remoteSSRC);
trackControl.setCodecChain(new Codec[] {
depack, playerScaler});
}
else
{
trackControl.setCodecChain(new Codec[] {playerScaler});
}
break;
}
}
@ -903,6 +954,16 @@ public void removeVideoListener(VideoListener listener)
videoNotifierSupport.removeVideoListener(listener);
}
/**
* Use or not RTCP feedback Picture Loss Indication.
*
* @param use use or not PLI
*/
public void setRtcpFeedbackPLI(boolean use)
{
usePLI = use;
}
/**
* Sets the size of the output video.
*
@ -913,6 +974,37 @@ public void setOutputSize(Dimension size)
outputSize = size;
}
/**
* Sets the <tt>RTPConnector</tt> that will be used to
* initialize some codec for RTCP feedback.
*
* @param rtpConnector the RTP connector
*/
public void setConnector(RTPConnectorImpl rtpConnector)
{
this.rtpConnector = rtpConnector;
}
/**
* Set the local SSRC.
*
* @param localSSRC local SSRC
*/
public void setLocalSSRC(long localSSRC)
{
this.localSSRC = localSSRC;
}
/**
* Set the remote SSRC.
*
* @param remoteSSRC remote SSRC
*/
public void setRemoteSSRC(long remoteSSRC)
{
this.remoteSSRC = remoteSSRC;
}
/**
* Sets the JMF <tt>Format</tt> in which a specific <tt>Processor</tt>
* producing media to be streamed to the remote peer is to output.
@ -933,8 +1025,8 @@ protected void setProcessorFormat(Processor processor, Format format)
*/
if(outputSize != null)
{
newFormat = new VideoFormat(tmp.getEncoding(), outputSize,
tmp.getMaxDataLength(), tmp.getDataType(),
newFormat = new VideoFormat(tmp.getEncoding(), outputSize,
tmp.getMaxDataLength(), tmp.getDataType(),
tmp.getFrameRate());
}
@ -965,27 +1057,69 @@ protected Format setProcessorFormat(
TrackControl trackControl,
Format format)
{
Codec codecs[] = null;
JNIEncoder encoder = null;
SwScaler scaler = null;
int nbCodec = 0;
/* for H264 we will monitor RTCP feedback
* for example if we receive a PLI/FIR message,
* we will send a keyframe
*/
if(format.getEncoding().equals("h264/rtp") && usePLI)
{
encoder = new JNIEncoder();
/* encoder need to be notified of RTCP feedback message */
try
{
((ControlTransformInputStream)rtpConnector.
getControlInputStream()).addRTCPFeedbackListener(encoder);
}
catch(Exception e)
{
logger.error("Error cannot get RTCP input stream", e);
}
nbCodec++;
}
if(outputSize != null)
{
/* We have been explicitly told to use a specified output size so
* create a custom SwScaler that will scale and convert color spaces
* in one call.
*/
SwScaler scaler = new SwScaler();
scaler = new SwScaler();
scaler.setOutputSize(outputSize);
nbCodec++;
}
/* Add our custom SwScaler to the codec chain so that it will be
* used instead of default.
*/
try
{
trackControl.setCodecChain(new Codec[] { scaler });
}
catch(UnsupportedPlugInException upiex)
{
logger.error("Failed to add SwScaler to codec chain", upiex);
}
codecs = new Codec[nbCodec];
nbCodec = 0;
if(scaler != null)
{
codecs[nbCodec] = scaler;
nbCodec++;
}
if(encoder != null)
{
codecs[nbCodec] = encoder;
}
/* Add our custom SwScaler and possibly RTCP aware codec to the codec
* chain so that it will be
* used instead of default.
*/
try
{
trackControl.setCodecChain(codecs);
}
catch(UnsupportedPlugInException upiex)
{
logger.error("Failed to add SwScaler/JNIEncoder to codec chain",
upiex);
}
return super.setProcessorFormat(trackControl, format);

@ -34,7 +34,7 @@ public class AudioMediaFormatImpl
*/
AudioMediaFormatImpl(AudioFormat format)
{
this(format, null);
this(format, null, null);
}
/**
@ -47,12 +47,15 @@ public class AudioMediaFormatImpl
* and provide an implementation of <tt>AudioMediaFormat</tt> for
* @param formatParameters the set of format-specific parameters of the new
* instance
* @param advancedParameters the set of format-specific parameters of the new
* instance
*/
AudioMediaFormatImpl(
AudioFormat format,
Map<String, String> formatParameters)
Map<String, String> formatParameters,
Map<String, String> advancedParameters)
{
super(fixChannels(format), formatParameters);
super(fixChannels(format), formatParameters, advancedParameters);
}
/**
@ -94,7 +97,7 @@ public class AudioMediaFormatImpl
*/
AudioMediaFormatImpl(String encoding, double clockRate, int channels)
{
this(encoding, clockRate, channels, null);
this(encoding, clockRate, channels, null, null);
}
/**
@ -112,9 +115,10 @@ public class AudioMediaFormatImpl
AudioMediaFormatImpl(
String encoding,
double clockRate,
Map<String, String> formatParameters)
Map<String, String> formatParameters,
Map<String, String> advancedParameters)
{
this(encoding, clockRate, 1, formatParameters);
this(encoding, clockRate, 1, formatParameters, advancedParameters);
}
/**
@ -130,12 +134,15 @@ public class AudioMediaFormatImpl
* stereo)
* @param formatParameters any codec-specific parameters that have been
* received via SIP/SDP or XMPP/Jingle
* @param advancedParameters any parameters that have been
* received via SIP/SDP or XMPP/Jingle
*/
AudioMediaFormatImpl(
String encoding,
double clockRate,
int channels,
Map<String, String> formatParameters)
Map<String, String> formatParameters,
Map<String, String> advancedParameters)
{
this(
new AudioFormat(
@ -143,7 +150,8 @@ public class AudioMediaFormatImpl
clockRate,
AudioFormat.NOT_SPECIFIED,
channels),
formatParameters);
formatParameters,
advancedParameters);
}
/**

@ -165,19 +165,21 @@ public MediaFormat createMediaFormat(
* <tt>AudioMediaFormat</tt> or a <tt>VideoMediaFormat</tt> instance if
* <tt>encoding</tt> is known to this <tt>MediaFormatFactory</tt>;
* otherwise, <tt>null</tt>
* @see MediaFormatFactory#createMediaFormat(String, double, Map)
* @see MediaFormatFactory#createMediaFormat(String, double, Map, Map)
*/
public MediaFormat createMediaFormat(
String encoding,
double clockRate,
Map<String, String> formatParams)
Map<String, String> formatParams,
Map<String, String> advancedParams)
{
return
createMediaFormat(
encoding,
clockRate,
1,
formatParams);
formatParams,
advancedParams);
}
/**
@ -196,18 +198,21 @@ public MediaFormat createMediaFormat(
* <tt>encoding</tt>; otherwise, ignored
* @param formatParams any codec specific parameters which have been
* received via SIP/SDP or XMPP/Jingle
* @param advancedParams any parameters which have been
* received via SIP/SDP or XMPP/Jingle
* @return a <tt>MediaFormat</tt> with the specified <tt>encoding</tt>,
* <tt>clockRate</tt>, <tt>channels</tt> and set of format parameters which
* is either an <tt>AudioMediaFormat</tt> or a <tt>VideoMediaFormat</tt>
* instance if <tt>encoding</tt> is known to this
* <tt>MediaFormatFactory</tt>; otherwise, <tt>null</tt>
* @see MediaFormatFactory#createMediaFormat(String, double, int, Map)
* @see MediaFormatFactory#createMediaFormat(String, double, int, Map, Map)
*/
public MediaFormat createMediaFormat(
String encoding,
double clockRate,
int channels,
Map<String, String> formatParams)
Map<String, String> formatParams,
Map<String, String> advancedParams)
{
MediaFormat mediaFormat
= createMediaFormat(encoding, clockRate, channels);
@ -218,17 +223,24 @@ public MediaFormat createMediaFormat(
{
Map<String, String> formatParameters
= new HashMap<String, String>();
Map<String, String> advancedParameters
= new HashMap<String, String>();
formatParameters.putAll(mediaFormat.getFormatParameters());
formatParameters.putAll(formatParams);
if(advancedParams != null)
{
advancedParameters.putAll(advancedParams);
}
switch (mediaFormat.getMediaType())
{
case AUDIO:
mediaFormat
= new AudioMediaFormatImpl(
((AudioMediaFormatImpl) mediaFormat).getFormat(),
formatParameters);
formatParameters, advancedParameters);
break;
case VIDEO:
VideoMediaFormatImpl videoMediaFormatImpl
@ -238,7 +250,8 @@ public MediaFormat createMediaFormat(
= new VideoMediaFormatImpl(
videoMediaFormatImpl.getFormat(),
videoMediaFormatImpl.getClockRate(),
formatParameters);
null,
advancedParameters);
break;
default:
mediaFormat = null;
@ -283,7 +296,8 @@ public MediaFormat createMediaFormat(
String encoding,
double clockRate,
int channels,
Map<String, String> formatParams)
Map<String, String> formatParams,
Map<String, String> advancedParams)
{
/*
@ -326,7 +340,8 @@ public MediaFormat createMediaFormat(
}
}
return createMediaFormat(encoding, clockRate, channels, formatParams);
return createMediaFormat(encoding, clockRate, channels, formatParams,
advancedParams);
}
/**

@ -75,7 +75,8 @@ else if (format instanceof VideoFormat)
public static MediaFormatImpl<? extends Format> createInstance(
Format format,
double clockRate,
Map<String, String> formatParameters)
Map<String, String> formatParameters,
Map<String, String> advancedParameters)
{
if (format instanceof AudioFormat)
{
@ -91,14 +92,16 @@ public static MediaFormatImpl<? extends Format> createInstance(
new AudioMediaFormatImpl(
(AudioFormat)
clockRateAudioFormat.intersects(audioFormat),
formatParameters);
formatParameters,
advancedParameters);
}
if (format instanceof VideoFormat)
return
new VideoMediaFormatImpl(
(VideoFormat) format,
clockRate,
formatParameters);
formatParameters,
advancedParameters);
return null;
}
@ -171,6 +174,12 @@ else if (!value1.equals(value2))
*/
private final Map<String, String> formatParameters;
/**
* The advanced parameters of this instance which have been received
* via SIP/SDP or XMPP/Jingle.
*/
private final Map<String, String> advancedParameters;
/**
* Initializes a new <tt>MediaFormatImpl</tt> instance which is to provide
* an implementation of <tt>MediaFormat</tt> for a specific <tt>Format</tt>.
@ -180,7 +189,7 @@ else if (!value1.equals(value2))
*/
protected MediaFormatImpl(T format)
{
this(format, null);
this(format, null, null);
}
/**
@ -192,8 +201,11 @@ protected MediaFormatImpl(T format)
* implementation of <tt>MediaFormat</tt> for
* @param formatParameters any codec-specific parameters that have been
* received via SIP/SDP or XMPP/Jingle
* @param advancedParameters any parameters that have been
* received via SIP/SDP or XMPP/Jingle
*/
protected MediaFormatImpl(T format, Map<String, String> formatParameters)
protected MediaFormatImpl(T format, Map<String, String> formatParameters,
Map<String, String> advancedParameters)
{
if (format == null)
throw new NullPointerException("format");
@ -203,6 +215,10 @@ protected MediaFormatImpl(T format, Map<String, String> formatParameters)
= ((formatParameters == null) || formatParameters.isEmpty())
? EMPTY_FORMAT_PARAMETERS
: new HashMap<String, String>(formatParameters);
this.advancedParameters
= ((advancedParameters == null) || advancedParameters.isEmpty())
? EMPTY_FORMAT_PARAMETERS
: new HashMap<String, String>(advancedParameters);
}
/**
@ -211,7 +227,7 @@ protected MediaFormatImpl(T format, Map<String, String> formatParameters)
*
* @param mediaFormat the object that we'd like to compare <tt>this</tt> one
* to.
*
8*
* @return <tt>true</tt> if the JMF <tt>Format</tt> instances encapsulated
* by this class are equal and <tt>false</tt> otherwise.
*/
@ -289,6 +305,22 @@ public Map<String, String> getFormatParameters()
: new HashMap<String, String>(formatParameters);
}
/**
* Implements MediaFormat#getAdvancedParameters(). Returns a copy of the
* attribute properties of this instance. Modifications to the returned Map
* do no affect the format properties of this instance.
*
* @return a copy of the attribute properties of this instance.
* Modifications to the returned Map do no affect the format properties of
* this instance.
*/
public Map<String, String> getAdvancedParameters()
{
return (advancedParameters == EMPTY_FORMAT_PARAMETERS)
? EMPTY_FORMAT_PARAMETERS
: new HashMap<String, String>(advancedParameters);
}
/**
* Gets the encoding of the JMF <tt>Format</tt> represented by this
* instance as it is known to JMF (in contrast to its RFC name).

@ -86,7 +86,7 @@ public class VideoMediaFormatImpl
*/
VideoMediaFormatImpl(VideoFormat format, double clockRate)
{
this(format, clockRate, null);
this(format, clockRate, null, null);
}
/**
@ -105,9 +105,10 @@ public class VideoMediaFormatImpl
VideoMediaFormatImpl(
VideoFormat format,
double clockRate,
Map<String, String> formatParameters)
Map<String, String> formatParameters,
Map<String, String> advancedParameters)
{
super(format, formatParameters);
super(format, formatParameters, advancedParameters);
this.clockRate = clockRate;
}

@ -0,0 +1,129 @@
/*
* SIP Communicator, the OpenSource Java VoIP and Instant Messaging client.
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package net.java.sip.communicator.impl.neomedia.transform;
import java.io.*;
import java.net.*;
import java.util.*;
import net.java.sip.communicator.service.neomedia.event.*;
/**
* Implement control channel (RTCP) for <tt>TransformInputStream</tt>
* which notify listeners when RTCP feedback messages are received.
*
* @author Bing SU (nova.su@gmail.com)
* @author Lubomir Marinov
* @author Sebastien Vincent
*/
public class ControlTransformInputStream extends TransformInputStream
{
/**
* List of feedback listeners;
*/
private List<RTCPFeedbackListener> listeners =
new ArrayList<RTCPFeedbackListener>();
/**
* Initializes a new <tt>ControlTransformInputStream</tt> which is to
* receive packet data from a specific UDP socket.
*
* @param socket the UDP socket the new instance is to receive data from
*/
public ControlTransformInputStream(DatagramSocket socket)
{
super(socket);
}
/**
* Add an <tt>RTCPFeedbackListener</tt>.
*
* @param listener listener to add
*/
public void addRTCPFeedbackListener(RTCPFeedbackListener listener)
{
if(!listeners.contains(listener))
{
listeners.add(listener);
}
}
/**
* Remove an <tt>RTCPFeedbackListener</tt>.
*
* @param listener listener to remove
*/
public void removeRTCPFeedbackListener(RTCPFeedbackListener listener)
{
if(listeners.contains(listener))
{
listeners.remove(listener);
}
}
/**
* Copies the content of the most recently received packet into
* <tt>inBuffer</tt>.
*
* @param inBuffer the <tt>byte[]</tt> that we'd like to copy the content
* of the packet to.
* @param offset the position where we are supposed to start writing in
* <tt>inBuffer</tt>.
* @param length the number of <tt>byte</tt>s available for writing in
* <tt>inBuffer</tt>.
*
* @return the number of bytes read
*
* @throws IOException if <tt>length</tt> is less than the size of the
* packet.
*/
public int read(byte[] inBuffer, int offset, int length)
throws IOException
{
if (ioError)
return -1;
int pktLength = pkt.getLength();
if (length < pktLength)
throw
new IOException("Input buffer not big enough for " + pktLength);
/* check if RTCP feedback message */
/* Feedback message size is minimum 12 bytes:
* Version/Padding/Feedback message type: 1 byte
* Payload type: 1 byte
* Length: 2 bytes
* SSRC of packet sender: 4 bytes
* SSRC of media source: 4 bytes
*/
if(pktLength >= 12)
{
byte data[] = pkt.getBuffer();
int fmt = 0;
int pt = 0;
/* get FMT field (last 5 bits of first byte) */
fmt = (data[0] & 0x1F);
pt |= (data[1] & 0xFF);
RTCPFeedbackEvent evt = new RTCPFeedbackEvent(this, fmt, pt);
/* notify feedback listeners */
for(RTCPFeedbackListener l : listeners)
{
l.feedbackReceived(evt);
}
}
System.arraycopy(
pkt.getBuffer(), pkt.getOffset(), inBuffer, offset, pktLength);
return pktLength;
}
}

@ -76,7 +76,7 @@ protected TransformInputStream createControlInputStream()
throws IOException
{
TransformInputStream controlInputStream
= new TransformInputStream(getControlSocket());
= new ControlTransformInputStream(getControlSocket());
controlInputStream.setTransformer(getRTCPTransformer());
return controlInputStream;

@ -868,10 +868,17 @@ private MediaStream initStream(StreamConnector connector,
//this is a reinit
}
/* set negociated output size for video stream */
if(device != null && device.getMediaType() == MediaType.VIDEO)
{
/* set negociated output size for video stream */
((VideoMediaStream)stream).setOutputSize(size);
/* set rtcp-fb */
if(format.getAdvancedParameters().containsKey("rtcp-fb"))
{
((VideoMediaStream)stream).setRtcpFeedbackPLI(true);
}
}
return configureAndStartStream(
@ -951,8 +958,8 @@ private MediaStream configureAndStartStream(
}
/**
* Send empty UDP packet to target destination in order to open port
* on NAT or RTP proxy if any.
* Send empty UDP packet to target destination data/control ports
* in order to open port on NAT or RTP proxy if any.
*
* @param target <tt>MediaStreamTarget</tt>
*/
@ -961,9 +968,15 @@ private void sendHolePunchPacket(MediaStreamTarget target)
logger.info("Try to open port on NAT if any");
try
{
/* data port (RTP) */
videoStreamConnector.getDataSocket().send(new DatagramPacket(
new byte[0], 0, target.getDataAddress().getAddress(),
target.getDataAddress().getPort()));
/* control port (RTCP) */
videoStreamConnector.getControlSocket().send(new DatagramPacket(
new byte[0], 0, target.getControlAddress().getAddress(),
target.getControlAddress().getPort()));
}
catch(Exception e)
{
@ -1470,6 +1483,7 @@ private void processAnswer(SessionDescription answer)
/* extract remote peer maximum supported resolution */
Dimension res[] = SdpUtils.extractSendRecvResolution(
mediaDescription);
if(res != null)
{
maxSendSize = res[0];

@ -304,25 +304,21 @@ private static MediaDescription removeMediaDesc(
}
/**
* Extracts and returns maximum resolution can receive from the image
* Extracts and returns maximum resolution can receive from the image
* attribute.
*
* @param mediaDesc the <tt>MediaDescription</tt> that we'd like to probe
* for maximum receive resolution.
* @return maximum resolution array (first element is send, second one is
* recv). Elements could be null if image attribute is not present or if
* @return maximum resolution array (first element is send, second one is
* recv). Elements could be null if image attribute is not present or if
* resoluion is a wildcard.
*/
@SuppressWarnings("unchecked") /* legacy code from jain-sdp */
public static java.awt.Dimension[] extractSendRecvResolution(
MediaDescription mediaDesc)
{
java.awt.Dimension res[] = new java.awt.Dimension[2];
String imgattr = null;
String token = null;
int idx = 0;
int idx2 = 0;
int len = 0;
Pattern pSendSingle = Pattern.compile("send \\[x=[0-9]+,y=[0-9]+\\]");
Pattern pRecvSingle = Pattern.compile("recv \\[x=[0-9]+,y=[0-9]+\\]");
Pattern pSendRange = Pattern.compile(
@ -330,10 +326,10 @@ public static java.awt.Dimension[] extractSendRecvResolution(
Pattern pRecvRange = Pattern.compile(
"recv \\[x=\\[[0-9]+-[0-9]+\\],y=\\[[0-9]+-[0-9]+\\]\\]");
Pattern pNumeric = Pattern.compile("[0-9]+");
Matcher mSingle = null;
Matcher mSingle = null;
Matcher mRange = null;
Matcher m = null;
try
{
imgattr = mediaDesc.getAttribute("imageattr");
@ -354,11 +350,11 @@ public static java.awt.Dimension[] extractSendRecvResolution(
* - single value [x=1920,y=1200]
* - range of values [x=[800-1024],y=[600-768]]
* - fixed range of values [x=[800,1024],y=[600,768]]
* - range of values with step (here 32) [x=[800:32:1024],y=[600:32:768]]
* - range of values with step [x=[800:32:1024],y=[600:32:768]]
*
* For the moment we only support the first two forms.
*/
/* send part */
mSingle = pSendSingle.matcher(imgattr);
mRange = pSendRange.matcher(imgattr);
@ -377,7 +373,7 @@ public static java.awt.Dimension[] extractSendRecvResolution(
res[0] = new java.awt.Dimension(val[0], val[1]);
}
else if(mRange.find()) /* try with range */
else if(mRange.find()) /* try with range */
{
/* have two value for width and two for height (min-max) */
int val[] = new int[4];
@ -409,7 +405,7 @@ else if(mRange.find()) /* try with range */
{
val[i] = Integer.parseInt(token.substring(m.start(), m.end()));
}
res[1] = new java.awt.Dimension(val[0], val[1]);
}
else if(mRange.find()) /* try with range */
@ -419,13 +415,13 @@ else if(mRange.find()) /* try with range */
int i = 0;
token = imgattr.substring(mRange.start(), mRange.end());
m = pNumeric.matcher(token);
while(m.find() && i < 4)
{
val[i] = Integer.parseInt(token.substring(m.start(), m.end()));
i++;
}
res[1] = new java.awt.Dimension(val[1], val[3]);
}
@ -437,7 +433,7 @@ else if(mRange.find()) /* try with range */
pSendSingle = null;
pRecvSingle = null;
pSendRange = null;
return res;
}
@ -468,8 +464,9 @@ public static List<MediaFormat> extractFormats(
DynamicPayloadTypeRegistry ptRegistry)
{
List<MediaFormat> mediaFmts = new ArrayList<MediaFormat>();
Vector<String> formatStrings;
List<Attribute> advp = new ArrayList<Attribute>();
try
{
formatStrings
@ -484,6 +481,20 @@ public static List<MediaFormat> extractFormats(
return mediaFmts;
}
try
{
Attribute rtcpFbAsterisk = findPayloadTypeSpecificAttribute(
mediaDesc.getAttributes(false), "rtcp-fb", "*");
if(rtcpFbAsterisk != null)
advp.add(rtcpFbAsterisk);
}
catch(SdpException e)
{
//there was a problem parsing the "*" rtcp-fb. try to ignore.
logger.debug("Does not seem like a valid rtcp-fb:* attribute", e);
}
for(String ptStr : formatStrings)
{
byte pt = -1;
@ -503,7 +514,8 @@ public static List<MediaFormat> extractFormats(
try
{
rtpmap = findPayloadTypeSpecificAttribute(
mediaDesc.getAttributes(false), SdpConstants.RTPMAP, pt);
mediaDesc.getAttributes(false), SdpConstants.RTPMAP,
Byte.toString(pt));
}
catch (SdpException e)
{
@ -513,10 +525,17 @@ public static List<MediaFormat> extractFormats(
}
Attribute fmtp = null;
Attribute rtcpFb = null;
try
{
fmtp = findPayloadTypeSpecificAttribute(
mediaDesc.getAttributes(false), "fmtp", pt);
mediaDesc.getAttributes(false), "fmtp", ptStr);
rtcpFb = findPayloadTypeSpecificAttribute(
mediaDesc.getAttributes(false), "rtcp-fb", ptStr);
if(rtcpFb != null)
advp.add(rtcpFb);
}
catch (SdpException exc)
{
@ -528,7 +547,7 @@ public static List<MediaFormat> extractFormats(
MediaFormat mediaFormat = null;
try
{
mediaFormat = createFormat(pt, rtpmap, fmtp, ptRegistry);
mediaFormat = createFormat(pt, rtpmap, fmtp, advp, ptRegistry);
}
catch (SdpException e)
{
@ -708,6 +727,7 @@ private static RTPExtension parseRTPExtensionAttribute(
* @param rtpmap an SDP <tt>Attribute</tt> mapping the <tt>payloadType</tt>
* to an encoding name.
* @param fmtp a list of format specific parameters
* @param advp list of advanced parameters (rtcp-fb, ...)
* @param ptRegistry the {@link DynamicPayloadTypeRegistry} that we are to
* use in case <tt>payloadType</tt> is dynamic and <tt>rtpmap</tt> is
* <tt>null</tt> (in which case we can hope its in the registry).
@ -725,6 +745,7 @@ private static MediaFormat createFormat(
byte payloadType,
Attribute rtpmap,
Attribute fmtp,
List<Attribute> advp,
DynamicPayloadTypeRegistry ptRegistry)
throws SdpException
{
@ -795,9 +816,20 @@ private static MediaFormat createFormat(
//Format parameters
Map<String, String> fmtParamsMap = null;
Map<String, String> advancedParamsMap = null;
if ( fmtp != null)
fmtParamsMap = parseFmtpAttribute(fmtp);
for(Attribute attr : advp)
{
/* RTCP feedback support */
if (attr.getName().equals("rtcp-fb"))
{
advancedParamsMap = parseRTCPFeedbackAttribute(attr);
}
}
//now create the format.
MediaFormat format
= SipActivator
@ -808,13 +840,14 @@ private static MediaFormat createFormat(
encoding,
clockRate,
numChannels,
fmtParamsMap);
fmtParamsMap,
advancedParamsMap);
/*
* We've just created a MediaFormat for the specified payloadType so we
* have to remember the mapping between the two so that we don't, for
* example, map the same payloadType to a different MediaFormat at a
* later time when we do automatic generatation of payloadType in
* later time when we do automatic generation of payloadType in
* DynamicPayloadTypeRegistry.
*/
/*
@ -832,6 +865,42 @@ private static MediaFormat createFormat(
return format;
}
/**
* Parses the value of <tt>rtcpAttr</tt> attribute.
*
* @param rtcpAttr SDP attribute containing rtcp-fb parameters.
* @return
*/
private static Map<String, String> parseRTCPFeedbackAttribute(
Attribute rtcpAttr)
{
Map<String, String> ret = new Hashtable<String, String>();
String attrVal = rtcpAttr.toString();
StringTokenizer tokenizer = new StringTokenizer(attrVal, " ", false);
/* valid rtcp-fb we support is on the form:
* a=rtcp-fb:100 nack pli
*/
if (tokenizer.countTokens() != 3)
return null;
/* skip rtcp-fb:pt */
tokenizer.nextToken();
String ackType = tokenizer.nextToken();
String ackExt = tokenizer.nextToken();
if(ackType.equals("nack") && (ackExt.equals("pli\r\n") ||
ackExt.equals("pli ")))
{
ret.put("rtcp-fb", "nack pli");
}
if (ret.size() == 0)
return null;
return ret;
}
/**
* Parses the value of the <tt>fmtpAttr</tt> attribute into a format
* parameters <tt>Map</tt> and returns it.
@ -900,7 +969,7 @@ private static Map<String, String> parseFmtpAttribute(Attribute fmtpAttr)
// No valid fmtp tokens found, just return null
if (fmtParamsMap.size() == 0)
return null;
return fmtParamsMap;
}
@ -924,14 +993,12 @@ private static Map<String, String> parseFmtpAttribute(Attribute fmtpAttr)
private static Attribute findPayloadTypeSpecificAttribute(
Vector<Attribute> mediaAttributes,
String attributeName,
byte payloadType)
String payloadType)
throws SdpException
{
if( mediaAttributes == null || mediaAttributes.size() == 0)
return null;
String ptStr = Byte.toString(payloadType);
for (Attribute attr : mediaAttributes)
{
if(!attributeName.equals(attr.getName()))
@ -944,7 +1011,7 @@ private static Attribute findPayloadTypeSpecificAttribute(
attrValue = attrValue.trim();
if(!attrValue.startsWith(ptStr + " "))
if(!attrValue.startsWith(payloadType + " "))
continue;
//that's it! we have the attribute we are looking for.
@ -1345,25 +1412,38 @@ public static MediaDescription createMediaDescription(
mediaAttributes.add(fmtp);
}
/* add extra attributes (rtcp-fb, ...) */
Iterator<Map.Entry<String, String>> iter = format
.getAdvancedParameters().entrySet().iterator();
while (iter.hasNext())
{
Map.Entry<String, String> ntry = iter.next();
Attribute adv = sdpFactory.createAttribute(ntry.getKey(),
payloadType + " " + ntry.getValue());
mediaAttributes.add(adv);
}
payloadTypesArray[i] = payloadType;
}
/* image attribute */
if(mediaType == MediaType.VIDEO)
{
/* one image attribute for all payload
/* one image attribute for all payload
*
* basically peer can send any size and can
* basically peer can send any size and can
* receive image size up to its display screen size
*/
ScreenDevice screen = SipActivator.getMediaService().getDefaultScreenDevice();
ScreenDevice screen = SipActivator.getMediaService().
getDefaultScreenDevice();
java.awt.Dimension res = null;
if(screen != null)
{
res = screen.getSize();
}
Attribute imgattr = createImageAttribute((byte)0, null, res);
mediaAttributes.add(imgattr);
}
@ -1493,7 +1573,7 @@ else if(MediaDirection.SENDRECV.equals(direction))
/**
* Creates an <tt>Attribute</tt> instance for send/recv image negociation.
*
*
* http://tools.ietf.org/html/draft-ietf-mmusic-image-attributes-04
*
* @param payloadType payload type
@ -1501,11 +1581,11 @@ else if(MediaDirection.SENDRECV.equals(direction))
* @param maxRecvSize maximum size peer can display
* @return image <tt>Attribute</tt>
*/
private static Attribute createImageAttribute(byte payloadType,
private static Attribute createImageAttribute(byte payloadType,
java.awt.Dimension sendSize, java.awt.Dimension maxRecvSize)
{
StringBuffer img = new StringBuffer("imageattr:");
if(payloadType != 0)
{
img.append(payloadType);
@ -1545,14 +1625,14 @@ private static Attribute createImageAttribute(byte payloadType,
/* can send "all" sizes */
img.append(" send *");
}
/* receive size */
if(maxRecvSize != null)
{
/* basically we can receive any size up to our
/* basically we can receive any size up to our
* screen display size
*/
/* recv [x=[min-max],y=[min-max]] */
img.append(" recv [x=[0-");
img.append((int)maxRecvSize.getWidth());
@ -1623,7 +1703,8 @@ public static MediaDescription createDisablingAnswer(
if(formatsVec == null)
{
formatsVec = new Vector<String>();
//add at least one format so that we could generate a valid offer.
//add at least one format so that we could generate a valid
//offer.
formatsVec.add(Integer.toString(0));
}

@ -313,7 +313,7 @@ public void actionPerformed(ActionEvent e)
/**
* The simple form for this wizard.
* @return
* @return the simple form for this wizard.
*/
public Object getSimpleForm()
{
@ -322,7 +322,8 @@ public Object getSimpleForm()
/**
* Whether is committed.
* @return
* @return <tt>true</tt> if the form is committed, <tt>false</tt>
* otherwise
*/
public boolean isCommitted()
{

@ -53,7 +53,7 @@ public static String getApplicationString(String key)
/**
* Returns the resource service.
* @return
* @return resource service.
*/
public static ResourceManagementService getResources()
{

@ -70,11 +70,18 @@ public interface VideoMediaStream
* <tt>VideoMediaStream</tt>
*/
public void removeVideoListener(VideoListener listener);
/**
* Set negociated output size.
*
* @param size output size of video stream
*/
public void setOutputSize(Dimension size);
/**
* Use or not RTCP feedback Picture Loss Indication.
*
* @param use use or not PLI
*/
public void setRtcpFeedbackPLI(boolean use);
}

@ -0,0 +1,85 @@
/*
* SIP Communicator, the OpenSource Java VoIP and Instant Messaging client.
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package net.java.sip.communicator.service.neomedia.event;
import java.util.*;
/**
* Represents an event coming from RTCP that meant to tell codec
* to do something (i.e send a keyframe, ...).
*
* @author Sebastien Vincent
*/
public class RTCPFeedbackEvent extends EventObject
{
/**
* Transport layer type (payload type).
*/
public static final int PT_TL = 205;
/**
* Payload-specific type (payload type).
*/
public static final int PT_PS = 206;
/**
* Picture Loss Indication message type.
*/
public static final int FMT_PLI = 1;
/**
* Full Intra-frame Request message type.
*/
public static final int FMT_FIR = 4;
/**
* Feedback message type.
*/
private int feedbackMessageType = 0;
/**
* Payload type.
*/
private int payloadType = 0;
/**
* Constructor.
*
* @param src source
* @param feedbackMessageType FMT
* @param payloadType PT
*/
public RTCPFeedbackEvent(Object src, int feedbackMessageType,
int payloadType)
{
super(src);
this.feedbackMessageType = feedbackMessageType;
this.payloadType = payloadType;
}
/**
* Get feedback message type.
*
* @return message type
*/
public int getFeedbackMessageType()
{
return feedbackMessageType;
}
/**
* Get payload type of RTCP packet.
*
* @return payload type
*/
public int getPayloadType()
{
return payloadType;
}
}

@ -0,0 +1,25 @@
/*
* SIP Communicator, the OpenSource Java VoIP and Instant Messaging client.
*
* Distributable under LGPL license.
* See terms of license at gnu.org.
*/
package net.java.sip.communicator.service.neomedia.event;
/**
* <tt>RTCPFeedbackListener</tt> is used by codec to be notified
* by RTCP feedback event such as PLI (Picture Loss Indication) or
* FIR (Full Intra-frame Request).
*
* @author Sebastien Vincent
*/
public interface RTCPFeedbackListener
{
/**
* Event fired when RTCP feedback message is received.
*
* @param event <tt>RTCPFeedbackEvent</tt>
*/
public void feedbackReceived(RTCPFeedbackEvent event);
}

@ -102,6 +102,16 @@ public interface MediaFormat
*/
public Map<String, String> getFormatParameters();
/**
* Returns a <tt>Map</tt> containing advanced parameters specific to this
* particular <MediaFormat</tt>. The parameters returned here are meant for
* use in SIP/SDP or XMPP session descriptions.
*
* @return a <tt>Map</tt> containing advanced parameters specific to this
* particular <tt>MediaFormat</tt>
*/
public Map<String, String> getAdvancedParameters();
/**
* Returns a <tt>String</tt> representation of this <tt>MediaFormat</tt>
* containing important format attributes such as the encoding for example.

@ -131,7 +131,8 @@ public MediaFormat createMediaFormat(
public MediaFormat createMediaFormat(
String encoding,
double clockRate,
Map<String, String> formatParams);
Map<String, String> formatParams,
Map<String, String> advancedParams);
/**
* Creates a <tt>MediaFormat</tt> for the specified <tt>encoding</tt>,
@ -159,7 +160,8 @@ public MediaFormat createMediaFormat(
String encoding,
double clockRate,
int channels,
Map<String, String> formatParams);
Map<String, String> formatParams,
Map<String, String> advancedParams);
/**
* Creates a <tt>MediaFormat</tt> either for the specified
@ -197,5 +199,6 @@ public MediaFormat createMediaFormat(
String encoding,
double clockRate,
int channels,
Map<String, String> formatParams);
Map<String, String> formatParams,
Map<String, String> advancedParams);
}

Loading…
Cancel
Save