sipwise
/
jitsi
mirror of https://github.com/sipwise/jitsi.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Attempts to use SwScaler for the scaling performed by the Player of remote video in a call.

Browse Source
cusax-fix
Lyubomir Marinov 16 years ago
parent 8b53d6d30f
commit 7f3bc17399
3 changed files with 459 additions and 102 deletions
Show Stats Download Patch File Download Diff File

64
src/net/java/sip/communicator/impl/neomedia/codec/video/SwScaler.java
Unescape View File

@ -62,6 +62,28 @@ public class SwScaler
new RGBFormat(null, -1, Format.shortArray, -1.0f, 24, -1, -1, -1),
};
/**
* Sets the <tt>Format</tt> in which this <tt>Codec</tt> is to output media
* data.
*
* @param format the <tt>Format</tt> in which this <tt>Codec</tt> is to
* output media data
* @return the <tt>Format</tt> in which this <tt>Codec</tt> is currently
* configured to output media data or <tt>null</tt> if <tt>format</tt> was
* found to be incompatible with this <tt>Codec</tt>
*/
@Override
public Format setOutputFormat(Format format)
{
Format outputFormat = super.setOutputFormat(format);
if (logger.isDebugEnabled() && (outputFormat != null))
logger.debug(
"SwScaler set to output with size "
+ ((VideoFormat) outputFormat).getSize());
return outputFormat;
}
/**
* Sets output size.
*
@ -90,6 +112,48 @@ public void setOutputSize(Dimension size)
-1.0f, 24, -1, -1, -1);
supportedOutputFormats[8] = new RGBFormat(size, -1, Format.shortArray,
-1.0f, 24, -1, -1, -1);
// Set the size to the outputFormat as well.
VideoFormat outputFormat = (VideoFormat) this.outputFormat;
if (outputFormat != null)
setOutputFormat(
new VideoFormat(
outputFormat.getEncoding(),
size,
outputFormat.getMaxDataLength(),
outputFormat.getDataType(),
outputFormat.getFrameRate())
.intersects(outputFormat));
}
/**
* Gets the <tt>Format</tt> in which this <tt>Codec</tt> is currently
* configured to accept input media data.
* <p>
* Make the protected super implementation public.
* </p>
*
* @return the <tt>Format</tt> in which this <tt>Codec</tt> is currently
* configured to accept input media data
* @see AbstractCodec#getInputFormat()
*/
@Override
public Format getInputFormat()
{
return super.getInputFormat();
}
public Dimension getOutputSize()
{
Format outputFormat = getOutputFormat();
if (outputFormat == null)
{
// They all have one and the same size.
outputFormat = supportedOutputFormats[0];
}
return ((VideoFormat) outputFormat).getSize();
}
/**

69
src/net/java/sip/communicator/impl/neomedia/device/MediaDeviceSession.java
Unescape View File

@ -6,7 +6,7 @@
*/
package net.java.sip.communicator.impl.neomedia.device;
import java.awt.Dimension;
import java.awt.Dimension; // disambiguation
import java.io.*;
import java.util.*;
@ -19,7 +19,6 @@
import net.java.sip.communicator.impl.neomedia.*;
import net.java.sip.communicator.impl.neomedia.format.*;
import net.java.sip.communicator.impl.neomedia.protocol.*;
import net.java.sip.communicator.impl.neomedia.codec.video.*;
import net.java.sip.communicator.service.neomedia.*;
import net.java.sip.communicator.service.neomedia.device.*;
import net.java.sip.communicator.service.neomedia.format.*;
@ -1085,7 +1084,7 @@ protected void processorControllerUpdate(ControllerEvent event)
}
if (format != null)
setFormat(processor, format);
setProcessorFormat(processor, format);
}
}
else if (event instanceof ControllerClosedEvent)
@ -1196,6 +1195,7 @@ public void setFormat(MediaFormat format)
* We need javax.media.Format and we know how to convert MediaFormat to
* it only for MediaFormatImpl so assert early.
*/
@SuppressWarnings("unchecked")
MediaFormatImpl<? extends Format> mediaFormatImpl
= (MediaFormatImpl<? extends Format>) format;
@ -1211,7 +1211,7 @@ public void setFormat(MediaFormat format)
int processorState = processor.getState();
if (processorState == Processor.Configured)
setFormat(processor, this.format);
setProcessorFormat(processor, this.format);
else if (processorIsPrematurelyClosed
|| ((processorState > Processor.Configured)
&& !format.equals(getFormat())))
@ -1220,14 +1220,14 @@ else if (processorIsPrematurelyClosed
}
/**
* Sets the JMF <tt>Format</tt> in which a specific <tt>Processor</tt> is to
* output media data.
* Sets the JMF <tt>Format</tt> in which a specific <tt>Processor</tt>
* producing media to be streamed to the remote peer is to output.
*
* @param processor the <tt>Processor</tt> to set the output <tt>Format</tt>
* of
* @param format the JMF <tt>Format</tt> to set to <tt>processor</tt>
*/
protected void setFormat(Processor processor, Format format)
protected void setProcessorFormat(Processor processor, Format format)
{
TrackControl[] trackControls = processor.getTrackControls();
MediaType mediaType = getMediaType();
@ -1277,35 +1277,6 @@ protected void setFormat(Processor processor, Format format)
case VIDEO:
if (supportedFormats[0] instanceof VideoFormat)
{
VideoFormat tmp = (VideoFormat)format;
Dimension size = tmp.getSize();
if(size != null)
{
/* We have been explictely told to use
* a specified output size so create a
* custom SwScaler that will rescale and
* change format in one call
*/
Codec ar[] = new Codec[1];
SwScaler scaler = new SwScaler();
scaler.setOutputSize(size);
ar[0] = scaler;
/* add our custom SwScaler to the codec chain so that
* it will be used instead of default SwScaler
*/
try
{
trackControl.setCodecChain(ar);
}
catch(Exception e)
{
System.out.println("Error setCodecChain: " + e);
}
}
supportedFormat
= findFirstMatchingFormat(supportedFormats, format);
@ -1328,7 +1299,8 @@ protected void setFormat(Processor processor, Format format)
trackControl.setEnabled(false);
else if (!supportedFormat.equals(trackControl.getFormat()))
{
Format setFormat = trackControl.setFormat(supportedFormat);
Format setFormat
= setProcessorFormat(trackControl, supportedFormat);
if (setFormat == null)
logger
@ -1361,6 +1333,29 @@ else if (logger.isTraceEnabled())
}
}
/**
* Sets the JMF <tt>Format</tt> of a specific <tt>TrackControl</tt> of the
* <tt>Processor</tt> which produces the media to be streamed by this
* <tt>MediaDeviceSession</tt> to the remote peer. Allows extenders to
* override the set procedure and to detect when the JMF <tt>Format</tt> of
* the specified <tt>TrackControl</tt> changes.
*
* @param trackControl the <tt>TrackControl</tt> to set the JMF
* <tt>Format</tt> of
* @param format the JMF <tt>Format</tt> to be set on the specified
* <tt>TrackControl</tt>
* @return the JMF <tt>Format</tt> set on <tt>TrackControl</tt> after the
* attempt to set the specified <tt>format</tt> or <tt>null</tt> if the
* specified <tt>format</tt> was found to be incompatible with
* <tt>trackControl</tt>
*/
protected Format setProcessorFormat(
TrackControl trackControl,
Format format)
{
return trackControl.setFormat(format);
}
/**
* Sets the indicator which determines whether this
* <tt>MediaDeviceSession</tt> is set to output "silence" instead of the

428
src/net/java/sip/communicator/impl/neomedia/device/VideoMediaDeviceSession.java
Unescape View File

@ -7,6 +7,7 @@
package net.java.sip.communicator.impl.neomedia.device;
import java.awt.*;
import java.awt.event.*;
import javax.media.*;
import javax.media.format.*;
@ -15,6 +16,7 @@
import javax.swing.*;
import net.java.sip.communicator.impl.neomedia.*;
import net.java.sip.communicator.impl.neomedia.codec.video.*;
import net.java.sip.communicator.impl.neomedia.imgstreaming.*;
import net.java.sip.communicator.service.neomedia.*;
import net.java.sip.communicator.service.neomedia.event.*;
@ -46,13 +48,6 @@ public class VideoMediaDeviceSession
private static final String DESKTOP_STREAMING_ICON
= "impl.media.DESKTOP_STREAMING_ICON";
/**
* The facility which aids this instance in managing a list of
* <tt>VideoListener</tt>s and firing <tt>VideoEvent</tt>s to them.
*/
private final VideoNotifierSupport videoNotifierSupport
= new VideoNotifierSupport(this);
/**
* Local <tt>Player</tt> for the local video.
*/
@ -68,6 +63,20 @@ public class VideoMediaDeviceSession
*/
private Dimension outputSize = null;
/**
* The <tt>SwScaler</tt> inserted into the codec chain of the
* <tt>Player</tt> rendering the media received from the remote peer and
* enabling the explicit setting of the video size.
*/
private SwScaler playerScaler;
/**
* The facility which aids this instance in managing a list of
* <tt>VideoListener</tt>s and firing <tt>VideoEvent</tt>s to them.
*/
private final VideoNotifierSupport videoNotifierSupport
= new VideoNotifierSupport(this);
/**
* Initializes a new <tt>VideoMediaDeviceSession</tt> instance which is to
* represent the work of a <tt>MediaStream</tt> with a specific video
@ -82,16 +91,6 @@ public VideoMediaDeviceSession(AbstractMediaDevice device)
super(device);
}
/**
* Set output size of video.
*
* @param size output size
*/
public void setOutputSize(Dimension size)
{
outputSize = size;
}
/**
* Adds a specific <tt>VideoListener</tt> to this instance in order to
* receive notifications when visual/video <tt>Component</tt>s are being
@ -179,33 +178,6 @@ protected DataSource createCaptureDevice()
return captureDevice;
}
/**
* Sets the JMF <tt>Format</tt> in which a specific <tt>Processor</tt> is to
* output media data.
*
* @param processor the <tt>Processor</tt> to set the output <tt>Format</tt>
* of
* @param format the JMF <tt>Format</tt> to set to <tt>processor</tt>
*/
@Override
protected void setFormat(Processor processor, Format format)
{
Format newFormat = null;
VideoFormat tmp = (VideoFormat)format;
/* add a size in the output format, as VideoFormat has no
* set accessors, we recreate the object
*/
if(outputSize != null)
{
newFormat = new VideoFormat(tmp.getEncoding(), outputSize,
tmp.getMaxDataLength(), tmp.getDataType(),
tmp.getFrameRate());
}
super.setFormat(processor, newFormat != null ? newFormat : format);
}
/**
* Asserts that a specific <tt>MediaDevice</tt> is acceptable to be set as
* the <tt>MediaDevice</tt> of this instance. Makes sure that its
@ -362,19 +334,25 @@ private void controllerUpdateForCreateLocalVisualComponent(
Player player = (Player) controllerEvent.getSourceController();
Component visualComponent = player.getVisualComponent();
if ((visualComponent != null)
&& !fireVideoEvent(
VideoEvent.VIDEO_ADDED,
visualComponent,
VideoEvent.LOCAL))
if (visualComponent != null)
{
// No listener interrested in our event so free resources.
if(localPlayer == player)
localPlayer = null;
player.stop();
player.deallocate();
player.close();
if (fireVideoEvent(
VideoEvent.VIDEO_ADDED,
visualComponent,
VideoEvent.LOCAL))
{
localVisualComponentConsumed(visualComponent, player);
}
else
{
// No listener interested in our event so free resources.
if(localPlayer == player)
localPlayer = null;
player.stop();
player.deallocate();
player.close();
}
}
}
}
@ -599,6 +577,102 @@ private static Component getVisualComponent(Player player)
return visualComponent;
}
/**
* Notifies this <tt>VideoMediaDeviceSession</tt> that a specific visual
* <tt>Component</tt> which depicts video streaming from the local peer to
* the remote peer and which has been created by a specific <tt>Player</tt>
* has been delivered to the registered <tt>VideoListener</tt>s and at least
* one of them has consumed it.
*
* @param visualComponent the visual <tt>Component</tt> depicting local
* video which has been consumed by the registered <tt>VideoListener</tt>s
* @param player the local <tt>Player</tt> which has created the specified
* visual <tt>Component</tt>
*/
private void localVisualComponentConsumed(
Component visualComponent,
Player player)
{
}
/**
* Notifies this instance that a specific <tt>Player</tt> of remote content
* has generated a <tt>ConfigureCompleteEvent</tt>.
*
* @param player the <tt>Player</tt> which is the source of a
* <tt>ConfigureCompleteEvent</tt>
* @see MediaDeviceSession#playerConfigureComplete(Processor)
*/
@Override
protected void playerConfigureComplete(final Processor player)
{
super.playerConfigureComplete(player);
TrackControl[] trackControls = player.getTrackControls();
SwScaler playerScaler = null;
if ((trackControls != null) && (trackControls.length != 0))
{
try
{
for (TrackControl trackControl : trackControls)
{
/*
* Since SwScaler will scale any input size into the
* configured output size, we may never get SizeChangeEvent
* from the player. We'll generate it ourselves then.
*/
playerScaler = new SwScaler()
{
/**
* The last size reported in the form of a
* SizeChangeEvent.
*/
private Dimension lastSize;
@Override
public int process(Buffer input, Buffer output)
{
int result = super.process(input, output);
if (result == BUFFER_PROCESSED_OK)
{
Format inputFormat = input.getFormat();
if (inputFormat != null)
{
Dimension inputSize
= ((VideoFormat) inputFormat).getSize();
if ((inputSize != null)
&& ((lastSize == null)
|| !lastSize
.equals(inputSize)))
{
lastSize = inputSize;
playerSizeChange(
player,
lastSize.width,
lastSize.height);
}
}
}
return result;
}
};
trackControl.setCodecChain(new Codec[] { playerScaler });
break;
}
}
catch (UnsupportedPlugInException upiex)
{
logger.error("Failed to add SwScaler to codec chain", upiex);
playerScaler = null;
}
}
this.playerScaler = playerScaler;
}
/**
* Gets notified about <tt>ControllerEvent</tt>s generated by a specific
* <tt>Player</tt> of remote content.
@ -614,7 +688,14 @@ protected void playerControllerUpdate(ControllerEvent event)
super.playerControllerUpdate(event);
if (event instanceof SizeChangeEvent)
playerSizeChange((SizeChangeEvent) event);
{
SizeChangeEvent sizeChangeEvent = (SizeChangeEvent) event;
playerSizeChange(
sizeChangeEvent.getSourceController(),
sizeChangeEvent.getWidth(),
sizeChangeEvent.getHeight());
}
}
/**
@ -626,31 +707,68 @@ protected void playerControllerUpdate(ControllerEvent event)
* @see MediaDeviceSession#playerRealizeComplete(Processor)
*/
@Override
protected void playerRealizeComplete(Processor player)
protected void playerRealizeComplete(final Processor player)
{
super.playerRealizeComplete(player);
Component visualComponent = getVisualComponent(player);
if (visualComponent != null)
{
/*
* SwScaler seems to be very good at scaling with respect to image
* quality so use it for the scaling in the player replacing the
* scaling it does upon rendering.
*/
visualComponent.addComponentListener(new ComponentAdapter()
{
@Override
public void componentResized(ComponentEvent e)
{
playerVisualComponentResized(player, e);
}
});
fireVideoEvent(
VideoEvent.VIDEO_ADDED,
visualComponent,
VideoEvent.REMOTE);
}
}
/**
* Notifies this instance that a specific <tt>Player</tt> of remote content
* has generated a <tt>SizeChangeEvent</tt>.
*
* @param event the <tt>SizeChangeEvent</tt> specifying the <tt>Player</tt>
* which is the source of the event and the additional details related to
* the event
* @param sourceController the <tt>Player</tt> which is the source of the
* event
* @param width the width reported in the event
* @param height the height reported in the event
* @see SizeChangeEvent
*/
protected void playerSizeChange(SizeChangeEvent event)
protected void playerSizeChange(
final Controller sourceController,
final int width,
final int height)
{
Player player = (Player) event.getSourceController();
/*
* Invoking anything that is likely to change the UI in the Player
* thread seems like a performance hit so bring it into the event
* thread.
*/
if (!SwingUtilities.isEventDispatchThread())
{
SwingUtilities.invokeLater(new Runnable()
{
public void run()
{
playerSizeChange(sourceController, width, height);
}
});
return;
}
Player player = (Player) sourceController;
Component visualComponent = getVisualComponent(player);
if (visualComponent != null)
@ -659,8 +777,97 @@ protected void playerSizeChange(SizeChangeEvent event)
this,
visualComponent,
SizeChangeVideoEvent.REMOTE,
event.getWidth(),
event.getHeight()));
width,
height));
}
/**
* Notifies this instance that the visual <tt>Component</tt> of a
* <tt>Player</tt> rendering remote content has been resized.
*
* @param player the <tt>Player</tt> rendering remote content the visual
* <tt>Component</tt> of which has been resized
* @param e a <tt>ComponentEvent</tt> which specifies the resized
* <tt>Component</tt>
*/
private void playerVisualComponentResized(
Processor player,
ComponentEvent e)
{
if (playerScaler == null)
return;
Component visualComponent = e.getComponent();
/*
* When the visualComponent is not in an UI hierarchy, its size isn't
* expected to be representative of what the user is seeing.
*/
if (visualComponent.getParent() == null)
return;
Dimension outputSize = visualComponent.getSize();
float outputWidth = outputSize.width;
float outputHeight = outputSize.height;
if ((outputWidth < 1) || (outputHeight < 1))
return;
/*
* The size of the output video will be calculated so that it fits into
* the visualComponent and the video aspect ratio is preserved. The
* presumption here is that the inputFormat holds the video size with
* the correct aspect ratio.
*/
Format inputFormat = playerScaler.getInputFormat();
if (inputFormat == null)
return;
Dimension inputSize = ((VideoFormat) inputFormat).getSize();
if (inputSize == null)
return;
int inputWidth = inputSize.width;
int inputHeight = inputSize.height;
if ((inputWidth < 1) || (inputHeight < 1))
return;
// Preserve the aspect ratio.
outputHeight = outputWidth * inputHeight / (float) inputWidth;
// Fit the output video into the visualComponent.
boolean scale = false;
float widthRatio;
float heightRatio;
if (Math.abs(outputWidth - inputWidth) < 1)
{
scale = true;
widthRatio = outputWidth / (float) inputWidth;
}
else
widthRatio = 1;
if (Math.abs(outputHeight - inputHeight) < 1)
{
scale = true;
heightRatio = outputHeight / (float) inputHeight;
}
else
heightRatio = 1;
if (scale)
{
float scaleFactor = Math.min(widthRatio, heightRatio);
outputWidth = inputWidth * scaleFactor;
outputHeight = inputHeight * scaleFactor;
}
outputSize.width = (int) outputWidth;
outputSize.height = (int) outputHeight;
playerScaler.setOutputSize(outputSize);
}
/**
@ -676,4 +883,95 @@ public void removeVideoListener(VideoListener listener)
{
videoNotifierSupport.removeVideoListener(listener);
}
/**
* Sets the size of the output video.
*
* @param size the size of the output video
*/
public void setOutputSize(Dimension size)
{
outputSize = size;
}
/**
* Sets the JMF <tt>Format</tt> in which a specific <tt>Processor</tt>
* producing media to be streamed to the remote peer is to output.
*
* @param processor the <tt>Processor</tt> to set the output <tt>Format</tt>
* of
* @param format the JMF <tt>Format</tt> to set to <tt>processor</tt>
* @see MediaDeviceSession#setProcessorFormat(Processor, Format)
*/
@Override
protected void setProcessorFormat(Processor processor, Format format)
{
Format newFormat = null;
VideoFormat tmp = (VideoFormat)format;
/* Add a size in the output format. As VideoFormat has no setter, we
* recreate the object.
*/
if(outputSize != null)
{
newFormat = new VideoFormat(tmp.getEncoding(), outputSize,
tmp.getMaxDataLength(), tmp.getDataType(),
tmp.getFrameRate());
}
super.setProcessorFormat(
processor,
newFormat != null ? newFormat : format);
}
/**
* Sets the JMF <tt>Format</tt> of a specific <tt>TrackControl</tt> of the
* <tt>Processor</tt> which produces the media to be streamed by this
* <tt>MediaDeviceSession</tt> to the remote peer. Allows extenders to
* override the set procedure and to detect when the JMF <tt>Format</tt> of
* the specified <tt>TrackControl</tt> changes.
*
* @param trackControl the <tt>TrackControl</tt> to set the JMF
* <tt>Format</tt> of
* @param format the JMF <tt>Format</tt> to be set on the specified
* <tt>TrackControl</tt>
* @return the JMF <tt>Format</tt> set on <tt>TrackControl</tt> after the
* attempt to set the specified <tt>format</tt> or <tt>null</tt> if the
* specified <tt>format</tt> was found to be incompatible with
* <tt>trackControl</tt>
* @see MediaDeviceSession#setProcessorFormat(TrackControl, Format)
*/
@Override
protected Format setProcessorFormat(
TrackControl trackControl,
Format format)
{
VideoFormat videoFormat = (VideoFormat) format;
Dimension size = videoFormat.getSize();
if(size != null)
{
/* We have been explicitly told to use a specified output size so
* create a custom SwScaler that will scale and convert color spaces
* in one call.
*/
SwScaler scaler = new SwScaler();
scaler.setOutputSize(size);
/* Add our custom SwScaler to the codec chain so that it will be
* used instead of default.
*/
try
{
trackControl.setCodecChain(new Codec[] { scaler });
}
catch(UnsupportedPlugInException upiex)
{
logger.error("Failed to add SwScaler to codec chain", upiex);
}
}
return super.setProcessorFormat(trackControl, format);
}
}

Write Preview
Loading…
Cancel
Save