Thanks for reading and sorry for my bad English in advance.
I'm working on Webcam Streaming with OpenCV. My final goal is making a Skype-like application. so I'm trying basic 1:1 TCP model first.
About the TCP 1:1 model,
After connection, client send its real-time Webcam frames and server receive and display it in on its jpanel.
I did receiving a picture and displaying it on jpanel so far.
I'm trying to receive successive frames.
At first, the problem was Server side socket seems like waiting until inputs from client are finished, i.e, it never stops because real-time frames are continuously sent.
So I sent every frame size before sending the frame to escape from unstoppable waiting.
But it doesn't work well. Client keeps sending frames, but server doesn't receive it well.
For example, if client send around 25k byte sized frames, the server only receive 1 to 3 bytes per one read even the buffer size is 512.
ClientThread.java
package client;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import javax.imageio.ImageIO;
import video.VideoCap;
public class ClientThread extends Thread
{
String serverIp;
int serverPort;
Socket socket;
VideoCap videoCap;
public ClientThread(Socket socket, String serverIp, int serverPort, VideoCap videoCap)
{
this.socket = socket;
this.serverIp = serverIp;
this.serverPort = serverPort;
this.videoCap = videoCap;
}
public void run()
{
while (ClientUI.calling)
{
try
{
InputStream in = socket.getInputStream();
DataInputStream dis = new DataInputStream(in);
OutputStream out = socket.getOutputStream();
DataOutputStream dos = new DataOutputStream(out);
// receive
int bufSize = dis.readInt();
while (ClientUI.calling)
{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(videoCap.getOneFrame(), "jpg", baos);
InputStream inputImage = new ByteArrayInputStream(baos.toByteArray());
// frame size
dos.writeInt(baos.size());
out(inputImage, baos, bufSize);
Thread.sleep(5000);
}
}
catch (IOException | InterruptedException e)
{
e.printStackTrace();
}
}
}
void out(InputStream in, OutputStream out, int bufSize)
{
long size = 0;
try
{
byte[] buf = new byte[bufSize];
int n;
while ((n = in.read(buf)) > 0)
{
out.write(buf, 0, n);
size += n;
System.out.println("size: " + size);
}
}
catch (IOException e)
{
e.printStackTrace();
}
finally
{
System.out.println(getClass().getName() + " :: out >>> sent size: " + size);
}
}
}
ServerThread.java
package server;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
import javax.imageio.ImageIO;
import javax.swing.JPanel;
public class ServerThread extends Thread
{
ServerSocket serverSocket;
Socket socket;
JPanel panel;
byte[] buf;
public ServerThread(ServerSocket serverSocket, JPanel panel, int bufSize)
{
this.serverSocket = serverSocket;
this.panel = panel;
buf = new byte[bufSize];
}
public void run()
{
try
{
System.out.println("waiting for client");
socket = serverSocket.accept();
System.out.println("client accepted");
InputStream in = socket.getInputStream();
DataInputStream dis = new DataInputStream(in);
OutputStream out = socket.getOutputStream();
DataOutputStream dos = new DataOutputStream(out);
dos.writeInt(buf.length);
while (ServerUI.calling)
{
int frameSize = dis.readInt();
ByteArrayOutputStream outImage = new ByteArrayOutputStream();
long size = 0;
int n;
while (frameSize >= size)
{
n = dis.read(buf);
if (n == -1)
break;
outImage.write(buf, 0, n);
size += n;
System.out.println(n);
}
InputStream inputImage = new ByteArrayInputStream(outImage.toByteArray());
BufferedImage bufferedImage = ImageIO.read(inputImage);
panel.getGraphics().drawImage(bufferedImage, 0, 0, null);
}
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
I changed DataOutput/InputStream to ObjectOutput/InputStream. I'm not sure why it did not go well, but I guess this is because of serializing problem. but byte is not necessary to be serialized so I don't know exactly.
I'll provide with anyhow working codes. Because of AudioServer I divided into two Thread so the previous codes and the codes below are pretty different.
VideoServerThread.java
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.net.ServerSocket;
import java.net.Socket;
import javax.imageio.ImageIO;
import javax.swing.JPanel;
public class VideoServerThread extends Thread
{
private ServerSocket serverSocket;
int videoServerPort;
private Socket socket;
private JPanel panel;
private boolean calling;
public VideoServerThread(ServerSocket serverSocket, int videoServerPort, JPanel panel, boolean calling)
{
this.serverSocket = serverSocket;
this.videoServerPort = videoServerPort;
this.panel = panel;
this.calling = calling;
}
@Override
public void run()
{
System.out.println("Video Server opened!");
try
{
serverSocket = new ServerSocket(videoServerPort);
socket = serverSocket.accept();
InputStream in = socket.getInputStream();
ObjectInputStream ois = new ObjectInputStream(in);
BufferedImage bufferedImage;
InputStream inputImage;
Frame f;
while (calling)
{
f = (Frame) ois.readObject();
inputImage = new ByteArrayInputStream(f.bytes);
bufferedImage = ImageIO.read(inputImage);
panel.getGraphics().drawImage(bufferedImage, 0, 0, panel.getWidth(), panel.getHeight(), null);
panel.getGraphics().drawImage(bufferedImage, 0, 0, null);
bufferedImage.flush();
inputImage.close();
f = null;
}
}
catch (IOException e)
{
e.printStackTrace();
}
catch (ClassNotFoundException e)
{
e.printStackTrace();
}
}
class Frame implements Serializable
{
public byte[] bytes;
public Frame(byte[] bytes)
{
this.bytes = bytes;
}
public int size()
{
return bytes.length;
}
}
}
VideoClientThread.java
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.net.Socket;
import javax.imageio.ImageIO;
import common.Frame;
import video.VideoCap;
public class VideoClientThread extends Thread
{
private final String formatType = "jpg";
private VideoCap videoCap;
private Socket socket;
private String ip;
private int port;
private boolean calling;
public VideoClientThread(VideoCap videoCap, Socket socket, String ip, int port, boolean calling)
{
this.videoCap = videoCap;
this.socket = socket;
this.ip = ip;
this.port = port;
this.calling = calling;
}
public void run()
{
try
{
socket = new Socket(ip, port);
socket.setSoTimeout(5000);
ObjectOutputStream oos = new ObjectOutputStream(socket.getOutputStream());
Frame f;
BufferedImage bufferedImage;
while (calling)
{
ByteArrayOutputStream fbaos = new ByteArrayOutputStream();
bufferedImage = videoCap.getOneFrame();
ImageIO.write(bufferedImage, formatType, fbaos);
f = new Frame(fbaos.toByteArray());
oos.writeObject(f);
oos.flush();
bufferedImage.flush();
// Thread.sleep(33);
}
}
catch (IOException e)
{
e.printStackTrace();
}
// catch (InterruptedException e)
// {
// e.printStackTrace();
// }
}
}