I'm trying to transmit large video files from clients to a server using java NIO. I seem to require using NIO because the files I want to send are much larger than the apparent file size limit with regular IO, which is about 2GB...my video files have been as large as 50GB each. Right now, i'm just trying to build a small program to understand the concepts. it will later be added to a larger program.
My problem lies in the fact that only the first few hundred kilobytes of the file get saved on the server. Each time I run it, a different about of data gets saved on the server. Can anyone help me with a solution? (and any other suggestions you might have...NIO is new to me) THANKS!
The client will have a collection of files to send to the server. The client will establish a connection with the server, and the server will reply saying it's ready. The client sends the file header information. The server then says its ready to receive the file contents. The client then sends the contents of the file. When the file is fully transmitted, it repeats over with the next file until no more files need to be sent.
public static void main(String[] args) throws Throwable {
FileSender fileSender = new FileSender("localhost", 7146);
fileSender.addFileToSend(new File("C:\\url\\to\\file1.jpg"));
fileSender.addFileToSend(new File("C:\\url\\to\\file2.jpg"));
fileSender.addFileToSend(new File("C:\\url\\to\\file3.jpg"));
fileSender.sendFiles();
}
private static String serverAddress;
private static int port;
private static Charset charSet = Charset.forName(System.getProperty("file.encoding"));
private SocketChannel server = null;
private File file;
private RandomAccessFile aFile;
private FileChannel fileChannel;
private long filesize, transmittedSoFar;
private int current;
private ByteBuffer buffer = ByteBuffer.allocate(131072); //128k
private ByteBuffer responseBuffer;
private CharBuffer charBuffer;
private CharsetDecoder charDecoder = charSet.newDecoder();
private Selector selector;
private ArrayList<File> filesToSend = new ArrayList<>(0);
private int fileCountTracker = 0;
FileSender(String serverAddress, int port) {
FileSender.serverAddress = serverAddress;
FileSender.port = port;
}
public void sendFiles() {
try {
server = SocketChannel.open();
server.connect(new InetSocketAddress(serverAddress, port));
server.configureBlocking(false);
System.out.println("Connected to Server");
selector = Selector.open();
server.register(selector, SelectionKey.OP_READ);
waitForResponse();
} catch (Exception e) {
e.printStackTrace();
}
}
void waitForResponse() throws Exception {
//TODO: track time. abort loop after 10 sec? 30 sec?
while (true) {
System.out.println("waiting for a response from server");
selector.select();
Set<SelectionKey> readyKeys = selector.selectedKeys();
Iterator<SelectionKey> iterator = readyKeys.iterator();
while (iterator.hasNext()) {
SelectionKey key = (SelectionKey) iterator.next();
iterator.remove();
if (key.isReadable()) {
responseBuffer = ByteBuffer.allocate(16);
server.read(responseBuffer);
responseBuffer.flip();
try {
charBuffer = charDecoder.decode(responseBuffer);
responseBuffer.clear();
String response = charBuffer.toString();
System.out.println(response);
if (response.startsWith("[readyForHeader]")) {
System.out.println("Received response: ready for header");
sendHeader();
}
else if (response.startsWith("[readyForBody]")) {
System.out.println("Received response: ready for body");
sendData();
}
else {
System.out.println("unknown response");
System.out.println(response);
}
} catch(Exception e) {
System.out.println("error decoding file info");
System.out.println(e.getMessage());
return;
}
}
}
}
}
public void addFileToSend(File file) {
filesToSend.add(file);
}
void sendHeader() {
System.out.println("Tracker: "+fileCountTracker);
try {
if (filesToSend.size() > fileCountTracker) { //still more files to send
System.out.println("a file exists at this array index");
this.file = filesToSend.get(fileCountTracker);
filesize = file.length();
aFile = new RandomAccessFile(file, "r");
transmittedSoFar = 0;
//generate file info buffers to send to server
byte[] fileInfoBytes = getFileMeta(file);
ByteBuffer lengthBuffer = ByteBuffer.allocate(4); //length of file info
lengthBuffer.putInt(0, fileInfoBytes.length);
System.out.println("Source info length: "+fileInfoBytes.length);
ByteBuffer infoBuffer = ByteBuffer.wrap(fileInfoBytes); //file info data
//send file info buffers
sendByteBuffer(lengthBuffer);
sendByteBuffer(infoBuffer);
} else {
System.out.println("sending zero to indicate no more files");
ByteBuffer lengthBuffer = ByteBuffer.allocate(4); //length of file info
lengthBuffer.putInt(0, 0); //tell server sending zero bytes. server will end connection
sendByteBuffer(lengthBuffer);
terminate();
}
}
catch (Exception e) {
e.getMessage();
terminate();
}
}
void sendData() {
try {
fileChannel = aFile.getChannel();
while ((current = fileChannel.read(buffer)) > 0 || buffer.position() > 0) {
transmittedSoFar = transmittedSoFar + (long)current;
System.out.println(Math.round(transmittedSoFar*100/filesize)+" "+transmittedSoFar);
buffer.flip();
server.write(buffer);
buffer.compact();
}
System.out.println("End of file reached..");
aFile.close();
} catch (FileNotFoundException e) {
System.out.println("FILE NOT FOUND EXCEPTION");
e.getMessage();
} catch (IOException e) {
System.out.println("IO EXCEPTION");
e.getMessage();
}
fileCountTracker++;
}
byte[] getFileMeta(File file) throws IOException {
StringBuffer fileInfo = new StringBuffer();
BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
fileInfo.append(file.getName() + "\n");
fileInfo.append(file.length() + "\n");
fileInfo.append(attr.creationTime() + "\n");
byte[] infoBytes = fileInfo.toString().getBytes();
return infoBytes;
}
void sendByteBuffer(ByteBuffer bb) throws IOException {
System.out.println("sending: "+bb.toString());
server.write(bb);
bb.rewind();
}
void terminate() {
try {
server.close();
System.out.println("Connection closed");
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Throwable {
FileReceiver fileReceiver = new FileReceiver(7146);
fileReceiver.initReceive();
}
static Charset charSet = Charset.forName(System.getProperty("file.encoding"));
static final Pattern pattern = Pattern.compile("[\n]");//new line
static int port;
static BytesTypeToReceive bytesType;
ServerSocketChannel server;
SocketChannel client;
ByteBuffer byteBuffer, responseBuffer;
CharBuffer charBuffer;
CharsetDecoder charDecoder = charSet.newDecoder();
RandomAccessFile aFile = null;
String fileInfo[];
int headerLength;
long remaining;
Selector selector;
public FileReceiver(int port) {
FileReceiver.port = port;
}
public void initReceive() {
try {
server = ServerSocketChannel.open();
server.configureBlocking(false);
server.socket().bind(new InetSocketAddress(port));
selector = Selector.open();
server.register(selector, SelectionKey.OP_ACCEPT);
waitForResponse();
} catch (Exception e) {
close();
e.printStackTrace();
}
}
void waitForResponse() throws Exception {
while (true) {
System.out.println("Waiting for data from client");
int selCount = selector.select();
System.out.println("selector count: "+selCount);
Set<SelectionKey> readyKeys = selector.selectedKeys();
Iterator<SelectionKey> iterator = readyKeys.iterator();
while (iterator.hasNext()) {
SelectionKey key = (SelectionKey) iterator.next();
iterator.remove();
if (key.isReadable()) {
if (bytesType == BytesTypeToReceive.HEADER) {
receiveHeader();
} else {
receiveBody();
}
} else if (key.isAcceptable()) {
client = server.accept();
System.out.println("Connection established...." + client.getRemoteAddress());
client.configureBlocking(false);
bytesType = BytesTypeToReceive.HEADER;
client.register(selector, SelectionKey.OP_READ);
sendResponse("[readyForHeader]");
}
}
Thread.sleep(250);
}
}
private void receiveHeader() {
System.out.println("Receiving header data");
byteBuffer = ByteBuffer.allocate(4);
try {
//read length
while (byteBuffer.remaining() > 0) client.read(byteBuffer);
System.out.println("what is this? "+byteBuffer.toString());
byteBuffer.rewind();
System.out.println("and this? "+byteBuffer.toString());
System.out.println("Info length is " + byteBuffer.getInt(0));
if (byteBuffer.getInt(0) == 0) {
System.out.println("no more files. end connection");
throw new IOException();
}
//resize to size indicated in first buffer
byteBuffer = ByteBuffer.allocate(byteBuffer.getInt(0));
//read file info
while (byteBuffer.remaining() > 0) client.read(byteBuffer);
byteBuffer.flip();
//decode file info
try {
charBuffer = charDecoder.decode(byteBuffer);
byteBuffer.clear();
System.out.println(charBuffer.toString());
} catch(Exception e) {
System.out.println("error decoding file info");
return;
}
fileInfo = pattern.split(charBuffer);
System.out.println("info0: "+fileInfo[0]);
System.out.println("info1: "+fileInfo[1]);
remaining = Long.parseLong(fileInfo[1]);
bytesType = BytesTypeToReceive.BODY;
//tell client ready for file data
sendResponse("[readyForBody]");
} catch (Exception e) {
System.out.println("Exception for checkForData. No more data?");
System.out.println(e.getMessage());
}
}
/**
* Reads the bytes from socket and writes to file
*
* @param socketChannel
*/
//private void readFileFromSocket(SocketChannel socketChannel, int infoLength) {
private void receiveBody() throws Exception {
int current;
System.out.println("About to receive "+remaining+" bytes.");
try {
//read file data
aFile = new RandomAccessFile("C:\\folder\\to\\save\\to\\"+fileInfo[0], "rw");
byteBuffer = ByteBuffer.allocate(131072);
FileChannel fileChannel = aFile.getChannel();
while (((current = client.read(byteBuffer)) > 0 || byteBuffer.position() > 0) && remaining > 0) {
remaining = remaining - (long)current;
System.out.println(current+" "+remaining);
byteBuffer.flip();
fileChannel.write(byteBuffer);
byteBuffer.compact();
}
fileChannel.close();
aFile.close();
System.out.println(current +" - End of file");
bytesType = BytesTypeToReceive.HEADER;
sendResponse("[readyForHeader]");
} catch (FileNotFoundException e) {
System.out.println("FILE NOT FOUND EXCEPTION");
e.getMessage();
} catch (IOException e) {
System.out.println("IO EXCEPTION");
e.getMessage();
} catch (InterruptedException e) {
System.out.println("INTERRUPTED EXCEPTION");
e.getMessage();
}
}
void sendResponse(String response) throws Exception {
System.out.println("Sending response: "+response);
byte[] data = response.getBytes("UTF-8");
responseBuffer = ByteBuffer.wrap(data);
client.write(responseBuffer);
responseBuffer.rewind();
}
public void close() {
try {
client.close();
server.close();
System.out.println("connection closed");
} catch (IOException e) {
e.printStackTrace();
}
}
This could be the problem:
while (((current = client.read(byteBuffer)) > 0
As your socket is configured non blocking and you don't have any selects on the input it'll quickly consume incoming data and stop as read will return -1 or 0.
Actually client hs similar problem but it'll just burn CPU while trying to push data to overloaded socket.