I'm trying to develop a module for Bonsai framework, it sends a stream of OpenCV.NET IplImage's to Max 6.1 via tcp stream connection via it's own TCP communication protocol / standard / module named jit.net.send and jit.net.recv
There is some documentation about jit.net.send and jit.net.recv on cycling74.com/sdk/MaxSDK-6.1.3/html/chapter_jit_networking.html
Some similar work done on C++, sending texture from OpenFrameworks to Max on github.com/bakercp/ofxJitterNetworkSender
And some 5 year old C# code sending texture from Max to Unity3D on disis.music.vt.edu/main/portfolio.php
What i'm after is sending a video stream (of OpenCV.NET IplImage's) from a C# app (under a module of the Bonsai framework) to Max.
My current sourcecode is at https://bitbucket.org/artica/bonsai-with-jitnetsend
I managed to port the C++ code and figure out the big endian issues (with some code from another stackoverflow thread). Now, when i run the app, Max flags that it is connected, and on the app side, when debugging, everything is being sent. But there is no data matrix image being displayed.
EDIT- managed to make some progress, sent some test jit.net.send packets between 2 machines and captured packets for analysis with wireshark. Here is a run-down of my analysis:
https://bitbucket.org/artica/bonsai-with-jitnetsend/wiki/Example%204x4
I rewrote the code to match the actual packet example and managed to get 4x4 images sending, but the image glitches up on 8x8 and starts throwing errors in higher resolutions. I'm guessing it's an IplImage format problem at this point. Here is the relevant code:
private string JIT_MATRIX_PACKET_ID = "JMTX";
public unsafe struct t_jit_net_packet_matrix {
public Int32 id;
public Int32 size;
public Int32 planecount;
public Int32 type; //0=char,1=long,2=float32,3=float64
public Int32 dimcount;
//public fixed Int32 dim[32];
//public fixed Int32 dimstride[32];
//public Int32 datasize;
//public double time; //64 bit double precision float
}
private t_jit_net_packet_matrix m_matrixHeader;
private void sendFrame(IplImage input) {
// setup matrix
Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
Int32 planecount = input.NumChannels;
Int32 typeSize = input.Depth;
Int32 type = JIT_MATRIX_TYPE_CHAR;
Int32 width = 16;// input.Width;
Int32 height = 16;// input.Height;
m_matrixHeader.id = matrix_id;
m_matrixHeader.size = 288;
m_matrixHeader.planecount = planecount;
m_matrixHeader.type = type;
m_matrixHeader.dimcount = 2;
Int32[] dim = new Int32[32];
dim[0] = width;
dim[1] = height;
int i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dim[i2] = 1;
i2++;
}
Int32[] dimstride = new Int32[32];
dimstride[0] = planecount;
dimstride[1] = width * height;
i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dimstride[i2] = 0;
i2++;
}
Int32 datasize = planecount * width * height;
m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
m_chunkHeader.size = sizeof(Int32) * (6 + 32 + 32) + sizeof(double); //should be 288 bytes
byte[] chunkHeader = StructToBytes(m_chunkHeader, Endianness.LittleEndian);
//Console.WriteLine(BitConverter.ToString(chunkHeader));
byte[] matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
//Console.WriteLine(BitConverter.ToString(matrixHeader));
byte[] dim_send = new byte[4 * 32];
byte[] dimstride_send = new byte[4 * 32];
for (int i = 0; i < 32; i++)
{
byte[] dimbytes = BitConverter.GetBytes(dim[i]);
Array.Reverse(dimbytes);
System.Buffer.BlockCopy(dimbytes, 0, dim_send, i * 4, dimbytes.Length);
byte[] dimstridebytes = BitConverter.GetBytes(dimstride[i]);
Array.Reverse(dimstridebytes);
System.Buffer.BlockCopy(dimstridebytes, 0, dimstride_send, i * 4, dimstridebytes.Length);
}
//Console.WriteLine(BitConverter.ToString(dim_send));
//Console.WriteLine(BitConverter.ToString(dimstride_send));
byte[] datasize_send = BitConverter.GetBytes(datasize);
Array.Reverse(datasize_send);
//Console.WriteLine(BitConverter.ToString(datasize_send));
double time = 0; //todo: should be elapsed time, not 0
byte[] time_send = BitConverter.GetBytes(time);
Array.Reverse(time_send);
//Console.WriteLine(BitConverter.ToString(time_send));
int size = width * height * 4 * 2;//input.Height * input.Width * input.NumChannels * input.Depth / 4;
byte[] managedArray = new byte[size];
Marshal.Copy(input.ImageData, managedArray, 0, size);
Array.Reverse(managedArray);
//Console.WriteLine(BitConverter.ToString(managedArray));
byte[] output = new byte[chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length + managedArray.Length];
// chunkheader
System.Buffer.BlockCopy(chunkHeader, 0, output, 0, chunkHeader.Length);
// matrixheader
System.Buffer.BlockCopy(matrixHeader, 0, output, chunkHeader.Length, matrixHeader.Length);
// dim
System.Buffer.BlockCopy(dim_send, 0, output, chunkHeader.Length + matrixHeader.Length, dim_send.Length);
// dimstride
System.Buffer.BlockCopy(dimstride_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length, dimstride_send.Length);
// datasize
System.Buffer.BlockCopy(datasize_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length, datasize_send.Length);
// time
System.Buffer.BlockCopy(time_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length, time_send.Length);
// matrix array
System.Buffer.BlockCopy(managedArray, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length, managedArray.Length);
Console.WriteLine(BitConverter.ToString(output));
if (myClient.Connected)
myStream.Write(output, 0, output.Length);
Console.WriteLine("looping");
//if (myClient.Connected)
// myStream.Write(managedArray, 0, managedArray.Length);
//Console.WriteLine(managedArray.ToString());
}
Managed to figure it out on my own.
private void sendFrame(IplImage input) {
try
{
// setup matrix
Int32 matrix_id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
Int32 planecount = input.NumChannels;
Int32 typeSize = input.Depth;
Int32 type = JIT_MATRIX_TYPE_CHAR;
Int32 width = input.Width;
Int32 height = input.Height;
m_matrixHeader.id = matrix_id;
m_matrixHeader.size = 288;
m_matrixHeader.planecount = planecount;
m_matrixHeader.type = type;
m_matrixHeader.dimcount = 2;
Int32[] dim = new Int32[32];
dim[0] = width;
dim[1] = height;
int i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dim[i2] = 1;
i2++;
}
Int32[] dimstride = new Int32[32];
dimstride[0] = planecount;
dimstride[1] = width * planecount;
i2 = 2;
while (i2 < JIT_MATRIX_MAX_DIMCOUNT)
{
dimstride[i2] = 0;
i2++;
}
Int32 datasize = planecount * width * height;
m_chunkHeader.id = BitConverter.ToInt32(Encoding.ASCII.GetBytes(JIT_MATRIX_PACKET_ID), 0);
m_chunkHeader.size = sizeof(Int32) * (6 + 32 + 32) + sizeof(double); //should be 288 bytes
byte[] chunkHeader = StructToBytes(m_chunkHeader, Endianness.LittleEndian);
//Console.WriteLine(BitConverter.ToString(chunkHeader));
byte[] matrixHeader = StructToBytes(m_matrixHeader, Endianness.BigEndian);
//Console.WriteLine(BitConverter.ToString(matrixHeader));
byte[] dim_send = new byte[4 * 32];
byte[] dimstride_send = new byte[4 * 32];
for (int i = 0; i < 32; i++)
{
byte[] dimbytes = BitConverter.GetBytes(dim[i]);
Array.Reverse(dimbytes);
System.Buffer.BlockCopy(dimbytes, 0, dim_send, i * 4, dimbytes.Length);
byte[] dimstridebytes = BitConverter.GetBytes(dimstride[i]);
Array.Reverse(dimstridebytes);
System.Buffer.BlockCopy(dimstridebytes, 0, dimstride_send, i * 4, dimstridebytes.Length);
}
//Console.WriteLine(BitConverter.ToString(dim_send));
//Console.WriteLine(BitConverter.ToString(dimstride_send));
byte[] datasize_send = BitConverter.GetBytes(datasize);
Array.Reverse(datasize_send);
//Console.WriteLine(BitConverter.ToString(datasize_send));
double time = 0; //todo: should be elapsed time, not 0
byte[] time_send = BitConverter.GetBytes(time);
Array.Reverse(time_send);
//Console.WriteLine(BitConverter.ToString(time_send));
int size = width * height * 4;//input.Height * input.Width * input.NumChannels * input.Depth / 4;
byte[] managedArray = new byte[size];
Marshal.Copy(input.ImageData, managedArray, 0, size);
Array.Reverse(managedArray);
byte[] output = new byte[chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length + managedArray.Length];
// chunkheader
System.Buffer.BlockCopy(chunkHeader, 0, output, 0, chunkHeader.Length);
// matrixheader
System.Buffer.BlockCopy(matrixHeader, 0, output, chunkHeader.Length, matrixHeader.Length);
// dim
System.Buffer.BlockCopy(dim_send, 0, output, chunkHeader.Length + matrixHeader.Length, dim_send.Length);
// dimstride
System.Buffer.BlockCopy(dimstride_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length, dimstride_send.Length);
// datasize
System.Buffer.BlockCopy(datasize_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length, datasize_send.Length);
// time
System.Buffer.BlockCopy(time_send, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length, time_send.Length);
// matrix array
System.Buffer.BlockCopy(managedArray, 0, output, chunkHeader.Length + matrixHeader.Length + dim_send.Length + dimstride_send.Length + datasize_send.Length + time_send.Length, managedArray.Length);
//Console.WriteLine(BitConverter.ToString(output));
if (myClient.Connected)
myStream.Write(output, 0, output.Length);
} catch ( Exception e )
{
Console.WriteLine("Exception: " + e.InnerException.Message);
}
}