Search code examples
c++rustredisutf-8

Encoded image sent over redis from Cpp app to Rust app, Invalid UTF-8- TypeError


I'm sending an encoded cv image over redis pubsub from Cpp app:

cv::Mat output_image;
output_image = cv::Mat(pic_height, pic_width * 2, CV_8UC3);
cv::imwrite(filename, output_image);
std::vector<uchar> encoded_image;
cv::imencode(".png", output_image, encoded_image);
std::string image_str(encoded_image.begin(), encoded_image.end());
string image_msg;
image_msg = to_string(pic_id) + "~" + metadata + "~" + image_str;
RedisCommunicator::GetInstance()->Publish(image_channel, image_msg);

I'm receiving the image in Rust app using redis-rs:

let mut redis_pubsub = redis_connection.as_pubsub();

redis_pubsub.subscribe(image_channel).expect("Failed to subscribe to channel");
info!("subscribed to channel: {}", image_channel);

redis_pubsub.set_read_timeout(Some(Duration::from_secs(5))).expect("Failed to set read timeout");

for _ in 0..3 {
    info!("###### Waiting for message...");
    let log = match redis_pubsub.get_message(){
        Ok(T) => T,
        Err(e) => {info!("Failed to get message: {}", e); continue;},
    };
    let payload : String = match log.get_payload() {
        Ok(T) => T,
        Err(e) => {info!("Failed to get payload: {}", e);continue;},
    };
    if log_enabled!(Level::Debug) {
        debug!("Got log from channel: '{}', log: {}", log.get_channel_name(), payload);
    }
    info!("{{\"{}\": {}}}", image_channel, payload);
    sleep(Duration::from_secs(1));
};

I'm receiving a format error in Rust. Logs:

[rusty_record] subscribed to channel: image_sender
[rusty_record] ###### Waiting for message...
[rusty_record] Failed to get payload: Invalid UTF-8- TypeError
[rusty_record] ###### Waiting for message...
[rusty_record] Failed to get payload: Invalid UTF-8- TypeError
[rusty_record] ###### Waiting for message...
[rusty_record] Failed to get payload: Invalid UTF-8- TypeError

Looking at redis-cli, there are messages in the pubsub channel:

redis-cli subscribe image_sender

1) "message"
2) "image_sender"
3) "3707~{\"pic_gain\":0,\"pic_exposure\":0,\"pic_time\":0,\"camera_id\":0,\"latitude\":0.000000,\"longitude\":0.000000,\"altitude\":0.000000,\"heading_degrees\":0.000000,\"auto_white_balance\":true}~\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xdb\x00C\x00\x02\x01\x01\x01\x01\x01\x02\x01\x01\x01\x02\x02\x02\x02\x02\x04\x03\x02\x02\x02\x02\x05\x04\x04\x03\x04\x06\x05\x06\x06\x06\x05\x06\x06\x06\a\t\b\x06\a\t\a\x06\x06\b\x0b\b\t\n\n\n\n\n\x06\b\x0b\x0c\x0b\n\x0c\t\n\n\n\xff\xdb\x00C\x01\x02\x02\x02\x02\x02\x02\x05\x03\x03\x05\n\a\x06\a\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\xff\xc0\x00\x11\b\x00\xc0\x05\xa0\x03\x01\"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\a\b\t\n\x0b\xff\xc4\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\a\"q\x142\x81\x91\xa1\b#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&'()*456789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\...

If I send a regular string it works fine.

Cpp:

string image_msg;
image_msg = to_string(pic_id) + "~" + metadata + "~" + "image_str";
RedisCommunicator::GetInstance()->Publish(image_channel, image_msg);

I'm receiving the messages in Rust. Logs:

[rusty_record] subscribed to channel: image_sender
[rusty_record] ###### Waiting for message...
[rusty_record] Got log from channel: 'image_handler_0', log: 7812~{"pic_gain":0,"pic_exposure":0,"pic_time":0,"camera_id":0,"latitude":0.000000,"longitude":0.000000,"altitude":0.000000,"heading_degrees":0.000000,"auto_white_balance":true}~image_str
[rusty_record] {"image_sender": 7812~{"pic_gain":0,"pic_exposure":0,"pic_time":0,"camera_id":0,"latitude":0.000000,"longitude":0.000000,"altitude":0.000000,"heading_degrees":0.000000,"auto_white_balance":true}~image_str}
[rusty_record] ###### Waiting for message...
[rusty_record] Got log from channel: 'image_handler_0', log: 7813~{"pic_gain":0,"pic_exposure":0,"pic_time":0,"camera_id":0,"latitude":0.000000,"longitude":0.000000,"altitude":0.000000,"heading_degrees":0.000000,"auto_white_balance":true}~image_str
[rusty_record] {"image_sender": 7813~{"pic_gain":0,"pic_exposure":0,"pic_time":0,"camera_id":0,"latitude":0.000000,"longitude":0.000000,"altitude":0.000000,"heading_degrees":0.000000,"auto_white_balance":true}~image_str}
[rusty_record] ###### Waiting for message...
[rusty_record] Got log from channel: 'image_handler_0', log: 7814~{"pic_gain":0,"pic_exposure":0,"pic_time":0,"camera_id":0,"latitude":0.000000,"longitude":0.000000,"altitude":0.000000,"heading_degrees":0.000000,"auto_white_balance":true}~image_str
[rusty_record] {"image_sender": 7814~{"pic_gain":0,"pic_exposure":0,"pic_time":0,"camera_id":0,"latitude":0.000000,"longitude":0.000000,"altitude":0.000000,"heading_degrees":0.000000,"auto_white_balance":true}~image_str}

** Still new to Rust please forgive me if there are some newbie mistakes.


Solution

  • Strings in Rust are expected to be UTF-8 encoded text. You should use Vec<u8> for binary data like your images. Most "string" manipulation functions also exist for byte slices &[u8], and types that dereference to byte slices like Vec<u8>, so you can parse a Vec<u8> as easily as a String (using e.g. splitn) and convert the parts that contain text with str::from_utf8:

    let mut redis_pubsub = redis_connection.as_pubsub();
    
    redis_pubsub.subscribe(image_channel).expect("Failed to subscribe to channel");
    info!("subscribed to channel: {}", image_channel);
    
    redis_pubsub.set_read_timeout(Some(Duration::from_secs(5))).expect("Failed to set read timeout");
    
    for _ in 0..3 {
        info!("###### Waiting for message...");
        let log = match redis_pubsub.get_message(){
            Ok(T) => T,
            Err(e) => {info!("Failed to get message: {}", e); continue;},
        };
        let payload : Vec<u8> = match log.get_payload() {
            Ok(T) => T,
            Err(e) => {info!("Failed to get payload: {}", e);continue;},
        };
        if log_enabled!(Level::Debug) {
            debug!("Got log from channel: '{}', log: {}", log.get_channel_name(), payload);
        }
        let parts: Vec<_> = payload.splitn (3, |&c| c == b'~').collect();
    
        info!("id: {:?}", str::from_utf8 (parts[0]));
        info!("metadata: {:?}", str::from_utf8 (parts[1]));
        info!("data: {:?}", parts[2]);
        sleep(Duration::from_secs(1));
    };