Search code examples
pythonc++windowsbinaryfilesifstream

How to debug reading a Catman binary file in C++, program reads incorrectly at a specific point

I am working in C++ with a program where I read a binary file received from a Catman's Data Acquisition Device. It has been working alright so far but I am stuck on a point where I think I either read too much or too little and I am out of ideas what differs from my C++ code compared to a python code that does exactly what I want to recreate.

There is a lot of repetitive lines of code involved so I want to preface with saying I have a sort of facit from a python library that does the exact thing I want. So I will show the parts where my C++ code seem to behave differently from the python code. I will print the entire reproducable code at the end of the post in a Post Scriptum.

Here is the python libary for what I want to do: https://github.com/leonbohmann/APReader/blob/9d76be8e94860a0711f2c7c4973641706c5ea479/apread/entries.py#L32

So my project is recreating the python library but for C++. I get stuck at reading an "extended channel header" where the "HPFilt" field starts filling with garbled/unintended data.

Here is what python reads from the "extended channel header":

{'T0': 44916.26666666667, 'dt': 3.3333333333333335, 'SensorType': 0, 'SupplyVoltage': 0, 'FiltChar': 0, 'FiltFreq': 0, 'TareVal': 0.0, 'ZeroVal': 0.0, 'MeasRange': 0.0, 'InChar': [0.0, 0.0, 0.0, 0.0], 'SerNo': ' ', 'PhysUnit': ' ', 'NativeUnit': ' ', 'Slot': 0, 'SubSlot': 0, 'AmpType': 0, 'APType': 0, 'kFactor': 0.0, 'bFactor': 0.0, 'MeasSig': 0, 'AmpInput': 0, 'HPFilt': 0, 'OLImportInfo': 0,

Here is that part of python code:

    exthdr['T0'] = rdr.read_double() # (pos0+) 8
    exthdr['dt'] = rdr.read_double() # 16
    exthdr['SensorType'] = rdr.read_int16() # 18
    exthdr['SupplyVoltage'] = rdr.read_int16() # 20
    
    exthdr['FiltChar'] = rdr.read_int16() # 22
    exthdr['FiltFreq'] = rdr.read_int16() # 24
    exthdr['TareVal'] = rdr.read_float() # 28
    exthdr['ZeroVal'] = rdr.read_float() # 32   
    exthdr['MeasRange'] = rdr.read_float() # 36
    exthdr['InChar'] = [rdr.read_float() for i in range(4)] # 40, 44, 48, 52
    
    exthdr['SerNo'] = rdr.read_string(32) # 84
    exthdr['PhysUnit'] = rdr.read_string(8) # 92
    exthdr['NativeUnit'] = rdr.read_string(8) # 100
    
    exthdr['Slot'] = rdr.read_int16() # 102
    exthdr['SubSlot'] = rdr.read_int16() # 104
    exthdr['AmpType'] = rdr.read_int16() # 106
    exthdr['APType'] = rdr.read_int16() # 108
    exthdr['kFactor'] = rdr.read_float() # 112
    exthdr['bFactor'] = rdr.read_float() # 116
    
    exthdr['MeasSig'] = rdr.read_int16() # 118
    exthdr['AmpInput'] = rdr.read_int16() # 120
    exthdr['HPFilt'] = rdr.read_int16() # 122
    exthdr['OLImportInfo'] = rdr.read_byte() # 123

Here is what my C++ program reads from the "extended channel header":

ch 1 Tzero : 44916.266667 ch 1 dt : 3.333333 ch 1 sensorType : 0 ch 1 supplyVoltage : 0 ch 1 filtChar : 0 ch 1 filtFreq : 0 ch 1 tareVal : 0.000000 ch 1 measRange : 0.000000 ch 1 inChar[0] : 0.000000 ch 1 inChar[1] : 0.000000 ch 1 inChar[2] : 0.000000 ch 1 inChar[3] : 0.000000 ch 1 serNo : ch 1 physUnit : ch 1 nativeUnit : ch 1 slot : 0 ch 1 subSlot : 0 ch 1 ampType : 0 ch 1 APType : 0 ch 1 kFactor : 0.000000 ch 1 bFactor : 0.000000 ch 1 measSig : 0 ch 1 ampInput : 0 ch 1 HPFilt : 538968832 ch 1 OLImport : 20

here's that part of C++ code:

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.TZero, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.TZero));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.dt, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.dt));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.sensorType, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.sensorType));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.supplyVoltage, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.supplyVoltage));

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtChar, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtChar));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtFreq, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtFreq));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.tareVal, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.tareVal));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.zeroVal, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.zeroVal));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.measRange, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.measRange));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[0], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[0]));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[1], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[1]));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[2], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[2]));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[3], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[3]));

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.serNo, 32);
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.physUnit, 8);
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.nativeUnit, 8);

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.slot, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.slot));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.subSlot, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.subSlot));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampType, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampType));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.APType, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.APType));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.kFactor, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.kFactor));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.bFactor, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.bFactor));

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.measSig, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.measSig));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampInput, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampInput));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.HPFilt, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.HPFilt));                  // Should read 0...
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.OLImport, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.OLImport));

So somewhere after the "dt" field it seems my code either reads too much or too little. I have checked the types I use so that they are not different but could be I misunderstood something there as well, so here they are:

struct catman_VB_DB_CHANHEADER
{
    double TZero;
    double dt;
    int sensorType;
    int supplyVoltage;
    int filtChar;
    int filtFreq;
    float tareVal;
    float zeroVal;
    float measRange;
    float inChar[4];
    char serNo[32];//std::string serNo;
    char physUnit[8];//std::string physUnit;
    char nativeUnit[8];//std::string nativeUnit;
    int slot;
    int subSlot;
    int ampType;
    int APType;
    float kFactor;
    float bFactor;
    int measSig;
    int ampInput;
    int HPFilt;
    byte OLImport;
};

My theory so far is that i might not be reading the empty fields properly of SerNo, physUnit and nativeUnit. Maybe something here is done differently between the languages? I have tried reading them, not reading them. Only difference is that I get different garbled fields.

PS. The entire reproducable code:

struct catman_VB_DB_CHANHEADER
{
    double TZero;
    double dt;
    int sensorType;
    int supplyVoltage;
    int filtChar;
    int filtFreq;
    float tareVal;
    float zeroVal;
    float measRange;
    float inChar[4];
    char serNo[32];//std::string serNo;
    char physUnit[8];//std::string physUnit;
    char nativeUnit[8];//std::string nativeUnit;
    int slot;
    int subSlot;
    int ampType;
    int APType;
    float kFactor;
    float bFactor;
    int measSig;
    int ampInput;
    int HPFilt;
    byte OLImport;
};

struct catmanGlobalSection
{
    short fileID;
    long dataOffset;
    short fileCommentLength;
    //byte fileComment[fileCommentLength];
    short additionalDataOffsetNoOfBytes;
    //byte additionalDataOffset[additionalDataOffsetNoOfBytes];
    short reserveStringNoOfBytes[32];
    //byte reserveString[reserveStringNoOfBytes];
    short noOfChannels;
    long maxChannelLength;
    long ChannelLengthOffset[16];
    long reductionFactor;
};

struct catmanChannelHeaderSection
{
    short channelLocation;
    long channelLength;
    short channelNameLength;
    //byte channelname[channelNameLength];
    short unitLength;
    //byte unit[unitLength];
    short channelCommentLength;
    //byte channelComment[channelCommentLength];
    short format;
    short dataWidth;
    double dateAndTimeOfMeasurement;
    long extendedChannelHeaderSize;
    catman_VB_DB_CHANHEADER extendedChannelHeader;
};

struct catmanFormat
{
    catmanGlobalSection globalSection;
    catmanChannelHeaderSection ChannelHeaderSection[16];
};

static catmanFormat catmanData;

std::ifstream rf("FTP Folder/Recorder 14_2022_12_21_06_24_00.bin", std::ios::in | std::ios::binary);

rf.read((char*)&catmanData.globalSection.fileID, sizeof(catmanData.globalSection.fileID));
rf.read((char*)&catmanData.globalSection.dataOffset, sizeof(catmanData.globalSection.dataOffset));
if(shortVersion)
    rf.ignore(catmanData.globalSection.dataOffset);
else
{
    rf.read((char*)&catmanData.globalSection.fileCommentLength, sizeof(catmanData.globalSection.fileCommentLength));
    rf.ignore(catmanData.globalSection.fileCommentLength);
    /* These lines misaligns the buffer... They should be included according to Catman's binary format document. Leave them commented for now.
    rf.read((char*)&catmanData.globalSection.additionalDataOffsetNoOfBytes, sizeof(catmanData.globalSection.additionalDataOffsetNoOfBytes));
    rf.ignore(catmanData.globalSection.additionalDataOffsetNoOfBytes);
    */
    for (int i = 0; i < 32; i++)
    {
        rf.read((char*)&catmanData.globalSection.reserveStringNoOfBytes[i], sizeof(catmanData.globalSection.reserveStringNoOfBytes[i]));
        rf.ignore(catmanData.globalSection.reserveStringNoOfBytes[i]);
    }
}
rf.read((char*)&catmanData.globalSection.noOfChannels, sizeof(catmanData.globalSection.noOfChannels));
rf.read((char*)&catmanData.globalSection.maxChannelLength, sizeof(catmanData.globalSection.maxChannelLength));
for (int i = 0; i < catmanData.globalSection.noOfChannels; i++)
{
    rf.read((char*)&catmanData.globalSection.ChannelLengthOffset[i], sizeof(catmanData.globalSection.ChannelLengthOffset[i]));
}
rf.read((char*)&catmanData.globalSection.reductionFactor, sizeof(catmanData.globalSection.reductionFactor));

for (int i = 0; i < catmanData.globalSection.noOfChannels; i++)
{
    rf.read((char*)&catmanData.ChannelHeaderSection[i].channelLocation, sizeof(catmanData.ChannelHeaderSection[i].channelLocation));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].channelLength, sizeof(catmanData.ChannelHeaderSection[i].channelLength));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].channelNameLength, sizeof(catmanData.ChannelHeaderSection[i].channelNameLength));
    rf.ignore(catmanData.ChannelHeaderSection[i].channelNameLength);
    rf.read((char*)&catmanData.ChannelHeaderSection[i].unitLength, sizeof(catmanData.ChannelHeaderSection[i].unitLength));
    rf.ignore(catmanData.ChannelHeaderSection[i].unitLength);
    rf.read((char*)&catmanData.ChannelHeaderSection[i].channelCommentLength, sizeof(catmanData.ChannelHeaderSection[i].channelCommentLength));
    rf.ignore(catmanData.ChannelHeaderSection[i].channelCommentLength);
    rf.read((char*)&catmanData.ChannelHeaderSection[i].format, sizeof(catmanData.ChannelHeaderSection[i].format));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].dataWidth, sizeof(catmanData.ChannelHeaderSection[i].dataWidth));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].dateAndTimeOfMeasurement, sizeof(catmanData.ChannelHeaderSection[i].dateAndTimeOfMeasurement));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeaderSize, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeaderSize));
    
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.TZero, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.TZero));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.dt, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.dt));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.sensorType, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.sensorType));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.supplyVoltage, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.supplyVoltage));

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtChar, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtChar));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtFreq, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.filtFreq));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.tareVal, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.tareVal));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.zeroVal, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.zeroVal));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.measRange, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.measRange));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[0], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[0]));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[1], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[1]));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[2], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[2]));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[3], sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.inChar[3]));

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.serNo, 32);
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.physUnit, 8);
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.nativeUnit, 8);

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.slot, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.slot));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.subSlot, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.subSlot));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampType, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampType));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.APType, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.APType));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.kFactor, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.kFactor));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.bFactor, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.bFactor));

    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.measSig, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.measSig));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampInput, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.ampInput));
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.HPFilt, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.HPFilt));                  // Should read 0...
    rf.read((char*)&catmanData.ChannelHeaderSection[i].extendedChannelHeader.OLImport, sizeof(catmanData.ChannelHeaderSection[i].extendedChannelHeader.OLImport));
}
Solution

  • After looking at the _Chcount field in the stream buffer I found my integer variables counted as 4 bytes when they should count as 2 bytes. I could have also checked directly but I didn't as it turned out.

    My integers were too large in other words and I changed them to short int instead. It works better now.