double buffering implementation in Intel PIN

Question

I have written a PIN tool for instruction tracing, where I spawned a separate thread for writing contents of the buffer to a file.

When run, most of the time an error is caused. Once in a while, it runs without any error.

➤ $PIN_ROOT/pin -t obj-intel64/chvp_tracer.so -- ./a.out                                                                                                                      (base) 
C: [tid:49808] Tool (or Pin) caused signal 11 at PC 0x78dd8c72a238
fish: Job 1, '$PIN_ROOT/pin -t obj-intel64/ch…' terminated by signal SIGSEGV (Address boundary error)

The code below is a part of my PIN tool that uses double buffering. WriteCurrentInstruction is an analysis routine that writes to the buffer and also swaps the buffers when full. ActuallyWriteInstruction is the thread that writes contents of a buffer to file.

Can someone help me find and fix the cause of SIGSEGV?

bool instrumentation_done = false;

constexpr uint32_t buffer_size = 2048;
std::array<std::vector<trace_instr_format_t>,2> double_buffer;
bool dump_first_buffer=false;

PIN_MUTEX buffer_locks[2];
PIN_SEMAPHORE sm;

#define READ_LOCK 0
#define WRITE_LOCK 1

void WriteCurrentInstruction()
{
  if(last_instr!=first_instr and instrCount > last_instr)
    PIN_ExitApplication(0);


  PIN_MutexLock(&buffer_locks[WRITE_LOCK]);
  int32_t idx = dump_first_buffer;
  auto &buff=double_buffer[idx];
  if(buff.size()>=buffer_size)
  {
    PIN_MutexLock(&buffer_locks[READ_LOCK]);
    dump_first_buffer=not dump_first_buffer;
    idx=dump_first_buffer;
    PIN_SemaphoreSet(&sm);
    PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
  }
  buff.push_back(curr_instr);
  PIN_MutexUnlock(&buffer_locks[WRITE_LOCK]);
}


VOID ActuallyWriteInstruction(VOID * args)
{
  uint64_t insts=0;
  while(not instrumentation_done)
  {
    PIN_SemaphoreWait(&sm);
    PIN_MutexLock(&buffer_locks[READ_LOCK]);
    PIN_SemaphoreClear(&sm);
    int32_t idx = not dump_first_buffer;
    auto &buff=double_buffer[idx];
    for(const auto &instr:buff)
    {
      outfile.write(reinterpret_cast<const char*>(&instr.ip),sizeof(instr.ip));
      outfile.write(reinterpret_cast<const char*>(&instr.inst_class),sizeof(instr.inst_class));
      if(instr.is_branch())
      {
        outfile.write(reinterpret_cast<const char*>(&instr.branch_taken),sizeof(instr.branch_taken));
        outfile.write(reinterpret_cast<const char*>(&instr.branch_target),sizeof(instr.branch_target));
      }
      outfile.write(reinterpret_cast<const char*>(&instr.num_operands),sizeof(instr.num_operands));
      for(const auto &op:instr.operands)
      {
        outfile.write(reinterpret_cast<const char*>(&op.header),sizeof(op.header));
        outfile.write(reinterpret_cast<const char*>(&op.element_size),sizeof(op.element_size));
        outfile.write(reinterpret_cast<const char*>(&op.num_elements),sizeof(op.num_elements));
        outfile.write(reinterpret_cast<const char*>(&op.base_reg),sizeof(op.base_reg));
        if(op.isMem())
        {
          outfile.write(reinterpret_cast<const char*>(&op.index_reg),sizeof(op.index_reg));
          outfile.write(reinterpret_cast<const char*>(&op.scale),sizeof(op.scale));
          outfile.write(reinterpret_cast<const char*>(&op.disp),sizeof(op.disp));
          outfile.write(reinterpret_cast<const char*>(&op.addr),sizeof(op.addr));
        }
        if(op.is_write())
        {
          const auto num_bytes = op.element_size*op.num_elements;
          outfile.write(reinterpret_cast<const char*>(&op.value),num_bytes);
          
        }
      }
    }
    insts+=buff.size();
    buff.clear();
    LOG("Instructions done "+std::to_string(insts)+"\n");
    PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
  }
}

VOID Fini(INT32 code, VOID* v) {
  LOG("Done, waiting for IO \n");
  {
    PIN_MutexLock(&buffer_locks[READ_LOCK]);
    dump_first_buffer=not dump_first_buffer;
    PIN_SemaphoreSet(&sm);
    PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
  }
  while(not PIN_SemaphoreIsSet(&sm)){;}
  instrumentation_done=true;
  PIN_WaitForThreadTermination(write_thid,PIN_INFINITE_TIMEOUT,NULL);
  LOG("Done, Exiting \n");
  PIN_MutexFini(&buffer_locks[0]);
  PIN_MutexFini(&buffer_locks[1]);
  PIN_SemaphoreFini(&sm);
  outfile.close();
}

int main(int argc, char* argv[])
{
  // Initialize PIN library. Print help message if -h(elp) is specified
  // in the command line or the command line is invalid
  if (PIN_Init(argc, argv))
    return Usage();

  outfile.open(KnobOutputFile.Value().c_str(), std::ios_base::binary | std::ios_base::trunc);
  if (!outfile) {
    std::cout << "Couldn't open output trace file. Exiting." << std::endl;
    exit(1);
  }

  // Register function to be called to instrument instructions
  INS_AddInstrumentFunction(Instruction, 0);

  first_instr=KnobSkipInstructions.Value();
  last_instr=first_instr+KnobTraceInstructions.Value();
  
  // Register function to be called when the application exits
  PIN_AddFiniFunction(Fini, 0);

  PIN_MutexInit(&buffer_locks[0]);
  PIN_MutexInit(&buffer_locks[1]);
  PIN_SemaphoreInit(&sm);
  /* THREADID thid = */ PIN_SpawnInternalThread((ROOT_THREAD_FUNC *)ActuallyWriteInstruction,NULL,0,&write_thid);
  // Start the program, never returns
  PIN_StartProgram();

  return 0;
}

Answer 1

I fixed it.

void WriteCurrentInstruction()
{
  if(last_instr!=first_instr and instrCount > last_instr)
    PIN_ExitApplication(0);


  PIN_MutexLock(&buffer_locks[WRITE_LOCK]);
  int32_t idx = dump_first_buffer;
  if(buffer_nb_instrs[idx]>=buffer_size)
  {
    PIN_MutexLock(&buffer_locks[READ_LOCK]);
    dump_first_buffer=not dump_first_buffer;
    idx=dump_first_buffer;
    PIN_SemaphoreSet(&sm);
    PIN_MutexUnlock(&buffer_locks[READ_LOCK]);
  }
  
  double_buffer[idx][buffer_nb_instrs[idx]++]=curr_instr;
  PIN_MutexUnlock(&buffer_locks[WRITE_LOCK]);
}

I have created a reference for double_buffer[idx], but idx could change after the reference is initialized due to the if condition.