How do I write a Tensorflow custom op containing a persistent C++ object?

Question

I'm developing a Tensorflow sequence model that uses a beam search through an OpenFST decoding graph (loaded from a binary file) over the logits output from a Tensorflow sequence model.

I've written a custom op that allows me to perform decoding over the logits, but each time, I'm having the op call fst::Read(BINARY_FILE) before performing the decoding. This might be fine as long as it stays small but I'd like to avoid the I/O overhead.

I've read through the Tensorflow custom op and tried to find similar examples but I'm still lost. Basically, what I want to do in the graph is:

FstDecodingOp.Initialize('BINARY_FILE.bin') #loads the BINARY_FILE.bin into memory
...
for o in output:
    FstDecodingOp.decode(o) # uses BINARY_FILE.bin to decode

This would of course be straightforward in Python outside of the tensorflow graph, but I need to eventually move this into a vanilla TF-Serving environment, so it needs to get frozen into an export graph. Has anyone encountered a similar problem before?

Solution:

Didn't realize that you could set private attributes using the "OpKernel(context)". Just initialized it using that function.

Edit: more detail on how I did it. Have yet to try serving.

REGISTER_OP("FstDecoder")
    .Input("log_likelihoods: float")
    .Attr("fst_decoder_path: string")
    ....

...

template <typename Device, typename T>
class FstDecoderOp : public OpKernel {

private:
   fst::Fst<fst::StdArc>* fst_;
   float beam_;

public:
  explicit FstDecoderOp(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("beam", &beam_));

    std::string fst_path;
    OP_REQUIRES_OK(context, context->GetAttr("fst_decoder_path", &fst_path));

    fst_ = fst::Fst<fst::StdArc>::Read(fst_path);
  }

  void Compute(OpKernelContext* context) override {
    // do some compute 
    const Tensor* log_likelihoods;

    OP_REQUIRES_OK(context, context->input("log_likelihoods", 
     &log_likelihoods));

    // simplified 
    compute_op(_fst, log_likelihoods);

  }
};

In python:

sess = tf.Session()
mat = tf.placeholder(tf.float32, shape=test_npy.shape)
res_ = decoder_op.fst_decoder(beam=30, fst_decoder_path="decoder_path.fst", log_likelihoods=mat)
res = sess.run(res_, {mat : test_npy} )

Answer 1

Solution:

Didn't realize that you could set private attributes using the "OpKernel(context)". Just initialized it using that function.

Edit: more detail on how I did it. Have yet to try serving.

REGISTER_OP("FstDecoder")
    .Input("log_likelihoods: float")
    .Attr("fst_decoder_path: string")
    ....

...

template <typename Device, typename T>
class FstDecoderOp : public OpKernel {

private:
   fst::Fst<fst::StdArc>* fst_;
   float beam_;

public:
  explicit FstDecoderOp(OpKernelConstruction* context) : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("beam", &beam_));

    std::string fst_path;
    OP_REQUIRES_OK(context, context->GetAttr("fst_decoder_path", &fst_path));

    fst_ = fst::Fst<fst::StdArc>::Read(fst_path);
  }

  void Compute(OpKernelContext* context) override {
    // do some compute 
    const Tensor* log_likelihoods;

    OP_REQUIRES_OK(context, context->input("log_likelihoods", 
     &log_likelihoods));

    // simplified 
    compute_op(_fst, log_likelihoods);

  }
};

In python:

sess = tf.Session()
mat = tf.placeholder(tf.float32, shape=test_npy.shape)
res_ = decoder_op.fst_decoder(beam=30, fst_decoder_path="decoder_path.fst", log_likelihoods=mat)
res = sess.run(res_, {mat : test_npy} )