Search code examples
c#neural-networkrecurrent-neural-networkcntk

How to create input data for a recurrent network in CNTK (C#)?

I need to create data for a mini-batch with shape: (5(input_vector), 50(sequence_length), 100(amount_of_input_sequences)), in order to run 100 instances of input data for a recurrent neural network in 1 run of model.Evaluate.

At the moment I am unable to create input data even for 1 run. My code produces the output for 50 runs of the neural network instead of using the data as a single sequence. I did not find an example of creating input data for RNN using the Value object. I also did not find an example of creating an input tensor with three axes.

I would be glad if someone tells me how to do this.

`var cpu_device = DeviceDescriptor.CPUDevice;
var gpu_device = DeviceDescriptor.GPUDevice(0);
var model_device = gpu_device;

int inputDim = 5;
int cellDim = 5;
int outputDim = 3;
int sequenceLength = 50;
int inputSequences = 100;

NDShape inputShape = NDShape.CreateNDShape(new int[] { inputDim });
NDShape outputShape = NDShape.CreateNDShape(new int[] { outputDim });

var inputVariable = Variable.InputVariable(inputShape, DataType.Float);
var outputVariable = Variable.InputVariable(outputShape, DataType.Float);

var lstmLayer = CntkWrapper.Layers.LSTM<float>(cellDim, inputVariable, model_device);
var model = CntkWrapper.Layers.Dense<float>(outputDim, lstmLayer, CNTKLib.Sigmoid, model_device);

Random random = new Random();
var inputs = new float[inputSequences][][];
for(int i = 0; i < inputs.Length; i++)
{
    inputs[i] = new float[sequenceLength][];
    for(int k = 0; k < inputs[i].Length; k++)
    {
        inputs[i][k] = new float[inputDim];
        for(int p = 0; p < inputs[i][k].Length; p++)
        {
            inputs[i][k][p] = (float)random.NextDouble();
        }
    }
}

NDArrayView[] sequenceNDArrayView = inputs[0].Select(o => new NDArrayView(inputShape, o, cpu_device)).ToArray();
var gpuInputsValue = Value.Create(inputShape, sequenceNDArrayView, gpu_device);

var inputDataMap = new Dictionary<Variable, Value>() { { inputVariable, gpuInputsValue } };
var outputDataMap = new Dictionary<Variable, Value>() { { model.Output, null } };
model.Evaluate(inputDataMap, outputDataMap, gpu_device);
var outputValue = outputDataMap[model.Output];
IList<IList<float>> actualLabelSoftMax = outputValue.GetDenseData<float>(model.Output);`

`public static class Layers
{
    public static uint Seed = 1;
    public static double NormalMean = 0.0;
    public static double StandartDev = 0.25;

    public static Function Dense<ElementType>(int dim, Variable previousLayer, Func<Variable, Function> activationFunction, DeviceDescriptor device)
    {
        int inputDimension = previousLayer.Shape.Dimensions[0];
        var weights = new Parameter(NDArrayView.RandomNormal<ElementType>(new[] { dim, inputDimension }, NormalMean, StandartDev, Seed++, device));
        var bias = new Parameter(NDArrayView.RandomNormal<ElementType>(new[] { dim }, NormalMean, StandartDev, Seed++, device));
        var layer = activationFunction(CNTKLib.Plus(bias, CNTKLib.Times(weights, previousLayer)));
        return layer;
    }
    public static Function LSTM<ElementType>(int cellDim, Variable previousLayer, DeviceDescriptor device)
    {
        Func<Variable, Function> pastValueRecurrenceHook = (x) => CNTKLib.PastValue(x);
        var dh = Variable.PlaceholderVariable(new int[] { cellDim }, previousLayer.DynamicAxes);
        var dc = Variable.PlaceholderVariable(new int[] { cellDim }, previousLayer.DynamicAxes);

        var prevOutput = dh;
        var prevCellState = dc;
        int outputDim = prevOutput.Shape[0];

        bool isFloatType = typeof(ElementType).Equals(typeof(float));
        DataType dataType = isFloatType ? DataType.Float : DataType.Double;

        Func<int, Parameter> createBiasParam;
        if (isFloatType)
            createBiasParam = (dim) => new Parameter(new int[] { dim }, 0.01f, device, "");
        else
            createBiasParam = (dim) => new Parameter(new int[] { dim }, 0.01, device, "");

        uint seed2 = 1;
        Func<int, Parameter> createProjectionParam = (oDim) => new Parameter(new int[] { oDim, NDShape.InferredDimension }, dataType, CNTKLib.GlorotUniformInitializer(1.0, 1, 0, seed2++), device);

        Func<int, Parameter> createDiagWeightParam = (dim) => new Parameter(new int[] { dim }, dataType, CNTKLib.GlorotUniformInitializer(1.0, 1, 0, seed2++), device);

        Func<Variable> projectInput = () => createBiasParam(cellDim) + (createProjectionParam(cellDim) * previousLayer);

        // Input gate
        Function it =
            CNTKLib.Sigmoid((Variable)(projectInput() + (createProjectionParam(cellDim) * prevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim), prevCellState));
        Function bit = CNTKLib.ElementTimes(it, CNTKLib.Tanh(projectInput() + (createProjectionParam(cellDim) * prevOutput)));

        // Forget-me-not gate
        Function ft = CNTKLib.Sigmoid((Variable)(projectInput() + (createProjectionParam(cellDim) * prevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim), prevCellState));
        Function bft = CNTKLib.ElementTimes(ft, prevCellState);

        Function ct = (Variable)bft + bit;

        // Output gate
        Function ot = CNTKLib.Sigmoid((Variable)(projectInput() + (createProjectionParam(cellDim) * prevOutput)) + CNTKLib.ElementTimes(createDiagWeightParam(cellDim), ct));
        Function ht = CNTKLib.ElementTimes(ot, CNTKLib.Tanh(ct));

        Function c = ct;
        Function h = (outputDim != cellDim) ? (createProjectionParam(outputDim) * ht) : ht;

        var LSTMCell = new Tuple<Function, Function>(h, c);

        var actualDh = pastValueRecurrenceHook(LSTMCell.Item1);
        var actualDc = pastValueRecurrenceHook(LSTMCell.Item2);

        (LSTMCell.Item1).ReplacePlaceholders(new Dictionary<Variable, Variable> { { dh, actualDh }, { dc, actualDc } });

        Function LSTMFunction = LSTMCell.Item1;

        Function layer = CNTKLib.SequenceLast(LSTMFunction);
        return layer;
    }
}`
Solution

  • I found this example which helped me a lot: https://github.com/albertalrisa/cntk-csharp-rnn/blob/master/playground/CNTKRefresh/CharRNNCNTK/Program.cs

    The input sequence is stored as a sequence of input vectors, for example, to store an input sequence of 10 input vectors of length 5, you need to combine all the input vectors into one vector of length 50 elements.

    Value.CreateSequence is used to create a single input sequence, and Value.CreateBatchOfSequences is used to create inputs that include multiple sequences.

    I thought that Value.Create only accepts IEnumerable<NDArrayView>, and NDArrayView in the constructor accepts a one-dimensional float[] array, that is, it will not be possible to pass several sequences into it. To do this, you need to use List<List<float>> and pass it to Value.CreateBatchOfSequences.

    Also, the sequence length axis is a dynamic axis because it can have a variable length. In the example it is created as:

    var axis = new Axis("inputAxis");
var inputVariable = Variable.InputVariable(inputShape, DataType.Float, "inputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });
var outputVariable = Variable.InputVariable(outputShape, DataType.Float, "outputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });

    But it also works without it, I don't know why.

    As a result, creating several input sequences with random values and running them through a recurrent neural network will look like this:

    var cpu_device = DeviceDescriptor.CPUDevice;
var gpu_device = DeviceDescriptor.GPUDevice(0);
var model_device = gpu_device;

int inputDim = 5;
int cellDim = 5;
int outputDim = 3;
int sequenceLength = 50;
int sequencesCount = 100;

NDShape inputShape = NDShape.CreateNDShape(new int[] { inputDim });
NDShape outputShape = NDShape.CreateNDShape(new int[] { outputDim });

var axis = new Axis("inputAxis");
var inputVariable = Variable.InputVariable(inputShape, DataType.Float, "inputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });
var outputVariable = Variable.InputVariable(outputShape, DataType.Float, "outputVariable", new List<Axis> { axis, Axis.DefaultBatchAxis() });

var lstmLayer = CntkWrapper.Layers.LSTM<float>(cellDim, inputVariable, model_device);
var model = CntkWrapper.Layers.Dense<float>(outputDim, lstmLayer, CNTKLib.Sigmoid, model_device);

Random random = new Random();
List<List<float>> inputSequences = new List<List<float>>();
for (int i = 0; i < sequencesCount; i++)
{
    List<float> sequence = new List<float>();
    for (int k = 0; k < sequenceLength; k++)
    {
        float[] input_vector = new float[inputDim];
        for (int n = 0; n < input_vector.Length; n++)
        {
            input_vector[n] = (float)random.NextDouble();
        }
        sequence.AddRange(input_vector);
    }
    inputSequences.Add(sequence);
}

var gpuInputSequences = Value.CreateBatchOfSequences(inputShape, inputSequences, gpu_device);

var inputDataMap = new Dictionary<Variable, Value>() { { inputVariable, gpuInputSequences } };
var outputDataMap = new Dictionary<Variable, Value>() { { model.Output, null } };
model.Evaluate(inputDataMap, outputDataMap, gpu_device);
var outputValue = outputDataMap[model.Output];
IList<IList<float>> actualLabelSoftMax = outputValue.GetDenseData<float>(model.Output);