Sustainable spread (RProp) slow

I am creating a library for Neural Networks; I have successfully implemented the Back Propagation algorithm, but I am having problems with Resilient Propagation.

I am using an XOR script to test my implementation; the error seems to decrease after a few epochs, then it increases and eventually stops changing.

The network I'm testing on has 3 levels - 2 inputs, 2 hidden and 1 output neurons / s. The synapse / dendrite weight is on the next neural layers, for example, the weights hidden on the scales will be stored on the hidden neurons, and the weights with the hidden output will be stored on the output neurons.

This is my gradient calculation (EDIT 4) :

private void CalculateGradient()
{
    for (int t = 0; t < TrainingInputs.Count; ++t) //loop through training data
    {
        Network.Run(TrainingInputs[t]);
        for (int l = Network.Layers.Count - 1; l > 0; l--)
        {
            for (int i = 0; i < Network.Layers[l].Neurons.Count; i++)
            {
                Network.Layers[l].Neurons[i].Delta = l < Network.Layers.Count - 1
                    ? CalculateNonLastGradient(l + 1, i, Network.Layers[l].Neurons[i].Value)
                    : CalculateLastGradient(TrainingOutputs[t][i], Network.Layers[l].Neurons[i].Value);
            }
        }
        for (int l = Network.Layers.Count - 1; l > 0; l--)
        {
            for (int j = 0; j < Network.Layers[l].Neurons.Count; ++j)
            {
                double grad = Network.Layers[l].Neurons[j].Delta;
                biasGrads[l][j] += grad;

                for (int i = 0; i < Network.Layers[l - 1].Neurons.Count; ++i)
                {
                    grad = Network.Layers[l].Neurons[j].Delta * Network.Layers[l - 1].Neurons[i].Value;
                    grads[l][j][i] += grad;
                }
            }
        }

        int o = 0;
        Layer layer = Network.Layers[Network.Layers.Count - 1];
        errors.Add(layer.Neurons.Sum(n => Math.Abs(TrainingOutputs[t][o++] - n.Value)));
    }
    Error = errors.Average();
}

private double CalculateLastGradient(double ideal, double nValue)
{
    return Network.Activation.Derivitive(nValue) * (ideal - nValue);
}
private double CalculateNonLastGradient(int nextLayer, int j, double nValue)
{
    double sum = 0.0;
    for (int i = 0; i < Network.Layers[nextLayer].Neurons.Count; i++)
    {
        sum += Network.Layers[nextLayer].Neurons[i].Delta * Network.Layers[nextLayer].Neurons[i].Dendrites[j].Weight;
    }
    return Network.Activation.Derivitive(nValue) * sum;
}

      

        
        
        
      

    

My RProp implementation (EDIT 4) :

public bool Algorithm()
{
    //initialise matrices
    if (!initializedMatrices)
    {
        InitializeMatrices();
    }

    ZeroOut();

    //calculate gradients
    CalculateGradient();

    for (int l = 1; l < Network.Layers.Count; l++) //layers
    {
        for (int i = 0; i < Network.Layers[l - 1].Neurons.Count; ++i) //prev layer neurons
        {
            for (int j = 0; j < Network.Layers[l].Neurons.Count; ++j) //current layer neurons
            {
                double delta = prevDeltas[l][j][i];
                int change = Math.Sign(prevGrads[l][j][i] * grads[l][j][i]);
                if (change > 0)
                {
                    delta = Math.Min(delta * etaPlus, deltaMax);
                    double deltaWeight = -Math.Sign(grads[l][j][i]) * delta;
                    Network.Layers[l].Neurons[j].Dendrites[i].Weight += deltaWeight;
                }
                else if (change < 0)
                {
                    delta = Math.Max(delta * etaMinus, deltaMin);
                    Network.Layers[l].Neurons[j].Dendrites[i].Weight -= prevDeltas[l][j][i];
                    prevGrads[l][j][i] = 0;
                }
                else if (change == 0)
                {
                    double deltaWeight = -Math.Sign(grads[l][j][i]) * delta;
                    Network.Layers[l].Neurons[j].Dendrites[i].Weight += deltaWeight;
                }
                prevGrads[l][j][i] = grads[l][j][i];
                prevDeltas[l][j][i] = delta;
            } //j
        } //i

        for (int i = 1; i < Network.Layers[l].Neurons.Count; ++i)
        {
            double delta = prevBiasDeltas[l][i];
            int change = Math.Sign(prevBiasGrads[l][i] * biasGrads[l][i]);
            if (change > 0)
            {
                delta = Math.Min(prevBiasDeltas[l][i] * etaPlus, deltaMax);
                double biasDeltaWeight = -Math.Sign(biasGrads[l][i]) * delta;
                Network.Layers[l].Neurons[i].Bias += biasDeltaWeight;
            }
            else if (change < 0)
            {
                delta = Math.Max(prevBiasDeltas[l][i] * etaMinus, deltaMin);
                Network.Layers[l].Neurons[i].Bias -= prevBiasDeltas[l][i];
                prevBiasGrads[l][i] = 0;
            }
            else if (change == 0)
            {
                double biasDeltaWeight = -Math.Sign(biasGrads[l][i]) * delta;
                Network.Layers[l].Neurons[i].Bias += biasDeltaWeight;
            }
            prevBiasGrads[l][i] = biasGrads[l][i];
            prevBiasDeltas[l][i] = delta;
        }
    }
    return true;
}

      

        
        
        
      

    

Can anyone point me to what might be wrong?

EDIT: The problem is the delta doesn't change.

EDIT 2: I fixed the delta without changing, initializing the first previous deltas to 0.01 and zero gradients before each call. The error increases very quickly, then decreases slowly with TANH; with Sigmoid he does the opposite.

EDIT 3: The loop for offset 'started at 0 when it should have been 1. Fixed this fix to the original problem, but appeared on a new one - the error stopped decreasing after a certain point.

EDIT 4: I realized that I was not accumulating gradients for each neuron over the training set. The error is now decreasing, but very slowly.