The accuracy reported by coffee and pickuff varies

Shown below is the train.Prototxt file that is used to train a pre-trained model.

    name: "TempWLDNET"
    layer {
      name: "data"
      type: "ImageData"
      top: "data"
      top: "label"
      include {
        phase: TRAIN
      }
      transform_param {
        mirror: true
        crop_size: 224 
        mean_file: "mean.binaryproto"
      }
      image_data_param {
        source: "train.txt"
        batch_size: 25
        new_height: 256 
        new_width: 256 
      }
    }
    layer {
      name: "data"
      type: "ImageData"
      top: "data"
      top: "label"
      include {
        phase: TEST
      }
      transform_param {
        mirror: false
        crop_size: 224 
        mean_file: "painmean.binaryproto"
      }
      image_data_param {
        source: "test.txt"
        batch_size: 25
        new_height: 256 
        new_width: 256 
      }
    }
    layer {
      name: "conv1"
      type: "Convolution"
      bottom: "data"
      top: "conv1"
      param {
        lr_mult: 1
        decay_mult: 1
      }
      param {
        lr_mult: 2
        decay_mult: 0
      }
      convolution_param {
        num_output: 96
        kernel_size: 7
        stride: 2
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layer {
      name: "relu1"
      type: "ReLU"
      bottom: "conv1"
      top: "conv1"
    }
    layer {
      name: "norm1"
      type: "LRN"
      bottom: "conv1"
      top: "norm1"
      lrn_param {
        local_size: 5
        alpha: 0.0005
        beta: 0.75
      }
    }
    layer {
      name: "pool1"
      type: "Pooling"
      bottom: "norm1"
      top: "pool1"
      pooling_param {
        pool: MAX
        kernel_size: 3
        stride: 3
      }
    }
    layer {
      name: "conv2"
      type: "Convolution"
      bottom: "pool1"
      top: "conv2"
      param {
        lr_mult: 1
        decay_mult: 1
      }
      param {
        lr_mult: 2
        decay_mult: 0
      }
      convolution_param {
        num_output: 256
        pad: 2
        kernel_size: 5
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layer {
      name: "relu2"
      type: "ReLU"
      bottom: "conv2"
      top: "conv2"
    }
    layer {
      name: "pool2"
      type: "Pooling"
      bottom: "conv2"
      top: "pool2"
      pooling_param {
        pool: MAX
        kernel_size: 2
        stride: 2
      }
    }
    layer {
      name: "conv3"
      type: "Convolution"
      bottom: "pool2"
      top: "conv3"
      param {
        lr_mult: 1
        decay_mult: 1
      }
      param {
        lr_mult: 2
        decay_mult: 0
      }
      convolution_param {
        num_output: 512
        pad: 1
        kernel_size: 3
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layer {
      name: "relu3"
      type: "ReLU"
      bottom: "conv3"
      top: "conv3"
    }
    layer {
      name: "conv4"
      type: "Convolution"
      bottom: "conv3"
      top: "conv4"
      param {
        lr_mult: 1
        decay_mult: 1
      }
      param {
        lr_mult: 2
        decay_mult: 0
      }
      convolution_param {
        num_output: 512
        pad: 1
        kernel_size: 3
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layer {
      name: "relu4"
      type: "ReLU"
      bottom: "conv4"
      top: "conv4"
    }
    layer {
      name: "conv5"
      type: "Convolution"
      bottom: "conv4"
      top: "conv5"
      param {
        lr_mult: 1
        decay_mult: 1
      }
      param {
        lr_mult: 2
        decay_mult: 0
      }
      convolution_param {
        num_output: 512
        pad: 1
        kernel_size: 3
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layer {
      name: "relu5"
      type: "ReLU"
      bottom: "conv5"
      top: "conv5"
    }
    layer {
      name: "pool5"
      type: "Pooling"
      bottom: "conv5"
      top: "pool5"
      pooling_param {
        pool: MAX
        kernel_size: 3
        stride: 3
      }
    }
    layer {
      name: "fc6"
      type: "InnerProduct"
      bottom: "pool5"
      top: "fc6"
      param {
        lr_mult: 1
        decay_mult: 1
      }
      param {
        lr_mult: 2
        decay_mult: 0
      }
      inner_product_param {
        num_output: 4048
        weight_filler {
          type: "gaussian"
          std: 0.005
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layer {
      name: "relu6"
      type: "ReLU"
      bottom: "fc6"
      top: "fc6"
    }
    layer {
      name: "drop6"
      type: "Dropout"
      bottom: "fc6"
      top: "fc6"
      dropout_param {
        dropout_ratio: 0.5
      }
    }
    layer {
      name: "fc7"
      type: "InnerProduct"
      bottom: "fc6"
      top: "fc7"
      # Note that lr_mult can be set to 0 to disable any fine-tuning of this, and any other, layer
      param {
        lr_mult: 1
        decay_mult: 1
      }
      param {
        lr_mult: 2
        decay_mult: 0
      }
      inner_product_param {
        num_output: 4048
        weight_filler {
          type: "gaussian"
          std: 0.005
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layer {
      name: "relu7"
      type: "ReLU"
      bottom: "fc7"
      top: "fc7"
    }
    layer {
      name: "drop7"
      type: "Dropout"
      bottom: "fc7"
      top: "fc7"
      dropout_param {
        dropout_ratio: 0.5
      }
    }
    layer {
      name: "fc8_temp"
      type: "InnerProduct"
      bottom: "fc7"
      top: "fc8_temp"
      # lr_mult is set to higher than for other layers, because this layer is starting from random while the others are already trained
      param {
        lr_mult: 10
        decay_mult: 1
      }
      param {
        lr_mult: 20
        decay_mult: 0
      }
      inner_product_param {
        num_output: 16
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layer {
      name: "accuracy"
      type: "Accuracy"
      bottom: "fc8_temp"
      bottom: "label"
      top: "accuracy"
      include {
        phase: TEST
      }
    }
    layer {
      name: "loss"
      type: "SoftmaxWithLoss"
      bottom: "fc8_temp"
      bottom: "label"
      top: "loss"
    }

      

Utilization of the above prototype file accuracy reported for the test set at the end of the training is 92%. For more information, see How to evaluate the accuracy and loss of a trained model as good or not in caffe?

I took a snapshot of the model at the end of iteration 13000 and using the below python script, I tried to plot a confusion matrix, the accuracy shown is 74%.

    #!/usr/bin/python
    # -*- coding: utf-8 -*-

    import sys
    import caffe
    import numpy as np
    import argparse
    from collections import defaultdict

    TRAIN_DATA_ROOT='/Images/test/'

    if __name__ == "__main__":
            parser = argparse.ArgumentParser()
            parser.add_argument('--proto', type=str, required=True)
            parser.add_argument('--model', type=str, required=True)
            parser.add_argument('--meanfile', type=str, required=True)
            parser.add_argument('--labelfile', type=str, required=True)
            args = parser.parse_args()

            proto_data = open(args.meanfile, 'rb').read()
            a = caffe.io.caffe_pb2.BlobProto.FromString(proto_data)
            mean  = caffe.io.blobproto_to_array(a)[0]


            caffe.set_mode_gpu()

            count = 0
            correct = 0
            matrix = defaultdict(int) # (real,pred) -> int
            labels_set = set()

            net = caffe.Net(args.proto, args.model, caffe.TEST)
            # load input and configure preprocessing    
            transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
            transformer.set_mean('data', mean)
            transformer.set_transpose('data', (2,0,1))
            transformer.set_channel_swap('data', (2,1,0))
            transformer.set_raw_scale('data', 1)


            #note we can change the batch size on-the-fly
            #since we classify only one image, we change batch size from 10 to 1
            net.blobs['data'].reshape(1,3,224,224)

            #load the image in the data layer
            f = open(args.labelfile, "r")
            for line in f.readlines():
                    parts = line.split()
                    example_image = parts[0]
                    label = int(parts[1])
                    im = caffe.io.load_image(TRAIN_DATA_ROOT + example_image)
                    print(im.shape)
                    net.blobs['data'].data[...] = transformer.preprocess('data', im)
                    out = net.forward()
                    plabel = int(out['prob'][0].argmax(axis=0))
                    count += 1
                    iscorrect = label == plabel
                    correct += (1 if iscorrect else 0)
                    matrix[(label, plabel)] += 1
                    labels_set.update([label, plabel])
                    if not iscorrect:
                            print("\rError: expected %i but predicted %i" \
                                        % (label, plabel))

                    sys.stdout.write("\rAccuracy: %.1f%%" % (100.*correct/count))
                    sys.stdout.flush()

            print(", %i/%i corrects" % (correct, count))

            print ("")
            print ("Confusion matrix:")
            print ("(r , p) | count")
            for l in labels_set:
                    for pl in labels_set:
                            print ("(%i , %i) | %i" % (l, pl, matrix[(l,pl)])) 

      

I am using deploy.protxt

    name: "CaffeNet"
    input: "data"
    input_shape {
      dim: 1
      dim: 3
      dim: 224
      dim: 224
    }
    layers {
      name: "conv1"
      type: CONVOLUTION
      bottom: "data"
      top: "conv1"

        blobs_lr: 1
        weight_decay: 1

        blobs_lr: 2
        weight_decay: 0


      convolution_param {
        num_output: 96
        kernel_size: 7
        stride: 2
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layers {
      name: "relu1"
      type: RELU
      bottom: "conv1"
      top: "conv1"
    }
    layers {
      name: "norm1"
      type: LRN
      bottom: "conv1"
      top: "norm1"
      lrn_param {
        local_size: 5
        alpha: 0.0005
        beta: 0.75
      }
    }
    layers {
      name: "pool1"
      type: POOLING
      bottom: "norm1"
      top: "pool1"
      pooling_param {
        pool: MAX
        kernel_size: 3
        stride: 3
      }
    }
    layers {
      name: "conv2"
      type: CONVOLUTION
      bottom: "pool1"
      top: "conv2"

        blobs_lr: 1
        weight_decay: 1


        blobs_lr: 2
        weight_decay: 0

      convolution_param {
        num_output: 256
        pad: 2
        kernel_size: 5
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layers {
      name: "relu2"
      type: RELU
      bottom: "conv2"
      top: "conv2"
    }
    layers {
      name: "pool2"
      type: POOLING
      bottom: "conv2"
      top: "pool2"
      pooling_param {
        pool: MAX
        kernel_size: 2
        stride: 2
      }
    }
    layers {
      name: "conv3"
      type: CONVOLUTION
      bottom: "pool2"
      top: "conv3"

        blobs_lr: 1
        weight_decay: 1

        blobs_lr: 2
        weight_decay: 0

      convolution_param {
        num_output: 512
        pad: 1
        kernel_size: 3
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layers {
      name: "relu3"
      type: RELU
      bottom: "conv3"
      top: "conv3"
    }
    layers {
      name: "conv4"
      type: CONVOLUTION
      bottom: "conv3"
      top: "conv4"

        blobs_lr: 1
        weight_decay: 1


        blobs_lr: 2
        weight_decay: 0

      convolution_param {
        num_output: 512
        pad: 1
        kernel_size: 3
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layers {
      name: "relu4"
      type: RELU
      bottom: "conv4"
      top: "conv4"
    }
    layers {
      name: "conv5"
      type: CONVOLUTION
      bottom: "conv4"
      top: "conv5"

        blobs_lr: 1
        weight_decay: 1


        blobs_lr: 2
        weight_decay: 0

      convolution_param {
        num_output: 512
        pad: 1
        kernel_size: 3
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layers {
      name: "relu5"
      type: RELU
      bottom: "conv5"
      top: "conv5"
    }
    layers {
      name: "pool5"
      type: POOLING
      bottom: "conv5"
      top: "pool5"
      pooling_param {
        pool: MAX
        kernel_size: 3
        stride: 3
      }
    }
    layers {
      name: "fc6"
      type: INNER_PRODUCT
      bottom: "pool5"
      top: "fc6"

        blobs_lr: 1
        weight_decay: 1

        blobs_lr: 2
        weight_decay: 0

      inner_product_param {
        num_output: 4048
        weight_filler {
          type: "gaussian"
          std: 0.005
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layers {
      name: "relu6"
      type: RELU
      bottom: "fc6"
      top: "fc6"
    }
    layers {
      name: "drop6"
      type: DROPOUT
      bottom: "fc6"
      top: "fc6"
      dropout_param {
        dropout_ratio: 0.5
      }
    }
    layers {
      name: "fc7"
      type: INNER_PRODUCT
      bottom: "fc6"
      top: "fc7"
      # Note that blobs_lr can be set to 0 to disable any fine-tuning of this, and any other, layers

        blobs_lr: 1
        weight_decay: 1

        blobs_lr: 2
        weight_decay: 0

      inner_product_param {
        num_output: 4048
        weight_filler {
          type: "gaussian"
          std: 0.005
        }
        bias_filler {
          type: "constant"
          value: 1
        }
      }
    }
    layers {
      name: "relu7"
      type: RELU
      bottom: "fc7"
      top: "fc7"
    }
    layers {
      name: "drop7"
      type: DROPOUT
      bottom: "fc7"
      top: "fc7"
      dropout_param {
        dropout_ratio: 0.5
      }
    }
    layers {
      name: "fc8_temp"
      type: INNER_PRODUCT
      bottom: "fc7"
      top: "fc8_temp"
      # blobs_lr is set to higher than for other layers, because this layers is starting from random while the others are already trained
        blobs_lr: 10
        weight_decay: 1

        blobs_lr: 20
        weight_decay: 0

      inner_product_param {
        num_output: 16
        weight_filler {
          type: "gaussian"
          std: 0.01
        }
        bias_filler {
          type: "constant"
          value: 0
        }
      }
    }
    layers {
      name: "prob"
      type: SOFTMAX
      bottom: "fc8_temp"
      top: "prob"
    }

      

The command used to run the script is

    python confusion.py --proto deploy.prototxt --model models/model_iter_13000.caffemodel --meanfile mean.binaryproto --labelfile NamesTest.txt

      

I doubt why there is a difference in accuracy when I use the same model and the same test suite. Am I doing something wrong? Thank you in advance.

+1


source to share


1 answer


There are differences between your validation step (TEST step) and the python code you are using:



  • You use a different middle file for training and testing (!): For is phase: TRAIN

    used mean_file: "mean.binaryproto"

    and for is phase: TEST

    used mean_file: "painmean.binaryproto"

    . Your python evaluation code uses the average learning file, not validation.
    It is not recommended to have different settings for training / validation.

  • Your input images have new_height: 256

    and copr_size: 224

    . These settings mean that caffe reads an image, scales it to 256x256

    , and then crops the center to size 224x224

    . Your Python code looks like a scale for input 224x224

    without cropping: you feed your network with different inputs.

  • Please make sure you have no other differences between your training prototype and your deployment prototype.

+1


source







All Articles