From 7200d07e968a0f0071d4794fb18933d2d3377ce4 Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Tue, 14 Oct 2014 20:27:17 +0000 Subject: [PATCH 1/9] allow two labels to be sent into contrastive loss --- include/caffe/loss_layers.hpp | 1 + src/caffe/layers/contrastive_loss_layer.cpp | 24 +++++++++++++++++++-- src/caffe/layers/contrastive_loss_layer.cu | 19 ++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/include/caffe/loss_layers.hpp b/include/caffe/loss_layers.hpp index 9fe58cd97bc..7f5c7fa251c 100644 --- a/include/caffe/loss_layers.hpp +++ b/include/caffe/loss_layers.hpp @@ -202,6 +202,7 @@ class ContrastiveLossLayer : public LossLayer { virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom); + Blob similar_; // either copied or calc'd from equality of two labels Blob diff_; // cached for backward pass Blob dist_sq_; // cached for backward pass Blob diff_sq_; // tmp storage for gpu forward pass diff --git a/src/caffe/layers/contrastive_loss_layer.cpp b/src/caffe/layers/contrastive_loss_layer.cpp index 0d0b443b66b..4bffa1966e3 100644 --- a/src/caffe/layers/contrastive_loss_layer.cpp +++ b/src/caffe/layers/contrastive_loss_layer.cpp @@ -20,6 +20,11 @@ void ContrastiveLossLayer::LayerSetUp( CHECK_EQ(bottom[2]->channels(), 1); CHECK_EQ(bottom[2]->height(), 1); CHECK_EQ(bottom[2]->width(), 1); + + CHECK_GE(bottom.size(), 3); + CHECK_LE(bottom.size(), 4); + + similar_.Reshape(bottom[0]->num(), 1, 1, 1); diff_.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); diff_sq_.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); dist_sq_.Reshape(bottom[0]->num(), 1, 1, 1); @@ -41,11 +46,26 @@ void ContrastiveLossLayer::Forward_cpu( diff_.mutable_cpu_data()); // a_i-b_i const int channels = bottom[0]->channels(); Dtype margin = this->layer_param_.contrastive_loss_param().margin(); + + for (int i = 0; i < bottom[0]->num(); ++i) { + if (bottom.size() == 3) { + // 1/0 label provided directly + similar_.mutable_cpu_data()[i] = + static_cast(bottom[2]->cpu_data()[i]); + } else if (bottom.size() == 4) { + // two labels in [0,N] are provided; are they equal? + similar_.mutable_cpu_data()[i] = + (static_cast(bottom[2]->cpu_data()[i]) == + static_cast(bottom[3]->cpu_data()[i])) ; + } + } + Dtype loss(0.0); for (int i = 0; i < bottom[0]->num(); ++i) { dist_sq_.mutable_cpu_data()[i] = caffe_cpu_dot(channels, diff_.cpu_data() + (i*channels), diff_.cpu_data() + (i*channels)); - if (static_cast(bottom[2]->cpu_data()[i])) { // similar pairs + + if (static_cast(similar_.cpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); @@ -68,7 +88,7 @@ void ContrastiveLossLayer::Backward_cpu(const vector*>& top, int channels = bottom[i]->channels(); for (int j = 0; j < num; ++j) { Dtype* bout = bottom[i]->mutable_cpu_diff(); - if (static_cast(bottom[2]->cpu_data()[j])) { // similar pairs + if (static_cast(similar_.cpu_data()[j])) { // similar pairs caffe_cpu_axpby( channels, alpha, diff --git a/src/caffe/layers/contrastive_loss_layer.cu b/src/caffe/layers/contrastive_loss_layer.cu index 78a55995a0a..9bdfd8fcfd1 100644 --- a/src/caffe/layers/contrastive_loss_layer.cu +++ b/src/caffe/layers/contrastive_loss_layer.cu @@ -32,9 +32,23 @@ void ContrastiveLossLayer::Forward_gpu( Dtype(0.0), dist_sq_.mutable_gpu_data()); // \Sum (a_i-b_i)^2 Dtype margin = this->layer_param_.contrastive_loss_param().margin(); + + for (int i = 0; i < bottom[0]->num(); ++i) { + if (bottom.size() == 3) { + // 1/0 label provided directly + similar_.mutable_cpu_data()[i] = + static_cast(bottom[2]->cpu_data()[i]); + } else if (bottom.size() == 4) { + // two labels in [0,N] are provided; are they equal? + similar_.mutable_cpu_data()[i] = + (static_cast(bottom[2]->cpu_data()[i]) == + static_cast(bottom[3]->cpu_data()[i])) ; + } + } + Dtype loss(0.0); for (int i = 0; i < bottom[0]->num(); ++i) { - if (static_cast(bottom[2]->cpu_data()[i])) { // similar pairs + if (static_cast(similar_.gpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); @@ -51,6 +65,7 @@ __global__ void CLLForward(const int count, const int channels, Dtype *bottom_diff) { CUDA_KERNEL_LOOP(i, count) { int n = i / channels; // the num index, to access y and dist_sq + if (static_cast(y[n])) { // similar pairs bottom_diff[i] = alpha * diff[i]; } else { // dissimilar pairs @@ -77,7 +92,7 @@ void ContrastiveLossLayer::Backward_gpu(const vector*>& top, // NOLINT_NEXT_LINE(whitespace/operators) CLLForward<<>>( count, channels, margin, alpha, - bottom[2]->gpu_data(), // pair similarity 0 or 1 + similar_.gpu_data(), diff_.gpu_data(), // the cached eltwise difference between a and b dist_sq_.gpu_data(), // the cached square distance between a and b bottom[i]->mutable_gpu_diff()); From bb5d54406e773ed0b2fb1cf5179f8e34c1caef31 Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Tue, 14 Oct 2014 20:57:13 +0000 Subject: [PATCH 2/9] fix lint --- src/caffe/layers/contrastive_loss_layer.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/caffe/layers/contrastive_loss_layer.cpp b/src/caffe/layers/contrastive_loss_layer.cpp index 4bffa1966e3..25bd1dd0d35 100644 --- a/src/caffe/layers/contrastive_loss_layer.cpp +++ b/src/caffe/layers/contrastive_loss_layer.cpp @@ -50,13 +50,13 @@ void ContrastiveLossLayer::Forward_cpu( for (int i = 0; i < bottom[0]->num(); ++i) { if (bottom.size() == 3) { // 1/0 label provided directly - similar_.mutable_cpu_data()[i] = + similar_.mutable_cpu_data()[i] = static_cast(bottom[2]->cpu_data()[i]); } else if (bottom.size() == 4) { // two labels in [0,N] are provided; are they equal? similar_.mutable_cpu_data()[i] = (static_cast(bottom[2]->cpu_data()[i]) == - static_cast(bottom[3]->cpu_data()[i])) ; + static_cast(bottom[3]->cpu_data()[i])); } } @@ -65,7 +65,7 @@ void ContrastiveLossLayer::Forward_cpu( dist_sq_.mutable_cpu_data()[i] = caffe_cpu_dot(channels, diff_.cpu_data() + (i*channels), diff_.cpu_data() + (i*channels)); - if (static_cast(similar_.cpu_data()[i])) { // similar pairs + if (static_cast(similar_.cpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); @@ -88,7 +88,7 @@ void ContrastiveLossLayer::Backward_cpu(const vector*>& top, int channels = bottom[i]->channels(); for (int j = 0; j < num; ++j) { Dtype* bout = bottom[i]->mutable_cpu_diff(); - if (static_cast(similar_.cpu_data()[j])) { // similar pairs + if (static_cast(similar_.cpu_data()[j])) { // similar pairs caffe_cpu_axpby( channels, alpha, From 134b44f1cfae70de4c9bf4a8d700324e550a9042 Mon Sep 17 00:00:00 2001 From: jackculpepper Date: Tue, 14 Oct 2014 14:19:43 -0700 Subject: [PATCH 3/9] Sigh..lint --- src/caffe/layers/contrastive_loss_layer.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/caffe/layers/contrastive_loss_layer.cu b/src/caffe/layers/contrastive_loss_layer.cu index 9bdfd8fcfd1..964e8e3cc23 100644 --- a/src/caffe/layers/contrastive_loss_layer.cu +++ b/src/caffe/layers/contrastive_loss_layer.cu @@ -42,7 +42,7 @@ void ContrastiveLossLayer::Forward_gpu( // two labels in [0,N] are provided; are they equal? similar_.mutable_cpu_data()[i] = (static_cast(bottom[2]->cpu_data()[i]) == - static_cast(bottom[3]->cpu_data()[i])) ; + static_cast(bottom[3]->cpu_data()[i])); } } From 5698756a44607bdbdae18182ba645c9430034d43 Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Wed, 15 Oct 2014 19:49:12 +0000 Subject: [PATCH 4/9] fix gpu/cpu bug, do not cast when setting similar_ --- src/caffe/layers/contrastive_loss_layer.cpp | 7 +++---- src/caffe/layers/contrastive_loss_layer.cu | 9 ++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/caffe/layers/contrastive_loss_layer.cpp b/src/caffe/layers/contrastive_loss_layer.cpp index 25bd1dd0d35..e75132bfb52 100644 --- a/src/caffe/layers/contrastive_loss_layer.cpp +++ b/src/caffe/layers/contrastive_loss_layer.cpp @@ -50,13 +50,12 @@ void ContrastiveLossLayer::Forward_cpu( for (int i = 0; i < bottom[0]->num(); ++i) { if (bottom.size() == 3) { // 1/0 label provided directly - similar_.mutable_cpu_data()[i] = - static_cast(bottom[2]->cpu_data()[i]); + similar_.mutable_cpu_data()[i] = bottom[2]->cpu_data()[i]; } else if (bottom.size() == 4) { // two labels in [0,N] are provided; are they equal? similar_.mutable_cpu_data()[i] = - (static_cast(bottom[2]->cpu_data()[i]) == - static_cast(bottom[3]->cpu_data()[i])); + (bottom[2]->cpu_data()[i] == + bottom[3]->cpu_data()[i] ? Dtype(1.0) : Dtype(0.0)); } } diff --git a/src/caffe/layers/contrastive_loss_layer.cu b/src/caffe/layers/contrastive_loss_layer.cu index 964e8e3cc23..227a1c8d615 100644 --- a/src/caffe/layers/contrastive_loss_layer.cu +++ b/src/caffe/layers/contrastive_loss_layer.cu @@ -36,19 +36,18 @@ void ContrastiveLossLayer::Forward_gpu( for (int i = 0; i < bottom[0]->num(); ++i) { if (bottom.size() == 3) { // 1/0 label provided directly - similar_.mutable_cpu_data()[i] = - static_cast(bottom[2]->cpu_data()[i]); + similar_.mutable_cpu_data()[i] = bottom[2]->cpu_data()[i]; } else if (bottom.size() == 4) { // two labels in [0,N] are provided; are they equal? similar_.mutable_cpu_data()[i] = - (static_cast(bottom[2]->cpu_data()[i]) == - static_cast(bottom[3]->cpu_data()[i])); + (bottom[2]->cpu_data()[i] == + bottom[3]->cpu_data()[i] ? Dtype(1.0) : Dtype(0.0)); } } Dtype loss(0.0); for (int i = 0; i < bottom[0]->num(); ++i) { - if (static_cast(similar_.gpu_data()[i])) { // similar pairs + if (static_cast(similar_.cpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); From ff0db9497b9cb904cab085d2b1031936e5da69e4 Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Wed, 15 Oct 2014 22:48:26 +0000 Subject: [PATCH 5/9] first pass at a prototxt file using new loss functionality changes include code to generated shared png files based storage current setup should give roughly the same performance as before seems to achieve approximately the same performance could 'shuffle' in image_data_layer to generate extra train pairs --- examples/mnist/convert_mnist_data.cpp | 26 +++++++ examples/siamese/create_mnist_siamese.sh | 44 ++++++++---- .../siamese/mnist_siamese_train_test.prototxt | 72 ++++++++++++------- examples/siamese/train_mnist_siamese.sh | 2 +- include/caffe/loss_layers.hpp | 4 +- src/caffe/layers/contrastive_loss_layer.cpp | 3 - 6 files changed, 108 insertions(+), 43 deletions(-) diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 2749e4521b6..732f4a20da1 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -18,6 +18,8 @@ #include // NOLINT(readability/streams) #include +#include "opencv2/core/core.hpp" +#include "opencv2/opencv.hpp" #include "caffe/proto/caffe.pb.h" using namespace caffe; // NOLINT(build/namespaces) @@ -94,6 +96,16 @@ void convert_dataset(const char* image_filename, const char* label_filename, << "mdb_txn_begin failed"; CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS) << "mdb_open failed. Does the lmdb already exist? "; + } else if (db_backend == "files") { // png files + CHECK_EQ(mkdir(db_path, 0744), 0) + << "mkdir " << db_path << "failed"; + for (int i = 0; i < 10; i++) { + int pathlen = strlen(db_path) + strlen("/0") + 1; + char path[pathlen]; + snprintf(path, pathlen, "%s/%c", db_path, '0' + i); + CHECK_EQ(mkdir(path, 0744), 0) + << "mkdir " << path << "failed"; + } } else { LOG(FATAL) << "Unknown db backend " << db_backend; } @@ -114,6 +126,7 @@ void convert_dataset(const char* image_filename, const char* label_filename, LOG(INFO) << "Rows: " << rows << " Cols: " << cols; for (int item_id = 0; item_id < num_items; ++item_id) { image_file.read(pixels, rows * cols); + label_file.read(&label, 1); datum.set_data(pixels, rows*cols); datum.set_label(label); @@ -131,6 +144,15 @@ void convert_dataset(const char* image_filename, const char* label_filename, mdb_key.mv_data = reinterpret_cast(&keystr[0]); CHECK_EQ(mdb_put(mdb_txn, mdb_dbi, &mdb_key, &mdb_data, 0), MDB_SUCCESS) << "mdb_put failed"; + } else if (db_backend == "files") { // png files + cv::Mat img = cv::Mat(28, 28, CV_8UC1, pixels); + + int pathlen = strlen(db_path) + strlen("/0/12345.png") + 1; + char filename[pathlen]; + + snprintf(filename, pathlen, "%s/%c/%05d.png", db_path, '0' + label, item_id); + //std::cout << "writing " << filename << std::endl; + cv::imwrite(filename, img); } else { LOG(FATAL) << "Unknown db backend " << db_backend; } @@ -146,6 +168,8 @@ void convert_dataset(const char* image_filename, const char* label_filename, << "mdb_txn_commit failed"; CHECK_EQ(mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn), MDB_SUCCESS) << "mdb_txn_begin failed"; + } else if (db_backend == "files") { // png files + } else { LOG(FATAL) << "Unknown db backend " << db_backend; } @@ -161,6 +185,8 @@ void convert_dataset(const char* image_filename, const char* label_filename, CHECK_EQ(mdb_txn_commit(mdb_txn), MDB_SUCCESS) << "mdb_txn_commit failed"; mdb_close(mdb_env, mdb_dbi); mdb_env_close(mdb_env); + } else if (db_backend == "files") { // files + } else { LOG(FATAL) << "Unknown db backend " << db_backend; } diff --git a/examples/siamese/create_mnist_siamese.sh b/examples/siamese/create_mnist_siamese.sh index 43ad6b184a7..4d6326304ad 100755 --- a/examples/siamese/create_mnist_siamese.sh +++ b/examples/siamese/create_mnist_siamese.sh @@ -1,21 +1,37 @@ #!/usr/bin/env sh -# This script converts the mnist data into leveldb format. +# This script converts the mnist data into lmdb/leveldb format, +# depending on the value assigned to $BACKEND. -EXAMPLES=./build/examples/siamese -DATA=./data/mnist +EXAMPLE=examples/siamese +DATA=data/mnist +BUILD=build/examples/mnist -echo "Creating leveldb..." +BACKEND="lmdb" +BACKEND="files" -rm -rf ./examples/siamese/mnist_siamese_train_leveldb -rm -rf ./examples/siamese/mnist_siamese_test_leveldb +echo "Creating ${BACKEND}..." -$EXAMPLES/convert_mnist_siamese_data.bin \ - $DATA/train-images-idx3-ubyte \ - $DATA/train-labels-idx1-ubyte \ - ./examples/siamese/mnist_siamese_train_leveldb -$EXAMPLES/convert_mnist_siamese_data.bin \ - $DATA/t10k-images-idx3-ubyte \ - $DATA/t10k-labels-idx1-ubyte \ - ./examples/siamese/mnist_siamese_test_leveldb +rm -rf $EXAMPLE/mnist_train_${BACKEND} +rm -rf $EXAMPLE/mnist_test_${BACKEND} + +$BUILD/convert_mnist_data.bin $DATA/train-images-idx3-ubyte \ + $DATA/train-labels-idx1-ubyte $EXAMPLE/mnist_train_${BACKEND} --backend=${BACKEND} +$BUILD/convert_mnist_data.bin $DATA/t10k-images-idx3-ubyte \ + $DATA/t10k-labels-idx1-ubyte $EXAMPLE/mnist_test_${BACKEND} --backend=${BACKEND} + +for t in train test +do + rm -f $EXAMPLE/mnist_${t}.txt + + for d in 0 1 2 3 4 5 6 7 8 9 + do + for f in $EXAMPLE/mnist_${t}_${BACKEND}/$d/*.png + do + echo "$f $d" >>$EXAMPLE/mnist_${t}.txt + done + done + + shuf $EXAMPLE/mnist_${t}.txt >$EXAMPLE/mnist_${t}_p.txt +done echo "Done." diff --git a/examples/siamese/mnist_siamese_train_test.prototxt b/examples/siamese/mnist_siamese_train_test.prototxt index 92361c31dc7..93ab26608fb 100644 --- a/examples/siamese/mnist_siamese_train_test.prototxt +++ b/examples/siamese/mnist_siamese_train_test.prototxt @@ -1,38 +1,61 @@ name: "mnist_siamese_train_test" + + layers { - name: "pair_data" - type: DATA - top: "pair_data" - top: "sim" - data_param { - source: "examples/siamese/mnist_siamese_train_leveldb" + name: "data" + type: IMAGE_DATA + top: "data" + top: "label" + image_data_param { + source: "examples/siamese/mnist_train.txt" + batch_size: 100 scale: 0.00390625 - batch_size: 64 + shuffle: false } include: { phase: TRAIN } } + layers { - name: "pair_data" - type: DATA - top: "pair_data" - top: "sim" - data_param { - source: "examples/siamese/mnist_siamese_test_leveldb" - scale: 0.00390625 + name: "data" + type: IMAGE_DATA + top: "data" + top: "label" + image_data_param { + source: "examples/siamese/mnist_test.txt" batch_size: 100 + scale: 0.00390625 + shuffle: false } include: { phase: TEST } } + + layers { - name: "slice_pair" - type: SLICE - bottom: "pair_data" - top: "data" - top: "data_p" - slice_param { - slice_dim: 1 - slice_point: 1 - } + name: "data_p" + type: IMAGE_DATA + top: "data_p" + top: "label_p" + image_data_param { + source: "examples/siamese/mnist_train_p.txt" + batch_size: 100 + scale: 0.00390625 + shuffle: false + } + include: { phase: TRAIN } +} + +layers { + name: "data_p" + type: IMAGE_DATA + top: "data_p" + top: "label_p" + image_data_param { + source: "examples/siamese/mnist_test_p.txt" + batch_size: 100 + scale: 0.00390625 + shuffle: false + } + include: { phase: TEST } } @@ -308,6 +331,7 @@ layers { } bottom: "feat" bottom: "feat_p" - bottom: "sim" + bottom: "label" + bottom: "label_p" top: "loss" } diff --git a/examples/siamese/train_mnist_siamese.sh b/examples/siamese/train_mnist_siamese.sh index 84a30a8ac44..5666a7c6d3e 100755 --- a/examples/siamese/train_mnist_siamese.sh +++ b/examples/siamese/train_mnist_siamese.sh @@ -2,4 +2,4 @@ TOOLS=./build/tools -$TOOLS/caffe train --solver=examples/siamese/mnist_siamese_solver.prototxt +GLOG_logtostderr=0 GLOG_log_dir=./examples/siamese/ $TOOLS/caffe train --solver=examples/siamese/mnist_siamese_solver.prototxt diff --git a/include/caffe/loss_layers.hpp b/include/caffe/loss_layers.hpp index 7f5c7fa251c..1f36987001c 100644 --- a/include/caffe/loss_layers.hpp +++ b/include/caffe/loss_layers.hpp @@ -153,7 +153,9 @@ class ContrastiveLossLayer : public LossLayer { virtual void LayerSetUp(const vector*>& bottom, const vector*>& top); - virtual inline int ExactNumBottomBlobs() const { return 3; } + virtual inline int ExactNumBottomBlobs() const { return -1; } + virtual inline int MinBottomBlobs() const { return 3; } + virtual inline int MaxBottomBlobs() const { return 4; } virtual inline LayerParameter_LayerType type() const { return LayerParameter_LayerType_CONTRASTIVE_LOSS; } diff --git a/src/caffe/layers/contrastive_loss_layer.cpp b/src/caffe/layers/contrastive_loss_layer.cpp index e75132bfb52..5d12690d12a 100644 --- a/src/caffe/layers/contrastive_loss_layer.cpp +++ b/src/caffe/layers/contrastive_loss_layer.cpp @@ -21,9 +21,6 @@ void ContrastiveLossLayer::LayerSetUp( CHECK_EQ(bottom[2]->height(), 1); CHECK_EQ(bottom[2]->width(), 1); - CHECK_GE(bottom.size(), 3); - CHECK_LE(bottom.size(), 4); - similar_.Reshape(bottom[0]->num(), 1, 1, 1); diff_.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); diff_sq_.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); From f2a0c9e73c02d36e696629fef07aad9a8a9defc5 Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Thu, 16 Oct 2014 21:34:56 +0000 Subject: [PATCH 6/9] shuffle training data --- examples/siamese/mnist_siamese_train_test.prototxt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/siamese/mnist_siamese_train_test.prototxt b/examples/siamese/mnist_siamese_train_test.prototxt index 93ab26608fb..6b71f5a0b62 100644 --- a/examples/siamese/mnist_siamese_train_test.prototxt +++ b/examples/siamese/mnist_siamese_train_test.prototxt @@ -10,7 +10,7 @@ layers { source: "examples/siamese/mnist_train.txt" batch_size: 100 scale: 0.00390625 - shuffle: false + shuffle: true } include: { phase: TRAIN } } @@ -39,7 +39,7 @@ layers { source: "examples/siamese/mnist_train_p.txt" batch_size: 100 scale: 0.00390625 - shuffle: false + shuffle: true } include: { phase: TRAIN } } From 4bbfea43d4586ffaacbfb81bb38a83da99d9be65 Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Fri, 17 Oct 2014 17:22:21 +0000 Subject: [PATCH 7/9] specify gpu --- examples/siamese/train_mnist_siamese.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/siamese/train_mnist_siamese.sh b/examples/siamese/train_mnist_siamese.sh index 5666a7c6d3e..6142ebc5d54 100755 --- a/examples/siamese/train_mnist_siamese.sh +++ b/examples/siamese/train_mnist_siamese.sh @@ -2,4 +2,4 @@ TOOLS=./build/tools -GLOG_logtostderr=0 GLOG_log_dir=./examples/siamese/ $TOOLS/caffe train --solver=examples/siamese/mnist_siamese_solver.prototxt +GLOG_logtostderr=0 GLOG_log_dir=./examples/siamese/ $TOOLS/caffe train --solver=examples/siamese/mnist_siamese_solver.prototxt --gpu=0 From fd97c2e9daa703bd488fe69d92d2adf599bfd0ac Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Thu, 23 Oct 2014 01:47:49 +0000 Subject: [PATCH 8/9] use boost to create directories --- Makefile | 2 +- examples/mnist/convert_mnist_data.cpp | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 35c37af50fb..9ac60c0caa6 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ ifneq ($(CPU_ONLY), 1) LIBRARIES := cudart cublas curand endif LIBRARIES += glog gflags protobuf leveldb snappy \ - lmdb boost_system hdf5_hl hdf5 m \ + lmdb boost_filesystem boost_system hdf5_hl hdf5 m \ opencv_core opencv_highgui opencv_imgproc PYTHON_LIBRARIES := boost_python python2.7 WARNINGS := -Wall -Wno-sign-compare diff --git a/examples/mnist/convert_mnist_data.cpp b/examples/mnist/convert_mnist_data.cpp index 732f4a20da1..b2c7a586e13 100644 --- a/examples/mnist/convert_mnist_data.cpp +++ b/examples/mnist/convert_mnist_data.cpp @@ -18,6 +18,8 @@ #include // NOLINT(readability/streams) #include +#include + #include "opencv2/core/core.hpp" #include "opencv2/opencv.hpp" #include "caffe/proto/caffe.pb.h" @@ -85,8 +87,9 @@ void convert_dataset(const char* image_filename, const char* label_filename, batch = new leveldb::WriteBatch(); } else if (db_backend == "lmdb") { // lmdb LOG(INFO) << "Opening lmdb " << db_path; - CHECK_EQ(mkdir(db_path, 0744), 0) - << "mkdir " << db_path << "failed"; + boost::filesystem::path p(db_path); + CHECK_EQ(boost::filesystem::create_directories(p), 0) + << "mkdir " << db_path << " failed"; CHECK_EQ(mdb_env_create(&mdb_env), MDB_SUCCESS) << "mdb_env_create failed"; CHECK_EQ(mdb_env_set_mapsize(mdb_env, 1099511627776), MDB_SUCCESS) // 1TB << "mdb_env_set_mapsize failed"; @@ -97,14 +100,19 @@ void convert_dataset(const char* image_filename, const char* label_filename, CHECK_EQ(mdb_open(mdb_txn, NULL, 0, &mdb_dbi), MDB_SUCCESS) << "mdb_open failed. Does the lmdb already exist? "; } else if (db_backend == "files") { // png files - CHECK_EQ(mkdir(db_path, 0744), 0) - << "mkdir " << db_path << "failed"; + boost::filesystem::path p(db_path); + if (! boost::filesystem::is_directory(p)) + CHECK_EQ(boost::filesystem::create_directories(p), 1) + << "mkdir " << p << " failed"; for (int i = 0; i < 10; i++) { - int pathlen = strlen(db_path) + strlen("/0") + 1; - char path[pathlen]; - snprintf(path, pathlen, "%s/%c", db_path, '0' + i); - CHECK_EQ(mkdir(path, 0744), 0) - << "mkdir " << path << "failed"; + char buf[2]; + snprintf(buf, 2, "%c", '0' + i); + string digit_dir_name(buf); + boost::filesystem::path digit_path = p / digit_dir_name; + + if (! boost::filesystem::is_directory(digit_path)) + CHECK_EQ(boost::filesystem::create_directories(digit_path), 1) + << "mkdir " << digit_path << " failed"; } } else { LOG(FATAL) << "Unknown db backend " << db_backend; From 52ec7469c148d56fb10054be617e89905e204239 Mon Sep 17 00:00:00 2001 From: Jack Culpepper Date: Thu, 23 Oct 2014 01:56:06 +0000 Subject: [PATCH 9/9] rename "similar_" field to "label_" add enum to params so that choice of label type is more explicit refactor label computation code into a private function add binary function generator and "caffe_cpu_same()" functions to math --- include/caffe/loss_layers.hpp | 6 ++- include/caffe/util/math_functions.hpp | 13 +++++++ src/caffe/layers/contrastive_loss_layer.cpp | 41 ++++++++++++++------- src/caffe/layers/contrastive_loss_layer.cu | 16 ++------ src/caffe/proto/caffe.proto | 10 ++++- 5 files changed, 57 insertions(+), 29 deletions(-) diff --git a/include/caffe/loss_layers.hpp b/include/caffe/loss_layers.hpp index 1f36987001c..b28c7b55607 100644 --- a/include/caffe/loss_layers.hpp +++ b/include/caffe/loss_layers.hpp @@ -204,11 +204,15 @@ class ContrastiveLossLayer : public LossLayer { virtual void Backward_gpu(const vector*>& top, const vector& propagate_down, const vector*>& bottom); - Blob similar_; // either copied or calc'd from equality of two labels + Blob label_; // either copied or calc'd from equality of two labels Blob diff_; // cached for backward pass Blob dist_sq_; // cached for backward pass Blob diff_sq_; // tmp storage for gpu forward pass Blob summer_vec_; // tmp storage for gpu forward pass + + private: + virtual void LabelSetUp(const vector*>& bottom); + }; /** diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp index 12823f8da0c..f2de1b1794c 100644 --- a/include/caffe/util/math_functions.hpp +++ b/include/caffe/util/math_functions.hpp @@ -112,6 +112,19 @@ inline int8_t caffe_sign(Dtype val) { return (Dtype(0) < val) - (val < Dtype(0)); } +#define DEFINE_CAFFE_CPU_BINARY_FUNC(name, operation) \ + template \ + void caffe_cpu_##name(const int n, const Dtype* a, const Dtype *b, Dtype* y) { \ + CHECK_GT(n, 0); CHECK(a); CHECK(b); CHECK(y); \ + for (int i = 0; i < n; ++i) { \ + operation; \ + } \ + } + +// output is 1 for the sames, 0 for not sames +DEFINE_CAFFE_CPU_BINARY_FUNC(same, y[i] = (a[i] == b[i])); + + // The following two macros are modifications of DEFINE_VSL_UNARY_FUNC // in include/caffe/util/mkl_alternate.hpp authored by @Rowland Depp. // Please refer to commit 7e8ef25c7 of the boost-eigen branch. diff --git a/src/caffe/layers/contrastive_loss_layer.cpp b/src/caffe/layers/contrastive_loss_layer.cpp index 5d12690d12a..d7b60ddba15 100644 --- a/src/caffe/layers/contrastive_loss_layer.cpp +++ b/src/caffe/layers/contrastive_loss_layer.cpp @@ -21,7 +21,7 @@ void ContrastiveLossLayer::LayerSetUp( CHECK_EQ(bottom[2]->height(), 1); CHECK_EQ(bottom[2]->width(), 1); - similar_.Reshape(bottom[0]->num(), 1, 1, 1); + label_.Reshape(bottom[0]->num(), 1, 1, 1); diff_.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); diff_sq_.Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); dist_sq_.Reshape(bottom[0]->num(), 1, 1, 1); @@ -31,6 +31,29 @@ void ContrastiveLossLayer::LayerSetUp( summer_vec_.mutable_cpu_data()[i] = Dtype(1); } +template +void ContrastiveLossLayer::LabelSetUp( + const vector*>& bottom) { + + switch (this->layer_param_.contrastive_loss_param().label_type()) { + case ContrastiveLossParameter_LabelType_PAIR_SIMILARITY: + CHECK_EQ(bottom.size(), 3); + caffe_copy(bottom[0]->num(), + bottom[2]->cpu_data(), + label_.mutable_cpu_data()); + break; + case ContrastiveLossParameter_LabelType_CATEGORY: + CHECK_EQ(bottom.size(), 4); + caffe_cpu_same(bottom[0]->num(), + bottom[2]->cpu_data(), + bottom[3]->cpu_data(), + label_.mutable_cpu_data()); + break; + default: + LOG(FATAL) << "Unknown label type"; + } +} + template void ContrastiveLossLayer::Forward_cpu( const vector*>& bottom, @@ -44,24 +67,14 @@ void ContrastiveLossLayer::Forward_cpu( const int channels = bottom[0]->channels(); Dtype margin = this->layer_param_.contrastive_loss_param().margin(); - for (int i = 0; i < bottom[0]->num(); ++i) { - if (bottom.size() == 3) { - // 1/0 label provided directly - similar_.mutable_cpu_data()[i] = bottom[2]->cpu_data()[i]; - } else if (bottom.size() == 4) { - // two labels in [0,N] are provided; are they equal? - similar_.mutable_cpu_data()[i] = - (bottom[2]->cpu_data()[i] == - bottom[3]->cpu_data()[i] ? Dtype(1.0) : Dtype(0.0)); - } - } + LabelSetUp(bottom); Dtype loss(0.0); for (int i = 0; i < bottom[0]->num(); ++i) { dist_sq_.mutable_cpu_data()[i] = caffe_cpu_dot(channels, diff_.cpu_data() + (i*channels), diff_.cpu_data() + (i*channels)); - if (static_cast(similar_.cpu_data()[i])) { // similar pairs + if (static_cast(label_.cpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); @@ -84,7 +97,7 @@ void ContrastiveLossLayer::Backward_cpu(const vector*>& top, int channels = bottom[i]->channels(); for (int j = 0; j < num; ++j) { Dtype* bout = bottom[i]->mutable_cpu_diff(); - if (static_cast(similar_.cpu_data()[j])) { // similar pairs + if (static_cast(label_.cpu_data()[j])) { // similar pairs caffe_cpu_axpby( channels, alpha, diff --git a/src/caffe/layers/contrastive_loss_layer.cu b/src/caffe/layers/contrastive_loss_layer.cu index 227a1c8d615..b7805d887c0 100644 --- a/src/caffe/layers/contrastive_loss_layer.cu +++ b/src/caffe/layers/contrastive_loss_layer.cu @@ -33,21 +33,11 @@ void ContrastiveLossLayer::Forward_gpu( dist_sq_.mutable_gpu_data()); // \Sum (a_i-b_i)^2 Dtype margin = this->layer_param_.contrastive_loss_param().margin(); - for (int i = 0; i < bottom[0]->num(); ++i) { - if (bottom.size() == 3) { - // 1/0 label provided directly - similar_.mutable_cpu_data()[i] = bottom[2]->cpu_data()[i]; - } else if (bottom.size() == 4) { - // two labels in [0,N] are provided; are they equal? - similar_.mutable_cpu_data()[i] = - (bottom[2]->cpu_data()[i] == - bottom[3]->cpu_data()[i] ? Dtype(1.0) : Dtype(0.0)); - } - } + LabelSetUp(bottom); Dtype loss(0.0); for (int i = 0; i < bottom[0]->num(); ++i) { - if (static_cast(similar_.cpu_data()[i])) { // similar pairs + if (static_cast(label_.cpu_data()[i])) { // similar pairs loss += dist_sq_.cpu_data()[i]; } else { // dissimilar pairs loss += std::max(margin-dist_sq_.cpu_data()[i], Dtype(0.0)); @@ -91,7 +81,7 @@ void ContrastiveLossLayer::Backward_gpu(const vector*>& top, // NOLINT_NEXT_LINE(whitespace/operators) CLLForward<<>>( count, channels, margin, alpha, - similar_.gpu_data(), + label_.gpu_data(), diff_.gpu_data(), // the cached eltwise difference between a and b dist_sq_.gpu_data(), // the cached square distance between a and b bottom[i]->mutable_gpu_diff()); diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index f0404a09b90..59def6a6c4a 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -380,8 +380,16 @@ message ConcatParameter { // Message that stores parameters used by ContrastiveLossLayer message ContrastiveLossParameter { - //margin for dissimilar pair + // margin for dissimilar pair optional float margin = 1 [default = 1.0]; + // types of labels on the bottom: + // PAIR_SIMILARITY -- there should be 3 bottoms total + // CATEGORY -- there should be 4 bottoms total + optional LabelType label_type = 2 [default = PAIR_SIMILARITY]; + enum LabelType { + PAIR_SIMILARITY = 0; // a single 0/1 label, where 1 indicates "same" + CATEGORY = 1; // a pair of category labels in [0,N-1] + } } // Message that stores parameters used by ConvolutionLayer