NextCloud/pdlib
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Face recognition

Browse Source 
This change adds support to retrieve 128D face descriptor for a given
landmark. Since now we have full pipeline, README.md has "general usage"
section and integration test is added. Also, return from
FaceLandmarkDetection is changed, so it can be given to FaceRecognition
without changes. All obtained values are crosschecked to match with values
from python versions (however, if num_jitters is > 1 in FaceRecognition,
values don't match between PHP and Python, I suspect it is related to usage
of dlib::rand, but still investigating)..
This commit is contained in:
Branko Kokanovic
2018-08-28 00:37:46 +02:00
parent 1a402fc63c
commit 1e830a3285
11 changed files with 468 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/cnn_face_detection.cc
View File

@@ -79,13 +79,13 @@ PHP_METHOD(CnnFaceDetection, detect)
auto dets = (*pnet)(img);
int rect_count = 0;
array_init(return_value);
// Scale the detection locations back to the original image size
// if the image was upscaled.
//
for (auto&& d: dets) {
d.rect = pyr.rect_down(d.rect, upsample_num);
// Create new assoc array with dimensions of found rectt and confidence
// Create new assoc array with dimensions of found rect and confidence
//
zval rect_arr;
array_init(&rect_arr);

38
src/face_landmark_detection.cc
View File

@@ -102,20 +102,8 @@ PHP_METHOD(FaceLandmarkDetection, __construct)
// Helper macro to automatically have parsing of "top"/"bottom"/"left"/"right"
#define PARSE_BOUNDING_BOX_EDGE(side) \
zval* data##side; \
/* Tries to find given key in array */ \
data##side = zend_hash_str_find(bounding_box_hash, #side, sizeof(#side)-1); \
if (data##side == nullptr) { \
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Bounding box (second argument) is missing " #side "key"); \
return; \
} \
\
/* We also need to check proper type of value in associative array */ \
if (Z_TYPE_P(data##side) != IS_LONG) { \
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Value of bounding box's (second argument) " #side " key is not long type"); \
return; \
} \
zend_long side = Z_LVAL_P(data##side); \
PARSE_LONG_FROM_ARRAY(bounding_box_hash, side, \
"Bounding box (second argument) is missing " #side "key", "Value of bounding box's (second argument) " #side " key is not long type")
PHP_METHOD(FaceLandmarkDetection, detect)
{
@@ -127,15 +115,15 @@ PHP_METHOD(FaceLandmarkDetection, detect)
// Parse path to image and bounding box. Bounding box is associative array of 4 elements - "top", "bottom", "left" and "right".
//
if (zend_parse_parameters(ZEND_NUM_ARGS(), "sa", &img_path, &img_path_len, &bounding_box) == FAILURE){
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Unable to parse detect arguments");
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Unable to parse detect arguments.");
return;
}
// Check that bounding box have exactly 4 elements
HashTable *bounding_box_hash = Z_ARRVAL_P(bounding_box);
uint32_t bounding_box_num_elements = zend_hash_num_elements(bounding_box_hash);
if (bounding_box_num_elements != 4) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Bounding box (second argument) needs to have exactly 4 elements");
if (bounding_box_num_elements < 4) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Bounding box (second argument) needs to have at least 4 elements");
return;
}
@@ -158,14 +146,28 @@ PHP_METHOD(FaceLandmarkDetection, detect)
// Each key is one part from shape. Value of each part is associative array of keys "x" and "y".
//
array_init(return_value);
zval rect_arr, parts_arr;
array_init(&rect_arr);
array_init(&parts_arr);
for (int i = 0; i < shape.num_parts(); i++) {
zval part;
array_init(&part);
dlib::point p = shape.part(i);
add_assoc_long(&part, "x", p.x());
add_assoc_long(&part, "y", p.y());
add_next_index_zval(return_value, &part);
add_next_index_zval(&parts_arr, &part);
}
const rectangle& r = shape.get_rect();
add_assoc_long(&rect_arr, "left", r.left());
add_assoc_long(&rect_arr, "top", r.top());
add_assoc_long(&rect_arr, "right", r.right());
add_assoc_long(&rect_arr, "bottom", r.bottom());
add_assoc_zval(return_value, "rect", &rect_arr);
add_assoc_zval(return_value, "parts", &parts_arr);
} catch (exception& e) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, e.what());
return;

189
src/face_recognition.cc Normal file
View File

@@ -0,0 +1,189 @@
#include "../php_pdlib.h"
#include "face_recognition.h"
#include <zend_exceptions.h>
#include <dlib/image_io.h>
using namespace std;
using namespace dlib;
static inline face_recognition *php_face_recognition_from_obj(zend_object *obj) {
return (face_recognition*)((char*)(obj) - XtOffsetOf(face_recognition, std));
}
#define Z_FACE_RECOGNITION_P(zv) php_face_recognition_from_obj(Z_OBJ_P((zv)))
PHP_METHOD(FaceRecognition, __construct)
{
char *sz_face_recognition_model_path;
size_t face_recognition_model_path_len;
face_recognition *fr = Z_FACE_RECOGNITION_P(getThis());
if (NULL == fr) {
php_error_docref(NULL TSRMLS_CC, E_ERROR, "Unable to find obj in FaceRecognition::__construct()");
return;
}
if (zend_parse_parameters(ZEND_NUM_ARGS(), "s",
&sz_face_recognition_model_path, &face_recognition_model_path_len) == FAILURE){
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Unable to parse face_recognition_model_path");
return;
}
try {
string face_recognition_model_path(sz_face_recognition_model_path, face_recognition_model_path_len);
fr->net = new anet_type;
deserialize(face_recognition_model_path) >> *(fr->net);
} catch (exception& e) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, e.what());
return;
}
}
std::vector<matrix<rgb_pixel>> pdlib_jitter_image(
const matrix<rgb_pixel>& img,
const int num_jitters,
dlib::rand& rnd) {
std::vector<matrix<rgb_pixel>> crops;
for (int i = 0; i < num_jitters; ++i)
crops.push_back(dlib::jitter_image(img,rnd));
return crops;
}
// Helper macro to automatically have parsing of "top"/"bottom"/"left"/"right"
//
#define PARSE_BOUNDING_BOX_EDGE(side) \
PARSE_LONG_FROM_ARRAY(rect_hash, side, \
"Shape's rect array is missing " #side "key", "Shape's rect array's " #side " key is not long type")
// Helper macro to parse "x"/"y"
//
#define PARSE_POINT(coord) \
PARSE_LONG_FROM_ARRAY(part_hash, coord, \
#coord " coordinate key is missing in parts array", #coord " coordinate key is not of long type")
PHP_METHOD(FaceRecognition, computeDescriptor)
{
char *img_path;
size_t img_path_len;
zval *shape;
long num_jitters = 1;
if (zend_parse_parameters(ZEND_NUM_ARGS(), "sa|l", &img_path, &img_path_len, &shape, &num_jitters) == FAILURE){
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Unable to parse computeDescriptor arguments");
return;
}
HashTable *shape_hash = Z_ARRVAL_P(shape);
uint32_t shape_hash_num_elements = zend_hash_num_elements(shape_hash);
if (shape_hash_num_elements != 2) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Shape (second argument) needs to have exactly 2 elements - keys \"rect\" and \"parts\"");
return;
}
zval *rect_zval = zend_hash_str_find(shape_hash, "rect", sizeof("rect")-1);
if (rect_zval == nullptr) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Shape (second argument) array needs to have \"rect\" key"); \
return;
}
if (Z_TYPE_P(rect_zval) != IS_ARRAY) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Value of shape's key \"rect\" must be array");
return;
}
HashTable *rect_hash = Z_ARRVAL_P(rect_zval);
PARSE_BOUNDING_BOX_EDGE(top)
PARSE_BOUNDING_BOX_EDGE(bottom)
PARSE_BOUNDING_BOX_EDGE(left)
PARSE_BOUNDING_BOX_EDGE(right)
rectangle rect(left, top, right, bottom);
zval *parts_zval = zend_hash_str_find(shape_hash, "parts", sizeof("parts")-1);
if (parts_zval == nullptr) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Shape (second argument) array needs to have \"parts\" key"); \
return;
}
if (Z_TYPE_P(parts_zval) != IS_ARRAY) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Value of shape's key \"parts\" must be array");
return;
}
HashTable *parts_hash = Z_ARRVAL_P(parts_zval);
HashPosition parts_pos;
uint32_t parts_count = zend_hash_num_elements(parts_hash);
point parts_points[parts_count];
if ((parts_count != 5) && (parts_count != 68)) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC,
"The full_object_detection must use the iBUG 300W 68 point face landmark style or dlib's 5 point style");
return;
}
for (zend_hash_internal_pointer_reset_ex(parts_hash, &parts_pos);
zend_hash_has_more_elements_ex(parts_hash, &parts_pos) == SUCCESS;
zend_hash_move_forward_ex(parts_hash, &parts_pos)
) {
zend_string* str_index = {0};
zend_ulong num_index;
zval *part_zval = zend_hash_get_current_data_ex(parts_hash, &parts_pos);
switch (zend_hash_get_current_key_ex(parts_hash, &str_index, &num_index, &parts_pos)) {
case HASH_KEY_IS_LONG:
if (Z_TYPE_P(part_zval) == IS_ARRAY)
{
HashTable *part_hash = Z_ARRVAL_P(part_zval);
PARSE_POINT(x)
PARSE_POINT(y)
if (num_index > parts_count) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Internal error, bad parsing of parts array");
return;
}
parts_points[num_index] = point(x, y);
} else {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Values from parts array must be arrays with \"x\" and \"y\" keys");
return;
}
break;
case HASH_KEY_IS_STRING:
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, "Parts array must be indexed and it contains string keys");
return;
break;
}
}
std::vector<point> parts;
for (unsigned int i = 0; i < parts_count; i++) {
parts.push_back(parts_points[i]);
}
try {
face_recognition *fr = Z_FACE_RECOGNITION_P(getThis());
full_object_detection fod(rect, parts);
matrix<rgb_pixel> img;
load_image(img, img_path);
std::vector<chip_details> dets;
dets.push_back(get_face_chip_details(fod, 150, 0.25));
dlib::array<matrix<rgb_pixel>> face_chips;
extract_image_chips(img, dets, face_chips);
array_init(return_value);
matrix<float,0,1> face_descriptor;
if (num_jitters <= 1) {
std::vector<matrix<float,0,1>> face_descriptors = fr->net->operator()(face_chips, 16);
face_descriptor = face_descriptors[0];
} else {
matrix<rgb_pixel>& face_chip = face_chips[0];
face_descriptor = mean(mat(fr->net->operator()(pdlib_jitter_image(face_chip, num_jitters, fr->rnd), 16)));
}
for (auto& d : face_descriptor) {
add_next_index_double(return_value, d);
}
} catch (exception& e) {
zend_throw_exception_ex(zend_ce_exception, 0 TSRMLS_CC, e.what());
return;
}
}

58
src/face_recognition.h Normal file
View File

@@ -0,0 +1,58 @@
//
// Created by branko at kokanovic dot org on 2018/8/26.
//
#ifndef PHP_DLIB_FACE_RECOGNITION_H
#define PHP_DLIB_FACE_RECOGNITION_H
#include <dlib/dnn.h>
using namespace dlib;
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual = add_prev1<block<N,BN,1,tag1<SUBNET>>>;
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual_down = add_prev2<avg_pool<2,2,2,2,skip1<tag2<block<N,BN,2,tag1<SUBNET>>>>>>;
template <int N, template <typename> class BN, int stride, typename SUBNET>
using block = BN<con<N,3,3,1,1,relu<BN<con<N,3,3,stride,stride,SUBNET>>>>>;
template <int N, typename SUBNET> using ares = relu<residual<block,N,affine,SUBNET>>;
template <int N, typename SUBNET> using ares_down = relu<residual_down<block,N,affine,SUBNET>>;
template <typename SUBNET> using alevel0 = ares_down<256,SUBNET>;
template <typename SUBNET> using alevel1 = ares<256,ares<256,ares_down<256,SUBNET>>>;
template <typename SUBNET> using alevel2 = ares<128,ares<128,ares_down<128,SUBNET>>>;
template <typename SUBNET> using alevel3 = ares<64,ares<64,ares<64,ares_down<64,SUBNET>>>>;
template <typename SUBNET> using alevel4 = ares<32,ares<32,ares<32,SUBNET>>>;
using anet_type = loss_metric<fc_no_bias<128,avg_pool_everything<
alevel0<
alevel1<
alevel2<
alevel3<
alevel4<
max_pool<3,3,2,2,relu<affine<con<32,7,7,2,2,
input_rgb_image_sized<150>
>>>>>>>>>>>>;
typedef struct _face_recognition {
anet_type *net;
zend_object std;
dlib::rand rnd;
} face_recognition;
ZEND_BEGIN_ARG_INFO_EX(face_recognition_ctor_arginfo, 0, 0, 1)
ZEND_ARG_INFO(0, face_recognition_model_path)
ZEND_END_ARG_INFO()
PHP_METHOD(FaceRecognition, __construct);
ZEND_BEGIN_ARG_INFO_EX(face_recognition_compute_descriptor_arginfo, 0, 0, 3)
ZEND_ARG_INFO(0, img_path)
ZEND_ARG_INFO(0, landmarks)
ZEND_ARG_INFO(0, num_jitters)
ZEND_END_ARG_INFO()
PHP_METHOD(FaceRecognition, computeDescriptor);
#endif //PHP_DLIB_FACE_RECOGNITION_H