diff --git a/envs/linux_desktop.py b/envs/linux_desktop.py index 6150623..e4e8e03 100644 --- a/envs/linux_desktop.py +++ b/envs/linux_desktop.py @@ -45,7 +45,7 @@ get_srcs('ext_modules/libmaix/components/third_party/imlib/src') + get_srcs('ext_modules/_maix_image'), libraries=[ - "opencv_videoio", "opencv_highgui", "opencv_core", "opencv_imgproc", "opencv_imgcodecs", "opencv_freetype" + "opencv_videoio", "opencv_highgui", "opencv_core", "opencv_imgproc", "opencv_imgcodecs", "opencv_freetype", "opencv_flann", "opencv_features2d", "opencv_calib3d" ], library_dirs=[ "/usr/lib/", diff --git a/envs/maix_r329.py b/envs/maix_r329.py index f97f070..3ea0e47 100644 --- a/envs/maix_r329.py +++ b/envs/maix_r329.py @@ -41,7 +41,7 @@ get_srcs('ext_modules/libmaix/components/maix_cv_image/src') + get_srcs('ext_modules/libmaix/components/third_party/imlib/src'), libraries=[ - "maix_image","maix_disp", "opencv_videoio", "opencv_highgui", "opencv_core", "opencv_imgproc", "opencv_imgcodecs", "opencv_freetype" + "maix_image","maix_disp", "opencv_videoio", "opencv_highgui", "opencv_core", "opencv_imgproc", "opencv_imgcodecs", "opencv_freetype", "opencv_flann", "opencv_features2d", "opencv_calib3d" ], library_dirs=["./ext_modules/libmaix/components/libmaix/lib/arch/r329/opencv4", ext_so, diff --git a/envs/maix_v83x.py b/envs/maix_v83x.py index 28d7d6c..751e273 100644 --- a/envs/maix_v83x.py +++ b/envs/maix_v83x.py @@ -211,8 +211,7 @@ get_srcs('ext_modules/libmaix/components/third_party/imlib/src'), libraries=[ "maix_utils", "maix_cam", "maix_image", - "opencv_videoio", "opencv_highgui", "opencv_core", "opencv_imgproc", "opencv_imgcodecs", "opencv_freetype", - # "opencv_xfeatures2d", "opencv_features2d", "opencv_flann", "opencv_ml", "opencv_calib3d", "opencv_shape", + "opencv_videoio", "opencv_highgui", "opencv_core", "opencv_imgproc", "opencv_imgcodecs", "opencv_freetype", "opencv_flann", "opencv_features2d", "opencv_calib3d" ], library_dirs=[ ext_so, diff --git a/ext_modules/_maix_image/include/cv_orb.hpp b/ext_modules/_maix_image/include/cv_orb.hpp new file mode 100644 index 0000000..2e2ac7f --- /dev/null +++ b/ext_modules/_maix_image/include/cv_orb.hpp @@ -0,0 +1,146 @@ +#ifndef cv_orb_hpp +#define cv_orb_hpp + +#include "maix_image.h" + +/* +from maix import * + +orb = image.orb() +print(orb) + +src = None +for i in range(10): + src = camera.capture() + display.show(src) + +while True: + img = camera.capture() + res = orb.match(src, img, limit=30, max=100, dump=0) + # print(len(res)) + for point in res: + # print(point) + img.draw_circle(point[3], point[4], 4, color=(0, 255, 0), thickness=1) + display.show(img) +*/ +struct cv_orb +{ + cv::Ptr detector; + std::vector queryPoints, trainPoints; + cv::Mat queryDescriptors, trainDescriptors; + void *cache_src; + + cv_orb(int nfeatures = 500, float scaleFactor = 1.2f, int nlevels = 8, int edgeThreshold = 31, int firstLevel = 0, int WTA_K = 2, int scoreType = 0, int patchSize = 31, int fastThreshold = 20) + { + // printf("cv_orb: %d %f %d %d %d %d %d %d %d\n", nfeatures, scaleFactor, nlevels, edgeThreshold, firstLevel, WTA_K, scoreType, patchSize, fastThreshold); + this->detector = cv::ORB::create(nfeatures, scaleFactor, nlevels, edgeThreshold, firstLevel, WTA_K, (cv::ORB::ScoreType)scoreType, patchSize, fastThreshold); + this->cache_src = NULL; + } + + py::list match(maix_image &src, maix_image &dst, int limit = 10, float min = 0.5f, float max = 100.0f, bool crossCheck = false, bool dump = 0) + { + + py::list ret; + if (NULL == src._img || NULL == dst._img) + return ret; + + if (src._img->width != dst._img->width && src._img->height != dst._img->height) + { + printf("[orb] match images of the same size are required\r\n"); + return ret; + } + + cv::Mat img_src(src._img->height, src._img->width, CV_8UC3, src._img->data); + if (this->cache_src != src._img) + { + cv::cvtColor(img_src, img_src, cv::COLOR_RGB2GRAY); + this->detector->detectAndCompute(img_src, cv::Mat(), queryPoints, queryDescriptors); + // printf("queryPoints: %d\n", queryPoints.size()); + if (queryPoints.size() != 0) + { + this->cache_src = src._img; + } + else + { + this->cache_src = NULL; + printf("[orb] this image get queryPoints.size() == 0, replace it with another image.\r\n"); + return ret; + } + } + + // printf("queryDescriptors: rows %d\n", queryDescriptors.rows); + // printf("queryDescriptors: cols %d\n" , queryDescriptors.cols); + // printf("queryDescriptors: type %d\n", queryDescriptors.type()); + // printf("queryPoints: %d\n", queryPoints.size()); + + cv::Mat img_dst(dst._img->height, dst._img->width, CV_8UC3, dst._img->data); + cv::cvtColor(img_dst, img_dst, cv::COLOR_RGB2GRAY); + this->detector->detectAndCompute(img_dst, cv::Mat(), trainPoints, trainDescriptors); + + // printf("trainDescriptors: rows %d\n", trainDescriptors.rows); + // printf("trainDescriptors: cols %d\n" , trainDescriptors.cols); + // printf("trainDescriptors: type %d\n", trainDescriptors.type()); + // printf("trainPoints: %d\n", trainPoints.size()); + + if (trainPoints.size() == 0) + return ret; + + std::vector matches; + cv::BFMatcher matcher(cv::NORM_HAMMING, crossCheck); + + //如果采用flannBased方法 那么 desp通过orb的到的类型不同需要先转换类型 + // cv::FlannBasedMatcher matcher(new cv::flann::LshIndexParams(table_number, key_size, multi_probe_level)); + // if (queryDescriptors.type() != CV_32F || trainDescriptors.type() != CV_32F) + // { + // queryDescriptors.convertTo(queryDescriptors, CV_32F); + // trainDescriptors.convertTo(trainDescriptors, CV_32F); + // } + + matcher.match(queryDescriptors, trainDescriptors, matches); + + if (matches.size() == 0) + return ret; + + if (dump) // 避免之后 matches 顺序改变 + { + if (queryPoints.size() > 0) + { + cv::Mat imageOutput; + cv::drawKeypoints(img_src, queryPoints, imageOutput, cv::Scalar::all(-1), cv::DrawMatchesFlags::DRAW_RICH_KEYPOINTS); + cv::imwrite("cv_surf_img_src.jpg", imageOutput); + } + if (trainPoints.size() > 0) + { + cv::Mat imageOutput; + cv::drawKeypoints(img_dst, trainPoints, imageOutput, cv::Scalar::all(-1), cv::DrawMatchesFlags::DRAW_RICH_KEYPOINTS); + cv::imwrite("cv_surf_img_dst.jpg", imageOutput); + } + if (trainPoints.size() > queryPoints.size()) + { + cv::Mat imageOutput; + drawMatches(img_src, queryPoints, img_dst, trainPoints, matches, imageOutput); + cv::imwrite("cv_surf_img_matches.jpg", imageOutput); + } + } + + //过滤出特征点距离 distance 小于 limit 以内的特征点 + for (int i = 0; i < matches.size(); i++) + { + // printf("distance: %f\n", matches[i].distance); + if (matches[i].distance < max) + { + max = matches[i].distance; + if (max < min) + max = min; + auto src_points = queryPoints[matches[i].queryIdx], dst_points = trainPoints[matches[i].trainIdx]; + auto tmp = py::make_tuple(matches[i].distance, (int)src_points.pt.x, (int)src_points.pt.y, (int)dst_points.pt.x, (int)dst_points.pt.y); + ret.append(tmp); + } + if (ret.size() >= limit) + break; + } + return ret; + } +}; + +#endif \ No newline at end of file diff --git a/ext_modules/_maix_image/include/v83x_image.h b/ext_modules/_maix_image/include/v83x_image.h deleted file mode 100644 index 7cddb3f..0000000 --- a/ext_modules/_maix_image/include/v83x_image.h +++ /dev/null @@ -1,126 +0,0 @@ - - -#ifdef CONFIG_ARCH_V83X - -#include -#include - -struct cv_surf -{ - cv::Ptr surf; - cv::Ptr matcher;; - std::vector queryPoints, trainPoints; - cv::Mat queryDescriptors, trainDescriptors; - void *cache_src; - - cv_surf(double hessianThreshold = 100, int nOctaves = 4, int nOctaveLayers = 3, bool extended = false, bool upright = false) - { - this->surf = cv::xfeatures2d::SURF::create(hessianThreshold, nOctaves, nOctaveLayers, extended, upright); - this->matcher = cv::DescriptorMatcher::create("FlannBased"); - this->cache_src = NULL; - // // ORB - // cv::Ptr orb = cv::ORB::create(); - // { - // //提取特征点 - // std::vector keypointsL, keypointsR; - // cv::Mat descriptorsL, descriptorsR; - // orb->detectAndCompute(imageL, cv::noArray(), keypointsL, descriptorsL); - // orb->detectAndCompute(imageR, cv::noArray(), keypointsR, descriptorsR); - // //匹配特征点 - // cv::BFMatcher matcher(cv::NORM_L2); - // std::vector matches; - // matcher.match(descriptorsL, descriptorsR, matches); - // //绘制匹配结果 - // cv::Mat imgMatches; - // cv::drawMatches(imageL, keypointsL, imageR, keypointsR, matches, imgMatches); - // } - } - - py::list match(maix_image &src, maix_image &dst, int limit = 20, float filter = 0.5, bool dump = 0) - { - if (NULL == src._img || NULL == dst._img) return py::list(); - - cv::Mat img_src(src._img->height, src._img->width, CV_8UC3, src._img->data); - if (this->cache_src != src._img) - { - this->surf->detectAndCompute(img_src, cv::Mat(), queryPoints, queryDescriptors); - this->cache_src = src._img; - } - - cv::Mat img_dst(dst._img->height, dst._img->width, CV_8UC3, dst._img->data); - this->surf->detectAndCompute(img_dst, cv::Mat(), trainPoints, trainDescriptors); - - // Struct for DMatch: query descriptor index, train descriptor index, train image index and distance between descriptors. - // int queryIdx –>是测试图像的特征点描述符（descriptor）的下标，同时也是描述符对应特征点（keypoint)的下标。 - // int trainIdx –> 是样本图像的特征点描述符的下标，同样也是相应的特征点的下标。 - // int imgIdx –>当样本是多张图像的话有用。 - // float distance –>代表这一对匹配的特征点描述符（本质是向量）的欧氏距离，数值越小也就说明两个特征点越相像。 - std::vector matches; - - //如果采用flannBased方法 那么 desp通过orb的到的类型不同需要先转换类型 - if (queryDescriptors.type() != CV_32F || trainDescriptors.type() != CV_32F) - { - queryDescriptors.convertTo(queryDescriptors, CV_32F); - trainDescriptors.convertTo(trainDescriptors, CV_32F); - } - - this->matcher->match(queryDescriptors, trainDescriptors, matches); - - //特征点按 distance 升序 - std::sort(matches.begin(), matches.end(), [](const cv::DMatch &m1, const cv::DMatch &m2) - { return m1.distance < m2.distance; }); - - //获取最大距离的特征点 - float max_dist = matches[matches.size() - 1].distance; - - //过滤出特征点距离小于 max_dist * filter 以内的特征点 - limit = std::min(limit, (int)matches.size()); - std::vector good_matches(limit); - if (filter < 1.0f) - { - for (int i = 0, stop = (int)(max_dist * filter); i < limit; i++) - { - if (matches[i].distance < stop) - { - good_matches.push_back(matches[i]); - break; - } - } - } - else - { - good_matches.assign(matches.begin(), matches.end()); - } - - if (dump) - { - { - cv::Mat imageOutput; - cv::drawKeypoints(img_src, queryPoints, imageOutput, cv::Scalar::all(-1), cv::DrawMatchesFlags::DRAW_RICH_KEYPOINTS); - cv::imwrite("cv_surf_img_src.jpg", imageOutput); - } - { - cv::Mat imageOutput; - cv::drawKeypoints(img_dst, queryPoints, imageOutput, cv::Scalar::all(-1), cv::DrawMatchesFlags::DRAW_RICH_KEYPOINTS); - cv::imwrite("cv_surf_img_dst.jpg", imageOutput); - } - { - cv::Mat imageOutput; - drawMatches(img_src, queryPoints, img_dst, trainPoints, good_matches, imageOutput); - cv::imwrite("cv_surf_img_matches.jpg", imageOutput); - } - } - - py::list ret; - for (auto &m : good_matches) - { - auto src_points = queryPoints[m.queryIdx], dst_points = trainPoints[m.trainIdx]; - auto tmp = py::make_tuple(src_points.pt.x, src_points.pt.y, src_points.response, m.distance, dst_points.pt.x, dst_points.pt.y, dst_points.response); - ret.append(tmp); - } - - return ret; - } -}; - -#endif diff --git a/ext_modules/_maix_image/py_maix_image.cpp b/ext_modules/_maix_image/py_maix_image.cpp index c4a45d2..89b1ce5 100644 --- a/ext_modules/_maix_image/py_maix_image.cpp +++ b/ext_modules/_maix_image/py_maix_image.cpp @@ -1,6 +1,6 @@ #include "maix_image.h" -#include "v83x_image.h" +#include "cv_orb.hpp" void _load_freetype(std::string path, int fontHeight) { @@ -153,10 +153,8 @@ PYBIND11_MODULE(_maix_image, mo) .def("find_template", &maix_image::_imlib_find_template, py::arg("template") = maix_image(), py::arg("thresh") = 0.5, py::arg("roi") = std::vector{0, 0, 0, 0}, py::arg("step") = 2, py::arg("search") = 1) .def("find_barcodes", &maix_image::_imlib_find_barcodes, py::arg("roi") = std::vector{0, 0, 0, 0}); // module end -#ifdef CONFIG_ARCH_V83X - pybind11::class_(mo, "cv_surf") - .def(pybind11::init(), pybind11::arg("hessianThreshold") = 100, pybind11::arg("nOctaves") = 4, pybind11::arg("nOctaveLayers") = 3, pybind11::arg("extended") = false, pybind11::arg("upright") = false) - .def("match", &cv_surf::match, pybind11::arg("src"), pybind11::arg("dst"), pybind11::arg("limit") = 20, pybind11::arg("filter") = 0.5, pybind11::arg("dump") = 0); -#endif - + pybind11::class_(mo, "orb") + .def(pybind11::init(), + py::arg("nfeatures") = 500, py::arg("scaleFactor") = 1.2f, py::arg("nlevels") = 8, py::arg("edgeThreshold") = 31, py::arg("firstLevel") = 0, py::arg("WTA_K") = 2, py::arg("scoreType") = 0, py::arg("patchSize") = 31, py::arg("fastThreshold") = 20) + .def("match", &cv_orb::match, pybind11::arg("src"), pybind11::arg("dst"), pybind11::arg("limit") = 30, pybind11::arg("max") = 0.5f, pybind11::arg("max") = 100.f, pybind11::arg("crossCheck") = false, pybind11::arg("dump") = 0); } diff --git a/maix/__init__.py b/maix/__init__.py index 3cf6550..5278267 100644 --- a/maix/__init__.py +++ b/maix/__init__.py @@ -1,5 +1,5 @@ -version='0.5.0' +version='0.5.1' __all__ = ['display', 'camera', 'image']