Skip to content

Commit

Permalink
it all works
Browse files Browse the repository at this point in the history
  • Loading branch information
raphaelchang committed Sep 19, 2019
1 parent 3a62f35 commit 13e38e0
Show file tree
Hide file tree
Showing 15 changed files with 238 additions and 20 deletions.
29 changes: 27 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ project(omni_slam_eval)

set(CMAKE_VERBOSE_MAKEFILE FALSE)

add_definitions("-std=c++17")
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE)
endif (NOT CMAKE_BUILD_TYPE)
Expand Down Expand Up @@ -109,6 +108,16 @@ add_executable(omni_slam_stereo_eval_node
src/omni_slam_stereo_eval_node.cc
)

add_executable(omni_slam_slam_eval_node
src/ros/slam_eval.cc
src/ros/stereo_eval.cc
src/ros/odometry_eval.cc
src/ros/reconstruction_eval.cc
src/ros/tracking_eval.cc
src/ros/eval_base.cc
src/omni_slam_slam_eval_node.cc
)

target_link_libraries(omni_slam_tracking_eval_node PUBLIC omni_slam_eval_lib
${catkin_LIBRARIES}
${OpenCV_LIBS}
Expand Down Expand Up @@ -154,12 +163,22 @@ target_link_libraries(omni_slam_stereo_eval_node PUBLIC omni_slam_eval_lib
${CERES_LIBRARIES}
)

target_link_libraries(omni_slam_slam_eval_node PUBLIC omni_slam_eval_lib
${catkin_LIBRARIES}
${OpenCV_LIBS}
${HDF5_CXX_LIBRARIES}
${OpenMP_CXX_FLAGS}
${PCL_LIBRARIES}
${CERES_LIBRARIES}
)

target_compile_options(omni_slam_tracking_eval_node PUBLIC ${OpenMP_CXX_FLAGS})
target_compile_options(omni_slam_matching_eval_node PUBLIC ${OpenMP_CXX_FLAGS})
target_compile_options(omni_slam_reconstruction_eval_node PUBLIC ${OpenMP_CXX_FLAGS})
target_compile_options(omni_slam_odometry_eval_node PUBLIC ${OpenMP_CXX_FLAGS})
target_compile_options(omni_slam_stereo_eval_node PUBLIC ${OpenMP_CXX_FLAGS})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -ggdb")
target_compile_options(omni_slam_slam_eval_node PUBLIC ${OpenMP_CXX_FLAGS})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -std=c++17")

install(TARGETS omni_slam_tracking_eval_node
ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
Expand Down Expand Up @@ -190,3 +209,9 @@ install(TARGETS omni_slam_stereo_eval_node
LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
)

install(TARGETS omni_slam_slam_eval_node
ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
)
52 changes: 52 additions & 0 deletions launch/slam_eval.launch
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
<launch>
<arg name="bag_file" default="" />
<arg name="results_file" default="$(arg bag_file).slam.hdf5" />
<arg name="camera_model" default="double_sphere" />
<arg name="camera_params" default="{fx: 295.936, fy: 295.936, cx: 512, cy: 512, chi: 0.3, alpha: 0.6666667}" />
<arg name="rate" default="1" />
<node pkg="omni_slam_eval" type="omni_slam_slam_eval_node" name="omni_slam_slam_eval_node" required="true" output="screen">
<param name="bag_file" value="$(arg bag_file)" />
<param name="results_file" value="$(arg results_file)" />
<param name="image_topic" value="/unity_ros/Sphere/FisheyeCamera/image_raw" />
<param name="stereo_image_topic" value="/unity_ros/Sphere/FisheyeCamera2/image_raw" />
<param name="depth_image_topic" value="/unity_ros/Sphere/FisheyeDepthCamera/image_raw" />
<param name="pose_topic" value="/unity_ros/Sphere/TrueState/pose" />
<param name="tracked_image_topic" value="/omni_slam/tracked" />
<param name="odometry_estimate_topic" value="/omni_slam/odometry" />
<param name="odometry_ground_truth_topic" value="/omni_slam/odometry_truth" />
<param name="path_estimate_topic" value="/omni_slam/odometry_path" />
<param name="path_ground_truth_topic" value="/omni_slam/odometry_path_truth" />
<param name="point_cloud_topic" value="/omni_slam/reconstructed" />
<param name="stereo_matched_topic" value="/omni_slam/stereo_matched" />
<param name="rate" value="$(arg rate)" />
<rosparam subst_value="true">
camera_model: '$(arg camera_model)'
camera_parameters: $(arg camera_params)
detector_type: 'GFTT'
detector_parameters: {maxCorners: 5000, qualityLevel: 0.001, minDistance: 5, blockSize: 5}
tracker_window_size: 128
tracker_num_scales: 4
tracker_checker_epipolar_threshold: 0.008
tracker_checker_iterations: 1000
tracker_delta_pixel_error_threshold: 0.0
tracker_error_threshold: 20.0
min_features_per_region: 100
pnp_inlier_threshold: 3.0
pnp_iterations: 3000
max_reprojection_error: 5.0
min_triangulation_angle: 3.0
bundle_adjustment_max_iterations: 1000
bundle_adjustment_loss_coefficient: 0.05
bundle_adjustment_logging: true
bundle_adjustment_num_threads: 20
stereo_matcher_window_size: 256
stereo_matcher_num_scales: 5
stereo_matcher_error_threshold: 20
stereo_matcher_epipolar_threshold: 0.008
stereo_tf_t: [2.0, 0.0, 0.0]
stereo_tf_r: [0.0, 0.0, 0.0, 1.0]
</rosparam>
</node>
</launch>


8 changes: 4 additions & 4 deletions src/data/frame.cc
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Frame::Frame(cv::Mat &image, cv::Mat &stereo_image, cv::Mat &depth_image, Matrix
depthImage_(depth_image.clone()),
pose_(pose),
invPose_(util::TFUtil::InversePoseMatrix(pose)),
stereoPose_(stereo_pose),
stereoPose_(util::TFUtil::InversePoseMatrix(stereo_pose)),
timeSec_(time),
cameraModel_(camera_model)
{
Expand All @@ -28,7 +28,7 @@ Frame::Frame(cv::Mat &image, cv::Mat &stereo_image, Matrix<double, 3, 4> &stereo
: id_(lastFrameId_++),
image_(image.clone()),
stereoImage_(stereo_image.clone()),
stereoPose_(stereo_pose),
stereoPose_(util::TFUtil::InversePoseMatrix(stereo_pose)),
timeSec_(time),
cameraModel_(camera_model)
{
Expand All @@ -54,7 +54,7 @@ Frame::Frame(cv::Mat &image, cv::Mat &stereo_image, cv::Mat &depth_image, Matrix
image_(image.clone()),
stereoImage_(stereo_image.clone()),
depthImage_(depth_image.clone()),
stereoPose_(stereo_pose),
stereoPose_(util::TFUtil::InversePoseMatrix(stereo_pose)),
timeSec_(time),
cameraModel_(camera_model)
{
Expand Down Expand Up @@ -82,7 +82,7 @@ Frame::Frame(cv::Mat &image, cv::Mat &stereo_image, Matrix<double, 3, 4> &pose,
stereoImage_(stereo_image.clone()),
pose_(pose),
invPose_(util::TFUtil::InversePoseMatrix(pose)),
stereoPose_(stereo_pose),
stereoPose_(util::TFUtil::InversePoseMatrix(stereo_pose)),
timeSec_(time),
cameraModel_(camera_model)
{
Expand Down
2 changes: 1 addition & 1 deletion src/module/reconstruction_module.cc
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ void ReconstructionModule::BundleAdjust(std::vector<data::Landmark> &landmarks)
{
bundleAdjuster_->Optimize(landmarks);

for (int i = 0; i < landmarks.size(); i++)
for (int i = 0; i < lastLandmarksSize_; i++)
{
if (landmarks[i].HasEstimatedPosition())
{
Expand Down
12 changes: 9 additions & 3 deletions src/module/stereo_module.cc
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#include "stereo_module.h"

#include <algorithm>

using namespace std;

namespace omni_slam
Expand Down Expand Up @@ -33,7 +35,8 @@ void StereoModule::Update(data::Frame &frame, std::vector<data::Landmark> &landm
{
Matrix<double, 3, 4> framePose = frame.HasEstimatedPose() ? frame.GetEstimatedPose() : frame.GetPose();
double depth = (landmark.GetEstimatedPosition() - framePose.block<3, 1>(0, 3)).norm();
visualization_.AddMatch(feat1->GetKeypoint().pt, feat2->GetKeypoint().pt, depth);
double depthGnd = (landmark.GetGroundTruth() - frame.GetPose().block<3, 1>(0, 3)).norm();
visualization_.AddMatch(feat1->GetKeypoint().pt, feat2->GetKeypoint().pt, depth, depthGnd);
}
}
frameNum_++;
Expand All @@ -55,12 +58,15 @@ void StereoModule::Visualization::Init(cv::Size img_size)
curDepth_ = cv::Mat::zeros(img_size, CV_8UC3);
}

void StereoModule::Visualization::AddMatch(cv::Point2f pt1, cv::Point2f pt2, double depth)
void StereoModule::Visualization::AddMatch(cv::Point2f pt1, cv::Point2f pt2, double depth, double depthGnd)
{
double err = std::abs(depth - depthGnd) / depthGnd;
err = min(err, 1.0);
depth = min(depth, maxDepth_);
cv::circle(curMask_, pt1, 3, cv::Scalar(255, 0, 0), -1);
cv::circle(curMask_, pt2 + cv::Point2f(curMask_.cols / 2., 0), 3, cv::Scalar(255, 0, 0), -1);
cv::line(curMask_, pt1, pt2 + cv::Point2f(curMask_.cols / 2., 0), cv::Scalar(255, 0, 0), 1);
cv::circle(curDepth_, pt1, 2, cv::Scalar(depth / maxDepth_ * 255, depth / maxDepth_ * 255, depth / maxDepth_ * 255));
cv::circle(curDepth_, pt1, 2, cv::Scalar((depth / maxDepth_ - err) * 255, (depth / maxDepth_ - err) * 255, depth / maxDepth_ * 255));
}

void StereoModule::Visualization::Draw(cv::Mat &img, const cv::Mat &stereo_img)
Expand Down
2 changes: 1 addition & 1 deletion src/module/stereo_module.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class StereoModule
{
public:
void Init(cv::Size img_size);
void AddMatch(cv::Point2f pt1, cv::Point2f pt2, double depth);
void AddMatch(cv::Point2f pt1, cv::Point2f pt2, double depth, double depthGnd);
void Draw(cv::Mat &img, const cv::Mat &stereo_img);

private:
Expand Down
8 changes: 6 additions & 2 deletions src/odometry/pnp.cc
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,15 @@ int PNP::Compute(const std::vector<data::Landmark> &landmarks, data::Frame &fram
{
if (landmark.IsObservedInFrame(frame.GetID()))
{
if (landmark.HasEstimatedPosition())
if (frame.HasStereoImage() && landmark.HasEstimatedPosition() && landmark.GetStereoObservationByFrameID(landmark.GetFirstFrameID()) != nullptr)
{
xs.push_back(landmark.GetEstimatedPosition());
}
else if (landmark.HasGroundTruth())
else if (!frame.HasStereoImage() && landmark.HasEstimatedPosition())
{
xs.push_back(landmark.GetEstimatedPosition());
}
else if (!frame.HasStereoImage() && landmark.HasGroundTruth())
{
xs.push_back(landmark.GetGroundTruth());
}
Expand Down
13 changes: 13 additions & 0 deletions src/omni_slam_slam_eval_node.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#include "ros/slam_eval.h"

int main(int argc, char *argv[])
{
ros::init(argc, argv, "omni_slam_slam_eval_node");

omni_slam::ros::SLAMEval slam_eval;
slam_eval.Run();

return 0;
}


4 changes: 4 additions & 0 deletions src/reconstruction/triangulator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ int Triangulator::Triangulate(std::vector<data::Landmark> &landmarks) const
for (auto it = landmarks.begin(); it < landmarks.end(); it++)
{
data::Landmark &landmark = *it;
if (landmark.HasEstimatedPosition() && landmark.GetStereoObservationByFrameID(landmark.GetFirstFrameID()) != nullptr)
{
continue;
}
Vector3d point;
if (TriangulateNViews(landmark.GetObservations(), point))
{
Expand Down
3 changes: 2 additions & 1 deletion src/ros/odometry_eval.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ class OdometryEval : public virtual TrackingEval<Stereo>
virtual void GetResultsData(std::map<std::string, std::vector<std::vector<double>>> &data);
virtual void Visualize(cv_bridge::CvImagePtr &base_img);

void PublishOdometry();

std::unique_ptr<module::OdometryModule> odometryModule_;

private:
void PublishOdometry();

::ros::Publisher odometryPublisher_;
::ros::Publisher odometryGndPublisher_;
Expand Down
63 changes: 63 additions & 0 deletions src/ros/slam_eval.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#include "slam_eval.h"

#include "odometry/pnp.h"
#include "optimization/bundle_adjuster.h"
#include "module/tracking_module.h"

#include "geometry_msgs/PoseStamped.h"
#include "nav_msgs/Path.h"

using namespace std;

namespace omni_slam
{
namespace ros
{

SLAMEval::SLAMEval(const ::ros::NodeHandle &nh, const ::ros::NodeHandle &nh_private)
: OdometryEval<true>(nh, nh_private), ReconstructionEval<true>(nh, nh_private), StereoEval(nh, nh_private)
{
}

void SLAMEval::InitPublishers()
{
OdometryEval<true>::InitPublishers();
ReconstructionEval<true>::InitPublishers();
StereoEval::InitPublishers();
}

void SLAMEval::ProcessFrame(unique_ptr<data::Frame> &&frame)
{
trackingModule_->Update(frame);
odometryModule_->Update(trackingModule_->GetLandmarks(), *trackingModule_->GetFrames().back());
reconstructionModule_->Update(trackingModule_->GetLandmarks());
trackingModule_->Redetect();
stereoModule_->Update(*trackingModule_->GetFrames().back(), trackingModule_->GetLandmarks());
}

void SLAMEval::Finish()
{
ReconstructionEval<true>::Finish();
PublishOdometry();
}

void SLAMEval::GetResultsData(std::map<std::string, std::vector<std::vector<double>>> &data)
{
}

void SLAMEval::Visualize(cv_bridge::CvImagePtr &base_img)
{
visualized_ = false;
ReconstructionEval<true>::Visualize(base_img);
OdometryEval<true>::Visualize(base_img);
}

void SLAMEval::Visualize(cv_bridge::CvImagePtr &base_img, cv_bridge::CvImagePtr &base_stereo_img)
{
Visualize(base_img);
StereoEval::Visualize(base_img, base_stereo_img);
}

}
}

34 changes: 34 additions & 0 deletions src/ros/slam_eval.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#ifndef _SLAM_EVAL_H_
#define _SLAM_EVAL_H_

#include "odometry_eval.h"
#include "reconstruction_eval.h"
#include "stereo_eval.h"
#include <ros/ros.h>
#include <vector>

namespace omni_slam
{
namespace ros
{

class SLAMEval : public OdometryEval<true>, ReconstructionEval<true>, StereoEval
{
public:
SLAMEval(const ::ros::NodeHandle &nh, const ::ros::NodeHandle &nh_private);
SLAMEval() : SLAMEval(::ros::NodeHandle(), ::ros::NodeHandle("~")) {}

private:
void InitPublishers();

void ProcessFrame(std::unique_ptr<data::Frame> &&frame);
void Finish();
void GetResultsData(std::map<std::string, std::vector<std::vector<double>>> &data);
void Visualize(cv_bridge::CvImagePtr &base_img);
void Visualize(cv_bridge::CvImagePtr &base_img, cv_bridge::CvImagePtr &base_stereo_img);
};

}
}

#endif /* _SLAM_EVAL_H_ */
2 changes: 2 additions & 0 deletions src/ros/stereo_eval.cc
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,10 @@ void StereoEval::GetResultsData(std::map<std::string, std::vector<std::vector<do

void StereoEval::Visualize(cv_bridge::CvImagePtr &base_img, cv_bridge::CvImagePtr &base_stereo_img)
{
cv::Mat orig = base_img->image.clone();
stereoModule_->Visualize(base_img->image, base_stereo_img->image);
matchedImagePublisher_.publish(base_img->toImageMsg());
base_img->image = orig;
}

}
Expand Down
Loading

0 comments on commit 13e38e0

Please sign in to comment.