diff --git a/cbox-diff/renders/cbox-diff.exr b/cbox-diff/renders/cbox-diff.exr index 45e81430..9cc6c964 100644 Binary files a/cbox-diff/renders/cbox-diff.exr and b/cbox-diff/renders/cbox-diff.exr differ diff --git a/cbox-diff/renders/cbox-ref.exr b/cbox-diff/renders/cbox-ref.exr index 1c525c0a..0307289c 100644 Binary files a/cbox-diff/renders/cbox-ref.exr and b/cbox-diff/renders/cbox-ref.exr differ diff --git a/cbox-diff/scenes/cbox-diff/cbox-diff-matte-test.luisa b/cbox-diff/scenes/cbox-diff/cbox-diff-matte-test.luisa new file mode 100644 index 00000000..c372d8ae --- /dev/null +++ b/cbox-diff/scenes/cbox-diff/cbox-diff-matte-test.luisa @@ -0,0 +1,142 @@ +// mat_cbox +Surface cbox-white : Matte { + Kd : Constant { + //v { 0.725, 0.71, 0.68 } + v { 0.5, 0.5, 0.5 } + requires_grad { true } + render_grad_map { true } + range { 0.00001, 0.99999 } + } +} + + +Surface cbox-green : Matte { + Kd : Constant { + //v { 0.14, 0.45, 0.091 } + v { 0.5, 0.5, 0.5 } + requires_grad { true } + range { 0.00001, 0.99999 } + } +} + +Surface cbox-red : Matte { + Kd : Constant { + //v { 0.63, 0.065, 0.05 } + v { 0.5, 0.5, 0.5 } + requires_grad { true } + range { 0.00001, 0.99999 } + } +} + +// shape_cbox +Shape cbox-ceiling : Mesh { + file { "../../meshes/cbox-parts/ceiling.obj" } + surface { @cbox-white } +} + +Shape cbox-floor : Mesh { + file { "../../meshes/cbox-parts/floor.obj" } + surface { @cbox-white } +} + +Shape cbox-left-wall : Mesh { + file { "../../meshes/cbox-parts/left-wall.obj" } + surface { @cbox-red } +} + +Shape cbox-right-wall : Mesh { + file { "../../meshes/cbox-parts/right-wall.obj" } + surface { @cbox-green } +} + +Shape cbox-back-wall : Mesh { + file { "../../meshes/cbox-parts/back-wall.obj" } + surface { @cbox-white } +} + + +Shape cbox-light : Mesh { + file { "../../meshes/cbox-parts/light.obj" } + light : Diffuse { + emission : Constant { + //v { 10, 10, 10 } + v { 45.0, 45.0, 45.0 } + } + } +} + +Shape cbox-tall-box : Mesh { + file { "../../meshes/cbox-parts/tall-box.obj" } + surface { @cbox-white } +} + +Shape cbox-short-box : Mesh { + file { "../../meshes/cbox-parts/short-box.obj" } + surface { @cbox-white } +} + + +Shape cbox : Group { + shapes { + @cbox-left-wall, + @cbox-ceiling, + @cbox-floor, + @cbox-right-wall, + @cbox-back-wall, + @cbox-tall-box, + @cbox-short-box, + @cbox-light + } +} + +Film film : Color { + resolution { 1024 } + exposure { 0 } +} + +FIlm display : Display { + base { @film } +} + +Camera camera : Pinhole { + position { 0.0, 1.0, 5.0 } + fov { 27.8 } + spp { 1024 } + film { @display } + file { "../../renders/cbox-diff.exr" } + filter : Gaussian { + radius { 1 } + } + target : Image { + file { "../../renders/cbox-ref.exr" } + } +} + +//Integrator pt : MegaRadiativeDiff { +Integrator pt : MegaReplayDiff { + sampler : Independent {} + depth { 10 } + rr_depth { 5 } + + display_camera_index { 0 } + save_process { true } + save_grad_map { true } + save_finite_diff { true } + loss : L2 {} + iterations { 10 } + + optimizer : Adam { + learning_rate { 0.03 } + } +} + +render { + spectrum : sRGB {} + cameras { + @camera + } + shapes { + @cbox + } + integrator { @pt } +} diff --git a/cbox-diff/scenes/cbox-diff/cbox-diff-matte.luisa b/cbox-diff/scenes/cbox-diff/cbox-diff-matte.luisa index bafe5f00..69aedb05 100644 --- a/cbox-diff/scenes/cbox-diff/cbox-diff-matte.luisa +++ b/cbox-diff/scenes/cbox-diff/cbox-diff-matte.luisa @@ -20,6 +20,7 @@ Surface cbox-white-2 : Matte { Kd : Constant { v { 0.3, 0.3, 0.9 } requires_grad { true } + render_grad_map { true } range { 0.00001, 0.99999 } } } @@ -150,6 +151,7 @@ Integrator pt : MegaReplayDiff { display_camera_index { 0 } save_process { true } + save_grad_map { true } loss : L2 {} iterations { 100 } diff --git a/src/apps/cli.cpp b/src/apps/cli.cpp index b7e986d8..73582eab 100644 --- a/src/apps/cli.cpp +++ b/src/apps/cli.cpp @@ -111,7 +111,7 @@ using namespace luisa::render; int main(int argc, char *argv[]) { log_level_info(); - luisa::log_level_verbose(); + // luisa::log_level_verbose(); luisa::compute::Context context{argv[0]}; auto macros = parse_cli_macros(argc, argv); for (auto &&[k, v] : macros) { diff --git a/src/base/camera.cpp b/src/base/camera.cpp index 2baf27a6..50049f85 100644 --- a/src/base/camera.cpp +++ b/src/base/camera.cpp @@ -16,6 +16,7 @@ namespace luisa::render { Camera::Camera(Scene *scene, const SceneNodeDesc *desc) noexcept : SceneNode{scene, desc, SceneNodeTag::CAMERA}, _film{scene->load_film(desc->property_node("film"))}, + _film_grad{scene->load_film(desc->property_node("film"))}, _filter{scene->load_filter(desc->property_node_or_default( "filter", SceneNodeDesc::shared_default_filter("Box")))}, _transform{scene->load_transform(desc->property_node_or_default("transform"))}, @@ -206,6 +207,7 @@ auto Camera::shutter_samples() const noexcept -> vector { Camera::Instance::Instance(Pipeline &pipeline, CommandBuffer &command_buffer, const Camera *camera) noexcept : _pipeline{&pipeline}, _camera{camera}, _film{camera->film()->build(pipeline, command_buffer)}, + _film_grad{camera->film_grad()->build(pipeline, command_buffer)}, _filter{pipeline.build_filter(command_buffer, camera->filter())}, _target{pipeline.build_texture(command_buffer, camera->target())} { pipeline.register_transform(camera->transform()); diff --git a/src/base/camera.h b/src/base/camera.h index b035f975..076af0ac 100644 --- a/src/base/camera.h +++ b/src/base/camera.h @@ -45,6 +45,7 @@ class Camera : public SceneNode { const Pipeline *_pipeline; const Camera *_camera; luisa::unique_ptr _film; + luisa::unique_ptr _film_grad; const Filter::Instance *_filter; const Texture::Instance *_target; @@ -70,6 +71,8 @@ class Camera : public SceneNode { [[nodiscard]] auto &pipeline() const noexcept { return *_pipeline; } [[nodiscard]] auto film() noexcept { return _film.get(); } [[nodiscard]] auto film() const noexcept { return _film.get(); } + [[nodiscard]] auto film_grad() noexcept { return _film_grad.get(); } + [[nodiscard]] auto film_grad() const noexcept { return _film_grad.get(); } [[nodiscard]] auto filter() noexcept { return _filter; } [[nodiscard]] auto filter() const noexcept { return _filter; } [[nodiscard]] auto target() const noexcept { return _target; } @@ -92,6 +95,7 @@ class Camera : public SceneNode { private: const Film *_film; + const Film *_film_grad; const Filter *_filter; const Transform *_transform; float2 _shutter_span; @@ -104,6 +108,7 @@ class Camera : public SceneNode { public: Camera(Scene *scene, const SceneNodeDesc *desc) noexcept; [[nodiscard]] auto film() const noexcept { return _film; } + [[nodiscard]] auto film_grad() const noexcept { return _film_grad; } [[nodiscard]] auto filter() const noexcept { return _filter; } [[nodiscard]] auto transform() const noexcept { return _transform; } [[nodiscard]] auto shutter_span() const noexcept { return _shutter_span; } diff --git a/src/base/integrator.cpp b/src/base/integrator.cpp index 563cafb9..5dd59572 100644 --- a/src/base/integrator.cpp +++ b/src/base/integrator.cpp @@ -123,6 +123,8 @@ DifferentiableIntegrator::DifferentiableIntegrator(Scene *scene, const SceneNode _iterations{std::max(desc->property_uint_or_default("iterations", 100u), 1u)}, _display_camera_index{desc->property_int_or_default("display_camera_index", -1)}, _save_process{desc->property_bool_or_default("save_process", false)}, + _save_grad_map{desc->property_bool_or_default("save_grad_map", false)}, + _save_finite_diff{desc->property_bool_or_default("save_finite_diff", false)}, _loss{scene->load_loss(desc->property_node_or_default( "loss", SceneNodeDesc::shared_default_loss("L2")))}, _optimizer{scene->load_optimizer(desc->property_node_or_default( diff --git a/src/base/integrator.h b/src/base/integrator.h index be446c9e..a699b401 100644 --- a/src/base/integrator.h +++ b/src/base/integrator.h @@ -117,6 +117,8 @@ class DifferentiableIntegrator : public Integrator { uint _iterations; int _display_camera_index; bool _save_process; + bool _save_grad_map; + bool _save_finite_diff; public: DifferentiableIntegrator(Scene *scene, const SceneNodeDesc *desc) noexcept; @@ -127,6 +129,8 @@ class DifferentiableIntegrator : public Integrator { [[nodiscard]] auto iterations() const noexcept { return _iterations; } [[nodiscard]] int display_camera_index() const noexcept { return _display_camera_index; } [[nodiscard]] bool save_process() const noexcept { return _save_process; } + [[nodiscard]] bool save_grad_map() const noexcept { return _save_grad_map; } + [[nodiscard]] bool save_finite_diff() const noexcept { return _save_finite_diff; } }; }// namespace luisa::render diff --git a/src/base/surface.cpp b/src/base/surface.cpp index 1c415e89..45c329ca 100644 --- a/src/base/surface.cpp +++ b/src/base/surface.cpp @@ -2,6 +2,7 @@ // Created by Mike on 2021/12/14. // +#include "util/spec.h" #include #include #include @@ -54,6 +55,17 @@ Surface::Evaluation Surface::Closure::evaluate( return eval; } +SampledSpectrum Surface::Closure::eval_grad( + Expr wo, Expr wi, TransportMode mode) const noexcept { + SampledSpectrum grad{swl().dimension()}; + $outline { + grad = _eval_grad(wo, wi, mode); + auto valid = validate_surface_sides(it().ng(), it().shading().n(), wo, wi); + grad = ite(valid, grad, 0.f); + }; + return grad; +} + Surface::Sample Surface::Closure::sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept { diff --git a/src/base/surface.h b/src/base/surface.h index 2e39f91f..ffccd109 100644 --- a/src/base/surface.h +++ b/src/base/surface.h @@ -78,6 +78,7 @@ class Surface : public SceneNode { template friend class OpacitySurfaceWrapper; [[nodiscard]] virtual Evaluation _evaluate(Expr wo, Expr wi, TransportMode mode) const noexcept = 0; + [[nodiscard]] virtual SampledSpectrum _eval_grad(Expr wo, Expr wi, TransportMode mode) const noexcept = 0; [[nodiscard]] virtual Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept = 0; virtual void _backward(Expr wo, Expr wi, const SampledSpectrum &df, TransportMode mode) const noexcept = 0; @@ -95,6 +96,8 @@ class Surface : public SceneNode { [[nodiscard]] auto instance() const noexcept { return static_cast(_instance); } [[nodiscard]] Evaluation evaluate(Expr wo, Expr wi, TransportMode mode = TransportMode::RADIANCE) const noexcept; + [[nodiscard]] SampledSpectrum eval_grad(Expr wo, Expr wi, + TransportMode mode = TransportMode::RADIANCE) const noexcept; [[nodiscard]] Sample sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode = TransportMode::RADIANCE) const noexcept; @@ -219,6 +222,11 @@ class OpacitySurfaceWrapper : public BaseSurface { TransportMode mode) const noexcept override { return _base->_evaluate(wo, wi, mode); } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, + Expr wi, + TransportMode mode) const noexcept override { + return _base->_eval_grad(wo, wi, mode); + } [[nodiscard]] Surface::Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept override { diff --git a/src/base/texture.cpp b/src/base/texture.cpp index 92af439c..65de7082 100644 --- a/src/base/texture.cpp +++ b/src/base/texture.cpp @@ -2,6 +2,7 @@ // Created by Mike Smith on 2022/1/25. // +#include "util/spec.h" #include #include @@ -12,9 +13,11 @@ Texture::Texture(Scene *scene, const SceneNodeDesc *desc) noexcept _range{desc->property_float2_or_default( "range", make_float2(std::numeric_limits::min(), std::numeric_limits::max()))}, - _requires_grad{desc->property_bool_or_default("requires_grad", false)} {} + _requires_grad{desc->property_bool_or_default("requires_grad", false)}, + _render_grad_map{desc->property_bool_or_default("render_grad_map", false)} {} bool Texture::requires_gradients() const noexcept { return _requires_grad; } +bool Texture::render_grad_map() const noexcept { return _render_grad_map; } void Texture::disable_gradients() noexcept { _requires_grad = false; } luisa::optional Texture::evaluate_static() const noexcept { return luisa::nullopt; } @@ -93,6 +96,14 @@ void Texture::Instance::backward_albedo_spectrum( backward(it, swl, time, dEnc); } +SampledSpectrum Texture::Instance::eval_grad_albedo_spectrum( + const Interaction &it, const SampledWavelengths &swl, + Expr time, const SampledSpectrum &dSpec) const noexcept { + auto dEnc = pipeline().spectrum()->backward_decode_albedo(swl, evaluate(it, swl, time), dSpec); + dEnc = make_float4(pipeline().spectrum()->backward_encode_srgb_albedo(dEnc), 1.f); + return eval_grad(it, swl, time, dEnc); +} + void Texture::Instance::backward_illuminant_spectrum( const Interaction &it, const SampledWavelengths &swl, Expr time, const SampledSpectrum &dSpec) const noexcept { diff --git a/src/base/texture.h b/src/base/texture.h index 428552c4..66bbc7d6 100644 --- a/src/base/texture.h +++ b/src/base/texture.h @@ -51,6 +51,8 @@ class Texture : public SceneNode { [[nodiscard]] auto &pipeline() const noexcept { return _pipeline; } [[nodiscard]] virtual Float4 evaluate( const Interaction &it, const SampledWavelengths &swl, Expr time) const noexcept = 0; + [[nodiscard]] virtual SampledSpectrum eval_grad( + const Interaction &it, const SampledWavelengths &swl, Expr time, Expr grad) const noexcept = 0; virtual void backward( const Interaction &it, const SampledWavelengths &swl, Expr time, Expr grad) const noexcept = 0; [[nodiscard]] virtual Spectrum::Decode evaluate_albedo_spectrum( @@ -59,9 +61,13 @@ class Texture : public SceneNode { const Interaction &it, const SampledWavelengths &swl, Expr time) const noexcept; [[nodiscard]] virtual Spectrum::Decode evaluate_illuminant_spectrum( const Interaction &it, const SampledWavelengths &swl, Expr time) const noexcept; + [[nodiscard]] SampledSpectrum eval_grad_albedo_spectrum( + const Interaction &it, const SampledWavelengths &swl, + Expr time, const SampledSpectrum &dSpec) const noexcept; void backward_albedo_spectrum( const Interaction &it, const SampledWavelengths &swl, Expr time, const SampledSpectrum &dSpec) const noexcept; + void backward_illuminant_spectrum( const Interaction &it, const SampledWavelengths &swl, Expr time, const SampledSpectrum &dSpec) const noexcept; @@ -82,11 +88,13 @@ class Texture : public SceneNode { private: float2 _range; bool _requires_grad; + bool _render_grad_map; public: Texture(Scene *scene, const SceneNodeDesc *desc) noexcept; [[nodiscard]] auto range() const noexcept { return _range; } [[nodiscard]] virtual bool requires_gradients() const noexcept; + [[nodiscard]] virtual bool render_grad_map() const noexcept; virtual void disable_gradients() noexcept; [[nodiscard]] virtual bool is_black() const noexcept = 0; [[nodiscard]] virtual bool is_constant() const noexcept = 0; diff --git a/src/integrators/mega_replay_diff.cpp b/src/integrators/mega_replay_diff.cpp index 29818c83..0595a6f9 100644 --- a/src/integrators/mega_replay_diff.cpp +++ b/src/integrators/mega_replay_diff.cpp @@ -15,7 +15,6 @@ namespace luisa::render { // #define LUISA_RENDER_PATH_REPLAY_DEBUG -// #define LUISA_RENDER_PATH_REPLAY_DEBUG_2 using namespace luisa::compute; @@ -42,10 +41,11 @@ class MegakernelReplayDiff final : public DifferentiableIntegrator { class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Instance { private: - luisa::unordered_map> - _render_shaders; + luisa::unordered_map> _render_shaders; luisa::unordered_map>> _bp_shaders, _render_1spp_shaders; + luisa::unordered_map> _render_grad_map_shaders; luisa::unordered_map> replay_Li; + luisa::unordered_map> last_time_rendered; private: void _render_one_camera( @@ -65,6 +65,7 @@ class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Inst auto camera = pipeline.camera(i); auto resolution = camera->film()->node()->resolution(); replay_Li.emplace(camera, pipeline.device().create_image(PixelStorage::FLOAT4, resolution)); + last_time_rendered.emplace(camera, pipeline.device().create_buffer(resolution.x * resolution.y)); } // handle output dir @@ -80,11 +81,11 @@ class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Inst CommandBuffer command_buffer{&stream}; #ifdef LUISA_RENDER_PATH_REPLAY_DEBUG command_buffer << pipeline().printer().reset(); -#endif -#ifdef LUISA_RENDER_PATH_REPLAY_DEBUG_2 - command_buffer << pipeline().printer().reset(); #endif luisa::vector rendered; + luisa::vector rendered_last; // for computing finite-difference + luisa::vector rendered_delta;// for computing finite-difference + luisa::vector rendered_grad; // for grad_map auto iteration_num = pt->iterations(); @@ -94,9 +95,17 @@ class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Inst // delete output buffer auto output_dir = std::filesystem::path("outputs") / luisa::format("output_buffer_camera_{:03}", i); + auto output_dir_2 = std::filesystem::path("outputs") / + luisa::format("output_finite_diff_camera_{:03}", i); + auto output_dir_3 = std::filesystem::path("outputs") / + luisa::format("output_grad_map_camera_{:03}", i); // std::cout << "Current Path: " << output_dir << std::endl; std::filesystem::remove_all(output_dir); + std::filesystem::remove_all(output_dir_2); + std::filesystem::remove_all(output_dir_3); std::filesystem::create_directories(output_dir); + std::filesystem::create_directories(output_dir_2); + std::filesystem::create_directories(output_dir_3); } for (auto k = 0u; k < iteration_num; ++k) { @@ -111,6 +120,12 @@ class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Inst auto output_path = std::filesystem::path("outputs") / luisa::format("output_buffer_camera_{:03}", i) / luisa::format("{:06}.exr", k); + auto output_path_gradmap = std::filesystem::path("outputs") / + luisa::format("output_grad_map_camera_{:03}", i) / + luisa::format("{:06}.exr", k); + auto output_path_finite_diff = std::filesystem::path("outputs") / + luisa::format("output_finite_diff_camera_{:03}", i) / + luisa::format("{:06}.exr", k); LUISA_INFO(""); LUISA_INFO("Camera {}", i); @@ -124,9 +139,115 @@ class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Inst if (pt->save_process()) { // save image rendered.resize(next_pow2(pixel_count)); + rendered_last.resize(next_pow2(pixel_count)); + rendered_delta.resize(next_pow2(pixel_count)); + rendered_grad.resize(next_pow2(pixel_count)); camera->film()->download(command_buffer, rendered.data()); command_buffer << synchronize(); save_image(output_path, (const float *)rendered.data(), resolution); + if (pt->save_grad_map()) { + camera->film_grad()->download(command_buffer, rendered_grad.data()); + + float max = -1000; + float min = 1000; + for (int i = 0; i < pixel_count; i++) { + if (rendered_grad[i].x > max) + max = rendered_grad[i].x; + if (rendered_grad[i].x < min) + min = rendered_grad[i].x; + } + auto max_abs = max > 0.f ? max : -max; + auto min_abs = min < 0.f ? -min : min; + auto bound = max_abs > min_abs ? max_abs : min_abs; + // LUISA_INFO("bound: {}", bound); + float satuation_up = 0.75; + float satuation_down = 0.1; + + float saturatin_delta = satuation_up - satuation_down; + for (int i = 0; i < pixel_count; i++) { + rendered_grad[i].x = rendered_grad[i].x / bound * 3 * saturatin_delta;// scaling + if (rendered_grad[i].x < -saturatin_delta) { + // blue + rendered_grad[i].y = rendered_grad[i].x + satuation_up + saturatin_delta; + if (rendered_grad[i].y < satuation_down) + rendered_grad[i].y = satuation_down; + rendered_grad[i].x = satuation_down; + rendered_grad[i].z = satuation_up; + } else if (rendered_grad[i].x < 0) { + // green + rendered_grad[i].z = satuation_down - rendered_grad[i].x; + rendered_grad[i].x = satuation_down; + rendered_grad[i].y = satuation_up; + } else if (rendered_grad[i].x < saturatin_delta) { + // yellow + rendered_grad[i].x = satuation_down + rendered_grad[i].x; + rendered_grad[i].y = satuation_up; + rendered_grad[i].z = satuation_down; + } else { + // red + rendered_grad[i].y = satuation_up + saturatin_delta - rendered_grad[i].x; + if (rendered_grad[i].y < satuation_down) + rendered_grad[i].y = satuation_down; + rendered_grad[i].x = satuation_up; + rendered_grad[i].z = satuation_down; + } + rendered_grad[i].w = 1; + } + save_image(output_path_gradmap, (const float *)rendered_grad.data(), resolution); + } + if (pt->save_finite_diff()) { + // just toy case + auto &&last_time = last_time_rendered[camera]; + if (k != 0) { + command_buffer << last_time.copy_to(rendered_last.data()); + command_buffer << synchronize(); + // float max = -1500; + // float min = 1500; + for (int i = 0; i < pixel_count; i++) { + auto pixel = rendered[i] - rendered_last[i]; + rendered_delta[i].x = 0;// use idx x as a temp buffer + for (int j = 0; j < 3; j++) { + rendered_delta[i].x += (rendered[i][j] - rendered_last[i][j]); + } + // if (i > 1024 * 130) { + // if (rendered_delta[i].x > max) + // max = rendered_delta[i].x; + // if (rendered_delta[i].x < min) + // min = rendered_delta[i].x; + // } + } + // LUISA_INFO("max: {}, min: {}", max, min); + for (int i = 0; i < pixel_count; i++) { + // if (rendered_delta[i].x > 20) { + // LUISA_INFO("maxindex: {},{}", int(i / resolution.x), i - int(i / resolution.x) * resolution.x); + // } + rendered_delta[i].x = rendered_delta[i].x * 4;// scaling + if (rendered_delta[i].x < -1) { + // blue + rendered_delta[i].y = rendered_delta[i].x + 2; + rendered_delta[i].x = 0; + rendered_delta[i].z = 1; + } else if (rendered_delta[i].x < 0) { + // green + rendered_delta[i].z = 0 - rendered_delta[i].x; + rendered_delta[i].x = 0; + rendered_delta[i].y = 1; + } else if (rendered_delta[i].x < 1) { + // yellow + rendered_delta[i].y = 1; + rendered_delta[i].z = 0; + } else { + // red + rendered_delta[i].y = 2 - rendered_delta[i].x; + rendered_delta[i].x = 1; + rendered_delta[i].z = 0; + } + rendered_delta[i].w = 1; + } + save_image(output_path_finite_diff, (const float *)rendered_delta.data(), resolution); + } + command_buffer << last_time.copy_from(rendered.data()) << synchronize(); + } } } @@ -148,6 +269,7 @@ class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Inst auto pixel_count = resolution.x * resolution.y; _render_one_camera(command_buffer, iteration_num, camera); + // _render_grad_map(command_buffer, iteration_num, camera); rendered.resize(next_pow2(pixel_count)); camera->film()->download(command_buffer, rendered.data()); @@ -158,9 +280,6 @@ class MegakernelReplayDiffInstance final : public DifferentiableIntegrator::Inst } #ifdef LUISA_RENDER_PATH_REPLAY_DEBUG command_buffer << pipeline().printer().retrieve() << synchronize(); -#endif -#ifdef LUISA_RENDER_PATH_REPLAY_DEBUG_2 - command_buffer << pipeline().printer().retrieve() << synchronize(); #endif LUISA_INFO("Finish saving results"); @@ -181,6 +300,7 @@ void MegakernelReplayDiffInstance::_integrate_one_camera( auto spp = camera->node()->spp(); auto resolution = camera->node()->film()->resolution(); + camera->film_grad()->prepare(command_buffer); LUISA_INFO("Start backward propagation."); @@ -380,6 +500,9 @@ void MegakernelReplayDiffInstance::_integrate_one_camera( SampledSpectrum Li{swl.dimension(), 0.f}; auto grad_weight = shutter_weight * static_cast(pt->node()->max_depth()); + // for rendering grad map + SampledSpectrum grad_map{swl.dimension(), 0.f}; + auto Li_last_pass = Li_1spp.read(pixel_id); Li[0u] = Li_last_pass[0u]; Li[1u] = Li_last_pass[1u]; @@ -395,33 +518,6 @@ void MegakernelReplayDiffInstance::_integrate_one_camera( auto pixel_uv_it = pixel_xy2uv(pixel_id, resolution); Float3 target = camera->target()->evaluate(pixel_uv_it, swl, 0.f).xyz(); -#ifdef LUISA_RENDER_PATH_REPLAY_DEBUG_2 - $if(all(pixel_id == make_uint2(80, 280))) { - $if(frame_index % 800 == 0) { - pipeline().printer().info(" "); - pipeline().printer().info("dloss of (80, 280): delta = ({}, {}, {})", d_loss[0u], d_loss[1u], d_loss[2u]); - pipeline().printer().info("rendered: delta = ({}, {}, {})", rendered[0u], rendered[1u], rendered[2u]); - pipeline().printer().info("target: delta = ({}, {}, {})", target[0u], target[1u], target[2u]); - }; - }; - $if(all(pixel_id == make_uint2(600, 750))) { - $if(frame_index % 800 == 0) { - pipeline().printer().info(" "); - pipeline().printer().info("dloss of (600, 750): delta = ({}, {}, {})", d_loss[0u], d_loss[1u], d_loss[2u]); - pipeline().printer().info("rendered: delta = ({}, {}, {})", rendered[0u], rendered[1u], rendered[2u]); - pipeline().printer().info("target: delta = ({}, {}, {})", target[0u], target[1u], target[2u]); - }; - }; - $if(all(pixel_id == make_uint2(280, 80))) { - $if(frame_index % 800 == 0) { - pipeline().printer().info(" "); - pipeline().printer().info("dloss of (280, 80): delta = ({}, {}, {})", d_loss[0u], d_loss[1u], d_loss[2u]); - pipeline().printer().info("rendered: delta = ({}, {}, {})", rendered[0u], rendered[1u], rendered[2u]); - pipeline().printer().info("target: delta = ({}, {}, {})", target[0u], target[1u], target[2u]); - }; - }; -#endif - auto ray = camera_ray; auto pdf_bsdf = def(1e16f); @@ -533,8 +629,9 @@ void MegakernelReplayDiffInstance::_integrate_one_camera( pipeline().printer().info("after -direct: Li = ({}, {}, {})", Li[0u], Li[1u], Li[2u]); }; #endif - closure->backward(wo, wi, d_loss * weight * light_sample.eval.L); + auto surface_grad = closure->eval_grad(wo, wi); + grad_map += surface_grad * weight * light_sample.eval.L; }; // sample material @@ -546,9 +643,11 @@ void MegakernelReplayDiffInstance::_integrate_one_camera( // path replay bp auto df = d_loss * grad_weight * Li; df = ite(surface_sample.eval.f == 0.f, 0.f, df / surface_sample.eval.f); - closure->backward(wo, surface_sample.wi, df); + auto surface_grad = closure->eval_grad(wo, surface_sample.wi); + grad_map += ite(surface_sample.eval.f == 0.f, 0.f, surface_grad * Li / surface_sample.eval.f); + beta *= w * surface_sample.eval.f; // apply eta scale @@ -570,6 +669,7 @@ void MegakernelReplayDiffInstance::_integrate_one_camera( beta *= ite(q < rr_threshold, 1.0f / q, 1.f); }; }; + camera->film_grad()->accumulate(pixel_id, spectrum->srgb(swl, grad_map * shutter_weight)); #ifdef LUISA_RENDER_PATH_REPLAY_DEBUG $if(all(pixel_id == pixel_checked)) { @@ -615,9 +715,6 @@ void MegakernelReplayDiffInstance::_integrate_one_camera( } #ifdef LUISA_RENDER_PATH_REPLAY_DEBUG command_buffer << pipeline().printer().retrieve() << synchronize(); -#endif -#ifdef LUISA_RENDER_PATH_REPLAY_DEBUG_2 - command_buffer << pipeline().printer().retrieve() << synchronize(); #endif } } diff --git a/src/surfaces/disney.cpp b/src/surfaces/disney.cpp index 41509aa8..6a7552f6 100644 --- a/src/surfaces/disney.cpp +++ b/src/surfaces/disney.cpp @@ -2,6 +2,7 @@ // Created by Mike Smith on 2022/1/30. // +#include "util/spec.h" #include #include #include @@ -987,6 +988,12 @@ class DisneySurfaceClosure : public Surface::Closure { TransportMode mode) const noexcept override { return _impl->evaluate(wo, wi, mode); } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, Expr wi, + TransportMode mode) const noexcept override { + // TODO + LUISA_WARNING_WITH_LOCATION("Not implemented."); + return {swl().dimension(), 0.f}; + } [[nodiscard]] Surface::Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept override { return _impl->sample(wo, u_lobe, u, mode); diff --git a/src/surfaces/glass.cpp b/src/surfaces/glass.cpp index 7b4e8e69..50e87e8a 100644 --- a/src/surfaces/glass.cpp +++ b/src/surfaces/glass.cpp @@ -199,6 +199,13 @@ class GlassClosure : public Surface::Closure { return {.f = f * abs_cos_theta(wi_local), .pdf = pdf}; } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, Expr wi, + TransportMode mode) const noexcept override { + // TODO + LUISA_WARNING_WITH_LOCATION("Not implemented."); + return {swl().dimension(), 0.f}; + } + [[nodiscard]] Surface::Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept override { diff --git a/src/surfaces/layered.cpp b/src/surfaces/layered.cpp index f3852fbe..34e37301 100644 --- a/src/surfaces/layered.cpp +++ b/src/surfaces/layered.cpp @@ -400,6 +400,14 @@ class LayeredSurfaceClosure : public Surface::Closure { return {.f = f / Float(ctx.samples), .pdf = lerp(1.f / (4.f * pi), pdf_sum / Float(ctx.samples), 0.9f)}; } + + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, Expr wi, + TransportMode mode) const noexcept override { + // TODO + LUISA_WARNING_WITH_LOCATION("Not implemented."); + return {swl().dimension(), 0.f}; + } + [[nodiscard]] Surface::Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept override { auto &&ctx = context(); diff --git a/src/surfaces/matte.cpp b/src/surfaces/matte.cpp index 85068e96..3b901f76 100644 --- a/src/surfaces/matte.cpp +++ b/src/surfaces/matte.cpp @@ -4,6 +4,7 @@ #include "core/logging.h" #include "dsl/stmt.h" +#include "util/spec.h" #include #include #include @@ -104,6 +105,26 @@ class MatteClosure : public Surface::Closure { return {.f = f * abs_cos_theta(wi_local), .pdf = pdf}; } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, + Expr wi, + TransportMode mode) const noexcept override { + auto &&ctx = context(); + auto _instance = instance(); + auto wo_local = ctx.it.shading().world_to_local(wo); + auto wi_local = ctx.it.shading().world_to_local(wi); + auto grad = _refl->eval_grad(wo_local, wi_local, mode); + SampledSpectrum spectrum_grad{swl().dimension(), 0.f}; + + if (auto kd = _instance->Kd()) { + spectrum_grad += kd->eval_grad_albedo_spectrum(ctx.it, swl(), time(), zero_if_any_nan(grad.dR)); + } + if (auto sigma = _instance->sigma()) { + auto dv = make_float4(ite(isnan(grad.dSigma), 0.f, grad.dSigma), 0.f, 0.f, 0.f); + spectrum_grad += sigma->eval_grad(ctx.it, swl(), time(), dv); + } + return spectrum_grad; + } + [[nodiscard]] Surface::Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept override { diff --git a/src/surfaces/metal.cpp b/src/surfaces/metal.cpp index c1eda407..ac4f4f9a 100644 --- a/src/surfaces/metal.cpp +++ b/src/surfaces/metal.cpp @@ -249,6 +249,12 @@ class MetalClosure : public Surface::Closure { auto pdf = lobe.pdf(wo_local, wi_local, mode); return {.f = f * abs_cos_theta(wi_local), .pdf = pdf}; } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, Expr wi, + TransportMode mode) const noexcept override { + // TODO + LUISA_WARNING_WITH_LOCATION("Not implemented."); + return {swl().dimension(), 0.f}; + } [[nodiscard]] Surface::Sample _sample(Expr wo, Expr, Expr u, TransportMode mode) const noexcept override { auto &&ctx = context(); diff --git a/src/surfaces/mirror.cpp b/src/surfaces/mirror.cpp index 5b7bc01d..3ef2e6bf 100644 --- a/src/surfaces/mirror.cpp +++ b/src/surfaces/mirror.cpp @@ -130,6 +130,12 @@ class MirrorClosure : public Surface::Closure { auto pdf = refl.pdf(wo_local, wi_local, mode); return {.f = f * abs_cos_theta(wi_local), .pdf = pdf}; } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, Expr wi, + TransportMode mode) const noexcept override { + // TODO + LUISA_WARNING_WITH_LOCATION("Not implemented."); + return {swl().dimension(), 0.f}; + } [[nodiscard]] Surface::Sample _sample(Expr wo, Expr, Expr u, TransportMode mode) const noexcept override { auto &&ctx = context(); diff --git a/src/surfaces/mix.cpp b/src/surfaces/mix.cpp index aa9be54f..270ec01d 100644 --- a/src/surfaces/mix.cpp +++ b/src/surfaces/mix.cpp @@ -164,6 +164,12 @@ class MixSurfaceClosure : public Surface::Closure { return _mix(eval_a, eval_b, ctx.ratio); } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, Expr wi, + TransportMode mode) const noexcept override { + // TODO + LUISA_WARNING_WITH_LOCATION("Not implemented."); + return {swl().dimension(), 0.f}; + } [[nodiscard]] Surface::Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept override { diff --git a/src/surfaces/plastic.cpp b/src/surfaces/plastic.cpp index 19abcc36..624e1bca 100644 --- a/src/surfaces/plastic.cpp +++ b/src/surfaces/plastic.cpp @@ -249,6 +249,12 @@ class PlasticClosure : public Surface::Closure { TransportMode mode) const noexcept override { return _impl->evaluate(wo, wi, mode); } + [[nodiscard]] SampledSpectrum _eval_grad(Expr wo, Expr wi, + TransportMode mode) const noexcept override { + // TODO + LUISA_WARNING_WITH_LOCATION("Not implemented."); + return {swl().dimension(), 0.f}; + } [[nodiscard]] Surface::Sample _sample(Expr wo, Expr u_lobe, Expr u, TransportMode mode) const noexcept override { diff --git a/src/textures/checkerboard.cpp b/src/textures/checkerboard.cpp index 76226df0..694f5c6e 100644 --- a/src/textures/checkerboard.cpp +++ b/src/textures/checkerboard.cpp @@ -80,6 +80,15 @@ class CheckerboardTextureInstance final : public Texture::Instance { }; return value; } + [[nodiscard]] SampledSpectrum eval_grad(const Interaction &it, + const SampledWavelengths &swl, + Expr time, + Expr grad) const noexcept override { + if (node()->requires_gradients()) { + LUISA_ERROR_WITH_LOCATION("Not supported."); + } + return {swl.dimension(), 0.f}; + } [[nodiscard]] Spectrum::Decode evaluate_albedo_spectrum( const Interaction &it, const SampledWavelengths &swl, Expr time) const noexcept override { Spectrum::Decode value{SampledSpectrum{swl.dimension()}, 0.f}; diff --git a/src/textures/constant.cpp b/src/textures/constant.cpp index edad900d..b52d3e82 100644 --- a/src/textures/constant.cpp +++ b/src/textures/constant.cpp @@ -2,6 +2,7 @@ // Created by Mike Smith on 2022/1/26. // +#include "util/spec.h" #include #include #include @@ -83,6 +84,24 @@ class ConstantTextureInstance final : public Texture::Instance { texture->should_inline()) { return texture->v(); } return pipeline().constant(_constant_slot); } + [[nodiscard]] SampledSpectrum eval_grad(const Interaction &it, + const SampledWavelengths &swl, + Expr time, + Expr grad) const noexcept override { + if (_diff_param) { + if (node()->render_grad_map()) { + // render grad map in a simple way: + // if texture param 3-dim, combine them. which means: add up each dim of grad, only eval_grad qualitatively. + // as for output: each dim of the sampledspectrum will be the same. + auto grads = (grad[0] + grad[1] + grad[2]); + return {swl.dimension(), ite(isnan(grads), 0.f, grads)}; + } else { + return {swl.dimension(), 0.f}; + } + } else { + return {swl.dimension(), 0.f}; + } + } [[nodiscard]] Spectrum::Decode evaluate_albedo_spectrum( const Interaction &it, const SampledWavelengths &swl, Expr time) const noexcept override { if (_diff_param) { return Instance::evaluate_albedo_spectrum(it, swl, time); } diff --git a/src/textures/image.cpp b/src/textures/image.cpp index a9edfbc9..cb270b84 100644 --- a/src/textures/image.cpp +++ b/src/textures/image.cpp @@ -203,6 +203,15 @@ class ImageTextureInstance final : public Texture::Instance { auto v = pipeline().tex2d(_texture_id).sample(uv);// TODO: LOD return _decode(v); } + [[nodiscard]] SampledSpectrum eval_grad(const Interaction &it, + const SampledWavelengths &swl, + Expr time, + Expr grad) const noexcept override { + if (node()->requires_gradients()) { + LUISA_ERROR_WITH_LOCATION("Not supported."); + } + return {swl.dimension(), 0.f}; + } void backward(const Interaction &it, const SampledWavelengths &swl, Expr time, Expr grad) const noexcept override { if (_diff_param) { diff --git a/src/textures/nishita_sky.cpp b/src/textures/nishita_sky.cpp index 5ab56380..c77f544d 100644 --- a/src/textures/nishita_sky.cpp +++ b/src/textures/nishita_sky.cpp @@ -171,6 +171,15 @@ class NishitaSkyInstance final : public Texture::Instance { } return make_float4((*_impl)(it.uv()), 1.f); } + [[nodiscard]] SampledSpectrum eval_grad(const Interaction &it, + const SampledWavelengths &swl, + Expr time, + Expr grad) const noexcept override { + if (node()->requires_gradients()) { + LUISA_ERROR_WITH_LOCATION("Not supported."); + } + return {swl.dimension(), 0.f}; + } void backward(const Interaction &it, const SampledWavelengths &swl, Expr time, Expr grad) const noexcept override { LUISA_WARNING_WITH_LOCATION("NishitaSkyInstance::backward() not implemented."); diff --git a/src/textures/swizzle.cpp b/src/textures/swizzle.cpp index 200cebbd..af586a72 100644 --- a/src/textures/swizzle.cpp +++ b/src/textures/swizzle.cpp @@ -100,6 +100,15 @@ class SwizzleTextureInstance final : public Texture::Instance { } return make_float4(); } + [[nodiscard]] SampledSpectrum eval_grad(const Interaction &it, + const SampledWavelengths &swl, + Expr time, + Expr grad) const noexcept override { + if (node()->requires_gradients()) { + LUISA_ERROR_WITH_LOCATION("Not supported."); + } + return {swl.dimension(), 0.f}; + } void backward(const Interaction &it, const SampledWavelengths &swl, Expr time, Expr grad) const noexcept override { if (node()->requires_gradients()) { auto g = def(make_float4()); diff --git a/src/util/scattering.cpp b/src/util/scattering.cpp index 0af65ad4..bfd06ad9 100644 --- a/src/util/scattering.cpp +++ b/src/util/scattering.cpp @@ -535,6 +535,26 @@ SampledSpectrum OrenNayar::evaluate( return forward_compute(wo, wi, mode, _a, _b, _r); } +OrenNayar::Gradient OrenNayar::eval_grad( + Expr wo, Expr wi, TransportMode mode) const noexcept { + auto d_r = _r; + auto d_sigma = _sigma; + $autodiff { + auto r = _r; + auto sigma = _sigma; + r.requires_grad(); + requires_grad(sigma); + auto sigma2 = sqr(radians(sigma)); + auto a = 1.f - (sigma2 / (2.f * sigma2 + 0.66f)); + auto b = 0.45f * sigma2 / (sigma2 + 0.09f); + auto y = forward_compute(wo, wi, mode, a, b, r); + y.backward(); + d_r = r.grad(); + d_sigma = grad(sigma); + }; + return {.dR = d_r, .dSigma = d_sigma}; +} + SampledSpectrum OrenNayar::forward_compute( Expr wo, Expr wi, TransportMode mode, Float a, Float b, SampledSpectrum r) const noexcept { auto valid = same_hemisphere(wo, wi); diff --git a/src/util/scattering.h b/src/util/scattering.h index 36b115ff..96c75588 100644 --- a/src/util/scattering.h +++ b/src/util/scattering.h @@ -254,6 +254,7 @@ class OrenNayar : public BxDF { public: OrenNayar(const SampledSpectrum &R, Expr sigma) noexcept; [[nodiscard]] SampledSpectrum evaluate(Expr wo, Expr wi, TransportMode mode) const noexcept override; + [[nodiscard]] Gradient eval_grad(Expr wo, Expr wi, TransportMode mode) const noexcept; [[nodiscard]] SampledSpectrum forward_compute(Expr wo, Expr wi, TransportMode mode, Float a, Float b, SampledSpectrum r) const noexcept; [[nodiscard]] Gradient backward(Expr wo, Expr wi, const SampledSpectrum &df, TransportMode mode) const noexcept; [[nodiscard]] SampledSpectrum albedo() const noexcept override { return _r; }