Various fixes and culling

Signed-off-by: Slendi <slendi@socopon.com>

src/Application.cpp

@@ -551,10 +551,9 @@ auto Application::run() -> void
 		ImGui::SetNextWindowSize({ 300, 100 });
 		ImGui::SetNextWindowPos({ 0, 0 });
 		ImGui::PushStyleColor(ImGuiCol_WindowBg, ImVec4 { 0, 0, 0, 0.5f });
-		if (ImGui::Begin("Debug Info", nullptr,
-		        ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoResize)) {
-			defer(ImGui::End());
-
+		bool debug_open { ImGui::Begin("Debug Info", nullptr,
+			ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoResize) };
+		if (debug_open) {
 			ImGui::Text("%s", std::format("FPS: {:.2f}", fps).c_str());
 			ImGui::Text("%s",
 			    std::format("Cam pos: ({:.2f}, {:.2f}, {:.2f})",
@@ -575,6 +574,7 @@ auto Application::run() -> void
 			        m_cursor.r, m_cursor.theta, m_cursor.phi)
 			        .c_str());
 		}
+		ImGui::End();
 		ImGui::PopStyleColor();
 
 		if (m_show_imgui) {
@@ -582,9 +582,8 @@ auto Application::run() -> void
 			ImGui::ShowDemoWindow();
 
 			ImGui::SetNextWindowSize({ 300, -1 }, ImGuiCond_Once);
-			if (ImGui::Begin("Fun menu")) {
-				defer(ImGui::End());
-
+			bool fun_menu_open { ImGui::Begin("Fun menu") };
+			if (fun_menu_open) {
 				static std::array<char const *, 4> const aa_items {
 					"None",
 					"MSAA 2X",
@@ -605,13 +604,60 @@ auto Application::run() -> void
 				        ImGuiTreeNodeFlags_Framed
 				            | ImGuiTreeNodeFlags_SpanAvailWidth
 				            | ImGuiTreeNodeFlags_DefaultOpen)) {
+					auto const camera_offset { m_camera.target
+						- m_camera.position };
+					auto const camera_distance { camera_offset.magnitude() };
+					auto const camera_direction {
+						camera_offset.normalized_safe()
+					};
+
 					ImGui::SliderFloat("Mouse sensitivity",
 					    &m_mouse_sensitivity, 0.0001f, 0.01f, "%.4f");
-					ImGui::DragFloat4("Pos", m_camera.position.data());
-					ImGui::DragFloat4("Target", m_camera.target.data());
-					ImGui::DragFloat4("Up", m_camera.up.data());
+					constexpr float position_step { 0.05f };
+					constexpr float target_step { 0.05f };
+					bool position_changed { ImGui::DragFloat3(
+						"Pos", m_camera.position.data(), position_step) };
+					bool target_changed { ImGui::DragFloat3(
+						"Target", m_camera.target.data(), target_step) };
+					ImGui::DragFloat3("Up", m_camera.up.data());
+
+					if (position_changed && !target_changed) {
+						auto offset { m_cursor.to_vec3() };
+						auto const preserve_distance { camera_distance > 0.0f
+							    ? camera_distance
+							    : offset.magnitude() };
+
+						if (offset.magnitude() == 0.0f
+						    && camera_direction.magnitude() > 0.0f) {
+							offset = camera_direction * preserve_distance;
+						}
+
+						if (offset.magnitude() > 0.0f) {
+							m_camera.target = m_camera.position + offset;
+						}
+					}
+
+					if (target_changed && !position_changed) {
+						auto const new_offset { m_camera.target
+							- m_camera.position };
+						auto new_direction { new_offset.normalized_safe() };
+						auto const preserve_distance { camera_distance > 0.0f
+							    ? camera_distance
+							    : new_offset.magnitude() };
+
+						if (new_direction.magnitude() == 0.0f) {
+							new_direction = camera_direction;
+						}
+
+						if (new_direction.magnitude() > 0.0f
+						    && preserve_distance > 0.0f) {
+							m_camera.position = m_camera.target
+							    - new_direction * preserve_distance;
+						}
+					}
 				}
 			}
+			ImGui::End();
 		}
 
 		ImGui::Render();
@@ -632,11 +678,6 @@ auto Application::run() -> void
 			projection[1][1] *= -1;
 			auto view_projection { projection * view };
 
-			// auto rect_model { smath::scale(
-			//	smath::translate(smath::Vec3 { 0.0f, 0.0f, -5.0f }),
-			//	smath::Vec3 { 5.0f, 5.0f, 1.0f }) };
-
-			// gl.set_transform(view_projection * rect_model);
 			gl.set_transform(view_projection);
 
 			gl.set_texture();
@@ -645,10 +686,17 @@ auto Application::run() -> void
 				auto const &surface = meshes[2]->surfaces[0];
 				gl.draw_mesh(meshes[2]->mesh_buffers,
 				    view_projection
-				        * smath::translate(smath::Vec3 { 0.0f, 0.0f, -10.0f }),
+				        * smath::translate(smath::Vec3 { 0.0f, 0.0f, -5.0f }),
 				    surface.count, surface.start_index);
 			}
 
+			gl.push_transform();
+			gl.set_transform(view_projection
+			    * smath::translate(smath::Vec3 { 0.0f, 0.0f, 5.0f })
+			    * smath::Quaternion<float>::from_axis_angle(
+			        smath::Vec3 { 0.0f, 1.0f, 0.0f }, smath::deg(180))
+			        .as_matrix());
+
 			gl.set_texture(&m_renderer->white_texture());
 
 			gl.begin(VulkanRenderer::GL::GeometryKind::Quads);

src/VulkanRenderer.cpp

@@ -46,7 +46,10 @@ auto VulkanRenderer::GL::begin_drawing(vk::CommandBuffer cmd,
 	m_indices.clear();
 	m_inside_primitive = false;
 	m_drawing = true;
-	m_active_pipeline = &m_renderer.m_vk.mesh_pipeline;
+	m_transform_stack.clear();
+	m_active_pipeline = m_culling_enabled
+	    ? &m_renderer.m_vk.mesh_pipeline_culled
+	    : &m_renderer.m_vk.mesh_pipeline;
 	m_transform = smath::Mat4::identity();
 	m_current_color = { 1.0f, 1.0f, 1.0f, 1.0f };
 	m_current_normal = { 0.0f, 0.0f, 1.0f };
@@ -164,6 +167,34 @@ auto VulkanRenderer::GL::set_texture(
 	m_bound_texture = texture.value_or(&m_renderer.m_vk.error_image);
 }
 
+auto VulkanRenderer::GL::set_culling(bool enabled) -> void
+{
+	if (m_culling_enabled == enabled) {
+		return;
+	}
+
+	if (m_drawing) {
+		flush();
+	}
+
+	m_culling_enabled = enabled;
+	if (!m_drawing) {
+		return;
+	}
+
+	if (m_active_pipeline == &m_renderer.m_vk.mesh_pipeline
+	    || m_active_pipeline == &m_renderer.m_vk.mesh_pipeline_culled) {
+		m_active_pipeline = enabled ? &m_renderer.m_vk.mesh_pipeline_culled
+		                            : &m_renderer.m_vk.mesh_pipeline;
+	} else if (m_active_pipeline == &m_renderer.m_vk.triangle_pipeline
+	    || m_active_pipeline == &m_renderer.m_vk.triangle_pipeline_culled) {
+		m_active_pipeline = enabled ? &m_renderer.m_vk.triangle_pipeline_culled
+		                            : &m_renderer.m_vk.triangle_pipeline;
+	}
+
+	bind_pipeline_if_needed();
+}
+
 auto VulkanRenderer::GL::end() -> void
 {
 	if (!m_inside_primitive)
@@ -235,7 +266,8 @@ auto VulkanRenderer::GL::flush() -> void
 
 	bind_pipeline_if_needed();
 
-	if (m_active_pipeline == &m_renderer.m_vk.mesh_pipeline) {
+	if (m_active_pipeline == &m_renderer.m_vk.mesh_pipeline
+	    || m_active_pipeline == &m_renderer.m_vk.mesh_pipeline_culled) {
 		auto const image_set {
 			m_renderer.m_vk.get_current_frame().frame_descriptors.allocate(
 			    m_renderer.m_logger, m_renderer.m_vkb.dev.device,
@@ -254,7 +286,7 @@ auto VulkanRenderer::GL::flush() -> void
 
 		auto vk_image_set = vk::DescriptorSet { image_set };
 		cmd.bindDescriptorSets(vk::PipelineBindPoint::eGraphics,
-		    m_renderer.m_vk.mesh_pipeline.get_layout(), 0, vk_image_set, {});
+		    m_active_pipeline->get_layout(), 0, vk_image_set, {});
 
 		GPUDrawPushConstants push_constants {};
 		push_constants.world_matrix = m_transform;
@@ -264,7 +296,7 @@ auto VulkanRenderer::GL::flush() -> void
 		push_constants.vertex_buffer
 		    = m_renderer.m_device.getBufferAddress(device_address_info);
 
-		cmd.pushConstants(m_renderer.m_vk.mesh_pipeline.get_layout(),
+		cmd.pushConstants(m_active_pipeline->get_layout(),
 		    vk::ShaderStageFlagBits::eVertex, 0, sizeof(push_constants),
 		    &push_constants);
 	}
@@ -283,13 +315,26 @@ auto VulkanRenderer::GL::flush() -> void
 
 auto VulkanRenderer::GL::use_pipeline(Pipeline &pipeline) -> void
 {
-	if (&pipeline == m_active_pipeline) {
+	Pipeline *resolved_pipeline = &pipeline;
+	if (&pipeline == &m_renderer.m_vk.mesh_pipeline
+	    || &pipeline == &m_renderer.m_vk.mesh_pipeline_culled) {
+		resolved_pipeline = m_culling_enabled
+		    ? &m_renderer.m_vk.mesh_pipeline_culled
+		    : &m_renderer.m_vk.mesh_pipeline;
+	} else if (&pipeline == &m_renderer.m_vk.triangle_pipeline
+	    || &pipeline == &m_renderer.m_vk.triangle_pipeline_culled) {
+		resolved_pipeline = m_culling_enabled
+		    ? &m_renderer.m_vk.triangle_pipeline_culled
+		    : &m_renderer.m_vk.triangle_pipeline;
+	}
+
+	if (resolved_pipeline == m_active_pipeline) {
 		return;
 	}
 
 	flush();
 
-	m_active_pipeline = &pipeline;
+	m_active_pipeline = resolved_pipeline;
 	bind_pipeline_if_needed();
 }
 
@@ -299,6 +344,22 @@ auto VulkanRenderer::GL::set_transform(smath::Mat4 const &transform) -> void
 	m_transform = transform;
 }
 
+auto VulkanRenderer::GL::push_transform() -> void
+{
+	m_transform_stack.push_back(m_transform);
+}
+
+auto VulkanRenderer::GL::pop_transform() -> void
+{
+	if (m_transform_stack.empty()) {
+		return;
+	}
+
+	flush();
+	m_transform = m_transform_stack.back();
+	m_transform_stack.pop_back();
+}
+
 auto VulkanRenderer::GL::draw_rectangle(smath::Vec2 pos, smath::Vec2 size,
     smath::Vec4 rect_color, float rotation) -> void
 {
@@ -415,7 +476,10 @@ auto VulkanRenderer::GL::draw_mesh(GPUMeshBuffers const &mesh,
 	assert(m_drawing && "begin_drawing must be called first");
 
 	flush();
-	use_pipeline(m_renderer.m_vk.mesh_pipeline);
+	Pipeline &mesh_pipeline = m_culling_enabled
+	    ? m_renderer.m_vk.mesh_pipeline_culled
+	    : m_renderer.m_vk.mesh_pipeline;
+	use_pipeline(mesh_pipeline);
 
 	auto const image_set {
 		m_renderer.m_vk.get_current_frame().frame_descriptors.allocate(
@@ -433,13 +497,13 @@ auto VulkanRenderer::GL::draw_mesh(GPUMeshBuffers const &mesh,
 
 	auto vk_image_set = vk::DescriptorSet { image_set };
 	m_cmd.bindDescriptorSets(vk::PipelineBindPoint::eGraphics,
-	    m_renderer.m_vk.mesh_pipeline.get_layout(), 0, vk_image_set, {});
+	    mesh_pipeline.get_layout(), 0, vk_image_set, {});
 
 	GPUDrawPushConstants push_constants {};
 	push_constants.world_matrix = transform;
 	push_constants.vertex_buffer = mesh.vertex_buffer_address;
 
-	m_cmd.pushConstants(m_renderer.m_vk.mesh_pipeline.get_layout(),
+	m_cmd.pushConstants(mesh_pipeline.get_layout(),
 	    vk::ShaderStageFlagBits::eVertex, 0, sizeof(push_constants),
 	    &push_constants);
 
@@ -563,7 +627,9 @@ VulkanRenderer::~VulkanRenderer()
 	m_vk.imm_command_pool.reset();
 	m_vk.imm_fence.reset();
 	m_vk.triangle_pipeline.reset();
+	m_vk.triangle_pipeline_culled.reset();
 	m_vk.mesh_pipeline.reset();
+	m_vk.mesh_pipeline_culled.reset();
 	m_vk.default_sampler_linear.reset();
 	m_vk.default_sampler_nearest.reset();
 
@@ -1030,6 +1096,25 @@ auto VulkanRenderer::triangle_pipeline_init() -> void
 		          .set_depth_format(
 		              static_cast<VkFormat>(m_vk.depth_image.format));
 	      });
+	m_vk.triangle_pipeline_culled
+	    = builder.build_graphics([&](GraphicsPipelineBuilder &pipeline_builder)
+	                                 -> GraphicsPipelineBuilder & {
+		      return pipeline_builder
+		          .set_shaders(
+		              triangle_vert_shader.get(), triangle_frag_shader.get())
+		          .set_input_topology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
+		          .set_polygon_mode(VK_POLYGON_MODE_FILL)
+		          .set_cull_mode(
+		              VK_CULL_MODE_BACK_BIT, VK_FRONT_FACE_COUNTER_CLOCKWISE)
+		          .set_multisampling(
+		              static_cast<VkSampleCountFlagBits>(m_vk.msaa_samples))
+		          .enable_blending_additive()
+		          .disable_depth_testing()
+		          .set_color_attachment_format(
+		              static_cast<VkFormat>(m_vk.draw_image.format))
+		          .set_depth_format(
+		              static_cast<VkFormat>(m_vk.depth_image.format));
+	      });
 }
 
 auto VulkanRenderer::mesh_pipeline_init() -> void
@@ -1086,6 +1171,25 @@ auto VulkanRenderer::mesh_pipeline_init() -> void
 		          .set_depth_format(
 		              static_cast<VkFormat>(m_vk.depth_image.format));
 	      });
+	m_vk.mesh_pipeline_culled
+	    = builder.build_graphics([&](GraphicsPipelineBuilder &pipeline_builder)
+	                                 -> GraphicsPipelineBuilder & {
+		      return pipeline_builder
+		          .set_shaders(
+		              triangle_vert_shader.get(), triangle_frag_shader.get())
+		          .set_input_topology(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
+		          .set_polygon_mode(VK_POLYGON_MODE_FILL)
+		          .set_cull_mode(
+		              VK_CULL_MODE_BACK_BIT, VK_FRONT_FACE_COUNTER_CLOCKWISE)
+		          .set_multisampling(
+		              static_cast<VkSampleCountFlagBits>(m_vk.msaa_samples))
+		          .disable_blending()
+		          .enable_depth_testing()
+		          .set_color_attachment_format(
+		              static_cast<VkFormat>(m_vk.draw_image.format))
+		          .set_depth_format(
+		              static_cast<VkFormat>(m_vk.depth_image.format));
+	      });
 }
 
 auto VulkanRenderer::imgui_init() -> void

src/VulkanRenderer.h

@@ -73,6 +73,7 @@ struct VulkanRenderer {
 		auto normal(smath::Vec3 const &normal) -> void;
 		auto set_texture(std::optional<AllocatedImage const *> texture
 		    = std::nullopt) -> void;
+		auto set_culling(bool enabled) -> void;
 		auto draw_rectangle(smath::Vec2 pos, smath::Vec2 size,
 		    smath::Vec4 color = smath::Vec4 { Colors::WHITE, 1.0f },
 		    float rotation = 0.0f) -> void;
@@ -84,6 +85,8 @@ struct VulkanRenderer {
 
 		auto use_pipeline(Pipeline &pipeline) -> void;
 		auto set_transform(smath::Mat4 const &transform) -> void;
+		auto push_transform() -> void;
+		auto pop_transform() -> void;
 		auto draw_mesh(GPUMeshBuffers const &mesh, smath::Mat4 const &transform,
 		    uint32_t index_count, uint32_t first_index = 0,
 		    int32_t vertex_offset = 0) -> void;
@@ -101,7 +104,9 @@ struct VulkanRenderer {
 		bool m_inside_primitive { false };
 		bool m_drawing { false };
 		Pipeline *m_active_pipeline { nullptr };
+		bool m_culling_enabled { true };
 		smath::Mat4 m_transform { smath::Mat4::identity() };
+		std::vector<smath::Mat4> m_transform_stack;
 		smath::Vec4 m_current_color { 1.0f, 1.0f, 1.0f, 1.0f };
 		smath::Vec3 m_current_normal { 0.0f, 0.0f, 1.0f };
 		smath::Vec2 m_current_uv { 0.0f, 0.0f };
@@ -256,7 +261,9 @@ private:
 		vk::DescriptorSetLayout single_image_descriptor_layout {};
 
 		Pipeline triangle_pipeline;
+		Pipeline triangle_pipeline_culled;
 		Pipeline mesh_pipeline;
+		Pipeline mesh_pipeline_culled;
 
 		GPUMeshBuffers rectangle;