diff --git a/engine/BUILD b/engine/BUILD index 6915f01..f953657 100644 --- a/engine/BUILD +++ b/engine/BUILD @@ -18,6 +18,7 @@ rust_binary( "src/render/vulkan/qfi.rs", "src/render/vulkan/shaders.rs", "src/render/vulkan/swapchains.rs", + "src/render/vulkan/worker.rs", "src/util/mod.rs", "src/util/counter.rs", ], diff --git a/engine/src/main.rs b/engine/src/main.rs index a3561c5..8eb7a02 100644 --- a/engine/src/main.rs +++ b/engine/src/main.rs @@ -1,6 +1,6 @@ use log; use env_logger; -use std::rc::Rc; +use std::sync::Arc; use cgmath as cgm; @@ -19,19 +19,19 @@ fn main() { let elapsed = 0.0; let transform = cgm::Matrix4::from_angle_z(cgm::Rad::from(cgm::Deg(elapsed as f32 * 0.180))) * cgm::Matrix4::from_translation(cgm::Vector3::new(0.0, 0.0, (i as f32)/1000.0)); - let vertices = Rc::new(vec![ + let vertices = Arc::new(vec![ data::Vertex::new([-0.5, -0.5, 0.0], [1.0, 0.0, 0.0]), data::Vertex::new([0.5, -0.5, 0.0], [0.0, 1.0, 0.0]), data::Vertex::new([0.5, 0.5, 0.0], [0.0, 0.0, 1.0]), data::Vertex::new([-0.5, 0.5, 0.0], [1.0, 1.0, 1.0]) ]); - let indices = Rc::new(vec![ + let indices = Arc::new(vec![ 0, 1, 2, 2, 3, 0, ]); let demo = render::renderable::Mesh { transform, vertices, indices }; - renderables.push(demo.data().unwrap()); + renderables.push(Arc::new(demo.data().unwrap())); } let mut renderer = render::Renderer::initialize(); diff --git a/engine/src/render/mod.rs b/engine/src/render/mod.rs index e8426f1..f89e756 100644 --- a/engine/src/render/mod.rs +++ b/engine/src/render/mod.rs @@ -22,7 +22,7 @@ pub struct Renderer { instance: vulkan::Instance, events_loop: EventsLoop, - render_data: Vec, + render_data: Vec>, } impl Renderer { @@ -38,7 +38,7 @@ impl Renderer { } } - pub fn set_render_data(&mut self, render_data: Vec) { + pub fn set_render_data(&mut self, render_data: Vec>) { self.render_data = render_data; } @@ -53,7 +53,7 @@ impl Renderer { } fn draw_frame(&mut self) { - self.instance.flip(&self.render_data); + self.instance.flip(self.render_data.clone()); } pub fn main_loop(&mut self) { diff --git a/engine/src/render/renderable.rs b/engine/src/render/renderable.rs index 86d8cd6..0afc458 100644 --- a/engine/src/render/renderable.rs +++ b/engine/src/render/renderable.rs @@ -1,11 +1,10 @@ -use std::cell::RefCell; use std::sync::Arc; -use std::rc::Rc; +use std::sync::Mutex; use cgmath as cgm; use vulkano::device as vd; use vulkano::buffer as vb; -use vulkano::sync::{FenceSignalFuture, GpuFuture}; +use vulkano::sync::GpuFuture; use crate::render::vulkan::data; @@ -15,30 +14,28 @@ pub trait Renderable { } } -#[derive(Clone)] struct VulkanData { vbuffer: Arc>, ibuffer: Arc>, } -#[derive(Clone)] pub struct Data { - vertices: Rc>, - indices: Rc>, + vertices: Arc>, + indices: Arc>, transform: cgm::Matrix4, - vulkan: RefCell>, + vulkan: Mutex>, } impl Data { pub fn new( - vertices: Rc>, - indices: Rc>, + vertices: Arc>, + indices: Arc>, transform: cgm::Matrix4, ) -> Data { Data { vertices, indices, transform, - vulkan: RefCell::new(None), + vulkan: Mutex::new(None), } } @@ -49,7 +46,7 @@ impl Data { Arc>, Arc>, ) { - let mut cache = self.vulkan.borrow_mut(); + let mut cache = self.vulkan.lock().unwrap(); match &mut *cache { Some(data) => (data.vbuffer.clone(), data.ibuffer.clone()), None => { @@ -82,8 +79,8 @@ impl Data { } pub struct Mesh { - pub vertices: Rc>, - pub indices: Rc>, + pub vertices: Arc>, + pub indices: Arc>, pub transform: cgm::Matrix4, } diff --git a/engine/src/render/vulkan/mod.rs b/engine/src/render/vulkan/mod.rs index cd726aa..2c98b21 100644 --- a/engine/src/render/vulkan/mod.rs +++ b/engine/src/render/vulkan/mod.rs @@ -16,6 +16,7 @@ mod pipeline; mod qfi; mod shaders; mod swapchains; +mod worker; use crate::render::renderable; @@ -31,6 +32,8 @@ pub struct Instance { debug_callback: vi::debug::DebugCallback, vulkan: Arc, + workers: Vec, + surface_binding: Option>, swapchain_binding: Option>, @@ -81,11 +84,16 @@ impl Instance { let vulkan = vulkanOpt.expect("could not create a vulkan instance"); let debug_callback = Self::init_debug_callback(&vulkan); + let workers = (0..4).map(|n| { + worker::Worker::new(n) + }).collect(); Self { debug_callback, vulkan, + workers, + surface_binding: None, swapchain_binding: None, @@ -132,13 +140,14 @@ impl Instance { self.armed = true; } - fn make_graphics_command( + fn make_graphics_commands( &mut self, - render_data: &Vec, - ) -> vc::AutoCommandBuffer { + render_data: &Vec>, + ) -> Vec> { let device = self.surface_binding().device.clone(); let rp = self.swapchain_binding().render_pass.clone(); - let qf = self.surface_binding().graphics_queue.family(); + let queue = self.surface_binding().graphics_queue.clone(); + let pipeline = self.pipeline.as_ref().unwrap().get_pipeline().clone(); let dimensions = self.dimensions(); let view = cgm::Matrix4::look_at( @@ -153,34 +162,32 @@ impl Instance { 10.0 ); + // Split work into N vectors (one per worker) + // This is not fair to workers, but good enough for now. + let nworkers = self.workers.len(); + let nwork = render_data.len(); + let nperworker = (nwork / nworkers) + 1; // last worker will be underloaded - let mut builder = vc::AutoCommandBufferBuilder::secondary_graphics_one_time_submit(device.clone(), qf, vf::Subpass::from(rp, 0).unwrap()).unwrap(); + let work = render_data.chunks(nperworker); + let futures = self.workers.iter().zip(work).map(|(w, c)| { + w.render(device.clone(), queue.clone(), rp.clone(), pipeline.clone(), view.clone(), proj.clone(), c.to_vec()) + }); - for d in render_data { - let (vbuffer, ibuffer) = d.vulkan_buffers(self.surface_binding().graphics_queue.clone()); - let ubo = data::UniformBufferObject { - model: proj.clone() * view.clone() * d.get_transform(), - }; - //let ub = self.uniform_pool.as_ref().unwrap().next(ubo.clone()).unwrap(); - //let ds = self.pipeline.as_mut().unwrap().make_descriptor_set(Box::new(ub)); - let pipeline = self.pipeline.as_ref().unwrap().get_pipeline(); - builder = builder.draw_indexed(pipeline, &vc::DynamicState::none(), - vec![vbuffer.clone()], - ibuffer.clone(), - (), - ubo).unwrap(); - } + let start = time::Instant::now(); + let res = futures.map(|r| { r.recv().unwrap() }).collect(); + let took = time::Instant::now().duration_since(start); + //log::info!("took {:?}", took); - builder.build().unwrap() + res } // (╯°□°)╯︵ ┻━┻ pub fn flip( &mut self, - render_data: &Vec, + render_data: Vec>, ) { // Build batch command buffer as early as possible. - let mut batch = self.make_graphics_command(render_data); + let mut batches = self.make_graphics_commands(&render_data); match &self.previous_frame_end { None => (), @@ -190,7 +197,7 @@ impl Instance { if !self.armed { self.arm(); // Rearming means the batch is invalid - rebuild it. - batch = self.make_graphics_command(render_data); + batches = self.make_graphics_commands(&render_data); } let chain = self.swapchain_binding().chain.clone(); @@ -205,7 +212,7 @@ impl Instance { }; let fb = self.swapchain_binding().framebuffers[image_index].clone(); - let command_buffer = self.make_command_buffer(fb, batch); + let command_buffer = self.make_command_buffer(fb, batches); let gq = self.surface_binding().graphics_queue.clone(); let pq = self.surface_binding().present_queue.clone(); @@ -240,7 +247,7 @@ impl Instance { fn make_command_buffer( &mut self, framebuffer: Arc, - batch: vc::AutoCommandBuffer, + batches: Vec>, ) -> Arc { let device = self.surface_binding().device.clone(); let qf = self.surface_binding().graphics_queue.family(); @@ -250,8 +257,11 @@ impl Instance { .begin_render_pass(framebuffer.clone(), false, vec![[0.0, 0.0, 0.0, 1.0].into()]) .unwrap(); - unsafe { - primary = primary.execute_commands(batch).unwrap(); + + for batch in batches { + unsafe { + primary = primary.execute_commands(batch).unwrap(); + } } Arc::new(primary.end_render_pass().unwrap().build().unwrap()) diff --git a/engine/src/render/vulkan/pipeline.rs b/engine/src/render/vulkan/pipeline.rs index 169dbd6..97eb647 100644 --- a/engine/src/render/vulkan/pipeline.rs +++ b/engine/src/render/vulkan/pipeline.rs @@ -14,7 +14,7 @@ use vulkano::pipeline::shader as vps; use crate::render::vulkan::data; use crate::render::vulkan::shaders; -type VulkanoPipeline = dyn vp::GraphicsPipelineAbstract + Send + Sync; +pub type VulkanoPipeline = dyn vp::GraphicsPipelineAbstract + Send + Sync; type VulkanoDescriptorSet = dyn vdd::DescriptorSet + Send + Sync; pub trait Pipeline { @@ -23,8 +23,6 @@ pub trait Pipeline { } pub struct Forward { - device: Arc, - pipeline: Arc, descriptor_set_pool: vdd::FixedSizeDescriptorSetsPool>, } @@ -121,8 +119,6 @@ impl Forward { let descriptor_set_pool = vdd::FixedSizeDescriptorSetsPool::new(pipeline.clone(), 0); Forward { - device, - pipeline, descriptor_set_pool } diff --git a/engine/src/render/vulkan/worker.rs b/engine/src/render/vulkan/worker.rs new file mode 100644 index 0000000..2d1c505 --- /dev/null +++ b/engine/src/render/vulkan/worker.rs @@ -0,0 +1,134 @@ +use log; + +use std::sync::Arc; +use std::sync::mpsc; +use std::thread; +use std::time; + +use cgmath as cgm; +use vulkano::command_buffer as vc; +use vulkano::device as vd; +use vulkano::framebuffer as vf; + +use crate::render::renderable; +use crate::render::vulkan::data; +use crate::render::vulkan::pipeline; + +enum Command { + Render(CommandRender), + Exit, +} + +struct CommandRender { + device: Arc, + queue: Arc, + render_pass: Arc, + pipeline: Arc, + + matrix_v: cgm::Matrix4, + matrix_p: cgm::Matrix4, + data: Vec>, + + result: mpsc::Sender>, +} + +pub struct Worker { + handle: thread::JoinHandle<()>, + control: mpsc::Sender, +} + +impl Worker { + pub fn new(id: u64) -> Worker { + let (control, rx) = mpsc::channel(); + + let handle = thread::spawn(move || { + log::info!("Worker {} starting...", id); + loop { + let mut done = false; + + match rx.recv() { + Err(err) => { + log::error!("Worker {} cannot receive, dying: {}", id, err); + done = true; + }, + Ok(cmd) => { + match cmd { + Command::Exit => { + log::info!("Worker {} exiting", id); + done = true; + } + Command::Render(r) => { + Worker::work_render(r); + } + } + }, + } + + if done { + break + } + } + }); + + Worker { + handle, control + } + } + + fn work_render(r: CommandRender) { + let qf = r.queue.family(); + + let mut builder = vc::AutoCommandBufferBuilder::secondary_graphics_one_time_submit( + r.device, qf, vf::Subpass::from(r.render_pass, 0).unwrap()).unwrap(); + + let start = time::Instant::now(); + for d in r.data { + let ubo = data::UniformBufferObject { + model: r.matrix_p.clone() * r.matrix_v.clone() * d.get_transform(), + }; + let (vbuffer, ibuffer) = d.vulkan_buffers(r.queue.clone()); + builder = builder.draw_indexed(r.pipeline.clone(), &vc::DynamicState::none(), + vec![vbuffer.clone()], + ibuffer.clone(), + (), + ubo).unwrap(); + } + let took = time::Instant::now().duration_since(start); + //log::info!("worker loop took {:?}", took); + + let buffer = builder.build().unwrap(); + r.result.send(Box::new(buffer)).unwrap(); + } + + pub fn render( + &self, + + device: Arc, + queue: Arc, + render_pass: Arc, + pipeline: Arc, + + matrix_v: cgm::Matrix4, + matrix_p: cgm::Matrix4, + data: Vec>, + ) -> mpsc::Receiver> { + let (result, rx) = mpsc::channel(); + + let req = Command::Render(CommandRender { + device, queue, render_pass, pipeline, + matrix_v, matrix_p, data, + + result + }); + + self.control.send(req).unwrap(); + return rx; + } +} + +impl Drop for Worker { + fn drop(&mut self) { + self.control.send(Command::Exit).unwrap(); + //self.handle.join().unwrap(); + } +}