multithreaded buffer building

engine/BUILD

@@ -18,6 +18,7 @@ rust_binary(
         "src/render/vulkan/qfi.rs",
         "src/render/vulkan/shaders.rs",
         "src/render/vulkan/swapchains.rs",
+        "src/render/vulkan/worker.rs",
         "src/util/mod.rs",
         "src/util/counter.rs",
     ],

engine/src/main.rs

@@ -1,6 +1,6 @@
 use log;
 use env_logger;
-use std::rc::Rc;
+use std::sync::Arc;
 
 use cgmath as cgm;
 
@@ -19,19 +19,19 @@ fn main() {
         let elapsed = 0.0;
         let transform = cgm::Matrix4::from_angle_z(cgm::Rad::from(cgm::Deg(elapsed as f32 * 0.180))) *
             cgm::Matrix4::from_translation(cgm::Vector3::new(0.0, 0.0, (i as f32)/1000.0));
-        let vertices = Rc::new(vec![
+        let vertices = Arc::new(vec![
             data::Vertex::new([-0.5, -0.5, 0.0], [1.0, 0.0, 0.0]),
             data::Vertex::new([0.5, -0.5, 0.0], [0.0, 1.0, 0.0]),
             data::Vertex::new([0.5, 0.5, 0.0], [0.0, 0.0, 1.0]),
             data::Vertex::new([-0.5, 0.5, 0.0], [1.0, 1.0, 1.0])
         ]);
-        let indices = Rc::new(vec![
+        let indices = Arc::new(vec![
             0, 1, 2, 2, 3, 0,
         ]);
         let demo = render::renderable::Mesh {
             transform, vertices, indices
         };
-        renderables.push(demo.data().unwrap());
+        renderables.push(Arc::new(demo.data().unwrap()));
     }
 
     let mut renderer = render::Renderer::initialize();

engine/src/render/mod.rs

@@ -22,7 +22,7 @@ pub struct Renderer {
     instance: vulkan::Instance<winit::Window>,
     events_loop: EventsLoop,
 
-    render_data: Vec<renderable::Data>,
+    render_data: Vec<Arc<renderable::Data>>,
 }
 
 impl Renderer {
@@ -38,7 +38,7 @@ impl Renderer {
         }
     }
 
-    pub fn set_render_data(&mut self, render_data: Vec<renderable::Data>) {
+    pub fn set_render_data(&mut self, render_data: Vec<Arc<renderable::Data>>) {
         self.render_data = render_data;
     }
 
@@ -53,7 +53,7 @@ impl Renderer {
     }
 
     fn draw_frame(&mut self) {
-        self.instance.flip(&self.render_data);
+        self.instance.flip(self.render_data.clone());
     }
 
     pub fn main_loop(&mut self) {

engine/src/render/renderable.rs

@@ -1,11 +1,10 @@
-use std::cell::RefCell;
 use std::sync::Arc;
-use std::rc::Rc;
+use std::sync::Mutex;
 
 use cgmath as cgm;
 use vulkano::device as vd;
 use vulkano::buffer as vb;
-use vulkano::sync::{FenceSignalFuture, GpuFuture};
+use vulkano::sync::GpuFuture;
 
 use crate::render::vulkan::data;
 
@@ -15,30 +14,28 @@ pub trait Renderable {
     }
 }
 
-#[derive(Clone)]
 struct VulkanData {
     vbuffer: Arc<vb::ImmutableBuffer<[data::Vertex]>>,
     ibuffer: Arc<vb::ImmutableBuffer<[u16]>>,
 }
 
-#[derive(Clone)]
 pub struct Data {
-    vertices: Rc<Vec<data::Vertex>>,
+    vertices: Arc<Vec<data::Vertex>>,
-    indices: Rc<Vec<u16>>,
+    indices: Arc<Vec<u16>>,
     transform: cgm::Matrix4<f32>,
 
-    vulkan: RefCell<Option<VulkanData>>,
+    vulkan: Mutex<Option<VulkanData>>,
 }
 
 impl Data {
     pub fn new(
-        vertices: Rc<Vec<data::Vertex>>,
+        vertices: Arc<Vec<data::Vertex>>,
-        indices: Rc<Vec<u16>>,
+        indices: Arc<Vec<u16>>,
         transform: cgm::Matrix4<f32>,
     ) -> Data {
         Data {
             vertices, indices, transform,
-            vulkan: RefCell::new(None),
+            vulkan: Mutex::new(None),
         }
     }
 
@@ -49,7 +46,7 @@ impl Data {
         Arc<vb::ImmutableBuffer<[data::Vertex]>>,
         Arc<vb::ImmutableBuffer<[u16]>>,
     ) {
-        let mut cache = self.vulkan.borrow_mut();
+        let mut cache = self.vulkan.lock().unwrap();
         match &mut *cache {
             Some(data) => (data.vbuffer.clone(), data.ibuffer.clone()),
             None => {
@@ -82,8 +79,8 @@ impl Data {
 }
 
 pub struct Mesh {
-    pub vertices: Rc<Vec<data::Vertex>>,
+    pub vertices: Arc<Vec<data::Vertex>>,
-    pub indices: Rc<Vec<u16>>,
+    pub indices: Arc<Vec<u16>>,
     pub transform: cgm::Matrix4<f32>,
 }
 

engine/src/render/vulkan/mod.rs

@@ -16,6 +16,7 @@ mod pipeline;
 mod qfi;
 mod shaders;
 mod swapchains;
+mod worker;
 
 use crate::render::renderable;
 
@@ -31,6 +32,8 @@ pub struct Instance<WT> {
     debug_callback: vi::debug::DebugCallback,
     vulkan: Arc<vi::Instance>,
 
+    workers: Vec<worker::Worker>,
+
     surface_binding: Option<binding::SurfaceBinding<WT>>,
     swapchain_binding: Option<swapchains::SwapchainBinding<WT>>,
 
@@ -81,11 +84,16 @@ impl<WT: 'static + Send + Sync> Instance<WT> {
         let vulkan = vulkanOpt.expect("could not create a vulkan instance");
         let debug_callback = Self::init_debug_callback(&vulkan);
 
+        let workers = (0..4).map(|n| {
+            worker::Worker::new(n)
+        }).collect();
 
         Self {
             debug_callback,
             vulkan,
 
+            workers,
+
             surface_binding: None,
             swapchain_binding: None,
 
@@ -132,13 +140,14 @@ impl<WT: 'static + Send + Sync> Instance<WT> {
         self.armed = true;
     }
 
-    fn make_graphics_command(
+    fn make_graphics_commands(
         &mut self,
-        render_data: &Vec<renderable::Data>,
+        render_data: &Vec<Arc<renderable::Data>>,
-    ) -> vc::AutoCommandBuffer {
+    ) -> Vec<Box<vc::AutoCommandBuffer>> {
         let device = self.surface_binding().device.clone();
         let rp = self.swapchain_binding().render_pass.clone();
-        let qf = self.surface_binding().graphics_queue.family();
+        let queue = self.surface_binding().graphics_queue.clone();
+        let pipeline = self.pipeline.as_ref().unwrap().get_pipeline().clone();
         let dimensions = self.dimensions();
 
         let view = cgm::Matrix4::look_at(
@@ -153,34 +162,32 @@ impl<WT: 'static + Send + Sync> Instance<WT> {
             10.0
         );
 
+        // Split work into N vectors (one per worker)
+        // This is not fair to workers, but good enough for now.
+        let nworkers = self.workers.len();
+        let nwork = render_data.len();
+        let nperworker = (nwork / nworkers) + 1; // last worker will be underloaded
 
-        let mut builder = vc::AutoCommandBufferBuilder::secondary_graphics_one_time_submit(device.clone(), qf, vf::Subpass::from(rp, 0).unwrap()).unwrap();
+        let work = render_data.chunks(nperworker);
+        let futures = self.workers.iter().zip(work).map(|(w, c)| {
+            w.render(device.clone(), queue.clone(), rp.clone(), pipeline.clone(), view.clone(), proj.clone(), c.to_vec())
+        });
 
-        for d in render_data {
+        let start = time::Instant::now();
-            let (vbuffer, ibuffer) = d.vulkan_buffers(self.surface_binding().graphics_queue.clone());
+        let res = futures.map(|r| { r.recv().unwrap() }).collect();
-            let ubo = data::UniformBufferObject {
+        let took = time::Instant::now().duration_since(start);
-                model: proj.clone() * view.clone() * d.get_transform(),
+        //log::info!("took {:?}", took);
-            };
-            //let ub = self.uniform_pool.as_ref().unwrap().next(ubo.clone()).unwrap();
-            //let ds = self.pipeline.as_mut().unwrap().make_descriptor_set(Box::new(ub));
-            let pipeline = self.pipeline.as_ref().unwrap().get_pipeline();
-            builder = builder.draw_indexed(pipeline, &vc::DynamicState::none(),
-                vec![vbuffer.clone()],
-                ibuffer.clone(),
-                (),
-                ubo).unwrap();
-        }
 
-        builder.build().unwrap()
+        res
     }
 
     // (╯°□°)╯︵ ┻━┻
     pub fn flip(
         &mut self,
-        render_data: &Vec<renderable::Data>,
+        render_data: Vec<Arc<renderable::Data>>,
     ) {
         // Build batch command buffer as early as possible.
-        let mut batch = self.make_graphics_command(render_data);
+        let mut batches = self.make_graphics_commands(&render_data);
 
         match &self.previous_frame_end {
             None => (),
@@ -190,7 +197,7 @@ impl<WT: 'static + Send + Sync> Instance<WT> {
         if !self.armed {
             self.arm();
             // Rearming means the batch is invalid - rebuild it.
-            batch = self.make_graphics_command(render_data);
+            batches = self.make_graphics_commands(&render_data);
         }
 
         let chain = self.swapchain_binding().chain.clone();
@@ -205,7 +212,7 @@ impl<WT: 'static + Send + Sync> Instance<WT> {
         };
 
         let fb = self.swapchain_binding().framebuffers[image_index].clone();
-        let command_buffer = self.make_command_buffer(fb, batch);
+        let command_buffer = self.make_command_buffer(fb, batches);
 
         let gq = self.surface_binding().graphics_queue.clone();
         let pq = self.surface_binding().present_queue.clone();
@@ -240,7 +247,7 @@ impl<WT: 'static + Send + Sync> Instance<WT> {
     fn make_command_buffer(
         &mut self,
         framebuffer: Arc<dyn vf::FramebufferAbstract + Send + Sync>,
-        batch: vc::AutoCommandBuffer,
+        batches: Vec<Box<vc::AutoCommandBuffer>>,
     ) -> Arc<vc::AutoCommandBuffer> {
         let device = self.surface_binding().device.clone();
         let qf = self.surface_binding().graphics_queue.family();
@@ -250,8 +257,11 @@ impl<WT: 'static + Send + Sync> Instance<WT> {
                  .begin_render_pass(framebuffer.clone(), false, vec![[0.0, 0.0, 0.0, 1.0].into()])
                  .unwrap();
 
-        unsafe {
+
-            primary = primary.execute_commands(batch).unwrap();
+        for batch in batches {
+            unsafe {
+                primary = primary.execute_commands(batch).unwrap();
+            }
         }
 
         Arc::new(primary.end_render_pass().unwrap().build().unwrap())

engine/src/render/vulkan/pipeline.rs

@@ -14,7 +14,7 @@ use vulkano::pipeline::shader as vps;
 use crate::render::vulkan::data;
 use crate::render::vulkan::shaders;
 
-type VulkanoPipeline = dyn vp::GraphicsPipelineAbstract + Send + Sync;
+pub type VulkanoPipeline = dyn vp::GraphicsPipelineAbstract + Send + Sync;
 type VulkanoDescriptorSet = dyn vdd::DescriptorSet + Send + Sync;
 
 pub trait Pipeline {
@@ -23,8 +23,6 @@ pub trait Pipeline {
 }
 
 pub struct Forward {
-    device: Arc<vd::Device>,
-
     pipeline: Arc<VulkanoPipeline>,
     descriptor_set_pool: vdd::FixedSizeDescriptorSetsPool<Arc<VulkanoPipeline>>,
 }
@@ -121,8 +119,6 @@ impl Forward {
         let descriptor_set_pool = vdd::FixedSizeDescriptorSetsPool::new(pipeline.clone(), 0);
 
         Forward {
-            device,
-
             pipeline,
             descriptor_set_pool
         }

engine/src/render/vulkan/worker.rs

@@ -0,0 +1,134 @@
+use log;
+
+use std::sync::Arc;
+use std::sync::mpsc;
+use std::thread;
+use std::time;
+
+use cgmath as cgm;
+use vulkano::command_buffer as vc;
+use vulkano::device as vd;
+use vulkano::framebuffer as vf;
+
+use crate::render::renderable;
+use crate::render::vulkan::data;
+use crate::render::vulkan::pipeline;
+
+enum Command {
+    Render(CommandRender),
+    Exit,
+}
+
+struct CommandRender {
+    device: Arc<vd::Device>,
+    queue: Arc<vd::Queue>,
+    render_pass: Arc<dyn vf::RenderPassAbstract + Send + Sync>,
+    pipeline: Arc<pipeline::VulkanoPipeline>,
+
+    matrix_v: cgm::Matrix4<f32>,
+    matrix_p: cgm::Matrix4<f32>,
+    data: Vec<Arc<renderable::Data>>,
+
+    result: mpsc::Sender<Box<vc::AutoCommandBuffer>>,
+}
+
+pub struct Worker {
+    handle: thread::JoinHandle<()>,
+    control: mpsc::Sender<Command>,
+}
+
+impl Worker {
+    pub fn new(id: u64) -> Worker {
+        let (control, rx) = mpsc::channel();
+
+        let handle = thread::spawn(move || {
+            log::info!("Worker {} starting...", id);
+            loop {
+                let mut done = false;
+
+                match rx.recv() {
+                    Err(err) => {
+                        log::error!("Worker {} cannot receive, dying: {}", id, err);
+                        done = true;
+                    },
+                    Ok(cmd) => {
+                        match cmd {
+                            Command::Exit => {
+                                log::info!("Worker {} exiting", id);
+                                done = true;
+                            }
+                            Command::Render(r) => {
+                                Worker::work_render(r);
+                            }
+                        }
+                    },
+                }
+
+                if done {
+                    break
+                }
+            }
+        });
+
+        Worker {
+            handle, control
+        }
+    }
+
+    fn work_render(r: CommandRender) {
+        let qf = r.queue.family();
+
+        let mut builder = vc::AutoCommandBufferBuilder::secondary_graphics_one_time_submit(
+            r.device, qf, vf::Subpass::from(r.render_pass, 0).unwrap()).unwrap();
+
+        let start = time::Instant::now();
+        for d in r.data {
+            let ubo = data::UniformBufferObject {
+                model: r.matrix_p.clone() * r.matrix_v.clone() * d.get_transform(),
+            };
+            let (vbuffer, ibuffer) = d.vulkan_buffers(r.queue.clone());
+            builder = builder.draw_indexed(r.pipeline.clone(), &vc::DynamicState::none(),
+                vec![vbuffer.clone()],
+                ibuffer.clone(),
+                (),
+                ubo).unwrap();
+        }
+        let took = time::Instant::now().duration_since(start);
+        //log::info!("worker loop took {:?}", took);
+
+        let buffer = builder.build().unwrap();
+        r.result.send(Box::new(buffer)).unwrap();
+    }
+
+    pub fn render(
+        &self,
+
+        device: Arc<vd::Device>,
+        queue: Arc<vd::Queue>,
+        render_pass: Arc<dyn vf::RenderPassAbstract + Send + Sync>,
+        pipeline: Arc<pipeline::VulkanoPipeline>,
+
+        matrix_v: cgm::Matrix4<f32>,
+        matrix_p: cgm::Matrix4<f32>,
+        data: Vec<Arc<renderable::Data>>,
+    ) -> mpsc::Receiver<Box<vc::AutoCommandBuffer>> {
+        let (result, rx) = mpsc::channel();
+
+        let req = Command::Render(CommandRender {
+            device, queue, render_pass, pipeline,
+            matrix_v, matrix_p, data,
+
+            result
+        });
+
+        self.control.send(req).unwrap();
+        return rx;
+    }
+}
+
+impl Drop for Worker {
+    fn drop(&mut self) {
+        self.control.send(Command::Exit).unwrap();
+        //self.handle.join().unwrap();
+    }
+}