Add textured OBJ model rendering

src/app.rs

@@ -1,6 +1,3 @@
-use eframe::{egui, egui_wgpu};
-use std::sync::Arc;
-
 use crate::connection::ConnectionManager;
 use crate::recording::Recorder;
 use crate::theme::{ONE_DARK_PRO, apply_fonts, apply_theme};
@@ -16,6 +13,8 @@ use crate::{
         draw_scene_panel, draw_stats_panel,
     },
 };
+use eframe::{egui, egui_wgpu};
+use std::sync::Arc;
 
 const DATA_LOG_EVERY_FRAMES: u64 = 30;
 
@@ -54,6 +53,7 @@ impl EskinDesktopApp {
             .callback_resources
             .insert(BackgroundRenderResources::new(
                 &wgpu_state.device,
+                &wgpu_state.queue,
                 &wgpu_state.target_format,
                 MATRIX_ROWS,
                 MATRIX_COLS,

src/main.rs

@@ -1,13 +1,15 @@
 mod app;
 mod connection;
 mod matrix;
+mod model;
 mod recording;
 mod render;
+mod resources;
 mod serial_core;
+mod texture;
 mod theme;
 mod ui;
 mod utils;
-mod model;
 use app::EskinDesktopApp;
 use eframe::egui;
 

src/model.rs

@@ -1,9 +1,7 @@
-use eframe::{
-    egui,
-    egui_wgpu::{self, wgpu},
-    wgpu::util::DeviceExt,
-};
+use eframe::egui_wgpu::wgpu;
+use std::ops::Range;
 
+use crate::texture;
 pub trait Vertex {
     fn desc<'a>() -> wgpu::VertexBufferLayout<'a>;
 }
@@ -12,4 +10,137 @@ pub trait Vertex {
 #[derive(Copy, Clone, Debug, bytemuck::Pod, bytemuck::Zeroable)]
 pub struct ModelVertex {
     pub position: [f32; 3],
-}
+    pub tex_coords: [f32; 2],
+    pub normal: [f32; 3],
+}
+
+impl Vertex for ModelVertex {
+    fn desc<'a>() -> wgpu::VertexBufferLayout<'a> {
+        use core::mem;
+        wgpu::VertexBufferLayout {
+            array_stride: mem::size_of::<ModelVertex>() as wgpu::BufferAddress,
+            step_mode: wgpu::VertexStepMode::Vertex,
+            attributes: &[
+                wgpu::VertexAttribute {
+                    offset: 0,
+                    shader_location: 0,
+                    format: wgpu::VertexFormat::Float32x3,
+                },
+                wgpu::VertexAttribute {
+                    offset: mem::size_of::<[f32; 3]>() as wgpu::BufferAddress,
+                    shader_location: 1,
+                    format: wgpu::VertexFormat::Float32x2,
+                },
+                wgpu::VertexAttribute {
+                    offset: mem::size_of::<[f32; 5]>() as wgpu::BufferAddress,
+                    shader_location: 2,
+                    format: wgpu::VertexFormat::Float32x3,
+                },
+            ],
+        }
+    }
+}
+
+pub struct Instance {
+    position: glam::Vec3,
+    rotation: glam::Quat,
+}
+
+impl Instance {
+    pub fn new(position: glam::Vec3, rotation: glam::Quat) -> Self {
+        Self { position, rotation }
+    }
+
+    pub fn to_raw(&self) -> InstanceRaw {
+        InstanceRaw {
+            model: (glam::Mat4::from_translation(self.position)
+                * glam::Mat4::from_quat(self.rotation))
+            .to_cols_array_2d(),
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)]
+pub struct InstanceRaw {
+    #[allow(dead_code)]
+    model: [[f32; 4]; 4],
+}
+
+impl InstanceRaw {
+    pub fn desc<'a>() -> wgpu::VertexBufferLayout<'a> {
+        use core::mem;
+        wgpu::VertexBufferLayout {
+            array_stride: mem::size_of::<InstanceRaw>() as wgpu::BufferAddress,
+            // We need to switch from using a step mode of Vertex to Instance
+            // This means that our shaders will only change to use the next
+            // instance when the shader starts processing a new instance
+            step_mode: wgpu::VertexStepMode::Instance,
+            attributes: &[
+                wgpu::VertexAttribute {
+                    offset: 0,
+                    // While our vertex shader only uses locations 0, and 1 now, in later tutorials we'll
+                    // be using 2, 3, and 4, for Vertex. We'll start at slot 5 not conflict with them later
+                    shader_location: 5,
+                    format: wgpu::VertexFormat::Float32x4,
+                },
+                // A mat4 takes up 4 vertex slots as it is technically 4 vec4s. We need to define a slot
+                // for each vec4. We don't have to do this in code though.
+                wgpu::VertexAttribute {
+                    offset: mem::size_of::<[f32; 4]>() as wgpu::BufferAddress,
+                    shader_location: 6,
+                    format: wgpu::VertexFormat::Float32x4,
+                },
+                wgpu::VertexAttribute {
+                    offset: mem::size_of::<[f32; 8]>() as wgpu::BufferAddress,
+                    shader_location: 7,
+                    format: wgpu::VertexFormat::Float32x4,
+                },
+                wgpu::VertexAttribute {
+                    offset: mem::size_of::<[f32; 12]>() as wgpu::BufferAddress,
+                    shader_location: 8,
+                    format: wgpu::VertexFormat::Float32x4,
+                },
+            ],
+        }
+    }
+}
+
+pub struct Model {
+    pub meshes: Vec<Mesh>,
+    pub materials: Vec<Material>,
+}
+
+pub struct Material {
+    pub name: String,
+    pub diffuse_texture: texture::Texture,
+    pub bind_group: wgpu::BindGroup,
+}
+
+pub struct Mesh {
+    pub name: String,
+    pub vertex_buffer: wgpu::Buffer,
+    pub index_buffer: wgpu::Buffer,
+    pub num_elements: u32,
+    pub material: usize,
+}
+
+pub trait DrawModel<'a> {
+    fn draw_mesh(&mut self, mesh: &'a Mesh);
+    fn draw_mesh_instanced(&mut self, mesh: &'a Mesh, instances: Range<u32>);
+}
+
+impl<'a, 'b> DrawModel<'b> for wgpu::RenderPass<'a>
+where
+    'b: 'a,
+{
+    fn draw_mesh(&mut self, mesh: &'b Mesh) {
+        self.draw_mesh_instanced(mesh, 0..1);
+    }
+
+    fn draw_mesh_instanced(&mut self, mesh: &'b Mesh, instances: Range<u32>) {
+        self.set_vertex_buffer(0, mesh.vertex_buffer.slice(..));
+        self.set_index_buffer(mesh.index_buffer.slice(..), wgpu::IndexFormat::Uint32);
+        self.draw_indexed(0..mesh.num_elements, 0, instances);
+    }
+}

src/render.rs

@@ -1,11 +1,14 @@
+use crate::{
+    matrix::{MatrixLayout, build_view_projection, glyph_world_position},
+    model::{Instance, InstanceRaw, Model, ModelVertex, Vertex},
+    resources,
+};
 use eframe::{
     egui,
     egui_wgpu::{self, wgpu},
     wgpu::util::DeviceExt,
 };
 
-use crate::matrix::{MatrixLayout, build_view_projection, glyph_world_position};
-
 pub const PRESSURE_CELL_COUNT: usize =
     (crate::matrix::MATRIX_ROWS * crate::matrix::MATRIX_COLS) as usize;
 pub type PressureFrame = [[f32; 2]; PRESSURE_CELL_COUNT];
@@ -65,6 +68,9 @@ pub struct BackgroundRenderResources {
     background_pipeline: wgpu::RenderPipeline,
     glyph_pipeline: wgpu::RenderPipeline,
     dot_pipeline: wgpu::RenderPipeline,
+    model_pipeline: wgpu::RenderPipeline,
+    model: Option<Model>,
+    model_instance_buffer: wgpu::Buffer,
     glyph_vertex_buffer: wgpu::Buffer,
     glyph_instance_buffer: wgpu::Buffer,
     glyph_instances: Vec<GlyphInstance>,
@@ -74,6 +80,7 @@ pub struct BackgroundRenderResources {
 impl BackgroundRenderResources {
     pub fn new(
         device: &wgpu::Device,
+        _queue: &wgpu::Queue,
         target_format: &wgpu::TextureFormat,
         rows: u32,
         cols: u32,
@@ -125,18 +132,72 @@ impl BackgroundRenderResources {
         //     source: wgpu::ShaderSource::Wgsl(include_str!("../static/wgsl/shader.wgsl").into()),
         // });
 
+        let texture_bind_group_layout =
+            device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+                label: Some("model_texture_bind_group_layout"),
+                entries: &[
+                    wgpu::BindGroupLayoutEntry {
+                        binding: 0,
+                        visibility: wgpu::ShaderStages::FRAGMENT,
+                        ty: wgpu::BindingType::Texture {
+                            sample_type: wgpu::TextureSampleType::Float { filterable: true },
+                            view_dimension: wgpu::TextureViewDimension::D2,
+                            multisampled: false,
+                        },
+                        count: None,
+                    },
+                    wgpu::BindGroupLayoutEntry {
+                        binding: 1,
+                        visibility: wgpu::ShaderStages::FRAGMENT,
+                        ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
+                        count: None,
+                    },
+                ],
+            });
+
         let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
             label: Some("Pressure Matrix Pipeline Layout"),
             bind_group_layouts: &[Some(&uniform_bind_group_layout)],
             immediate_size: 0,
         });
+        let model_pipeline_layout =
+            device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
+                label: Some("Hand Model Pipeline Layout"),
+                bind_group_layouts: &[
+                    Some(&uniform_bind_group_layout),
+                    Some(&texture_bind_group_layout),
+                ],
+                immediate_size: 0,
+            });
 
         let background_pipeline =
             create_background_pipeline(device, target_format, &shader, &pipeline_layout);
         let glyph_pipeline =
             create_glyph_pipeline(device, target_format, &shader, &pipeline_layout);
-        let dot_pipeline =
-            create_dot_pipeline(device, target_format, &shader, &pipeline_layout);
+        let dot_pipeline = create_dot_pipeline(device, target_format, &shader, &pipeline_layout);
+
+        let model_pipeline =
+            create_model_pipeline(device, target_format, &shader, &model_pipeline_layout);
+
+        let model =
+            match resources::load_model("cube.obj", device, _queue, &texture_bind_group_layout) {
+                Ok(model) => Some(model),
+                Err(err) => {
+                    log::warn!("failed to load cube.obj: {err:#}");
+                    None
+                }
+            };
+
+        let model_instance = Instance::new(
+            glam::Vec3::new(0.0, -2.0, 12.0),
+            glam::Quat::from_rotation_y(0.65) * glam::Quat::from_rotation_x(-0.35),
+        )
+        .to_raw();
+        let model_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+            label: Some("Hand Model Instance Buffer"),
+            contents: bytemuck::cast_slice(&[model_instance]),
+            usage: wgpu::BufferUsages::VERTEX,
+        });
 
         let glyph_vertices = [
             GlyphVertex {
@@ -172,6 +233,9 @@ impl BackgroundRenderResources {
             background_pipeline,
             glyph_pipeline,
             dot_pipeline,
+            model_pipeline,
+            model,
+            model_instance_buffer,
             glyph_vertex_buffer,
             glyph_instance_buffer,
             glyph_instances,
@@ -217,6 +281,20 @@ impl BackgroundRenderResources {
         render_pass.set_pipeline(&self.background_pipeline);
         render_pass.draw(0..3, 0..1);
 
+        if let Some(model) = &self.model {
+            render_pass.set_pipeline(&self.model_pipeline);
+            render_pass.set_vertex_buffer(1, self.model_instance_buffer.slice(..));
+            for mesh in &model.meshes {
+                if let Some(material) = model.materials.get(mesh.material) {
+                    render_pass.set_bind_group(1, &material.bind_group, &[]);
+                }
+                render_pass.set_vertex_buffer(0, mesh.vertex_buffer.slice(..));
+                render_pass
+                    .set_index_buffer(mesh.index_buffer.slice(..), wgpu::IndexFormat::Uint32);
+                render_pass.draw_indexed(0..mesh.num_elements, 0, 0..1);
+            }
+        }
+
         // let marker_pipeline = match self.marker_mode {
         //     MarkerMode::Number => &self.glyph_pipeline,
         //     MarkerMode::Dot => &self.dot_pipeline,
@@ -296,6 +374,54 @@ fn create_glyph_pipeline(
     })
 }
 
+fn create_model_pipeline(
+    device: &wgpu::Device,
+    target_format: &wgpu::TextureFormat,
+    shader: &wgpu::ShaderModule,
+    layout: &wgpu::PipelineLayout,
+) -> wgpu::RenderPipeline {
+    device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
+        label: Some("Hand Model Pipeline"),
+        layout: Some(layout),
+        vertex: wgpu::VertexState {
+            module: shader,
+            entry_point: Some("vs_model"),
+            compilation_options: Default::default(),
+            buffers: &[ModelVertex::desc(), InstanceRaw::desc()],
+        },
+        fragment: Some(wgpu::FragmentState {
+            module: shader,
+            entry_point: Some("fs_model"),
+            compilation_options: Default::default(),
+            targets: &[Some(wgpu::ColorTargetState {
+                format: *target_format,
+                blend: Some(wgpu::BlendState {
+                    color: wgpu::BlendComponent::REPLACE,
+                    alpha: wgpu::BlendComponent::REPLACE,
+                }),
+                write_mask: wgpu::ColorWrites::ALL,
+            })],
+        }),
+        primitive: wgpu::PrimitiveState {
+            topology: wgpu::PrimitiveTopology::TriangleList,
+            strip_index_format: None,
+            front_face: wgpu::FrontFace::Ccw,
+            cull_mode: None,
+            polygon_mode: wgpu::PolygonMode::Fill,
+            unclipped_depth: false,
+            conservative: false,
+        },
+        depth_stencil: None,
+        multisample: wgpu::MultisampleState {
+            count: 1,
+            mask: !0,
+            alpha_to_coverage_enabled: false,
+        },
+        multiview_mask: None,
+        cache: None,
+    })
+}
+
 fn create_dot_pipeline(
     device: &wgpu::Device,
     target_format: &wgpu::TextureFormat,

src/resources.rs

@@ -0,0 +1,128 @@
+use crate::{model, texture};
+use eframe::wgpu;
+use eframe::wgpu::util::DeviceExt;
+use std::path::{Path, PathBuf};
+
+fn resource_path(file_name: &str) -> PathBuf {
+    Path::new(env!("RESOURCE_DIR")).join(file_name)
+}
+
+pub fn load_string(file_name: &str) -> anyhow::Result<String> {
+    Ok(std::fs::read_to_string(resource_path(file_name))?)
+}
+
+pub fn load_binary(file_name: &str) -> anyhow::Result<Vec<u8>> {
+    Ok(std::fs::read(resource_path(file_name))?)
+}
+
+pub fn load_texture(
+    file_name: &str,
+    device: &wgpu::Device,
+    queue: &wgpu::Queue,
+) -> anyhow::Result<texture::Texture> {
+    let data = load_binary(file_name)?;
+    texture::Texture::from_bytes(device, queue, &data, file_name)
+}
+
+pub fn load_model(
+    file_name: &str,
+    device: &wgpu::Device,
+    queue: &wgpu::Queue,
+    texture_bind_group_layout: &wgpu::BindGroupLayout,
+) -> anyhow::Result<model::Model> {
+    let path = resource_path(file_name);
+    let (models, obj_materials) = tobj::load_obj(
+        &path,
+        &tobj::LoadOptions {
+            single_index: true,
+            triangulate: true,
+            ..Default::default()
+        },
+    )?;
+
+    let mut materials = Vec::new();
+    for material in obj_materials? {
+        let diffuse_texture = match material.diffuse_texture.as_deref() {
+            Some(texture_path) => load_texture(texture_path, device, queue)?,
+            None => texture::Texture::from_rgba8(
+                device,
+                queue,
+                &[255, 255, 255, 255],
+                1,
+                1,
+                &material.name,
+            )?,
+        };
+
+        let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
+            label: Some(&format!("{} Texture Bind Group", material.name)),
+            layout: texture_bind_group_layout,
+            entries: &[
+                wgpu::BindGroupEntry {
+                    binding: 0,
+                    resource: wgpu::BindingResource::TextureView(&diffuse_texture.view),
+                },
+                wgpu::BindGroupEntry {
+                    binding: 1,
+                    resource: wgpu::BindingResource::Sampler(&diffuse_texture.sampler),
+                },
+            ],
+        });
+
+        materials.push(model::Material {
+            name: material.name,
+            diffuse_texture,
+            bind_group,
+        });
+    }
+
+    let meshes = models
+        .into_iter()
+        .map(|m| {
+            let vertices = (0..m.mesh.positions.len() / 3)
+                .map(|i| model::ModelVertex {
+                    position: [
+                        m.mesh.positions[i * 3],
+                        m.mesh.positions[i * 3 + 1],
+                        m.mesh.positions[i * 3 + 2],
+                    ],
+                    tex_coords: if m.mesh.texcoords.len() >= i * 2 + 2 {
+                        [m.mesh.texcoords[i * 2], m.mesh.texcoords[i * 2 + 1]]
+                    } else {
+                        [0.0, 0.0]
+                    },
+                    normal: if m.mesh.normals.len() >= i * 3 + 3 {
+                        [
+                            m.mesh.normals[i * 3],
+                            m.mesh.normals[i * 3 + 1],
+                            m.mesh.normals[i * 3 + 2],
+                        ]
+                    } else {
+                        [0.0, 1.0, 0.0]
+                    },
+                })
+                .collect::<Vec<_>>();
+
+            let vertex_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+                label: Some(&format!("{} Vertex Buffer", m.name)),
+                contents: bytemuck::cast_slice(&vertices),
+                usage: wgpu::BufferUsages::VERTEX,
+            });
+            let index_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
+                label: Some(&format!("{} Index Buffer", m.name)),
+                contents: bytemuck::cast_slice(&m.mesh.indices),
+                usage: wgpu::BufferUsages::INDEX,
+            });
+
+            model::Mesh {
+                name: m.name,
+                vertex_buffer,
+                index_buffer,
+                num_elements: m.mesh.indices.len() as u32,
+                material: m.mesh.material_id.unwrap_or(0),
+            }
+        })
+        .collect::<Vec<_>>();
+
+    Ok(model::Model { meshes, materials })
+}

src/texture.rs

@@ -1,8 +1,12 @@
-use anyhow::*;
-use eframe::egui_wgpu::{self, wgpu};
-use image::GenericImageView;
+#![allow(dead_code)]
 
-#[allow(dead_code)]
+use anyhow::*;
+use eframe::{
+    egui,
+    egui_wgpu::{self, wgpu},
+    wgpu::util::DeviceExt,
+};
+use image::GenericImageView;
 pub struct Texture {
     pub texture: wgpu::Texture,
     pub view: wgpu::TextureView,
@@ -10,6 +14,51 @@ pub struct Texture {
 }
 
 impl Texture {
+    pub const DEPTH_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Depth32Float;
+
+    pub fn create_depth_texture(
+        device: &wgpu::Device,
+        config: &wgpu::SurfaceConfiguration,
+        label: &str,
+    ) -> Self {
+        let size = wgpu::Extent3d {
+            width: config.width,
+            height: config.height,
+            depth_or_array_layers: 1,
+        };
+        let desc = wgpu::TextureDescriptor {
+            label: Some(label),
+            size,
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: Self::DEPTH_FORMAT,
+            usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING,
+            view_formats: &[],
+        };
+        let texture = device.create_texture(&desc);
+        let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
+        let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
+            address_mode_u: wgpu::AddressMode::ClampToEdge,
+            address_mode_v: wgpu::AddressMode::ClampToEdge,
+            address_mode_w: wgpu::AddressMode::ClampToEdge,
+            mag_filter: wgpu::FilterMode::Linear,
+            min_filter: wgpu::FilterMode::Linear,
+            mipmap_filter: wgpu::MipmapFilterMode::Nearest,
+            compare: Some(wgpu::CompareFunction::LessEqual),
+            lod_min_clamp: 0.0,
+            lod_max_clamp: 200.0,
+            ..Default::default()
+        });
+
+        Self {
+            texture,
+            view,
+            sampler,
+        }
+    }
+
+    #[allow(dead_code)]
     pub fn from_bytes(
         device: &wgpu::Device,
         queue: &wgpu::Queue,
@@ -26,8 +75,8 @@ impl Texture {
         img: &image::DynamicImage,
         label: Option<&str>,
     ) -> Result<Self> {
-        let rgba = img.to_rgba8();
         let dimensions = img.dimensions();
+        let rgba = img.to_rgba8();
 
         let size = wgpu::Extent3d {
             width: dimensions.0,
@@ -78,4 +127,62 @@ impl Texture {
             sampler,
         })
     }
+
+    pub fn from_rgba8(
+        device: &wgpu::Device,
+        queue: &wgpu::Queue,
+        rgba: &[u8],
+        width: u32,
+        height: u32,
+        label: &str,
+    ) -> Result<Self> {
+        let size = wgpu::Extent3d {
+            width,
+            height,
+            depth_or_array_layers: 1,
+        };
+        let texture = device.create_texture(&wgpu::TextureDescriptor {
+            label: Some(label),
+            size,
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: wgpu::TextureFormat::Rgba8UnormSrgb,
+            usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+            view_formats: &[],
+        });
+
+        queue.write_texture(
+            wgpu::TexelCopyTextureInfo {
+                aspect: wgpu::TextureAspect::All,
+                texture: &texture,
+                mip_level: 0,
+                origin: wgpu::Origin3d::ZERO,
+            },
+            rgba,
+            wgpu::TexelCopyBufferLayout {
+                offset: 0,
+                bytes_per_row: Some(4 * width),
+                rows_per_image: Some(height),
+            },
+            size,
+        );
+
+        let view = texture.create_view(&wgpu::TextureViewDescriptor::default());
+        let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
+            address_mode_u: wgpu::AddressMode::ClampToEdge,
+            address_mode_v: wgpu::AddressMode::ClampToEdge,
+            address_mode_w: wgpu::AddressMode::ClampToEdge,
+            mag_filter: wgpu::FilterMode::Linear,
+            min_filter: wgpu::FilterMode::Nearest,
+            mipmap_filter: wgpu::MipmapFilterMode::Nearest,
+            ..Default::default()
+        });
+
+        Ok(Self {
+            texture,
+            view,
+            sampler,
+        })
+    }
 }