use crate::{ matrix::{MatrixLayout, build_view_projection, glyph_world_position}, model::{AlphaMode, InstanceRaw, ModelVertex, Vertex}, resources, texture, }; use eframe::{ egui, egui_wgpu::{self, wgpu}, wgpu::util::DeviceExt, }; use std::ops::Range; pub const PRESSURE_CELL_COUNT: usize = (crate::matrix::MATRIX_ROWS * crate::matrix::MATRIX_COLS) as usize; pub type PressureFrame = [[f32; 2]; PRESSURE_CELL_COUNT]; pub type PressureSamples = Vec<[f32; 2]>; pub struct WgpuBackgroundCallback { pub width: f32, pub height: f32, pub pressure: PressureFrame, pub hand_pressure: PressureSamples, pub active_mode: ActiveMode, } #[derive(Clone, PartialEq, Eq, Debug)] pub enum ActiveMode { Finger(FingerMode), Hand(HandGatewayMode), } #[derive(Clone, PartialEq, Eq, Debug)] pub struct FingerMode { pub rows: u32, pub cols: u32, pub range: Range, pub dot: bool, } #[derive(Clone, PartialEq, Eq, Debug)] pub struct HandGatewayMode { pub range: Range, } const HAND_TIP_MATRICES: [HandTipMatrix; 5] = [ HandTipMatrix { center_px: [260.0, 490.0], size_px: [70.0, 120.0], angle_rad: -0.45, }, HandTipMatrix { center_px: [360.0, 255.0], size_px: [70.0, 120.0], angle_rad: -0.05, }, HandTipMatrix { center_px: [485.0, 225.0], size_px: [70.0, 120.0], angle_rad: 0.0, }, HandTipMatrix { center_px: [595.0, 260.0], size_px: [70.0, 120.0], angle_rad: 0.12, }, HandTipMatrix { center_px: [705.0, 385.0], size_px: [70.0, 120.0], angle_rad: 0.28, }, ]; const HAND_PALM_CHIPS: [HandPalmChip; 2] = [ HandPalmChip { center_px: [538.0, 608.0], size_px: [248.0, 82.0], angle_rad: 0.06, rows: 5, cols: 14, }, HandPalmChip { center_px: [606.0, 780.0], size_px: [72.0, 214.0], angle_rad: 0.05, rows: 11, cols: 4, }, ]; const HAND_FINGER_SENSOR_CELLS: usize = 12 * 7; const HAND_PALM_HORIZONTAL_OFFSET: usize = HAND_FINGER_SENSOR_CELLS * 5; const HAND_PALM_VERTICAL_OFFSET: usize = HAND_PALM_HORIZONTAL_OFFSET + 5 * 14; // Each entry pins one miniature matrix to a fingertip in hand.png. // Coordinates are authored in source-image pixels so they are easy to tune by eye. // size_px keeps the same 7:12 aspect as the Finger-mode 7 columns x 12 rows matrix. struct HandTipMatrix { center_px: [f32; 2], size_px: [f32; 2], angle_rad: f32, } // Palm chips follow the hand layout: one horizontal 5x14 matrix and one vertical // 11x4 matrix, rendered as dark inset chip tiles on the palm. struct HandPalmChip { center_px: [f32; 2], size_px: [f32; 2], angle_rad: f32, rows: u32, cols: u32, } impl egui_wgpu::CallbackTrait for WgpuBackgroundCallback { fn prepare( &self, _device: &wgpu::Device, queue: &wgpu::Queue, _screen_descriptor: &egui_wgpu::ScreenDescriptor, _egui_encoder: &mut wgpu::CommandEncoder, resources: &mut egui_wgpu::CallbackResources, ) -> Vec { let resources: &mut BackgroundRenderResources = resources.get_mut().unwrap(); resources.prepare( queue, self.width, self.height, &self.pressure, &self.hand_pressure, ); Vec::new() } fn paint( &self, _info: egui::PaintCallbackInfo, render_pass: &mut wgpu::RenderPass<'static>, resources: &egui_wgpu::CallbackResources, ) { let resources: &BackgroundRenderResources = resources.get().unwrap(); resources.paint(render_pass, &self.active_mode); } } pub struct BackgroundRenderResources { layout: MatrixLayout, rows: u32, cols: u32, surface_is_srgb: bool, uniform: MatrixUniform, uniform_buffer: wgpu::Buffer, uniform_bind_group: wgpu::BindGroup, background_pipeline: wgpu::RenderPipeline, hand_image_pipeline: wgpu::RenderPipeline, glyph_pipeline: wgpu::RenderPipeline, dot_pipeline: wgpu::RenderPipeline, hand_membrane_pipeline: wgpu::RenderPipeline, hand_dot_pipeline: wgpu::RenderPipeline, hand_palm_chip_pipeline: wgpu::RenderPipeline, hand_palm_dot_pipeline: wgpu::RenderPipeline, hand_image_bind_group: wgpu::BindGroup, hand_image_texture: texture::Texture, glyph_vertex_buffer: wgpu::Buffer, glyph_instance_buffer: wgpu::Buffer, glyph_instances: Vec, hand_membrane_instance_buffer: wgpu::Buffer, hand_membrane_instances: Vec, hand_dot_instance_buffer: wgpu::Buffer, hand_dot_instances: Vec, hand_palm_chip_instance_buffer: wgpu::Buffer, hand_palm_chip_instances: Vec, hand_palm_dot_instance_buffer: wgpu::Buffer, hand_palm_dot_instances: Vec, render_options: RenderOptions, } struct ModelPipelines { opaque: ModelCullPipelines, mask: ModelCullPipelines, blend: ModelCullPipelines, } struct ModelCullPipelines { single_sided: wgpu::RenderPipeline, double_sided: wgpu::RenderPipeline, } #[derive(Copy, Clone, Debug)] struct RenderOptions { debug_mode: u32, ambient_enabled: bool, tone_mapping_enabled: bool, exposure: f32, } impl RenderOptions { fn from_env() -> Self { let debug_mode = std::env::var("ESKIN_MATERIAL_DEBUG") .ok() .and_then(|value| value.parse().ok()) .unwrap_or(0); let ambient_enabled = env_bool("ESKIN_AMBIENT", true); let tone_mapping_enabled = env_bool("ESKIN_TONEMAP", true); let exposure = std::env::var("ESKIN_EXPOSURE") .ok() .and_then(|value| value.parse().ok()) .unwrap_or(1.0); log::warn!( "render options material_debug={debug_mode} ambient_enabled={ambient_enabled} tone_mapping_enabled={tone_mapping_enabled} exposure={exposure:.3}" ); Self { debug_mode, ambient_enabled, tone_mapping_enabled, exposure, } } fn as_uniform(self) -> [f32; 4] { [ self.debug_mode as f32, if self.ambient_enabled { 1.0 } else { 0.0 }, if self.tone_mapping_enabled { 1.0 } else { 0.0 }, self.exposure, ] } } fn env_bool(name: &str, default: bool) -> bool { std::env::var(name) .ok() .map(|value| { let value = value.to_ascii_lowercase(); matches!(value.as_str(), "1" | "true" | "yes" | "on") }) .unwrap_or(default) } impl BackgroundRenderResources { pub fn new( device: &wgpu::Device, _queue: &wgpu::Queue, target_format: &wgpu::TextureFormat, rows: u32, cols: u32, ) -> Self { let layout = MatrixLayout::new(rows, cols); let surface_is_srgb = target_format.is_srgb(); let render_options = RenderOptions::from_env(); log::warn!("wgpu target surface format: {target_format:?}, srgb={surface_is_srgb}"); let hand_image_texture = match resources::load_texture("hand.png", device, _queue) { Ok(texture) => texture, Err(err) => { log::warn!("failed to load hand.png background: {err:#}"); texture::Texture::from_rgba8( device, _queue, &[0, 0, 0, 255], 1, 1, "Fallback Hand Background Texture", ) .expect("fallback hand texture should be valid") } }; log::warn!( "using hand.png as render background: {}x{}", hand_image_texture.width, hand_image_texture.height ); let uniform = MatrixUniform::new( 1.0, 1.0, build_view_projection(1.0, &layout), surface_is_srgb, render_options, [ hand_image_texture.width as f32, hand_image_texture.height as f32, ], ); let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Pressure Matrix Uniform Buffer"), contents: bytemuck::cast_slice(&[uniform]), usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, }); let uniform_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("pressure_matrix_bind_group_layout"), entries: &[wgpu::BindGroupLayoutEntry { binding: 0, visibility: wgpu::ShaderStages::VERTEX_FRAGMENT, ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None, }, count: None, }], }); let uniform_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("pressure_matrix_bind_group"), layout: &uniform_bind_group_layout, entries: &[wgpu::BindGroupEntry { binding: 0, resource: uniform_buffer.as_entire_binding(), }], }); let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("Pressure Matrix Background Shader"), source: wgpu::ShaderSource::Wgsl(include_str!("../static/wgsl/shader.wgsl").into()), }); // let glyph_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { // label: Some("Pressure Matrix Glyph Shader"), // source: wgpu::ShaderSource::Wgsl(include_str!("../static/wgsl/shader.wgsl").into()), // }); // let dot_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { // label: Some("Pressure Matrix Dot Shader"), // source: wgpu::ShaderSource::Wgsl(include_str!("../static/wgsl/shader.wgsl").into()), // }); let hand_image_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("hand_image_bind_group_layout"), entries: &[ wgpu::BindGroupLayoutEntry { binding: 0, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 1, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, ], }); let hand_image_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("hand_image_bind_group"), layout: &hand_image_bind_group_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(&hand_image_texture.view), }, wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&hand_image_texture.sampler), }, ], }); let texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("model_texture_bind_group_layout"), entries: &[ wgpu::BindGroupLayoutEntry { binding: 0, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 1, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, wgpu::BindGroupLayoutEntry { binding: 2, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 3, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 4, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, wgpu::BindGroupLayoutEntry { binding: 5, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 6, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, wgpu::BindGroupLayoutEntry { binding: 7, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 8, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, wgpu::BindGroupLayoutEntry { binding: 9, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { sample_type: wgpu::TextureSampleType::Float { filterable: true }, view_dimension: wgpu::TextureViewDimension::D2, multisampled: false, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 10, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, ], }); let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("Pressure Matrix Pipeline Layout"), bind_group_layouts: &[Some(&uniform_bind_group_layout)], immediate_size: 0, }); let hand_image_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("Hand Image Pipeline Layout"), bind_group_layouts: &[ Some(&uniform_bind_group_layout), Some(&hand_image_bind_group_layout), ], immediate_size: 0, }); let model_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("Hand Model Pipeline Layout"), bind_group_layouts: &[ Some(&uniform_bind_group_layout), Some(&texture_bind_group_layout), ], immediate_size: 0, }); let background_pipeline = create_background_pipeline(device, target_format, &shader, &pipeline_layout); let hand_image_pipeline = create_hand_image_pipeline(device, target_format, &shader, &hand_image_pipeline_layout); let glyph_pipeline = create_glyph_pipeline(device, target_format, &shader, &pipeline_layout); let dot_pipeline = create_dot_pipeline(device, target_format, &shader, &pipeline_layout); let hand_membrane_pipeline = create_hand_membrane_pipeline(device, target_format, &shader, &pipeline_layout); let hand_dot_pipeline = create_hand_dot_pipeline(device, target_format, &shader, &pipeline_layout); let hand_palm_chip_pipeline = create_hand_palm_chip_pipeline(device, target_format, &shader, &pipeline_layout); let hand_palm_dot_pipeline = create_hand_palm_dot_pipeline(device, target_format, &shader, &pipeline_layout); let _model_pipelines = create_model_pipelines(device, target_format, &shader, &model_pipeline_layout); let glyph_vertices = [ GlyphVertex { local: [-1.0, -1.0], }, GlyphVertex { local: [1.0, -1.0] }, GlyphVertex { local: [-1.0, 1.0] }, GlyphVertex { local: [-1.0, 1.0] }, GlyphVertex { local: [1.0, -1.0] }, GlyphVertex { local: [1.0, 1.0] }, ]; let glyph_vertex_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Pressure Glyph Vertex Buffer"), contents: bytemuck::cast_slice(&glyph_vertices), usage: wgpu::BufferUsages::VERTEX, }); let glyph_instances = build_glyph_instances(rows, cols, &layout, &[[0.0, 0.0]; PRESSURE_CELL_COUNT]); let glyph_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Pressure Glyph Instance Buffer"), contents: bytemuck::cast_slice(&glyph_instances), usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, }); let hand_membrane_instances = build_hand_membrane_instances( hand_image_texture.width as f32, hand_image_texture.height as f32, ); let hand_membrane_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Hand Fingertip Membrane Instance Buffer"), contents: bytemuck::cast_slice(&hand_membrane_instances), usage: wgpu::BufferUsages::VERTEX, }); let hand_dot_instances = build_hand_dot_instances( rows, cols, hand_image_texture.width as f32, hand_image_texture.height as f32, &[[0.0, 0.0]; PRESSURE_CELL_COUNT], ); let hand_dot_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Hand Fingertip Dot Matrix Instance Buffer"), contents: bytemuck::cast_slice(&hand_dot_instances), usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, }); let hand_palm_chip_instances = build_hand_palm_chip_instances( hand_image_texture.width as f32, hand_image_texture.height as f32, ); let hand_palm_chip_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Hand Palm Chip Instance Buffer"), contents: bytemuck::cast_slice(&hand_palm_chip_instances), usage: wgpu::BufferUsages::VERTEX, }); let hand_palm_dot_instances = build_hand_palm_dot_instances( rows, cols, hand_image_texture.width as f32, hand_image_texture.height as f32, &[[0.0, 0.0]; PRESSURE_CELL_COUNT], ); let hand_palm_dot_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Hand Palm Chip Dot Instance Buffer"), contents: bytemuck::cast_slice(&hand_palm_dot_instances), usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, }); Self { layout, rows, cols, surface_is_srgb, uniform, uniform_buffer, uniform_bind_group, background_pipeline, hand_image_pipeline, glyph_pipeline, dot_pipeline, hand_membrane_pipeline, hand_dot_pipeline, hand_palm_chip_pipeline, hand_palm_dot_pipeline, hand_image_bind_group, hand_image_texture, glyph_vertex_buffer, glyph_instance_buffer, glyph_instances, hand_membrane_instance_buffer, hand_membrane_instances, hand_dot_instance_buffer, hand_dot_instances, hand_palm_chip_instance_buffer, hand_palm_chip_instances, hand_palm_dot_instance_buffer, hand_palm_dot_instances, render_options, } } fn prepare( &mut self, queue: &wgpu::Queue, width: f32, height: f32, pressure: &PressureFrame, hand_pressure: &[[f32; 2]], ) { let aspect = width / height.max(1.0); self.uniform = MatrixUniform::new( width, height, build_view_projection(aspect, &self.layout), self.surface_is_srgb, self.render_options, [ self.hand_image_texture.width as f32, self.hand_image_texture.height as f32, ], ); queue.write_buffer( &self.uniform_buffer, 0, bytemuck::cast_slice(&[self.uniform]), ); update_glyph_instances( &mut self.glyph_instances, self.rows, self.cols, &self.layout, pressure, ); queue.write_buffer( &self.glyph_instance_buffer, 0, bytemuck::cast_slice(&self.glyph_instances), ); let hand_pressure = if hand_pressure.is_empty() { pressure.as_slice() } else { hand_pressure }; // Hand mode uses UV-anchored fingertip matrices over hand.png. // Rebuild their instance positions here so pressure colors update every frame. self.hand_dot_instances = build_hand_dot_instances( self.rows, self.cols, self.hand_image_texture.width as f32, self.hand_image_texture.height as f32, hand_pressure, ); queue.write_buffer( &self.hand_dot_instance_buffer, 0, bytemuck::cast_slice(&self.hand_dot_instances), ); // Palm chips reuse the same live 12x7 pressure frame, but draw it as // embedded micro-pixels inside dark chip tiles. self.hand_palm_dot_instances = build_hand_palm_dot_instances( self.rows, self.cols, self.hand_image_texture.width as f32, self.hand_image_texture.height as f32, hand_pressure, ); queue.write_buffer( &self.hand_palm_dot_instance_buffer, 0, bytemuck::cast_slice(&self.hand_palm_dot_instances), ); } fn paint(&self, render_pass: &mut wgpu::RenderPass<'_>, active_mode: &ActiveMode) { render_pass.set_bind_group(0, &self.uniform_bind_group, &[]); render_pass.set_pipeline(&self.background_pipeline); render_pass.draw(0..3, 0..1); match active_mode { ActiveMode::Finger(mode) => self.paint_finger(render_pass, mode), ActiveMode::Hand(mode) => { render_pass.set_pipeline(&self.hand_image_pipeline); render_pass.set_bind_group(1, &self.hand_image_bind_group, &[]); render_pass.draw(0..6, 0..1); self.paint_hand(render_pass, mode); } } } fn paint_finger(&self, render_pass: &mut wgpu::RenderPass<'_>, mode: &FingerMode) { let _range = mode.range.clone(); let marker_pipeline = if mode.dot { &self.dot_pipeline } else { &self.glyph_pipeline }; let draw_count = self.visible_instance_count(mode.rows, mode.cols); render_pass.set_pipeline(marker_pipeline); render_pass.set_vertex_buffer(0, self.glyph_vertex_buffer.slice(..)); render_pass.set_vertex_buffer(1, self.glyph_instance_buffer.slice(..)); render_pass.draw(0..6, 0..draw_count); } fn paint_hand(&self, render_pass: &mut wgpu::RenderPass<'_>, mode: &HandGatewayMode) { let _range = mode.range.clone(); // First draw the translucent sensor membranes, then draw live pressure beads on their grid. render_pass.set_pipeline(&self.hand_membrane_pipeline); render_pass.set_vertex_buffer(0, self.glyph_vertex_buffer.slice(..)); render_pass.set_vertex_buffer(1, self.hand_membrane_instance_buffer.slice(..)); render_pass.draw(0..6, 0..self.hand_membrane_instances.len() as u32); render_pass.set_pipeline(&self.hand_dot_pipeline); render_pass.set_vertex_buffer(0, self.glyph_vertex_buffer.slice(..)); render_pass.set_vertex_buffer(1, self.hand_dot_instance_buffer.slice(..)); render_pass.draw(0..6, 0..self.hand_dot_instances.len() as u32); render_pass.set_pipeline(&self.hand_palm_chip_pipeline); render_pass.set_vertex_buffer(0, self.glyph_vertex_buffer.slice(..)); render_pass.set_vertex_buffer(1, self.hand_palm_chip_instance_buffer.slice(..)); render_pass.draw(0..6, 0..self.hand_palm_chip_instances.len() as u32); render_pass.set_pipeline(&self.hand_palm_dot_pipeline); render_pass.set_vertex_buffer(0, self.glyph_vertex_buffer.slice(..)); render_pass.set_vertex_buffer(1, self.hand_palm_dot_instance_buffer.slice(..)); render_pass.draw(0..6, 0..self.hand_palm_dot_instances.len() as u32); } fn visible_instance_count(&self, rows: u32, cols: u32) -> u32 { let requested = rows.saturating_mul(cols); requested.min(self.glyph_instances.len() as u32) } } fn create_background_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Pressure Matrix Background Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_background"), compilation_options: Default::default(), buffers: &[], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_background"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::REPLACE), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn create_hand_image_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Hand Image Background Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_hand_image"), compilation_options: Default::default(), buffers: &[], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_hand_image"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::ALPHA_BLENDING), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn create_glyph_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Pressure Matrix Glyph Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_glyph"), compilation_options: Default::default(), buffers: &[GlyphVertex::desc(), GlyphInstance::desc()], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_glyph"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::ALPHA_BLENDING), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn create_model_pipelines( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> ModelPipelines { ModelPipelines { opaque: ModelCullPipelines { single_sided: create_model_pipeline( device, target_format, shader, layout, AlphaMode::Opaque, false, ), double_sided: create_model_pipeline( device, target_format, shader, layout, AlphaMode::Opaque, true, ), }, mask: ModelCullPipelines { single_sided: create_model_pipeline( device, target_format, shader, layout, AlphaMode::Mask, false, ), double_sided: create_model_pipeline( device, target_format, shader, layout, AlphaMode::Mask, true, ), }, blend: ModelCullPipelines { single_sided: create_model_pipeline( device, target_format, shader, layout, AlphaMode::Blend, false, ), double_sided: create_model_pipeline( device, target_format, shader, layout, AlphaMode::Blend, true, ), }, } } fn create_model_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, alpha_mode: AlphaMode, double_sided: bool, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some(&format!( "Hand Model {} {} Pipeline", alpha_mode.as_str(), if double_sided { "Double Sided" } else { "Single Sided" } )), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_model"), compilation_options: Default::default(), buffers: &[ModelVertex::desc(), InstanceRaw::desc()], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_model"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: if alpha_mode == AlphaMode::Blend { Some(wgpu::BlendState::ALPHA_BLENDING) } else { None }, write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState { topology: wgpu::PrimitiveTopology::TriangleList, strip_index_format: None, front_face: wgpu::FrontFace::Ccw, cull_mode: if double_sided { None } else { Some(wgpu::Face::Back) }, polygon_mode: wgpu::PolygonMode::Fill, unclipped_depth: false, conservative: false, }, depth_stencil: None, multisample: wgpu::MultisampleState { count: 1, mask: !0, alpha_to_coverage_enabled: false, }, multiview_mask: None, cache: None, }) } fn create_dot_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Pressure Matrix Dot Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_dot"), compilation_options: Default::default(), buffers: &[GlyphVertex::desc(), GlyphInstance::desc()], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_dot"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::ALPHA_BLENDING), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn create_hand_membrane_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Hand Fingertip Sensor Membrane Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_hand_membrane"), compilation_options: Default::default(), buffers: &[GlyphVertex::desc(), GlyphInstance::desc()], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_hand_membrane"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::ALPHA_BLENDING), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn create_hand_dot_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Hand Fingertip Dot Matrix Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_hand_dot"), compilation_options: Default::default(), buffers: &[GlyphVertex::desc(), GlyphInstance::desc()], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_hand_dot"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::ALPHA_BLENDING), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn create_hand_palm_chip_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Hand Palm Embedded Chip Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_hand_palm_chip"), compilation_options: Default::default(), buffers: &[GlyphVertex::desc(), GlyphInstance::desc()], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_hand_palm_chip"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::ALPHA_BLENDING), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn create_hand_palm_dot_pipeline( device: &wgpu::Device, target_format: &wgpu::TextureFormat, shader: &wgpu::ShaderModule, layout: &wgpu::PipelineLayout, ) -> wgpu::RenderPipeline { device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Hand Palm Embedded Chip Dot Pipeline"), layout: Some(layout), vertex: wgpu::VertexState { module: shader, entry_point: Some("vs_hand_palm_dot"), compilation_options: Default::default(), buffers: &[GlyphVertex::desc(), GlyphInstance::desc()], }, fragment: Some(wgpu::FragmentState { module: shader, entry_point: Some("fs_hand_palm_dot"), compilation_options: Default::default(), targets: &[Some(wgpu::ColorTargetState { format: *target_format, blend: Some(wgpu::BlendState::ALPHA_BLENDING), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState::default(), multiview_mask: None, cache: None, }) } fn build_hand_membrane_instances(image_width: f32, image_height: f32) -> Vec { HAND_TIP_MATRICES .iter() .map(|tip| { let uv_x = tip.center_px[0] / image_width.max(1.0); let uv_y = tip.center_px[1] / image_height.max(1.0); let membrane_size = [tip.size_px[0] * 1.62, tip.size_px[1] * 1.56]; GlyphInstance { // Membrane shaders read xy as hand.png UV. world_position: [uv_x, uv_y, 0.0, 1.0], // Store angle and an oversized source-image pixel size for the floating sensor film. style: [tip.angle_rad, membrane_size[0], membrane_size[1], 0.0], } }) .collect() } fn build_hand_palm_chip_instances(image_width: f32, image_height: f32) -> Vec { HAND_PALM_CHIPS .iter() .map(|chip| { let uv_x = chip.center_px[0] / image_width.max(1.0); let uv_y = chip.center_px[1] / image_height.max(1.0); GlyphInstance { // Palm chip shaders read xy as hand.png UV. world_position: [uv_x, uv_y, 0.0, 1.0], // Store chip angle, source-image pixel size, and matrix shape for the shader grid. style: [ chip.angle_rad, chip.size_px[0], chip.size_px[1], (chip.rows * 100 + chip.cols) as f32, ], } }) .collect() } fn build_hand_dot_instances( rows: u32, cols: u32, image_width: f32, image_height: f32, pressure: &[[f32; 2]], ) -> Vec { let mut instances = Vec::with_capacity(HAND_TIP_MATRICES.len() * rows as usize * cols as usize); for (tip_index, tip) in HAND_TIP_MATRICES.into_iter().enumerate() { let cos = tip.angle_rad.cos(); let sin = tip.angle_rad.sin(); for row in 0..rows { for col in 0..cols { let index = (row * cols + col) as usize; let [normalized, display_value] = sample_pressure_at( pressure, tip_index * HAND_FINGER_SENSOR_CELLS + index, index, ); // Lay out a rows x cols matrix in fingertip-local pixel space. let local_x = (col as f32 - cols as f32 / 2.0 + 0.5) / cols as f32 * tip.size_px[0]; let local_y = (row as f32 - rows as f32 / 2.0 + 0.5) / rows as f32 * tip.size_px[1]; // Rotate the local matrix so it follows the direction of the finger. let x = tip.center_px[0] + local_x * cos - local_y * sin; let y = tip.center_px[1] + local_x * sin + local_y * cos; let uv_x = x / image_width.max(1.0); let uv_y = y / image_height.max(1.0); instances.push(GlyphInstance { world_position: [uv_x, uv_y, 0.0, 1.0], style: [normalized, display_value, 0.0, 0.0], }); } } } instances } fn build_hand_palm_dot_instances( _rows: u32, _cols: u32, image_width: f32, image_height: f32, pressure: &[[f32; 2]], ) -> Vec { let chip_dot_count: usize = HAND_PALM_CHIPS .iter() .map(|chip| (chip.rows * chip.cols) as usize) .sum(); let mut instances = Vec::with_capacity(chip_dot_count); for (chip_index, chip) in HAND_PALM_CHIPS.into_iter().enumerate() { let cos = chip.angle_rad.cos(); let sin = chip.angle_rad.sin(); // Leave a bevel around the chip so the matrix reads as embedded pixels. let active_size = [chip.size_px[0] * 0.72, chip.size_px[1] * 0.76]; for row in 0..chip.rows { for col in 0..chip.cols { let index = (row * chip.cols + col) as usize; let offset = match chip_index { 0 => HAND_PALM_HORIZONTAL_OFFSET, _ => HAND_PALM_VERTICAL_OFFSET, }; let [normalized, display_value] = sample_pressure_at(pressure, offset + index, index); let local_x = (col as f32 - chip.cols as f32 / 2.0 + 0.5) / chip.cols as f32 * active_size[0]; let local_y = (row as f32 - chip.rows as f32 / 2.0 + 0.5) / chip.rows as f32 * active_size[1]; let x = chip.center_px[0] + local_x * cos - local_y * sin; let y = chip.center_px[1] + local_x * sin + local_y * cos; instances.push(GlyphInstance { world_position: [ x / image_width.max(1.0), y / image_height.max(1.0), 0.0, 1.0, ], style: [normalized, display_value, 0.0, 0.0], }); } } } instances } fn sample_pressure_at(pressure: &[[f32; 2]], index: usize, fallback_index: usize) -> [f32; 2] { pressure .get(index) .or_else(|| pressure.get(fallback_index)) .copied() .unwrap_or([0.0, 0.0]) } fn build_glyph_instances( rows: u32, cols: u32, layout: &MatrixLayout, pressure: &PressureFrame, ) -> Vec { let mut instances = Vec::with_capacity((rows * cols) as usize); for row in 0..rows { for col in 0..cols { let index = (row * cols + col) as usize; let [normalized, display_value] = pressure.get(index).copied().unwrap_or([0.0, 0.0]); let (world_position, normalized) = glyph_world_position(row, col, rows, cols, layout, normalized); instances.push(GlyphInstance { world_position, style: [normalized, display_value, 0.0, 0.0], }); } } instances } fn update_glyph_instances( instances: &mut [GlyphInstance], rows: u32, cols: u32, layout: &MatrixLayout, pressure: &PressureFrame, ) { for row in 0..rows { for col in 0..cols { let index = (row * cols + col) as usize; let [normalized, display_value] = pressure.get(index).copied().unwrap_or([0.0, 0.0]); let (world_position, normalized) = glyph_world_position(row, col, rows, cols, layout, normalized); if let Some(instance) = instances.get_mut(index) { instance.world_position = world_position; instance.style = [normalized, display_value, 0.0, 0.0]; } } } } #[repr(C)] #[derive(Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] struct MatrixUniform { view_proj: [[f32; 4]; 4], viewport: [f32; 4], glyph: [f32; 4], color: [f32; 4], render_options: [f32; 4], image: [f32; 4], } impl MatrixUniform { fn new( width: f32, height: f32, view_proj: [[f32; 4]; 4], surface_is_srgb: bool, render_options: RenderOptions, image_size: [f32; 2], ) -> Self { Self { view_proj, viewport: [width.max(1.0), height.max(1.0), 0.0, 0.0], glyph: [16.0, 0.0, 0.0, 0.0], color: [0.05, 0.92, 0.32, if surface_is_srgb { 1.0 } else { 0.0 }], render_options: render_options.as_uniform(), image: [image_size[0].max(1.0), image_size[1].max(1.0), 0.0, 0.0], } } } #[repr(C)] #[derive(Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] struct GlyphVertex { local: [f32; 2], } impl GlyphVertex { fn desc<'a>() -> wgpu::VertexBufferLayout<'a> { wgpu::VertexBufferLayout { array_stride: std::mem::size_of::() as wgpu::BufferAddress, step_mode: wgpu::VertexStepMode::Vertex, attributes: &[wgpu::VertexAttribute { offset: 0, shader_location: 0, format: wgpu::VertexFormat::Float32x2, }], } } } #[repr(C)] #[derive(Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)] struct GlyphInstance { world_position: [f32; 4], style: [f32; 4], } impl GlyphInstance { fn desc<'a>() -> wgpu::VertexBufferLayout<'a> { wgpu::VertexBufferLayout { array_stride: std::mem::size_of::() as wgpu::BufferAddress, step_mode: wgpu::VertexStepMode::Instance, attributes: &[ wgpu::VertexAttribute { offset: 0, shader_location: 1, format: wgpu::VertexFormat::Float32x4, }, wgpu::VertexAttribute { offset: std::mem::size_of::<[f32; 4]>() as wgpu::BufferAddress, shader_location: 2, format: wgpu::VertexFormat::Float32x4, }, ], } } }