yanjie/eskin-player
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use hand texture as render background

Browse Source
This commit is contained in:
lenn
2026-06-26 16:59:58 +08:00
parent 0b658d11c5
commit b3fa3c8341
13 changed files with 1962 additions and 1375180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

441
src/render.rs
View File

@@ -1,7 +1,7 @@
use crate::{
matrix::{MatrixLayout, build_view_projection, glyph_world_position},
model::{Instance, InstanceRaw, Model, ModelVertex, Vertex},
resources,
model::{AlphaMode, InstanceRaw, ModelVertex, Vertex},
resources, texture,
};
use eframe::{
egui,
@@ -69,20 +69,86 @@ pub struct BackgroundRenderResources {
layout: MatrixLayout,
rows: u32,
cols: u32,
surface_is_srgb: bool,
uniform: MatrixUniform,
uniform_buffer: wgpu::Buffer,
uniform_bind_group: wgpu::BindGroup,
background_pipeline: wgpu::RenderPipeline,
hand_image_pipeline: wgpu::RenderPipeline,
glyph_pipeline: wgpu::RenderPipeline,
dot_pipeline: wgpu::RenderPipeline,
model_pipeline: wgpu::RenderPipeline,
hand_image_bind_group: wgpu::BindGroup,
hand_image_texture: texture::Texture,
model: Option<Model>,
model_instance_buffer: wgpu::Buffer,
glyph_vertex_buffer: wgpu::Buffer,
glyph_instance_buffer: wgpu::Buffer,
glyph_instances: Vec<GlyphInstance>,
render_options: RenderOptions,
}
struct ModelPipelines {
opaque: ModelCullPipelines,
mask: ModelCullPipelines,
blend: ModelCullPipelines,
}
struct ModelCullPipelines {
single_sided: wgpu::RenderPipeline,
double_sided: wgpu::RenderPipeline,
}
#[derive(Copy, Clone, Debug)]
struct RenderOptions {
debug_mode: u32,
ambient_enabled: bool,
tone_mapping_enabled: bool,
exposure: f32,
}
impl RenderOptions {
fn from_env() -> Self {
let debug_mode = std::env::var("ESKIN_MATERIAL_DEBUG")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(0);
let ambient_enabled = env_bool("ESKIN_AMBIENT", true);
let tone_mapping_enabled = env_bool("ESKIN_TONEMAP", true);
let exposure = std::env::var("ESKIN_EXPOSURE")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(1.0);
log::warn!(
"render options material_debug={debug_mode} ambient_enabled={ambient_enabled} tone_mapping_enabled={tone_mapping_enabled} exposure={exposure:.3}"
);
Self {
debug_mode,
ambient_enabled,
tone_mapping_enabled,
exposure,
}
}
fn as_uniform(self) -> [f32; 4] {
[
self.debug_mode as f32,
if self.ambient_enabled { 1.0 } else { 0.0 },
if self.tone_mapping_enabled { 1.0 } else { 0.0 },
self.exposure,
]
}
}
fn env_bool(name: &str, default: bool) -> bool {
std::env::var(name)
.ok()
.map(|value| {
let value = value.to_ascii_lowercase();
matches!(value.as_str(), "1" | "true" | "yes" | "on")
})
.unwrap_or(default)
}
impl BackgroundRenderResources {
@@ -94,7 +160,40 @@ impl BackgroundRenderResources {
cols: u32,
) -> Self {
let layout = MatrixLayout::new(rows, cols);
let uniform = MatrixUniform::new(1.0, 1.0, build_view_projection(1.0, &layout));
let surface_is_srgb = target_format.is_srgb();
let render_options = RenderOptions::from_env();
log::warn!("wgpu target surface format: {target_format:?}, srgb={surface_is_srgb}");
let hand_image_texture = match resources::load_texture("hand.png", device, _queue) {
Ok(texture) => texture,
Err(err) => {
log::warn!("failed to load hand.png background: {err:#}");
texture::Texture::from_rgba8(
device,
_queue,
&[0, 0, 0, 255],
1,
1,
"Fallback Hand Background Texture",
)
.expect("fallback hand texture should be valid")
}
};
log::warn!(
"using hand.png as render background: {}x{}",
hand_image_texture.width,
hand_image_texture.height
);
let uniform = MatrixUniform::new(
1.0,
1.0,
build_view_projection(1.0, &layout),
surface_is_srgb,
render_options,
[
hand_image_texture.width as f32,
hand_image_texture.height as f32,
],
);
let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("Pressure Matrix Uniform Buffer"),
contents: bytemuck::cast_slice(&[uniform]),
@@ -140,9 +239,9 @@ impl BackgroundRenderResources {
// source: wgpu::ShaderSource::Wgsl(include_str!("../static/wgsl/shader.wgsl").into()),
// });
let texture_bind_group_layout =
let hand_image_bind_group_layout =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("model_texture_bind_group_layout"),
label: Some("hand_image_bind_group_layout"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
@@ -163,11 +262,132 @@ impl BackgroundRenderResources {
],
});
let hand_image_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("hand_image_bind_group"),
layout: &hand_image_bind_group_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::TextureView(&hand_image_texture.view),
},
wgpu::BindGroupEntry {
binding: 1,
resource: wgpu::BindingResource::Sampler(&hand_image_texture.sampler),
},
],
});
let texture_bind_group_layout =
device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("model_texture_bind_group_layout"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 3,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 4,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 5,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 6,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 7,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 8,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 9,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
sample_type: wgpu::TextureSampleType::Float { filterable: true },
view_dimension: wgpu::TextureViewDimension::D2,
multisampled: false,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 10,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
],
});
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Pressure Matrix Pipeline Layout"),
bind_group_layouts: &[Some(&uniform_bind_group_layout)],
immediate_size: 0,
});
let hand_image_pipeline_layout =
device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Hand Image Pipeline Layout"),
bind_group_layouts: &[
Some(&uniform_bind_group_layout),
Some(&hand_image_bind_group_layout),
],
immediate_size: 0,
});
let model_pipeline_layout =
device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("Hand Model Pipeline Layout"),
@@ -180,32 +400,14 @@ impl BackgroundRenderResources {
let background_pipeline =
create_background_pipeline(device, target_format, &shader, &pipeline_layout);
let hand_image_pipeline =
create_hand_image_pipeline(device, target_format, &shader, &hand_image_pipeline_layout);
let glyph_pipeline =
create_glyph_pipeline(device, target_format, &shader, &pipeline_layout);
let dot_pipeline = create_dot_pipeline(device, target_format, &shader, &pipeline_layout);
let model_pipeline =
create_model_pipeline(device, target_format, &shader, &model_pipeline_layout);
let model =
match resources::load_model("hand.obj", device, _queue, &texture_bind_group_layout) {
Ok(model) => Some(model),
Err(err) => {
log::warn!("failed to load hand.obj: {err:#}");
None
}
};
let model_instance = Instance::new(
glam::Vec3::new(0.0, -2.0, 12.0),
glam::Quat::from_rotation_y(0.65) * glam::Quat::from_rotation_x(-0.35),
)
.to_raw();
let model_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("Hand Model Instance Buffer"),
contents: bytemuck::cast_slice(&[model_instance]),
usage: wgpu::BufferUsages::VERTEX,
});
let _model_pipelines =
create_model_pipelines(device, target_format, &shader, &model_pipeline_layout);
let glyph_vertices = [
GlyphVertex {
@@ -235,25 +437,36 @@ impl BackgroundRenderResources {
layout,
rows,
cols,
surface_is_srgb,
uniform,
uniform_buffer,
uniform_bind_group,
background_pipeline,
hand_image_pipeline,
glyph_pipeline,
dot_pipeline,
model_pipeline,
model,
model_instance_buffer,
hand_image_bind_group,
hand_image_texture,
glyph_vertex_buffer,
glyph_instance_buffer,
glyph_instances,
render_options,
}
}
fn prepare(&mut self, queue: &wgpu::Queue, width: f32, height: f32, pressure: &PressureFrame) {
let aspect = width / height.max(1.0);
self.uniform =
MatrixUniform::new(width, height, build_view_projection(aspect, &self.layout));
self.uniform = MatrixUniform::new(
width,
height,
build_view_projection(aspect, &self.layout),
self.surface_is_srgb,
self.render_options,
[
self.hand_image_texture.width as f32,
self.hand_image_texture.height as f32,
],
);
queue.write_buffer(
&self.uniform_buffer,
0,
@@ -280,6 +493,10 @@ impl BackgroundRenderResources {
render_pass.set_pipeline(&self.background_pipeline);
render_pass.draw(0..3, 0..1);
render_pass.set_pipeline(&self.hand_image_pipeline);
render_pass.set_bind_group(1, &self.hand_image_bind_group, &[]);
render_pass.draw(0..6, 0..1);
match active_mode {
ActiveMode::Finger(mode) => self.paint_finger(render_pass, mode),
ActiveMode::Hand(mode) => self.paint_hand(render_pass, mode),
@@ -303,20 +520,7 @@ impl BackgroundRenderResources {
fn paint_hand(&self, render_pass: &mut wgpu::RenderPass<'_>, mode: &HandGatewayMode) {
let _range = mode.range.clone();
if let Some(model) = &self.model {
render_pass.set_pipeline(&self.model_pipeline);
render_pass.set_vertex_buffer(1, self.model_instance_buffer.slice(..));
for mesh in &model.meshes {
if let Some(material) = model.materials.get(mesh.material) {
render_pass.set_bind_group(1, &material.bind_group, &[]);
}
render_pass.set_vertex_buffer(0, mesh.vertex_buffer.slice(..));
render_pass
.set_index_buffer(mesh.index_buffer.slice(..), wgpu::IndexFormat::Uint32);
render_pass.draw_indexed(0..mesh.num_elements, 0, 0..1);
}
}
let _ = render_pass;
}
fn visible_instance_count(&self, rows: u32, cols: u32) -> u32 {
@@ -358,6 +562,39 @@ fn create_background_pipeline(
})
}
fn create_hand_image_pipeline(
device: &wgpu::Device,
target_format: &wgpu::TextureFormat,
shader: &wgpu::ShaderModule,
layout: &wgpu::PipelineLayout,
) -> wgpu::RenderPipeline {
device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("Hand Image Background Pipeline"),
layout: Some(layout),
vertex: wgpu::VertexState {
module: shader,
entry_point: Some("vs_hand_image"),
compilation_options: Default::default(),
buffers: &[],
},
fragment: Some(wgpu::FragmentState {
module: shader,
entry_point: Some("fs_hand_image"),
compilation_options: Default::default(),
targets: &[Some(wgpu::ColorTargetState {
format: *target_format,
blend: Some(wgpu::BlendState::ALPHA_BLENDING),
write_mask: wgpu::ColorWrites::ALL,
})],
}),
primitive: wgpu::PrimitiveState::default(),
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview_mask: None,
cache: None,
})
}
fn create_glyph_pipeline(
device: &wgpu::Device,
target_format: &wgpu::TextureFormat,
@@ -391,14 +628,88 @@ fn create_glyph_pipeline(
})
}
fn create_model_pipelines(
device: &wgpu::Device,
target_format: &wgpu::TextureFormat,
shader: &wgpu::ShaderModule,
layout: &wgpu::PipelineLayout,
) -> ModelPipelines {
ModelPipelines {
opaque: ModelCullPipelines {
single_sided: create_model_pipeline(
device,
target_format,
shader,
layout,
AlphaMode::Opaque,
false,
),
double_sided: create_model_pipeline(
device,
target_format,
shader,
layout,
AlphaMode::Opaque,
true,
),
},
mask: ModelCullPipelines {
single_sided: create_model_pipeline(
device,
target_format,
shader,
layout,
AlphaMode::Mask,
false,
),
double_sided: create_model_pipeline(
device,
target_format,
shader,
layout,
AlphaMode::Mask,
true,
),
},
blend: ModelCullPipelines {
single_sided: create_model_pipeline(
device,
target_format,
shader,
layout,
AlphaMode::Blend,
false,
),
double_sided: create_model_pipeline(
device,
target_format,
shader,
layout,
AlphaMode::Blend,
true,
),
},
}
}
fn create_model_pipeline(
device: &wgpu::Device,
target_format: &wgpu::TextureFormat,
shader: &wgpu::ShaderModule,
layout: &wgpu::PipelineLayout,
alpha_mode: AlphaMode,
double_sided: bool,
) -> wgpu::RenderPipeline {
device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("Hand Model Pipeline"),
label: Some(&format!(
"Hand Model {} {} Pipeline",
alpha_mode.as_str(),
if double_sided {
"Double Sided"
} else {
"Single Sided"
}
)),
layout: Some(layout),
vertex: wgpu::VertexState {
module: shader,
@@ -412,10 +723,11 @@ fn create_model_pipeline(
compilation_options: Default::default(),
targets: &[Some(wgpu::ColorTargetState {
format: *target_format,
blend: Some(wgpu::BlendState {
color: wgpu::BlendComponent::REPLACE,
alpha: wgpu::BlendComponent::REPLACE,
}),
blend: if alpha_mode == AlphaMode::Blend {
Some(wgpu::BlendState::ALPHA_BLENDING)
} else {
None
},
write_mask: wgpu::ColorWrites::ALL,
})],
}),
@@ -423,7 +735,11 @@ fn create_model_pipeline(
topology: wgpu::PrimitiveTopology::TriangleList,
strip_index_format: None,
front_face: wgpu::FrontFace::Ccw,
cull_mode: None,
cull_mode: if double_sided {
None
} else {
Some(wgpu::Face::Back)
},
polygon_mode: wgpu::PolygonMode::Fill,
unclipped_depth: false,
conservative: false,
@@ -524,15 +840,26 @@ struct MatrixUniform {
viewport: [f32; 4],
glyph: [f32; 4],
color: [f32; 4],
render_options: [f32; 4],
image: [f32; 4],
}
impl MatrixUniform {
fn new(width: f32, height: f32, view_proj: [[f32; 4]; 4]) -> Self {
fn new(
width: f32,
height: f32,
view_proj: [[f32; 4]; 4],
surface_is_srgb: bool,
render_options: RenderOptions,
image_size: [f32; 2],
) -> Self {
Self {
view_proj,
viewport: [width.max(1.0), height.max(1.0), 0.0, 0.0],
glyph: [16.0, 0.0, 0.0, 0.0],
color: [0.05, 0.92, 0.32, 1.0],
color: [0.05, 0.92, 0.32, if surface_is_srgb { 1.0 } else { 0.0 }],
render_options: render_options.as_uniform(),
image: [image_size[0].max(1.0), image_size[1].max(1.0), 0.0, 0.0],
}
}
}