From 466bb5dec7bf212a28f325c2031018ddabfd0f6e Mon Sep 17 00:00:00 2001 From: lenn Date: Tue, 14 Jul 2026 11:08:00 +0800 Subject: [PATCH] Tune hand pressure visualization --- src/app.rs | 229 ++++++++++++++++++++++++----- src/render.rs | 58 ++++++-- src/resources.rs | 5 +- src/serial_core/multi_dim_force.rs | 2 +- static/wgsl/shader.wgsl | 39 +++-- 5 files changed, 273 insertions(+), 60 deletions(-) diff --git a/src/app.rs b/src/app.rs index 7a452d3..20cd29f 100644 --- a/src/app.rs +++ b/src/app.rs @@ -62,6 +62,7 @@ pub struct EskinDesktopApp { latest_matrix_rows: u32, latest_matrix_cols: u32, last_sample_at: Option, + last_color_debug_at: Option, breakout_visible: bool, breakout_game: BreakoutGame, live_rates: LiveRateMeters, @@ -175,6 +176,7 @@ impl EskinDesktopApp { latest_matrix_rows: MATRIX_ROWS, latest_matrix_cols: MATRIX_COLS, last_sample_at: None, + last_color_debug_at: None, breakout_visible: false, breakout_game: BreakoutGame::default(), live_rates: LiveRateMeters::new(), @@ -339,6 +341,7 @@ impl EskinDesktopApp { self.latest_raw_matrix.extend_from_slice(&sample.matrix); self.latest_matrix_rows = sample.rows; self.latest_matrix_cols = sample.cols; + self.log_hand_color_mapping_debug(&sample.matrix); self.analyze_spatial_force(&sample.matrix); @@ -580,6 +583,7 @@ impl EskinDesktopApp { self.pressure_matrix.fill([0.0, 0.0]); self.hand_pressure.clear(); self.last_sample_at = None; + self.last_color_debug_at = None; self.force_estimator.reset(); self.latest_spatial_force = None; self.latest_hand_spatial_forces = [None; HAND_FINGERTIP_COUNT]; @@ -599,6 +603,50 @@ impl EskinDesktopApp { SerialMode::Hand => ActiveMode::Hand(HandGatewayMode { range: 0..7000 }), } } + + fn log_hand_color_mapping_debug(&mut self, raw: &[u32]) { + let now = Instant::now(); + if self + .last_color_debug_at + .is_some_and(|last| now.duration_since(last) < Duration::from_secs(1)) + { + return; + } + self.last_color_debug_at = Some(now); + + let mut parts = Vec::with_capacity(HAND_FORCE_PANEL_COUNT); + let mut start = 0; + for (panel_index, (range, cell_count)) in HAND_SENSOR_PANEL_COLOR_RANGES + .iter() + .copied() + .zip(HAND_SENSOR_PANEL_CELL_COUNTS) + .enumerate() + { + let end = start + cell_count; + let peak = raw + .get(start..end) + .unwrap_or(&[]) + .iter() + .copied() + .max() + .unwrap_or(0); + let cell_range = range.per_cell(cell_count); + let [intensity, _] = cell_range.normalize(peak); + parts.push(format!( + "{} raw_peak={} range_total_max={:.0} cell_range_max={:.0} intensity={:.3}", + match panel_index { + 0..=4 => format!("F{}", panel_index + 1), + 5 => "Palm-H".to_owned(), + _ => "Palm-V".to_owned(), + }, + peak, + range.max, + cell_range.max, + intensity + )); + start = end; + } + } } fn hand_image_pixel_to_screen(rect: egui::Rect, point_px: [f32; 2]) -> egui::Pos2 { @@ -693,18 +741,18 @@ fn raw_to_hand_segment_g(index: usize, raw: u32) -> f32 { 2 => raw_to_g3(raw), 3 => raw_to_g4(raw), 4 => raw_to_g5(raw), - 5 => raw_to_g6(raw), - 6 => raw_to_g7(raw), + // 5 => raw_to_g6(raw), + // 6 => raw_to_g7(raw), _ => raw_to_gd(raw), } } fn raw_to_gd(raw: u32) -> f32 { - const RAW: [u32; 12] = [ - 0, 21382, 108507, 123183, 147405, 171105, 192395, 250443, 231423, 350560, 396616, 429444, + const RAW: [u32; 11] = [ + 0, 486, 5310, 8130, 10005, 12883, 14700, 18141, 19877, 26270, 32648, ]; - const FORCE_CENTI_N: [f32; 12] = [ - 0.0, 57.0, 257.0, 357.0, 457.0, 557.0, 657.0, 857.0, 1057.0, 1557.0, 2057.0, 2557.0, + const FORCE_CENTI_N: [f32; 11] = [ + 0.0, 57.0, 257.0, 357.0, 457.0, 557.0, 657.0, 857.0, 1057.0, 1557.0, 2057.0, ]; if raw <= RAW[0] { @@ -736,7 +784,7 @@ fn raw_to_gd(raw: u32) -> f32 { fn raw_to_g1(raw: u32) -> f32 { const RAW: [u32; 11] = [ - 0, 115181, 365308, 485911, 598964, 681811, 761756, 877545, 976003, 1168784, 1309378, + 0, 106443, 365678, 449286, 532425, 608187, 685522, 797058, 886997, 1082570, 1244793, ]; const FORCE_CENTI_N: [f32; 11] = [ 0.0, 57.0, 257.0, 357.0, 457.0, 557.0, 657.0, 857.0, 1057.0, 1557.0, 2057.0, @@ -771,7 +819,7 @@ fn raw_to_g1(raw: u32) -> f32 { fn raw_to_g2(raw: u32) -> f32 { const RAW: [u32; 11] = [ - 0, 116825, 371568, 475160, 567645, 651220, 714733, 832157, 938778, 1144649, 1290773, + 0, 111143, 310153, 375593, 427920, 497433, 548676, 648392, 715118, 877487, 994887, ]; const FORCE_CENTI_N: [f32; 11] = [ 0.0, 57.0, 257.0, 357.0, 457.0, 557.0, 657.0, 857.0, 1057.0, 1557.0, 2057.0, @@ -806,7 +854,7 @@ fn raw_to_g2(raw: u32) -> f32 { fn raw_to_g3(raw: u32) -> f32 { const RAW: [u32; 11] = [ - 0, 113320, 367656, 457415, 538259, 613913, 666943, 791771, 877385, 1035434, 1196014, + 0, 66898, 271692, 354094, 443663, 534994, 603246, 721942, 837520, 1030111, 1216809, ]; const FORCE_CENTI_N: [f32; 11] = [ 0.0, 57.0, 257.0, 357.0, 457.0, 557.0, 657.0, 857.0, 1057.0, 1557.0, 2057.0, @@ -841,7 +889,7 @@ fn raw_to_g3(raw: u32) -> f32 { fn raw_to_g4(raw: u32) -> f32 { const RAW: [u32; 11] = [ - 0, 106394, 383401, 495623, 590405, 681715, 767392, 906395, 1030218, 1254130, 1385368, + 0, 122337, 381708, 480527, 577445, 677301, 762306, 906751, 1019182, 1254601, 1415051, ]; const FORCE_CENTI_N: [f32; 11] = [ 0.0, 57.0, 257.0, 357.0, 457.0, 557.0, 657.0, 857.0, 1057.0, 1557.0, 2057.0, @@ -876,7 +924,7 @@ fn raw_to_g4(raw: u32) -> f32 { fn raw_to_g5(raw: u32) -> f32 { const RAW: [u32; 11] = [ - 0, 150954, 469433, 613171, 717043, 817483, 963155, 1129123, 1327873, 1668629, 1885354, + 0, 82466, 287752, 356167, 433286, 506855, 604014, 671789, 770692, 887287, 1032058, ]; const FORCE_CENTI_N: [f32; 11] = [ 0.0, 57.0, 257.0, 357.0, 457.0, 557.0, 657.0, 857.0, 1057.0, 1557.0, 2057.0, @@ -1027,6 +1075,79 @@ fn split_viewport_body(rect: egui::Rect) -> egui::Rect { ) } +#[derive(Clone, Copy)] +struct PressureColorRange { + min: f32, + max: f32, + gamma: f32, +} + +// The shader begins its orange-to-red transition at intensity 0.66. With this +// gamma, a raw cell value at 80% of its configured range maps to that point. +const PRESSURE_COLOR_GAMMA: f32 = 1.86; + +impl PressureColorRange { + const fn new(min: u32, max: u32) -> Self { + Self { + min: min as f32, + max: max as f32, + gamma: PRESSURE_COLOR_GAMMA, + } + } + + fn normalize(self, value: u32) -> [f32; 2] { + let raw = value as f32; + let span = (self.max - self.min).max(1.0); + let mapped = ((raw - self.min) / span).clamp(0.0, 1.0); + + let intensity = if raw <= self.min + 4.0 { + 0.0 + } else { + mapped.powf(self.gamma) + }; + + let display_value = if raw <= self.min + 4.0 { + 0.0 + } else { + raw.round().min(9999.0) + }; + + [intensity, display_value] + } + + fn per_cell(self, cell_count: usize) -> Self { + let cells = cell_count.max(1) as f32; + Self { + min: self.min / cells, + max: self.max / cells, + gamma: self.gamma, + } + } +} + +const DEFAULT_COLOR_RANGE: PressureColorRange = PressureColorRange::new(0, 7000); + +const HAND_SENSOR_PANEL_COLOR_RANGES: [PressureColorRange; HAND_FORCE_PANEL_COUNT] = [ + PressureColorRange::new(0, 1244793), + PressureColorRange::new(0, 994887), + PressureColorRange::new(0, 1216809), + PressureColorRange::new(0, 1415051), + PressureColorRange::new(0, 1032058), + PressureColorRange::new(0, 32648), + PressureColorRange::new(0, 32648), +]; + +const HAND_FINGER_SENSOR_CELLS: usize = 12 * 7; +const HAND_SENSOR_PANEL_CELL_COUNTS: [usize; HAND_FORCE_PANEL_COUNT] = [ + HAND_FINGER_SENSOR_CELLS, + HAND_FINGER_SENSOR_CELLS, + HAND_FINGER_SENSOR_CELLS, + HAND_FINGER_SENSOR_CELLS, + HAND_FINGER_SENSOR_CELLS, + 5 * 14, + 11 * 4, +]; + fn normalize_pressure_sample(raw: &[u32], rows: u32, cols: u32, normalized: &mut PressureFrame) { normalized.fill([0.0, 0.0]); @@ -1046,28 +1167,37 @@ fn normalize_pressure_sample(raw: &[u32], rows: u32, cols: u32, normalized: &mut } fn normalize_pressure_samples(raw: &[u32]) -> PressureSamples { - raw.iter().copied().map(normalize_pressure_value).collect() + raw.iter() + .copied() + .enumerate() + .map(|(index, value)| { + hand_sensor_panel_for_index(index) + .map(|(range, cell_count)| range.per_cell(cell_count).normalize(value)) + .unwrap_or_else(|| DEFAULT_COLOR_RANGE.normalize(value)) + }) + .collect() +} + +fn hand_sensor_panel_for_index(index: usize) -> Option<(PressureColorRange, usize)> { + let mut start = 0; + + for (range, cell_count) in HAND_SENSOR_PANEL_COLOR_RANGES + .iter() + .copied() + .zip(HAND_SENSOR_PANEL_CELL_COUNTS) + { + let end = start + cell_count; + if index < end { + return Some((range, cell_count)); + } + start = end; + } + + None } fn normalize_pressure_value(value: u32) -> [f32; 2] { - const RANGE_MIN: f32 = 0.0; - const RANGE_MAX: f32 = 7000.0; - const DISPLAY_GAMMA: f32 = 0.45; - - let raw_value = value as f32; - let mapped = ((raw_value - RANGE_MIN) / (RANGE_MAX - RANGE_MIN)).clamp(0.0, 1.0); - let intensity = if raw_value <= RANGE_MIN + 4.0 { - 0.0 - } else { - mapped.powf(DISPLAY_GAMMA) - }; - let display_value = if raw_value <= RANGE_MIN + 4.0 { - 0.0 - } else { - raw_value.round().min(9999.0) - }; - - [intensity, display_value] + DEFAULT_COLOR_RANGE.normalize(value) } #[cfg(test)] @@ -1082,8 +1212,8 @@ mod tests { let [low_intensity, low_label] = normalize_pressure_value(100); assert!( - low_intensity > 0.10, - "low hand-gateway values should be visible in the wgpu dot matrix" + low_intensity > 0.0, + "low hand-gateway values should retain a non-zero response" ); assert_eq!(low_label, 100.0); @@ -1091,6 +1221,40 @@ mod tests { assert_eq!(max_intensity, 1.0); assert_eq!(max_label, 7000.0); } + + #[test] + fn pressure_normalization_starts_red_transition_at_eighty_percent() { + let range = PressureColorRange::new(0, 10_000); + let [intensity, _] = range.normalize(8_000); + + assert!( + (intensity - 0.66).abs() < 0.002, + "80% of the configured raw range should map to the shader's red-transition point" + ); + } + + #[test] + fn hand_panel_ranges_cover_both_palm_boards() { + let total_cells: usize = HAND_SENSOR_PANEL_CELL_COUNTS.iter().sum(); + let mut raw = vec![0; total_cells]; + let mut start = 0; + + for (range, cell_count) in HAND_SENSOR_PANEL_COLOR_RANGES + .iter() + .copied() + .zip(HAND_SENSOR_PANEL_CELL_COUNTS) + { + raw[start] = (range.max / cell_count as f32).ceil() as u32; + start += cell_count; + } + + let normalized = normalize_pressure_samples(&raw); + let palm_horizontal_start = HAND_SENSOR_PANEL_CELL_COUNTS[..5].iter().sum::(); + let palm_vertical_start = palm_horizontal_start + HAND_SENSOR_PANEL_CELL_COUNTS[5]; + + assert_eq!(normalized[palm_horizontal_start][0], 1.0); + assert_eq!(normalized[palm_vertical_start][0], 1.0); + } } impl eframe::App for EskinDesktopApp { @@ -1109,4 +1273,3 @@ impl eframe::App for EskinDesktopApp { ctx.request_repaint(); } } - diff --git a/src/render.rs b/src/render.rs index cd6cdaf..996ee8d 100644 --- a/src/render.rs +++ b/src/render.rs @@ -44,22 +44,22 @@ pub struct HandGatewayMode { const HAND_TIP_MATRICES: [HandTipMatrix; 5] = [ HandTipMatrix { - center_px: [265.0, 495.0], + center_px: [263.5, 495.0], size_px: [70.0, 120.0], angle_rad: -0.40, }, HandTipMatrix { - center_px: [359.0, 260.0], + center_px: [362.0, 260.0], size_px: [70.0, 120.0], - angle_rad: -0.17, + angle_rad: -0.158, }, HandTipMatrix { - center_px: [482.0, 228.0], + center_px: [480.5, 228.0], size_px: [70.0, 120.0], - angle_rad: -0.03, + angle_rad: -0.02, }, HandTipMatrix { - center_px: [596.0, 265.0], + center_px: [594.0, 265.0], size_px: [70.0, 120.0], angle_rad: 0.10, }, @@ -532,12 +532,13 @@ impl BackgroundRenderResources { let hand_membrane_instances = build_hand_membrane_instances( hand_image_texture.width as f32, hand_image_texture.height as f32, + &[], ); let hand_membrane_instance_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: Some("Hand Fingertip Membrane Instance Buffer"), contents: bytemuck::cast_slice(&hand_membrane_instances), - usage: wgpu::BufferUsages::VERTEX, + usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST, }); let hand_dot_instances = build_hand_dot_instances( rows, @@ -654,6 +655,17 @@ impl BackgroundRenderResources { hand_pressure }; + self.hand_membrane_instances = build_hand_membrane_instances( + self.hand_image_texture.width as f32, + self.hand_image_texture.height as f32, + hand_pressure, + ); + queue.write_buffer( + &self.hand_membrane_instance_buffer, + 0, + bytemuck::cast_slice(&self.hand_membrane_instances), + ); + // Hand mode uses UV-anchored fingertip matrices over hand.png. // Rebuild their instance positions here so pressure colors update every frame. self.hand_dot_instances = build_hand_dot_instances( @@ -1139,24 +1151,48 @@ fn create_hand_palm_dot_pipeline( }) } -fn build_hand_membrane_instances(image_width: f32, image_height: f32) -> Vec { +fn build_hand_membrane_instances( + image_width: f32, + image_height: f32, + pressure: &[[f32; 2]], +) -> Vec { HAND_TIP_MATRICES .iter() - .map(|tip| { + .enumerate() + .map(|(tip_index, tip)| { let uv_x = tip.center_px[0] / image_width.max(1.0); let uv_y = tip.center_px[1] / image_height.max(1.0); let membrane_size = [tip.size_px[0] * 1.44, tip.size_px[1] * 1.36]; + let tip_intensity = hand_tip_peak_intensity(pressure, tip_index); GlyphInstance { // Membrane shaders read xy as hand.png UV. world_position: [uv_x, uv_y, 0.0, 1.0], - // Store angle and an oversized source-image pixel size for the floating sensor film. - style: [tip.angle_rad, membrane_size[0], membrane_size[1], 0.0], + // Store angle, source-image pixel size, and the live fingertip peak intensity. + style: [ + tip.angle_rad, + membrane_size[0], + membrane_size[1], + tip_intensity, + ], } }) .collect() } +fn hand_tip_peak_intensity(pressure: &[[f32; 2]], tip_index: usize) -> f32 { + let start = tip_index * HAND_FINGER_SENSOR_CELLS; + let end = start + HAND_FINGER_SENSOR_CELLS; + + pressure + .get(start..end) + .unwrap_or(&[]) + .iter() + .map(|sample| sample[0]) + .fold(0.0, f32::max) + .clamp(0.0, 1.0) +} + fn build_hand_palm_chip_instances(image_width: f32, image_height: f32) -> Vec { HAND_PALM_CHIPS .iter() diff --git a/src/resources.rs b/src/resources.rs index 8b16e0c..ac4f8dd 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -38,8 +38,9 @@ pub fn load_texture( device: &wgpu::Device, queue: &wgpu::Queue, ) -> anyhow::Result { - let data = load_binary(file_name)?; - texture::Texture::from_bytes(device, queue, &data, file_name) + // let data = load_binary(file_name)?; + let data = include_bytes!("../res/hand.png"); + texture::Texture::from_bytes(device, queue, data, file_name) } pub fn load_model( diff --git a/src/serial_core/multi_dim_force.rs b/src/serial_core/multi_dim_force.rs index e0530e6..3878204 100644 --- a/src/serial_core/multi_dim_force.rs +++ b/src/serial_core/multi_dim_force.rs @@ -5,7 +5,7 @@ const SENSOR_COUNT: usize = SENSOR_ROWS * SENSOR_COLS; const TOTAL_PRESSURE_LOW_THRESHOLD: f32 = 500.0; const COP_STABILITY_FRAMES_REQUIRED: usize = 15; -const POST_INIT_WINDOW_CNT: usize = 100; +const POST_INIT_WINDOW_CNT: usize = 50; const POST_INIT_STABLE_CNT: usize = 50; const POST_INIT_STABLE_THRESH: f32 = 0.1; diff --git a/static/wgsl/shader.wgsl b/static/wgsl/shader.wgsl index 7fe6ba8..b16f563 100644 --- a/static/wgsl/shader.wgsl +++ b/static/wgsl/shader.wgsl @@ -101,7 +101,7 @@ fn sample_range_color(value: f32) -> vec3f { return mix(range_stop_color(1u), range_stop_color(2u), local); } - let local = smoothstep(0.66, 1.0, t); + let local = smoothstep(0.66, 0.92, t); return mix(range_stop_color(2u), range_stop_color(3u), local); } @@ -368,6 +368,7 @@ fn hand_image_uv_to_clip(image_uv: vec2f) -> vec2f { struct HandMembraneVertexOutput { @builtin(position) clip_position: vec4f, @location(0) local: vec2f, + @location(1) intensity: f32, } fn rotate_2d(point: vec2f, angle: f32) -> vec2f { @@ -417,12 +418,17 @@ fn vs_hand_membrane(vertex: DotVertexInput, instance: DotInstanceInput) -> HandM var out: HandMembraneVertexOutput; out.clip_position = vec4f(hand_image_uv_to_clip(image_uv), 0.0, 1.0); out.local = vertex.local; + out.intensity = saturate(instance.style.w); return out; } @fragment fn fs_hand_membrane(in: HandMembraneVertexOutput) -> @location(0) vec4f { let p = in.local; + let live = saturate(in.intensity); + let response = smoothstep(0.03, 0.96, live); + let hot = smoothstep(0.72, 0.96, live); + let live_color = sample_range_color(live); let halo_shape = fingertip_film_alpha(p, 0.72, -0.20, 0.055); let panel = fingertip_film_alpha(p, 0.66, -0.22, 0.035); @@ -472,20 +478,21 @@ fn fs_hand_membrane(in: HandMembraneVertexOutput) -> @location(0) vec4f { let glass_base = vec3f(0.006, 0.055, 0.105); let glass_cyan = vec3f(0.025, 0.42, 0.58); - let rim_color = vec3f(0.10, 0.88, 1.00); + let glass_live = mix(glass_cyan, live_color * 0.58, response); + let rim_color = mix(vec3f(0.10, 0.88, 1.00), live_color, response); let extrusion_color = vec3f(0.004, 0.080, 0.110); - let extrusion_edge_color = vec3f(0.015, 0.30, 0.38); - let bevel_highlight = vec3f(0.48, 1.00, 1.00); + let extrusion_edge_color = mix(vec3f(0.015, 0.30, 0.38), live_color * 0.72, response); + let bevel_highlight = mix(vec3f(0.48, 1.00, 1.00), live_color, response * 0.82); let bevel_dark_color = vec3f(0.004, 0.035, 0.060); - let dot_color = vec3f(0.08, 0.48, 0.58); - let pressure_color = dot_color; - let pressure_hot_color = dot_color; + let dot_color = mix(vec3f(0.08, 0.48, 0.58), live_color, response); + let pressure_color = mix(dot_color, live_color, response); + let pressure_hot_color = live_color; - let color = extrusion_color * extrusion * 0.90 + let membrane_color = extrusion_color * extrusion * 0.90 + extrusion_edge_color * extrusion * halo_shape * 0.32 + glass_base * panel * 0.26 - + glass_cyan * edge_fade * 0.46 - + glass_cyan * broad_bevel * 0.24 + + glass_live * edge_fade * 0.46 + + glass_live * broad_bevel * 0.24 + rim_color * rim * 0.88 + bevel_highlight * bevel_light * 1.35 + bevel_dark_color * bevel_dark * 0.95 @@ -495,6 +502,10 @@ fn fs_hand_membrane(in: HandMembraneVertexOutput) -> @location(0) vec4f { + pressure_color * dots * pressure * 0.82 + pressure_hot_color * dots * pressure_hot * 0.90; + let live_wash = live_color * response * (inner * 0.12 + dots * 0.72 + rim * 0.30) + + live_color * hot * (inner * 0.08 + dots * 0.22); + let color = membrane_color + live_wash; + let alpha = clamp( extrusion * 0.44 + panel * 0.055 @@ -508,9 +519,9 @@ fn fs_hand_membrane(in: HandMembraneVertexOutput) -> @location(0) vec4f { + dots * pressure * 0.10, 0.0, 0.90, - ); + ) + response * (dots * 0.07 + rim * 0.035) + hot * dots * 0.04; - return output_color(color, alpha); + return output_color(color, clamp(alpha, 0.0, 0.96)); } @vertex @@ -593,7 +604,9 @@ fn vs_hand_palm_dot(vertex: DotVertexInput, instance: DotInstanceInput) -> DotVe let shaped = smoothstep(0.0, 1.0, intensity); let center = hand_image_uv_to_clip(instance.world_position.xy); - let pixel_size = u.glyph.x * mix(0.22, 0.34, shaped); + // Palm boards have more tightly packed cells than the fingertips, so use + // a larger circular mask to keep each sensor visibly readable. + let pixel_size = u.glyph.x * mix(0.40, 0.54, shaped); let ndc_offset = vertex.local * vec2f(pixel_size / u.viewport.x, pixel_size / u.viewport.y) * 2.0; var out: DotVertexOutput;