feat: 新增 Playwright 测试 + ModelStage UI 更新与 serial 小修

.github/workflows/playwright.yml

@@ -0,0 +1,27 @@
+name: Playwright Tests
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: lts/*
+    - name: Install dependencies
+      run: npm ci
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+    - name: Run Playwright tests
+      run: npx playwright test
+    - uses: actions/upload-artifact@v4
+      if: ${{ !cancelled() }}
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30

.gitignore

@@ -37,3 +37,11 @@ vite.config.ts.timestamp-*
 npm-debug.log*
 yarn-debug.log*
 pnpm-debug.log*
+
+# Playwright
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+/playwright/.auth/

package-lock.json

@@ -16,10 +16,12 @@
         "three": "^0.183.2"
       },
       "devDependencies": {
+        "@playwright/test": "^1.61.0",
         "@sveltejs/adapter-static": "^3.0.6",
         "@sveltejs/kit": "^2.9.0",
         "@sveltejs/vite-plugin-svelte": "^5.0.0",
         "@tauri-apps/cli": "^2",
+        "@types/node": "^26.0.0",
         "@types/three": "^0.183.1",
         "svelte": "^5.0.0",
         "svelte-check": "^4.0.0",
@@ -526,6 +528,22 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.61.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.0.tgz",
+      "integrity": "sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.61.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@polka/url": {
       "version": "1.0.0-next.29",
       "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
@@ -1321,6 +1339,16 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/node": {
+      "version": "26.0.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.0.tgz",
+      "integrity": "sha512-vf2YFi1iY9lHGwNJMs01biZFbKJkrZR1T6/MlzjhJLPdntOHLhTrDSnSVcdtvjihi4VQNlrFRIxLsDBlQpAipA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~8.3.0"
+      }
+    },
     "node_modules/@types/stats.js": {
       "version": "0.17.4",
       "resolved": "https://registry.npmjs.org/@types/stats.js/-/stats.js-0.17.4.tgz",
@@ -1686,6 +1714,53 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.61.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz",
+      "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.61.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.61.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz",
+      "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/postcss": {
       "version": "8.5.10",
       "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz",
@@ -1918,6 +1993,13 @@
         "node": ">=14.17"
       }
     },
+    "node_modules/undici-types": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
+      "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/vite": {
       "version": "6.4.2",
       "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.2.tgz",

package.json

@@ -22,10 +22,12 @@
     "three": "^0.183.2"
   },
   "devDependencies": {
+    "@playwright/test": "^1.61.0",
     "@sveltejs/adapter-static": "^3.0.6",
     "@sveltejs/kit": "^2.9.0",
     "@sveltejs/vite-plugin-svelte": "^5.0.0",
     "@tauri-apps/cli": "^2",
+    "@types/node": "^26.0.0",
     "@types/three": "^0.183.1",
     "svelte": "^5.0.0",
     "svelte-check": "^4.0.0",

playwright.config.ts

@@ -0,0 +1,79 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://localhost:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});

src-tauri/src/commands/serial.rs

@@ -24,7 +24,7 @@ const DEFAULT_TACTILE_REPLY_TIMEOUT_MS: u64 = 140;
 
 // MCU currently uses a fixed node layout. Update this list to match the firmware:
 // one entry per Config bit, containing that node's number of u16 samples.
-const DEFAULT_HAND_GATEWAY_NODE_SAMPLE_COUNTS: &[u16] = &[84];
+const DEFAULT_HAND_GATEWAY_NODE_SAMPLE_COUNTS: &[u16] = &[84, 84, 84, 84, 84, 70, 44];
 
 type SharedTactileRecording = Arc<Mutex<TactileARecording>>;
 

src-tauri/src/serial_core/serial.rs

@@ -241,13 +241,28 @@ pub async fn run_hand_gateway_serial(
                         frame.valid_config,
                         frame.block_count,
                     );
-                    for node in frame.nodes {
+                    let mut pressure_matrix = Vec::new();
+                    for node in &frame.nodes {
                         info!(
                             "[hand_gateway][node] mask=0x{:08X} valid={} payload={:02X?}",
                             node.config_mask,
                             node.valid,
                             node.payload,
                         );
+                        if node.valid {
+                            pressure_matrix.extend(
+                                HandGatewayCodec::parse_node_payload(&node.payload)?
+                                    .into_iter()
+                                    .map(f32::from),
+                            );
+                        } else {
+                            let sample_count = node.payload.len() / 2;
+                            pressure_matrix.extend(std::iter::repeat(0.0).take(sample_count));
+                        }
+                    }
+
+                    if !pressure_matrix.is_empty() {
+                        app.emit("hand_gateway_pressure_matrix", pressure_matrix)?;
                     }
                 }
             }

src/lib/components/CenterStage.svelte

@@ -188,7 +188,14 @@
       {#if isModelStage}
         <div class="canvas-wrap">
           {#key modelUrl}
-            <ModelStage {locale} {modelUrl} />
+            <ModelStage
+              {locale}
+              {modelUrl}
+              {pressureMatrix}
+              {rangeMin}
+              {rangeMax}
+              {colorMapPreset}
+            />
           {/key}
         </div>
       {:else if showPrecisionTestPanel}

src/lib/components/ModelStage.svelte

@@ -4,12 +4,27 @@
   import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js";
   import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
   import type { GLTF } from "three/examples/jsm/loaders/GLTFLoader.js";
-  import type { LocaleCode } from "$lib/types/hud";
+  import { pressureColorPalettes } from "$lib/config/color-map";
+  import type { LocaleCode, PressureColorMapPreset } from "$lib/types/hud";
 
   type ModelLoadState = "loading" | "ready" | "missing" | "error";
+  type SensorRegion = {
+    id: string;
+    rows: number;
+    cols: number;
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    rotation: number;
+  };
 
   export let locale: LocaleCode = "zh-CN";
   export let modelUrl = "/models/je-skin-model.glb";
+  export let pressureMatrix: number[] | null = null;
+  export let rangeMin = 0;
+  export let rangeMax = 4095;
+  export let colorMapPreset: PressureColorMapPreset = "emerald";
 
   let rootEl: HTMLDivElement | undefined;
   let canvasEl: HTMLCanvasElement | undefined;
@@ -22,13 +37,29 @@
   const MODEL_TARGET_HEIGHT = 6.55;
   const MODEL_MIN_SCALE = 0.02;
   const MODEL_MAX_SCALE = 80;
+  const SENSOR_Z = 0.91;
   const FIXED_CAMERA_POSITION = new THREE.Vector3(-0.113, -0.149, 10.911);
   const FIXED_CAMERA_TARGET = new THREE.Vector3(0, 2.35, 0);
   const MODEL_FRONT_ROTATION_Y = -Math.PI / 2;
   const MODEL_UPRIGHT_ROTATION_Z = Math.PI * 1.5 - THREE.MathUtils.degToRad(2.2);
+  const sensorRegions: SensorRegion[] = [
+    { id: "thumb", rows: 12, cols: 7, x: -1.48, y: 1.95, width: 0.54, height: 1.02, rotation: THREE.MathUtils.degToRad(35) },
+    { id: "index", rows: 12, cols: 7, x: -0.72, y: 3.84, width: 0.54, height: 1.02, rotation: THREE.MathUtils.degToRad(6) },
+    { id: "middle", rows: 12, cols: 7, x: 0.03, y: 4.18, width: 0.6, height: 1.14, rotation: 0 },
+    { id: "ring", rows: 12, cols: 7, x: 0.8, y: 4, width: 0.54, height: 1, rotation: THREE.MathUtils.degToRad(-5) },
+    { id: "pinky", rows: 12, cols: 7, x: 1.58, y: 3.36, width: 0.48, height: 0.92, rotation: THREE.MathUtils.degToRad(-8) },
+    { id: "palm-horizontal", rows: 5, cols: 14, x: 0.18, y: 1.34, width: 1.84, height: 0.54, rotation: 0 },
+    { id: "palm-vertical", rows: 11, cols: 4, x: 1.02, y: 0.2, width: 0.44, height: 1.46, rotation: 0 }
+  ];
 
   $: canvasLabel = locale === "zh-CN" ? "3D 模型" : "3D Model";
 
+  $: updateSensorColors();
+
+  let sensorGeometry: THREE.BufferGeometry | null = null;
+  let sensorColors: Float32Array | null = null;
+  let sensorPointCount = 0;
+
   function disposeObject3D(object: THREE.Object3D): void {
     object.traverse((child) => {
       const mesh = child as THREE.Mesh;
@@ -100,6 +131,86 @@
     return Math.min(max, Math.max(min, value));
   }
 
+  function mixColors(low: THREE.Color, high: THREE.Color, amount: number): THREE.Color {
+    return low.clone().lerp(high, clamp(amount, 0, 1));
+  }
+
+  function buildSensorOverlay(): THREE.Points {
+    const positions: number[] = [];
+
+    for (const region of sensorRegions) {
+      const sin = Math.sin(region.rotation);
+      const cos = Math.cos(region.rotation);
+      const colSpan = Math.max(region.cols - 1, 1);
+      const rowSpan = Math.max(region.rows - 1, 1);
+
+      for (let row = 0; row < region.rows; row += 1) {
+        for (let col = 0; col < region.cols; col += 1) {
+          const localX = (col / colSpan - 0.5) * region.width;
+          const localY = (0.5 - row / rowSpan) * region.height;
+          const x = region.x + localX * cos - localY * sin;
+          const y = region.y + localX * sin + localY * cos;
+
+          positions.push(x, y, SENSOR_Z);
+        }
+      }
+    }
+
+    sensorPointCount = positions.length / 3;
+    sensorColors = new Float32Array(sensorPointCount * 3);
+    sensorGeometry = new THREE.BufferGeometry();
+    sensorGeometry.setAttribute("position", new THREE.Float32BufferAttribute(positions, 3));
+    sensorGeometry.setAttribute("color", new THREE.BufferAttribute(sensorColors, 3));
+
+    const points = new THREE.Points(
+      sensorGeometry,
+      new THREE.PointsMaterial({
+        size: 0.052,
+        vertexColors: true,
+        transparent: true,
+        opacity: 0.92,
+        depthTest: false,
+        depthWrite: false,
+        sizeAttenuation: true
+      })
+    );
+    points.renderOrder = 20;
+    points.name = "hand-sensor-dot-overlay";
+    updateSensorColors();
+    return points;
+  }
+
+  function updateSensorColors(): void {
+    if (!sensorGeometry || !sensorColors || sensorPointCount === 0) {
+      return;
+    }
+
+    const palette = pressureColorPalettes[colorMapPreset] ?? pressureColorPalettes.emerald;
+    const base = new THREE.Color(palette.labelZero);
+    const mid = new THREE.Color(palette.surfaceHigh);
+    const hot = new THREE.Color(palette.surfaceHot);
+    const span = Math.max(rangeMax - rangeMin, 1);
+    const dimFactor = pressureMatrix ? 0.9 : 0.42;
+
+    for (let index = 0; index < sensorPointCount; index += 1) {
+      const value = pressureMatrix?.[index];
+      const normalized = Number.isFinite(value) ? clamp((Number(value) - rangeMin) / span, 0, 1) : 0;
+      const color =
+        normalized <= 0.5
+          ? mixColors(base, mid, normalized * 2)
+          : mixColors(mid, hot, (normalized - 0.5) * 2);
+
+      sensorColors[index * 3] = color.r * dimFactor;
+      sensorColors[index * 3 + 1] = color.g * dimFactor;
+      sensorColors[index * 3 + 2] = color.b * dimFactor;
+    }
+
+    const colorAttribute = sensorGeometry.getAttribute("color");
+    if (colorAttribute) {
+      colorAttribute.needsUpdate = true;
+    }
+  }
+
   function materialToUnlit(material: THREE.Material): THREE.Material {
     const source = material as THREE.MeshStandardMaterial & THREE.MeshPhysicalMaterial;
     const unlit = new THREE.MeshBasicMaterial({
@@ -247,6 +358,8 @@
     applyUnlitMaterials(activeModel);
     scene.add(activeModel);
     frameObject(activeModel, camera, controls);
+    const sensorOverlay = buildSensorOverlay();
+    scene.add(sensorOverlay);
 
     const loader = new GLTFLoader();
     loader.load(
@@ -311,6 +424,7 @@
       renderer.setAnimationLoop(null);
       controls.dispose();
       disposeObject3D(activeModel);
+      disposeObject3D(sensorOverlay);
       disposeObject3D(labGroup);
       renderer.dispose();
     };

src/routes/+page.svelte

@@ -1849,6 +1849,7 @@
     let disposed = false;
     let unlistenHudStream: UnlistenFn | null = null;
     let unlistenHandGatewayRaw: UnlistenFn | null = null;
+    let unlistenHandGatewayPressureMatrix: UnlistenFn | null = null;
     let unlistenDevkitPztAngle: UnlistenFn | null = null;
     let stopMockFeed: (() => void) | null = null;
 
@@ -1890,6 +1891,25 @@
         .catch((error) => {
           console.error("Failed to listen for hand_gateway_raw:", error);
         });
+      void listen<number[]>("hand_gateway_pressure_matrix", (event) => {
+        if (replayHasData) {
+          return;
+        }
+
+        pressureMatrix = event.payload;
+        hasSignalData = event.payload.length > 0;
+      })
+        .then((unlisten) => {
+          if (disposed) {
+            unlisten();
+            return;
+          }
+
+          unlistenHandGatewayPressureMatrix = unlisten;
+        })
+        .catch((error) => {
+          console.error("Failed to listen for hand_gateway_pressure_matrix:", error);
+        });
       void listen<DevKitPztAngleEvent>("devkit_pzt_angle", (event) => {
         const angleDeg = Number(event.payload.angle);
         if (!Number.isFinite(angleDeg)) {
@@ -1931,6 +1951,7 @@
       stopMockFeed?.();
       unlistenHudStream?.();
       unlistenHandGatewayRaw?.();
+      unlistenHandGatewayPressureMatrix?.();
       unlistenDevkitPztAngle?.();
       if (devkitStatusTimer != null) {
         window.clearInterval(devkitStatusTimer);

tests/example.spec.ts

@@ -0,0 +1,18 @@
+import { test, expect } from '@playwright/test';
+
+test('has title', async ({ page }) => {
+  await page.goto('https://playwright.dev/');
+
+  // Expect a title "to contain" a substring.
+  await expect(page).toHaveTitle(/Playwright/);
+});
+
+test('get started link', async ({ page }) => {
+  await page.goto('https://playwright.dev/');
+
+  // Click the get started link.
+  await page.getByRole('link', { name: 'Get started' }).click();
+
+  // Expects page to have a heading with the name of Installation.
+  await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible();
+});

vite.config.js

@@ -1,7 +1,6 @@
 import { defineConfig } from "vite";
 import { sveltekit } from "@sveltejs/kit/vite";
 
-// @ts-expect-error process is a nodejs global
 const host = process.env.TAURI_DEV_HOST;
 
 // https://vite.dev/config/