当前位置：首页 > news >正文

浏览器中使用AI模型实现绘制人体骨架

news 2025/8/21 6:49:10

浏览器中实现绘制人体骨架

最近在为后续项目做技术预研遇到了一个问题，需要再视频上绘制人体骨架，并在相应的关节点上显示相关数据。
开始方案是python通过模型识别人体关键点位置并将位置推送到mqtt服务上并将视频在推送到视频服务器上，然后前端订阅mqtt的数据并拉取python推送的流数据
这样实现后发现视频流会存在很大的延迟，并且没法和响应时间的数据对应上。后面再不停地找新的实现方案的时候，发现现在浏览器已经能使用一些模型了，
后就预研了一下这个方案

方案说明

前端通过chrome浏览器提供的运行模型的能力来进行人体骨架的一个识别，识别完成后，在video元素上叠加一个canvas元素，
在canvas通过识别出来的人体骨架关键点进行人体骨架的绘制

安装插件

pnpm i @mediapipe/tasks-vision 官网文档地址在文章末尾

引用插件

import {DrawingUtils, PoseLandmarker,} from "@mediapipe/tasks-vision";

使用

先创建模型

const createPoseLandmarker = async () => {poseLandmarker = await PoseLandmarker.createFromOptions({wasmBinaryPath: 'http://my.zsyou.top/2024/vision_wasm_internal.wasm', //运行模型需要相关的文件，原始官网是通过调用一个网址获取这两个文件地址，原地址需要科学上网，wasmLoaderPath: 'http://my.zsyou.top/2024/vision_wasm_internal.js'}, {baseOptions: {modelAssetPath: `http://my.zsyou.top/2024/pose_landmarker_lite.task`, //模型文件delegate: "GPU"},runningMode: 'VIDEO',numPoses: 2});console.log('poseLandmarker=====>', poseLandmarker)
};

获取视频流


const start = async () => {await createPoseLandmarker() //因为下载文件需要时间所以提前下载const constraints = {video: true};if (navigator.mediaDevices.getUserMedia) {navigator.mediaDevices.getUserMedia(constraints).then((stream) => {videoElement1.value.srcObject = stream;videoElement1.value.addEventListener("loadeddata", predictWebcam);});} else {message.error('调用摄像头失败')}
}

绘制人体骨架


async function predictWebcam() {const render = () => {const canvasCtx = canvasElement1.value.getContext("2d");const drawingUtils = new DrawingUtils(canvasCtx);let startTimeMs = performance.now();if (lastVideoTime !== videoElement1.value.currentTime) {lastVideoTime = videoElement1.value.currentTime;poseLandmarker.detectForVideo(videoElement1.value, startTimeMs, (result) => {canvasCtx.save();canvasCtx.clearRect(0, 0, canvasElement1.value.width, canvasElement1.value.height);for (const landmark of result.landmarks) {drawingUtils.drawLandmarks(landmark, {color: '#FF0000',fillColor: '#FF0000',radius: 2})drawingUtils.drawConnectors(landmark, PoseLandmarker.POSE_CONNECTIONS, {lineWidth: 2,color: '#00FF00',fillColor: '#00FF00',});}canvasCtx.restore();})}requestAnimationFrame(render);}render();
}

完整代码

<!--------------------------------------------* @Author: shiyzhang zsyou0911@126.com* @Date: 2024/8/23 下午2:31* @Description: * 
------------------------------------------------>
<template><div style="width: 100%"><ASpace><AButton @click="start">开 始</AButton><AButton @click="stop"> 关闭摄像头</AButton></ASpace><div class="pose_land_marker"><video ref="videoElement1" autoplay class="local_video" style="height: 480px;width: 640px"></video><canvas ref="canvasElement1" class="canvas_element" height="480" style="height: 480px;width: 640px;right: 0"width="640"></canvas></div></div>
</template>
<script lang="ts" setup>
import {onMounted, ref} from 'vue'
import {message} from 'ant-design-vue'
import dayjs from 'dayjs'
import {DrawingUtils, PoseLandmarker,} from "@mediapipe/tasks-vision";
import {appendWriteFile, getUserBaseDirHandle} from './untis/index'let poseLandmarker: PoseLandmarker = undefined;
let lastVideoTime = -1;
const videoElement1 = ref(null)
const canvasElement1 = ref(null)
const height = 480
const width = 640const createPoseLandmarker = async () => {poseLandmarker = await PoseLandmarker.createFromOptions({wasmBinaryPath: 'http://my.zsyou.top/2024/vision_wasm_internal.wasm',wasmLoaderPath: 'http://my.zsyou.top/2024/vision_wasm_internal.js'}, {baseOptions: {modelAssetPath: `http://my.zsyou.top/2024/pose_landmarker_lite.task`,delegate: "GPU"},runningMode: 'VIDEO',numPoses: 2});console.log('poseLandmarker=====>', poseLandmarker)
};onMounted(() => {})
//停止
const stop = () => {if (videoElement1.value.srcObject) {videoElement1.value.srcObject?.getTracks().forEach((track) => track.stop());}
}
//开始
const start = async () => {await createPoseLandmarker()const constraints = {video: true};if (navigator.mediaDevices.getUserMedia) {navigator.mediaDevices.getUserMedia(constraints).then((stream) => {videoElement1.value.srcObject = stream;videoElement1.value.addEventListener("loadeddata", predictWebcam);});} else {message.error('调用摄像头失败')}}async function predictWebcam() {const render = () => {const canvasCtx = canvasElement1.value.getContext("2d");const drawingUtils = new DrawingUtils(canvasCtx);let startTimeMs = performance.now();if (lastVideoTime !== videoElement1.value.currentTime) {lastVideoTime = videoElement1.value.currentTime;poseLandmarker.detectForVideo(videoElement1.value, startTimeMs, (result) => {canvasCtx.save();canvasCtx.clearRect(0, 0, canvasElement1.value.width, canvasElement1.value.height);for (const landmark of result.landmarks) {drawingUtils.drawLandmarks(landmark, {color: '#FF0000',fillColor: '#FF0000',radius: 2})drawingUtils.drawConnectors(landmark, PoseLandmarker.POSE_CONNECTIONS, {lineWidth: 2,color: '#00FF00',fillColor: '#00FF00',});}canvasCtx.restore();})}requestAnimationFrame(render);}render();
}</script><style scoped>
.pose_land_marker {position: relative;margin-top: 10px;height: 480px;.canvas_element, .local_video {position: absolute;left: 0;top: 0;}
}
</style>