web端实现AR人脸特效 | 社区征文 - 文章 - 开发者社区

预览

https://code.juejin.cn/pen/7175016747785781308

https://code.juejin.cn/pen/7175016747785781308

前言

直播、短视频、在线会议等应用越来越多地进入人们的生活，随之诞生的是丰富的各类创意玩法与新鲜体验，其中大量应用了以AI检测和图形渲染为基础的AR技术。

而随着Web技术的不断成熟，AR技术在Web上的实现成为了一种可能。今天就总结了在Web端实现此功能的几个技术要点，跟大家一起探讨一下。

架构和概念

抽象整体的实现思路如下

graph TD
A[调取Camera获得相机画面] --> B[使用tensorflow加载人脸识别模型生成FaceMesh] --> C[根据FaceMesh生成三角网格并进行UV贴图]

FaceMesh

MediaPipe Face Mesh是一种脸部几何解决方案，即使在移动设备上，也可以实时估计468个3D脸部界标。它采用机器学习（ML）来推断3D表面几何形状，只需要单个摄像机输入，而无需专用的深度传感器。该解决方案利用轻量级的模型架构以及整个管线中的GPU加速，可提供对实时体验至关重要的实时性能。

UVMap

UV是二维纹理坐标，U代表水平方向，V代表垂直方向。UV Map用来描述三维物体表面与图像纹理(Texture) 的映射关系，有了UV Map，我们就可以将二维的图像纹理粘贴到三维的物体表面。

矩形贴图和球面的映射图

技术实现

调取Camera获得相机画面

通过navigator.mediaDevices.getUserMedia获取stream，放到video查看。

async function setupWebcam() {
    return new Promise( ( resolve, reject ) => {
        const webcamElement = document.getElementById( "webcam" );
        const navigatorAny = navigator;
        navigator.getUserMedia = navigator.getUserMedia ||
        navigatorAny.webkitGetUserMedia || navigatorAny.mozGetUserMedia ||
        navigatorAny.msGetUserMedia;
        if( navigator.getUserMedia ) {
            navigator.getUserMedia( { video: true },
                stream => {
                    webcamElement.srcObject = stream;
                    webcamElement.addEventListener( "loadeddata", resolve, false );
                },
            error => reject());
        }
        else {
            reject();
        }
    });
}

人脸识别

//创建模型
createModel() {
    return new Promise(async resolve => {
        await tf.setBackend('webgl')
        const model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
        const detectorConfig = {
            maxFaces: 1, //检测到的最大面部数量
            refineLandmarks: true, //可以完善眼睛和嘴唇周围的地标坐标，并在虹膜周围输出其他地标
            runtime: 'mediapipe',
            solutionPath: 'https://unpkg.com/@mediapipe/face_mesh', //WASM二进制文件和模型文件所在的路径
        };
        this.model = await faceLandmarksDetection.createDetector(model, detectorConfig);
        resolve(this.model);
    })
},
//识别
async recognition() {
    try {
        const video = this.$refs.video;
        const faces = await this.model.estimateFaces(video, {
            flipHorizontal: false, //镜像
        });
        if (faces.length > 0) {
            const keypoints = faces[0].keypoints;
            this.render3D({
                scaledMesh:keypoints.reduce((acc, pos) =>{
                    acc.push([pos.x,pos.y,pos.z])
                    return acc
                }, [])
            });
        }else{
            this.render3D({scaledMesh:[]})
        }
    } catch (error) {
        console.log(error);
    }
}

3D场景贴图

    //3D场景
    const scene = new THREE.Scene();

    //添加一些光照
    scene.add( new THREE.AmbientLight( 0xcccccc, 0.4 ) );
    camera.add( new THREE.PointLight( 0xffffff, 0.8 ) );
    
    //正交相机
    scene camera = new THREE.PerspectiveCamera( 45, 1, 0.1, 2000 );
    camera.position.x = videoWidth / 2;
    camera.position.y = -videoHeight / 2;
    camera.position.z = -( videoHeight / 2 ) / Math.tan( 45 / 2 )
    scene.add( camera ); 
    
    //渲染器
    const renderer = new THREE.WebGLRenderer({
        canvas: document.getElementById( "overlay" ),
        alpha: true
    });
    
    //创建geometry，将468个人脸特征点按照一定的顺序(TRIANGULATION)组成三角网格，并加载UV_COORDS
    const geometry = new THREE.BufferGeometry()
    geometry.setIndex(TRIANGULATION)
    geometry.setAttribute('uv', new THREE.Float32BufferAttribute(UV_COORDS.map((item, index) => index % 2 ? item : 1 - item), 2))
    geometry.computeVertexNormals()
                
    //创建material
    const textureLoader = new THREE.TextureLoader();
    const meshImg = this.meshList[meshIndex].src;//材质图片地址
    textureLoader.load(meshImg,texture=>{
        texture.encoding = THREE.sRGBEncoding
        texture.anisotropy = 16
        const material = new THREE.MeshBasicMaterial({
            map: texture,
            transparent: true,
            color: new THREE.Color(0xffffff),
            reflectivity: 0.5
        });
        const mesh = new THREE.Mesh(geometry, material)
        scene.add(mesh)
    })
    // 根据face mesh实时更新geometry
    updateGeometry(prediction){
        let w = canvasWidth;
        let h = canvasWidth;
        const faceMesh = resolveMesh(prediction.scaledMesh, w, h)
        const positionBuffer = faceMesh.reduce((acc, pos) => acc.concat(pos), [])
        geometry.setAttribute('position', new THREE.Float32BufferAttribute(positionBuffer, 3))
        geometry.attributes.position.needsUpdate = true
    }
   resolveMesh(faceMesh, vw, vh){
       return faceMesh.map(p => [p[0] - vw / 2, vh / 2 - p[1], -p[2]])
   }
   
   //渲染
   render3D(prediction){
        if (prediction) {
            updateGeometry(prediction)
        }
        renderer.render(scene, threeCamera)
    }

加载3D模型

//加载3D模型
const loader = new GLTFLoader();
const Object3D = new THREE.Object3D();
loader.load(modelUrl, (gltf) => {
    const object = gltf.scene
    const box = new THREE.Box3().setFromObject(object)
    const size = box.getSize(new THREE.Vector3()).length()
    const center = box.getCenter(new THREE.Vector3())
    object.position.x += (object.position.x - center.x);
    object.position.y += (object.position.y - center.y + 1);
    object.position.z += (object.position.z - center.z - 15);
    Object3D.add(object)
    this.scene.add(Object3D)
})

//计算Matrix
const position = prediction.midwayBetweenEyes[0]
const scale = this.getScale(prediction.scaledMesh, 234, 454)
const rotation = this.getRotation(prediction.scaledMesh, 10, 50, 280)
object.position.set(...position)
object.scale.setScalar(scale / 20)
object.scale.x *= -1
object.rotation.setFromRotationMatrix(rotation)
object.rotation.y = -object.rotation.y
object.rotateZ(Math.PI)
object.rotateX(-Math.PI * .05)
if (this.morphTarget) {
    // flipped
    this.morphTarget['leftEye'] && this.morphTarget['leftEye'](1 - prediction.faceRig.eye.r)
    this.morphTarget['rightEye'] && this.morphTarget['rightEye'](1 - prediction.faceRig.eye.l)
    this.morphTarget['mouth'] && this.morphTarget['mouth'](prediction.faceRig.mouth.shape.A)
}