HoloLens 어드벤트 캘린더 24 일째 기사입니다!

HoloLens의 탭 조작에 의해, 사진을 찍으면 손가락이 비쳐 버려, 회수하는 경우가 자주 있습니다. 이번에는 HoloLens의 카메라 화각을 3D 맵에 시각화해 보았습니다.

개발 환경

Unity 2017.1.2f1

Visual Studio 2017(15.4.4)

HoloToolkit-Unity-v1.2017.2.0.unitypackage

HoloPointCloud

Point Cloud Free Viewer

구현

이 기사 HoloLens에서 방의 3D 스캔을 시도했습니다. 을 기반으로 만들어 갑니다.
HoloPointCloud을 다운로드하세요.
Unity에서 열고 Project 보기에서 saco->Scenes->PhotoCaptureRaw 장면을 선택합니다.
HoloToolkit 및 Point Cloud Free Viewer의 Shader 폴더만 가져옵니다.
SpatialMapping의 Inspector 보기에서 PhotoCaptureRaw.cs를 다음과 같이 편집합니다.

using HoloToolkit.Unity.InputModule;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
using UnityEngine.VR.WSA.WebCam;

public class PhotoCaptureRaw : MonoBehaviour, IInputClickHandler
{

    private PhotoCapture photoCaptureObject = null;
    private bool isCapturing = false;

    private int Width;
    private int Height;

    public int lengthOfLineRenderer;


    // Use this for initialization
    void Start()
    {

        InputManager.Instance.PushFallbackInputHandler(gameObject);

    }

    // Update is called once per frame
    void Update()
    {
        //if (Input.GetKeyDown("space"))
        //{
        //    FrameDisplay();
        //}
    }

    void /*FrameDisplay()*/IInputClickHandler.OnInputClicked(InputClickedEventData eventData)
    {
        Debug.Log("Tapped");
        if (!isCapturing)
        {
            if (photoCaptureObject != null)
            {
                photoCaptureObject.Dispose();
                photoCaptureObject = null;
            }
            PhotoCapture.CreateAsync(false, OnPhotoCaptureCreated);
            isCapturing = true;
        }
    }

    void OnPhotoCaptureCreated(PhotoCapture captureObject)
    {
        photoCaptureObject = captureObject;

        CameraParameters c = new CameraParameters();
        Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();

        c.hologramOpacity = 0.0f;
        c.cameraResolutionWidth = cameraResolution.width;
        c.cameraResolutionHeight = cameraResolution.height;
        c.pixelFormat = CapturePixelFormat.BGRA32;


        Width = cameraResolution.width;
        Height = cameraResolution.height;
        Debug.Log(String.Format("width={0}, height={1}", Width, Height));

        //captureObject.StartPhotoModeAsync(c, OnPhotoModeStarted);
        // Activate the camera
        photoCaptureObject.StartPhotoModeAsync(c, delegate (PhotoCapture.PhotoCaptureResult result) {
            // Take a picture
            photoCaptureObject.TakePhotoAsync(OnCapturedPhotoToMemory);
        });
    }

    void CreateFrame(PhotoCaptureFrame photoCaptureFrame)
    {
        // ------------------------------------------------------
        // make xyxrgb
        //
        int texWidth = Width / 8;
        int texHeight = Height / 8;
        int numPoints = texWidth * texHeight;
        List<Vector3> points = new List<Vector3>();
        List<int> indecies = new List<int>();

        // ------------------------------------------------------
        // pos & rot
        //
        Matrix4x4 cameraToWorldMatrix;
        photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
        Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;

        Matrix4x4 projectionMatrix;
        photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix);

        // 0 right
        // 1 up
        // 2 forward
        // 3 position

        // Position the canvas object slightly in front
        // of the real world web camera.
#if UNITY_EDITOR
        Vector3 position = cameraToWorldMatrix.GetColumn(3) + cameraToWorldMatrix.GetColumn(2);
#else
        Vector3 position = cameraToWorldMatrix.GetColumn(3) - cameraToWorldMatrix.GetColumn(2);
#endif

        // Rotate the canvas object so that it faces the user.
        Quaternion rotation = Quaternion.LookRotation(cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));


        // ------------------------------------------------------
        // pos -> ray
        //
        int pointCount = 0;
        {
            int y = texHeight - 1;
            for (int x = texWidth - 1; x >= 0; --x)
            {
                //int i = y * texWidth + x;
                points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
                                                    (y - (texHeight / 2)) / 200.0F, 0.0F));
#if false
                Vector3 point = points[pointCount];
#else
                // SetTRS
                Vector3 translation = position;
                //Vector3 eulerAngles;
                Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
                Matrix4x4 m = Matrix4x4.identity;
                //Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);

                m.SetTRS(translation, rotation, scale);
                Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif

                Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
                                            point.y - cameraToWorldMatrix.GetColumn(3).y,
                                            point.z - cameraToWorldMatrix.GetColumn(3).z);

                RaycastHit hit;

                if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
                {
                    points[pointCount] = hit.point;
                    indecies.Add(pointCount);
                    pointCount++;
                }
                else
                {
                    points.RemoveAt(pointCount);
                }
            }
        }

        {
            int x = 0;
            for (int y = texHeight - 2; y >= 1; --y)
            {
                points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
                                    (y - (texHeight / 2)) / 200.0F, 0.0F));
                //int i = y * texWidth + x;
#if false
                Vector3 point = points[pointCount];
#else
                // SetTRS
                Vector3 translation = position;
                //Vector3 eulerAngles;
                Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
                Matrix4x4 m = Matrix4x4.identity;
                //Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);

                m.SetTRS(translation, rotation, scale);
                Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif

                Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
                                            point.y - cameraToWorldMatrix.GetColumn(3).y,
                                            point.z - cameraToWorldMatrix.GetColumn(3).z);

                RaycastHit hit;

                if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
                {
                    points[pointCount] = hit.point;
                    indecies.Add(pointCount);
                    pointCount++;
                }
                else
                {
                    points.RemoveAt(pointCount);
                }
            }
        }
        {
            int y = 0;
            for (int x = 0; x <= texWidth - 1; ++x)
            {
                //int i = y * texWidth + x;
                points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
                                    (y - (texHeight / 2)) / 200.0F, 0.0F));
#if false
                Vector3 point = points[pointCount];
#else
                // SetTRS
                Vector3 translation = position;
                //Vector3 eulerAngles;
                Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
                Matrix4x4 m = Matrix4x4.identity;
                //Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);

                m.SetTRS(translation, rotation, scale);
                Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif

                Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
                                            point.y - cameraToWorldMatrix.GetColumn(3).y,
                                            point.z - cameraToWorldMatrix.GetColumn(3).z);

                RaycastHit hit;

                if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
                {
                    points[pointCount] = hit.point;
                    indecies.Add(pointCount);
                    pointCount++;
                }
                else
                {
                    points.RemoveAt(pointCount);
                }
            }

        }
        {
            int x = texWidth - 1;
            for (int y = 1; y < texHeight - 1; ++y)
            {
                //int i = y * texWidth + x;
                points.Add(new Vector3(((x - 1) - (texWidth / 2)) / 200.0F,
                                    (y - (texHeight / 2)) / 200.0F, 0.0F));
#if false
                Vector3 point = points[pointCount];
#else
                // SetTRS
                Vector3 translation = position;
                //Vector3 eulerAngles;
                Vector3 scale = new Vector3(1.0F, 1.0F, 1.0F);
                Matrix4x4 m = Matrix4x4.identity;
                //Quaternion rotation = Quaternion.Euler(eulerAngles.x, eulerAngles.y, eulerAngles.z);

                m.SetTRS(translation, rotation, scale);
                Vector3 point = m.MultiplyPoint3x4(points[pointCount]);
#endif

                Vector3 front = new Vector3(point.x - cameraToWorldMatrix.GetColumn(3).x,
                                            point.y - cameraToWorldMatrix.GetColumn(3).y,
                                            point.z - cameraToWorldMatrix.GetColumn(3).z);

                RaycastHit hit;

                if (Physics.Raycast(cameraToWorldMatrix.GetColumn(3), front, out hit, 50.0F))
                {
                    points[pointCount] = hit.point;
                    indecies.Add(pointCount);
                    pointCount++;
                }
                else
                {
                    points.RemoveAt(pointCount);
                }
            }
        }

        Debug.Log(pointCount);

        LineRenderer lineRenderer = GetComponent<LineRenderer>();
        lengthOfLineRenderer = pointCount;
        lineRenderer.positionCount = pointCount;

        var t = Time.time;
        for (int i = 0; i < lengthOfLineRenderer; i++)
        {
            lineRenderer.SetPosition(i, points[i]);
        }
    }

    void OnStoppedPhotoMode(PhotoCapture.PhotoCaptureResult result)
    {
        photoCaptureObject.Dispose();
        photoCaptureObject = null;
        isCapturing = false;
    }

    void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
    {
        if (result.success)
        {
            string filename = string.Format(@"CapturedImage{0}_n.jpg", Time.time);
            string filePath = System.IO.Path.Combine(Application.persistentDataPath, filename);

            photoCaptureObject.TakePhotoAsync(filePath, PhotoCaptureFileOutputFormat.JPG, OnCapturedPhotoToDisk);
            CreateFrame(photoCaptureFrame);
        }
        else
        {
            Debug.LogError("Unable to start photo mode!");
        }
        photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
        isCapturing = false;
    }

    void OnCapturedPhotoToDisk(PhotoCapture.PhotoCaptureResult result)
    {
        if (result.success)
        {
            Debug.Log("Saved Photo to disk!");
        }
        else
        {
            Debug.Log("Failed to save Photo to disk");
        }
    }

}

이미지 프레임에 대해 RayCast의 HitPoint를 LineRenderer로 연결하여 표시하고 있습니다.

SpatialMapping에 LineRenderer를 추가하고 Width를 0.01로 설정합니다.


평소처럼 빌드하고 실행하십시오.
탭하면 촬영한 범위의 화각이 표시됩니다.

실행 동영상은 여기입니다.


손가락이 비치고 있으면 소리 등으로 알려주면 좋을지도 모릅니다.

PhotoCaptureFrame을 얻고 나서가 아니라면 카메라 Matrix를 얻을 수 없으므로 항상 촬영해야합니다 ...


그림 1 MRC로 촬영한 영상


그림 2 PhotoCapture로 촬영한 이미지

참고 기사

HoloLens에서 방의 3D 스캔을 시도했습니다.

LineRenderer.SetPosition