EEC-693/793 Applied Computer Vision with Depth Cameras

1. EEC-693/793Applied Computer Vision with Depth Cameras Lecture 17 Wenbing Zhao wenbing@ieee.org

2. Outline Skeleton logging Replaying in Unity with logged skeleton data

3. Skeleton Joint Data Logging You can add logging for skeleton joint data into any of your existing project with skeleton tracking enabled Make sure you add the System.IO into your namespace We will use StreamWriter class for logging to a comma separated value file (.csv). Add the following member variable to your MainWindow class: • using System.IO; StreamWriter sw = File.CreateText("kinect.csv");

4. Skeleton Joint Data Logging Add captions for columns. Add the following line in WindowLoaded() Log per frame information, including Timestamp Frame number Floor clip plane sw.WriteLine("FrameNo,timestsamp,MC-X,MC-Y,MC-Z,MC-D,HipCenter-X,HipCenter-Y,HipCenter-Z,HipCenter-D,Spine-X,Spine-Y,Spine-Z,Spine-D,ShoulderCenter-X,ShoulderCenter-Y,ShoulderCenter-Z,ShoulderCenter-D,Head-X,Head-Y,Head-Z,Head-D,ShoulderLeft-X,ShoulderLeft-Y,ShoulderLeft-Z,ShoulderLeft-D,ElbowLeft-X,ElbowLeft-Y,ElbowLeft-Z,ElbowLeft-D,WristLeft-X,WristLeft-Y,WristLeft-Z,WristLeft-D,HandLeft-X,HandLeft-Y,HandLeft-Z,HandLeft-D,ShoulderRight-X,ShoulderRight-Y,ShoulderRight-Z,ShoulderRight-D,ElbowRight-X,ElbowRight-Y,ElbowRight-Z,ElbowRight-D,WristRight-X,WristRight-Y,WristRight-Z,WristRight-D,HandRight-X,HandRight-Y,HandRight-Z,HandRight-D,HipLeft-X,HipLeft-Y,HipLeft-Z,HipLeft-D,KneeLeft-X,KneeLeft-Y,KneeLeft-Z,KneeLeft-D,AnkleLeft-X,AnkleLeft-Y,AnkleLeft-Z,AnkleLeft-D,FootLeft-X,FootLeft-Y,FootLeft-Z,FootLeft-D,HipRight-X,HipRight-Y,HipRight-Z,HipRight-D,KneeRight-X,KneeRigt-Y,KneeRight-Z,KneeRight-D,AnkleRight-X,AnkleRight-Y,AnkleRight-Z,AnkleRight-D,FootRight-X,FootRight-Y,FootRight-Z,FootRight-D");

5. Per Frame Information void skeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e) { canvas1.Children.Clear(); long timestamp = -1; int frameno = -1; float A = 0; float B = 0; float C = 0; float D = 0; using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame()) { // check for frame drop. if (skeletonFrame == null) { return; }

6. Per Frame Info timestamp = skeletonFrame.Timestamp; frameno = skeletonFrame.FrameNumber; A = skeletonFrame.FloorClipPlane.Item1; B = skeletonFrame.FloorClipPlane.Item2; C = skeletonFrame.FloorClipPlane.Item3; D = skeletonFrame.FloorClipPlane.Item4; // copy the frame data in to the collection skeletonFrame.CopySkeletonDataTo(totalSkeleton); // get the first Tracked skeleton skeleton = (from trackskeleton in totalSkeleton where trackskeleton.TrackingState == SkeletonTrackingState.Tracked select trackskeleton).FirstOrDefault(); // if the first skeleton returns null if (skeleton == null) return;

7. Log Per Frame Info if (skeleton != null && this.currentSkeletonID != skeleton.TrackingId) { this.currentSkeletonID = skeleton.TrackingId; int totalTrackedJoints = skeleton.Joints.Where(item => item.TrackingState == JointTrackingState.Tracked).Count(); string TrackedTime = DateTime.Now.ToString("hh:mm:ss"); string status = "Skeleton Id: " + this.currentSkeletonID + ", total tracked joints: " + totalTrackedJoints + ", TrackTime: " + TrackedTime+"\n"; this.textBlock1.Text += status; } DrawSkeleton(skeleton); sw.Write(frameno+”,”); sw.Write(timestamp+",");

8. Log Joint Info float xc = skeleton.Position.X; float yc = skeleton.Position.Y; float zc = skeleton.Position.Z; float cmd = A * xc + B * yc + C * zc + D; sw.Write(xc + "," + yc + "," + zc + "," + cmd + ","); for (int i = 0; i < 20; i++) { float x = skeleton.Joints[(JointType)i].Position.X; float y = skeleton.Joints[(JointType)i].Position.Y; float z = skeleton.Joints[(JointType)i].Position.Z; float dist = A * x + B * y + C * z + D; sw.Write(x + "," + y + "," + z + "," + dist + ","); } sw.WriteLine(""); } }

9. A Sample Log If you have Excel in your computer, you can double click the csv file to open it

10. Replay Logged Data in Unity Create a new Unity project. It is based on the KinectPointMan example as part of the ZDK download Add the floor the same way as before Create an empty game object, name it KinectPointMan Under the KinectPointMan, add 20 3D spheres, and name them properly using the 20 Kinect joints Create a new C# script, I named it BackScript.cs Attach the script to the KinectPointMan Connect the 20 spheres in the scene with 20 joints listed in BackScript.cs Save the scene as KinectPlayer

11. Replay Logged Data in Unity

12. Connect the 20 Joints to the Script

13. BackScript.cs using UnityEngine;using System.Collections;using System.Xml;using System.ComponentModel;using System.Collections.Generic;using System.Linq;using System.Text;using System;using System.IO;public class BackScript : MonoBehaviour {    public GameObject Hip_Center;    public GameObject Spine;    public GameObject Shoulder_Center;    public GameObject Head;    public GameObject Shoulder_Left;    public GameObject Elbow_Left;    public GameObject Wrist_Left;    public GameObject Hand_Left;

14. BackScript.cs     public GameObject Shoulder_Right;    public GameObject Elbow_Right;    public GameObject Wrist_Right;    public GameObject Hand_Right;    public GameObject Hip_Left;    public GameObject Knee_Left;    public GameObject Ankle_Left;    public GameObject Foot_Left;    public GameObject Hip_Right;    public GameObject Knee_Right;    public GameObject Ankle_Right;    public GameObject Foot_Right;    private GameObject[] _bones; //internal handle for the bones of the model     GameObject hip2spine;    GameObject spine2shoulder;    GameObject shoulder2head;    GameObject leftShoulder;    GameObject rightShoulder;    GameObject leftUpperArm;

15. BackScript.cs     GameObject leftArm;    GameObject leftHand;    GameObject rightUpperArm;    GameObject rightArm;    GameObject rightHand;    GameObject leftHip;    GameObject rightHip;    GameObject leftUpperLeg;    GameObject leftLeg;    GameObject leftFoot;    GameObject rightUpperLeg;    GameObject rightLeg;    GameObject rightFoot;     public GUIText statusDisplay;    List<string> exitems = new List<string>();    List<string> exfileitems = new List<string>();    List<List<int>> exjointsitems = new List<List<int>>();    List<float[][]> exdata = new List<float[][]>();

16. BackScript.cs     private int guideCounter = 0;    bool skipframe = false;    public Vector3 Scale = new Vector3(0.001f, 0.001f, 0.001f);     float lastTs;    private Vector3 rootPosition = new Vector3();// Use this for initialization    void Start () {//store bones in a list for easier access        _bones = new GameObject[] {Hip_Center, Spine, Shoulder_Center, Head,            Shoulder_Left, Elbow_Left, Wrist_Left, Hand_Left,            Shoulder_Right, Elbow_Right, Wrist_Right, Hand_Right,            Hip_Left, Knee_Left, Ankle_Left, Foot_Left,            Hip_Right, Knee_Right, Ankle_Right, Foot_Right};        connectBones();    }     void Update () {        if (exdata.Count == 0)            return;    }

17. BackScript.cs     void OnGUI()    {        if (GUI.Button (new Rect (10, 70, 150, 30), "Load Kinect v1 Data")) {            loadConfig(true);            Debug.Log ("Kinect v1 data is loaded");        }         if (Event.current.Equals(Event.KeyboardEvent("escape"))) {            print("Quitting");            Application.Quit();        }        if (Event.current.Equals(Event.KeyboardEvent("p"))) {            print ("Previous Frame");            if(this.guideCounter > 2)                this.guideCounter = this.guideCounter - 2;            DisplayNextFrame();        }        if (Event.current.Equals(Event.KeyboardEvent("n"))) {            print ("Next Frame");            DisplayNextFrame();        }        if (Event.current.Equals(Event.KeyboardEvent("s"))) {            string filename = GetScreenshotFilename();            print("Writing screenshot to " + filename);            Application.CaptureScreenshot(filename);        }    }

18. BackScript.cs     private void loadConfig(bool kinectv1data)    {                        XmlDocument xmlDoc = new XmlDocument ();        xmlDoc.Load ("test.xml");        XmlNodeList fileName = xmlDoc.GetElementsByTagName ("GuideFile");        if (1 != fileName.Count) {                Debug.Log ("expecting exactly one guidefile, got: " + fileName.Count);                return;        }        XmlNodeList innerlist = fileName [0].ChildNodes;        if (1 != innerlist.Count) {                Debug.Log ("guidefile should have only one child, got: " + innerlist.Count);        }        String fn = innerlist [0].InnerText;        float[][] guideData = getGuideData (fn);        exdata.Add (guideData);    }

19. BackScript.cs     private float[][] getGuideData(String filename)    {        String[] dataStrings = null;                if (File.Exists(filename))        {            dataStrings = File.ReadAllLines(filename);            Debug.Log("Replay: done loading data");        }        else        {            Debug.Log("file not found: "+filename);            return null;        }// the first row is the caption, skip it, hence the -1        int size = dataStrings.Length-1;        float[][] allSamples = new float[size][]; 20 joints + misc. Frame Time Jagged Array: allSamples

20. BackScript.cs // the first row is the caption, skip it        for (int i = 0; i < allSamples.Length; i++)        {            String[] row = dataStrings[i + 1].Split(',');            int rowlen = 1+20*3; // 1 column for ts, 20 sets of joints (x,y,z)            allSamples[i] = new float[rowlen];            int k = 0;            for(int j=0; j<row.Length; j++) {// column 0 is for frame numberif(j==0) continue;// column 2,3,4,5 are for center of massif(j >= 2 && j <=5) continue;// for each joint, the 4th column is for vertical distance, skip it tooif(j > 6 && ((j-5) % 4)==0) continue;allSamples[i][k++]=(float)Convert.ToDouble(row[j]);if(k == 61) break; }    } return allSamples; }

21. BackScript.cs     string GetScreenshotFilename()    {        System.IO.Directory.CreateDirectory("Screenshots");        int i=1;        while (System.IO.File.Exists(System.IO.Path.Combine("Screenshots", "Screenshot" + i + ".png"))) {            i++;        }        return System.IO.Path.Combine("Screenshots", "Screenshot" + i + ".png");    }

22. BackScript.cs: DisplayNextFrame()     void DisplayNextFrame() {        float[][] guideData = exdata[0];        float[] frame;        if(guideCounter < guideData.Length)         {            frame = guideData[guideCounter];            if(null == frame)  {  return; } // first column is frame number; second column is timestamp;  first 3 are for rootposition            Vector3 jpt = new Vector3();            int jointCount = 20;            for(int i=0; i<jointCount; i++)            {                jpt.x = 1200 - frame[1+3*i]*1000;                jpt.y = frame[2+3*i]*1000+500;                jpt.z = frame[3+3*i]*1000;                 Debug.Log ("x="+jpt.x+",y="+jpt.y+",z="+jpt.z);                Vector3 dest = Vector3.Scale(jpt, Scale);                  _bones[i].transform.position = dest;            }                updateBones(); guideCounter++;        } if(guideCounter >= guideData.Length) guideCounter = 0;    }

23. BackScript.cs: connectBones()     void connectBones()    {        this.hip2spine = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        this.spine2shoulder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        shoulder2head = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftShoulder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightShoulder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftUpperArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftHand = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightUpperArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightHand = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftHip = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightHip = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftUpperLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        leftFoot = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightUpperLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);        rightFoot = GameObject.CreatePrimitive(PrimitiveType.Cylinder);

24. BackScript.cs: connectBones()         hip2spine.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        spine2shoulder.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        shoulder2head.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);        leftShoulder.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        rightShoulder.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        leftUpperArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        leftArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        leftHand.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);        rightUpperArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        rightArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        rightHand.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);        leftHip.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);        rightHip.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);        leftUpperLeg.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        leftLeg.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);        leftFoot.transform.localScale = new Vector3(0.05f, 0.15f, 0.05f);        rightUpperLeg.transform.localScale = new Vector3(0.15f, 0.25f, 0.15f);        rightLeg.transform.localScale = new Vector3(0.15f, 0.25f, 0.15f);        rightFoot.transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);    }

25. BackScript.cs     void updateBones()    {        updateOneBone(hip2spine, Hip_Center, Spine);        updateOneBone(spine2shoulder, Spine, Shoulder_Center);        updateOneBone(shoulder2head, Shoulder_Center, Head);        updateOneBone(leftShoulder, Shoulder_Center, Shoulder_Left);        updateOneBone(rightShoulder, Shoulder_Center, Shoulder_Right);        updateOneBone(leftUpperArm, Shoulder_Left, Elbow_Left);        updateOneBone(leftArm, Elbow_Left, Wrist_Left);        updateOneBone(leftHand, Wrist_Left, Hand_Left);        updateOneBone(rightUpperArm, Shoulder_Right, Elbow_Right);        updateOneBone(rightArm, Elbow_Right, Wrist_Right);        updateOneBone(rightHand, Wrist_Right, Hand_Right);        updateOneBone(leftHip, Hip_Center, Hip_Left);        updateOneBone(rightHip, Hip_Center, Hip_Right);        updateOneBone(leftUpperLeg, Hip_Left, Knee_Left);        updateOneBone(leftLeg, Knee_Left, Ankle_Left);        updateOneBone(leftFoot, Ankle_Left, Foot_Left);        updateOneBone(rightUpperLeg, Hip_Right, Knee_Right);        updateOneBone(rightLeg, Knee_Right, Ankle_Right);        updateOneBone(rightFoot, Ankle_Right, Foot_Right);     }  

26. BackScript.cs     void updateOneBone(GameObject bone, GameObject start, GameObject end)    {        Vector3 v3End = end.transform.position;        Vector3 v3Start = start.transform.position;        bone.transform.position = (v3End-v3Start)/2.0f + v3Start;         bone.transform.rotation = Quaternion.FromToRotation(Vector3.up, v3End-v3Start);            }     

27. To Run The Replay Unity App Make sure in the Unity app folder you have the following two files: test.xml and kinect.csv You need to locate kinect.csv you saved previously for the logging part and copy it here Kinect.csv may have to be cleaned up a little before feeding to the Unity app Make sure the last row is complete. Remove the row if incomplete test.xml content: <?xml version="1.0" encoding="utf-8"?> <ETatHome> <GuideFile>kinect.csv</GuideFile> </ETatHome>

28. Challenge Tasks For logging, add a start and stop button for starting and stopping the logging For replaying, also add a start and stop button, and a target-frame-rate button, for automated replaying (instead of manually display frame by frame). Tweak parameters so that different frame rates can be used: default, 30 frames/second, 20, and 10, etc. For replaying, add display for joint angle for a joint that you choose