3. Creating a virtual touch surface

Download this project

In this hacked code (there are things that should be saved as constants/variables… probably some other ugly things lying around… to be cleaned up one day) I create a virtual surface that corresponds to positions within a rectangle on my wall so that when touch the rectangle on the wall, the cursor on my screen moves correspondingly. If I touch the middle of the rectangle on my wall, the cursor moves to the middle. However, as I am more concerned with progressing quickly on a more general way of positioning my Kinect to achieve this, I only made the cursor move horizontally and left its vertical position to be 100 pixels from the top of the screen. It should be pretty easy to fix that.

The following program depends on the Kinect being in a certain position relative to the screen. If I were facing my virtual screen, the Kinect will be to my right. The right of the screen must be 0.77m from the Kinect and the left of the screen should be 1.11m from the Kinect. Of course, you can change the relevant parts of the code to change the screen position.

Here’s the main code:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;
using System.Runtime.InteropServices;
using Microsoft.Kinect;
//using Microsoft.Kinect.Toolkit;
using KinectMouseController;

namespace WpfApp1
{

    public partial class MainWindow : Window
    {

        KinectSensor sensor;
        private byte[] colorPixels;
        List<int> list;

        public MainWindow()
        {
            InitializeComponent();
        }

        private void loaded(object sender, RoutedEventArgs e)
        {
            if (KinectSensor.KinectSensors.Count > 0)
            {
                sensor = KinectSensor.KinectSensors[0];



                if (sensor.Status == KinectStatus.Connected)
                {
                    sensor.ColorStream.Enable();
                    sensor.DepthStream.Enable(DepthImageFormat.Resolution320x240Fps30);
                    sensor.SkeletonStream.Enable();
                    sensor.AllFramesReady += Sensor_AllFramesReady;
                    sensor.Start();
                }

                list = new List<int>(10);
                //sensor.ElevationAngle += 20;
            }
        }

        private void Sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
        {
            

            
            using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
            {
                if(depthFrame == null)
                {
                    return;
                }

                byte[] pixels = GenerateColouredBytes(depthFrame);
                int stride = depthFrame.Width * 4;
                image1.Source = BitmapSource.Create(depthFrame.Width, depthFrame.Height, 96, 96, PixelFormats.Bgr32, null, pixels, stride);
            }
            
        }

        private byte[] GenerateColouredBytes(DepthImageFrame depthFrame)
        {
            DepthImagePixel[] rawDepthData = new DepthImagePixel[depthFrame.PixelDataLength];
            depthFrame.CopyDepthImagePixelDataTo(rawDepthData);

            SkeletonPoint[] skelectonPoints = new SkeletonPoint[rawDepthData.Length];
            CoordinateMapper cm = new CoordinateMapper(sensor);
            cm.MapDepthFrameToSkeletonFrame(DepthImageFormat.Resolution320x240Fps30, rawDepthData, skelectonPoints);

            byte[] colourPixels = new byte[depthFrame.Height * depthFrame.Width * 4];
            const int BlueIndex = 0;
            const int GreenIndex = 1;
            const int RedIndex = 2;

            bool first = true;
            SkeletonPoint firstPoint;

            for (int depthIndex = 0, colorIndex = 0;
                depthIndex < rawDepthData.Length && colorIndex < colourPixels.Length;
                depthIndex++, colorIndex += 4)
            {
                float x = skelectonPoints[depthIndex].X * 100;
                float y = skelectonPoints[depthIndex].Y * 100;
                float z = skelectonPoints[depthIndex].Z * 100;

                if (x > -16 && x < -14 && y > -10 && y < 20 && z > 50 && z < 250)
                {
                    colourPixels[colorIndex + BlueIndex] = (byte)0;
                    colourPixels[colorIndex + GreenIndex] = (byte)0;
                    colourPixels[colorIndex + RedIndex] = (byte)255;

                    //use first position from top of screen as easy way to 
                    //get finger position
                    if (first)
                    {
                        firstPoint = skelectonPoints[depthIndex];
                        first = false;
                        

                        if(firstPoint.Z < 1.11 && firstPoint.Z > 0.77)
                        {
                            double finger = firstPoint.Z - 0.77;
                            double fingerFraction = finger / 0.34;
                            int fingerPixel = (int) (System.Windows.SystemParameters.PrimaryScreenWidth - (System.Windows.SystemParameters.PrimaryScreenWidth * fingerFraction));

                            //remove jitter from incorrect IR data by using median data
                            if (list.Count() < 8)
                            {
                                list.Add(fingerPixel);
                            }
                            else
                            {
                                list.Add(fingerPixel);
                                list.RemoveAt(0);

                                int[] sortedCopy = list.OrderBy(i => i).ToArray();
                                fingerPixel = sortedCopy[3];
                            }

                            string text = string.Join(",", list);

                            this.label1.Content = text;

                            KinectMouseController.KinectMouseMethods.SendMouseInput(fingerPixel, 100, (int)System.Windows.SystemParameters.PrimaryScreenWidth, (int)System.Windows.SystemParameters.PrimaryScreenHeight, false);
                        }

                    }
                }
                else if (x > -20 && x < 0 && y > -10 && y < 20 && z > 50 && z < 250)
                {
                    colourPixels[colorIndex + BlueIndex] = (byte)0;
                    colourPixels[colorIndex + GreenIndex] = (byte)0;
                    colourPixels[colorIndex + RedIndex] = (byte)0;                   
                }
                else
                {
                    colourPixels[colorIndex + BlueIndex] = (byte)255;
                    colourPixels[colorIndex + GreenIndex] = (byte)255;
                    colourPixels[colorIndex + RedIndex] = (byte)255;
                }
            }

            return colourPixels;
        }

        private void closing(object sender, System.ComponentModel.CancelEventArgs e)
        {
            sensor.Stop();
            sensor.AudioSource.Stop();
        }

        private void MouseOver(object sender, MouseEventArgs e)
        {
            //this.label1.Content = "X " + e.GetPosition(this.image1).X;
        }
    }
}

 

//