erickulcyk's picture

OpenTK With Kinect

Hello,

I am new to opentk and opengl, and I am looking to render a point cloud using Kinect depth data. I found a sample on the internet which will display a static point cloud, but I am having difficulty extending it to use the Kinect. For some reason if I put the Kinect code in the sample class or thread as the point cloud display, the depth frame ready event is never fired. So I made thread just for the graphics rendering and I am trying to pass in the data whenever I get a new frame. However, GL.DrawArrays throws an System.System.AccessViolationException when I try to do so. Any help would be appreciated. Here is my c# code:
Thanks,
Eric

using System;
using System.Collections.Generic;
using System.Text;
using System.Runtime.InteropServices;
using System.Threading;
 
using OpenTK;
using OpenTK.Graphics;
using OpenTK.Graphics.OpenGL;
using OpenTK.Platform;
using System.Drawing;
using Microsoft.Kinect;
 
namespace Examples.Tutorial
{
    public class KinectInterface
    {
        KinectSensor a;
        SkeletonPoint[] depthPoints;
        Vector3[] depthVector;
        CloudExample cloudEx;
 
        public KinectInterface(CloudExample cloudEx)
        {
            this.cloudEx = cloudEx;
            Console.WriteLine("Number of sensors: " + KinectSensor.KinectSensors.Count);
            if (KinectSensor.KinectSensors.Count > 0)
            {
                a = KinectSensor.KinectSensors[0];
                a.DepthFrameReady += new EventHandler<DepthImageFrameReadyEventArgs>(KinectInterface_DepthFrameReady);
                a.DepthStream.Enable(DepthImageFormat.Resolution640x480Fps30);
                a.ColorStream.Enable();
 
                try
                {
                    a.Start();
                }
                catch
                {
                    Console.WriteLine("Unable to start the Kinect Sensor");
                }
 
                a.ElevationAngle = 0;
                Console.WriteLine("Status: " + a.Status.ToString());
 
            }
        }
 
        void KinectInterface_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
        {
            //Console.WriteLine("Depth Frame Ready!");
            using (var depthFrame = e.OpenDepthImageFrame())
            {
                if (depthPoints == null)
                {
                    if (cloudEx == null)
                        return;
 
                    depthPoints = new SkeletonPoint[depthFrame.PixelDataLength];
                    depthVector = new Vector3[depthFrame.PixelDataLength];
                    cloudEx.SetGLData(depthVector);
                }
 
                a.CoordinateMapper.MapDepthFrameToSkeletonFrame(depthFrame.Format, depthFrame.GetRawPixelData(), depthPoints);
                lock (depthVector)
                {
                    for (int i = 0; i < depthFrame.PixelDataLength; i++)
                    {
                        depthVector[i] = new Vector3(depthPoints[i].X, depthPoints[i].Y, depthPoints[i].Z);
                    }
                }
            }
        }
    }
 
    public class CloudExample : GameWindow
    {
        float rotation_speed = 90.0f;
        float angle,angley;
        float scale = 4;
        float scale_speed = .99f;
        int vbo_id;
        int vbo_size;
        OpenTK.Matrix4 projection;
 
        const int CloudSize = 32;
        const bool HighQuality = true;
 
 
        public CloudExample()
            : base(800, 600)
        {
            Load += LoadHandler;
            Resize += ResizeHandler;
            UpdateFrame += UpdateHandler;
            RenderFrame += RenderHandler;
        }
 
 
        void LoadHandler(object sender, EventArgs e)
        {
            GL.Enable(EnableCap.DepthTest);
 
            // Improve visual quality at the expense of performance
            if (HighQuality)
            {
                int max_size;
                GL.GetInteger(GetPName.PointSizeMax, out max_size);
                GL.Enable(EnableCap.PointSmooth);
            }
 
            //var vertices = GenerateSampleGLData();
            //SetGLData(vertices);
        }
 
        private static Vector3[] GenerateSampleGLData()
        {
            // Imagine that the cloud is a bool[CloudSize, CloudSize, CloudSize] array.
            // This code translates the point cloud into vertex coordinates
            var vertices = new Vector3[CloudSize * CloudSize * CloudSize];
            Console.WriteLine("Setting Data");
            int index = 0;
            for (int i = 0; i < CloudSize; i++)
                for (int j = 0; j < CloudSize; j++)
                    for (int k = 0; k < CloudSize; k++)
                        if (Math.Sqrt(i * i + j * j + k * k) < CloudSize) // Point cloud shaped like a sphere
                        {
                            vertices[index++] = new Vector3(
                                -CloudSize / 2 + i,
                                -CloudSize / 2 + j,
                                -CloudSize / 2 + k);
                        }
            return vertices;
        }
 
        public void SetGLData(Vector3[] vertices)
        {
            // Load those vertex coordinates into a VBO
            vbo_size = vertices.Length; // Necessary for rendering later on
            GL.GenBuffers(1, out vbo_id);
            GL.BindBuffer(BufferTarget.ArrayBuffer, vbo_id);
            GL.BufferData(BufferTarget.ArrayBuffer,
                          new IntPtr(vertices.Length * BlittableValueType.StrideOf(vertices)),
                          vertices,
                          BufferUsageHint.DynamicDraw);
        }
 
        void ResizeHandler(object sender, EventArgs e)
        {
            GL.Viewport(ClientRectangle);
 
            float aspect_ratio = Width / (float)Height;
            projection = OpenTK.Matrix4.CreatePerspectiveFieldOfView(MathHelper.PiOver4, aspect_ratio, 1, 512);
            GL.MatrixMode(MatrixMode.Projection);
            GL.LoadMatrix(ref projection);
        }
 
        void UpdateHandler(object sender, FrameEventArgs e)
        {
            if (Keyboard[OpenTK.Input.Key.Escape])
                this.Exit();
        }
 
        void RenderHandler(object sender, FrameEventArgs e)
        {
            GL.Clear(ClearBufferMask.ColorBufferBit |
                     ClearBufferMask.DepthBufferBit |
                     ClearBufferMask.StencilBufferBit);
 
            if (HighQuality)
            {
                // See http://www.opengl.org/discussion_boards/ubbthreads.php?ubb=showflat&Number=263583#Post263583
                GL.PointParameter(PointParameterName.PointDistanceAttenuation,
                    new float[] { 0, 0, (float)Math.Pow(1 / (projection.M11 * Width / 2), 2) });
            }
 
            //if (!Keyboard[OpenTK.Input.Key.Space])
            if (Keyboard[OpenTK.Input.Key.Left])
                angle += rotation_speed * (float)e.Time;
            else if (Keyboard[OpenTK.Input.Key.Right])
                angle -= rotation_speed * (float)e.Time;
            if (Keyboard[OpenTK.Input.Key.Up])
                angley += rotation_speed * (float)e.Time;
            else if (Keyboard[OpenTK.Input.Key.Down])
                angley -= rotation_speed * (float)e.Time;
            else if (Keyboard[OpenTK.Input.Key.A])
                scale /= scale_speed;
            else if (Keyboard[OpenTK.Input.Key.Z])
                scale *= scale_speed;
 
            OpenTK.Matrix4 lookat = OpenTK.Matrix4.LookAt(0, 128, 256, 0, 0, 0, 0, 1, 0);
            Vector3 scalev = new Vector3(scale, scale, scale);
            GL.MatrixMode(MatrixMode.Modelview);
            GL.LoadMatrix(ref lookat);
            GL.Scale(scalev);
            GL.Rotate(angle, 0f, 1.0f, 0.0f);
            GL.Rotate(angley, 1f, 0f, 0.0f);
 
            // To draw a VBO:
            // 1) Ensure that the VertexArray client state is enabled.
            // 2) Bind the vertex and element buffer handles.
            // 3) Set up the data pointers (vertex, normal, color) according to your vertex format.
 
            GL.EnableClientState(ArrayCap.VertexArray);
            GL.BindBuffer(BufferTarget.ArrayBuffer, vbo_id);
            GL.VertexPointer(3, VertexPointerType.Float, Vector3.SizeInBytes, new IntPtr(0));
            GL.DrawArrays(BeginMode.Points, 0, vbo_size);
 
            SwapBuffers();
        }
 
 
        static CloudExample b;
        public static void RunCloudExample()
        {
            b = new CloudExample();
            b.Run(30.0, 0.0);
        }
 
        public static void Main()
        {
            Thread t = new Thread(RunCloudExample);
            t.Start();
            while (b == null) { }
            KinectInterface a = new KinectInterface(b);
 
            while (true) { }
        }
    }
}

Comments

Comment viewing options

Select your preferred way to display the comments and click "Save settings" to activate your changes.
the Fiddler's picture

You need to have a "current" GraphicsContext on any thread that is issuing OpenGL commands. Take a look at the GraphicsContext documentation under the [Remarks] and [Methods] sections.