Commits

Cliff Biffle  committed 82cb6cd

Un-projection is now a single matrix multiply on the GPU. Having re-learned matrix math, which I hadn't used since high school, I was able to rearrange all the linearization and projection calculations so that I could describe the process as a single multiply in homogeneous space. Unfortunately, this disables normal calculation for now -- and thus lighting. Will fix all this shortly.

  • Participants
  • Parent commits 32563c5
  • Branches shaders

Comments (0)

Files changed (5)

File KVDepthView.m

 - (void) resizeGL;
 - (void) updateProjectionWithOffset: (float) offset;
 - (void) updateModelViewWithOffset: (float) offset;
+- (void) updateTextureMatrix;
 - (GLuint) loadShaderOfType: (GLenum) type fromFiles: (NSArray *) names;
 
 @property(assign, nonatomic) BOOL updateQueued;
 #pragma mark --- GL Stuff ---
 
 - (void) prepareOpenGL {
-  NSArray *vertexShaderFiles = [NSArray arrayWithObjects: @"kinect_project", @"lighting", @"main", nil];
+  NSArray *vertexShaderFiles = [NSArray arrayWithObjects: @"lighting", @"main", nil];
   GLint vertexShader = [self loadShaderOfType: GL_VERTEX_SHADER
                                     fromFiles: vertexShaderFiles];
   
   glRotatef(spinX, 1, 0, 0);
 }
 
+- (void) updateTextureMatrix {
+  [[self openGLContext] makeCurrentContext];
+  
+  glMatrixMode(GL_TEXTURE);
+  
+  // Since we pass the depth field to the card in Texture 0,
+  // why not pass the depth inverse-projection matrix in
+  // the corresponding matrix?
+  glActiveTexture(GL_TEXTURE0);
+  
+  GLdouble fovDegrees = 57.;
+  GLdouble halfFovRadians = fovDegrees / 2. / 180. * M_PI;
+  // The width of the projected depth field at Z=1
+  GLdouble widthAt1 = tan(halfFovRadians);
+  // Ratio of screen height : width.
+  GLdouble aspectRatio = 3. / 4.;
+  
+  // Linearization constants per CCNY ROS folks:
+  //  Z = ca / (depth + cb)
+  GLdouble ca = -325.616, cb = -1084.61;
+
+  // This matrix is the product of the following pipeline:
+  // - Convert texel-space (aka clip-space, 0 ≤ x,y ≤ 1)
+  //   into perspective-space (-1 ≤ x,y ≤ 1).
+  // - Convert camera's depth samples (converted to floating
+  //   point by OpenGL) into 1/Z.
+  // - Convert perspective-space into view-space by
+  //   multiplying xy by tan(fov/2).
+  // - Correct aspect ratio in Y.
+  // - Exchange Z and W to correct inversion of Z.
+  GLdouble matrix[16] = {
+    -2 * widthAt1, 0, 0, 0,
+    0, -2 * widthAt1 * aspectRatio, 0, 0,
+    0, 0, 0, 65535. / ca,
+    widthAt1, widthAt1 * aspectRatio, 1, cb / ca,
+  };
+  glLoadMatrixd(matrix);
+}
+
 #define INTEROCULAR_DISTANCE (0.0635F/2)
 - (void) drawRect: (NSRect)dirtyRect {
   [self resizeGL];
+  [self updateTextureMatrix];
   
   glUniform1i(textureEnableUniform, textured);
     

File KVRenderer.m

   int vidx = 0;
   for (int y = 0; y < 480; y++) {
     for (int x = 0; x < 640; x++, vidx++) {
-      rays[vidx].x = -(float) (x - 320) / 320.f;
-      rays[vidx].y = -(float) (y - 240) / 320.f;
+      rays[vidx].x = (x / 640.);
+      rays[vidx].y = (y / 480.);
+      rays[vidx].z = 4;
     }
   }
 }

File KinectViewer.xcodeproj/project.pbxproj

 		C7759B4612A8B2D3003479EF /* KVKinectHardware.m in Sources */ = {isa = PBXBuildFile; fileRef = C7759B4512A8B2D3003479EF /* KVKinectHardware.m */; };
 		C7759C0712A97873003479EF /* KVDepthRecorder.m in Sources */ = {isa = PBXBuildFile; fileRef = C7759C0612A97873003479EF /* KVDepthRecorder.m */; };
 		C7759C1012A97A3B003479EF /* KVDepthPlayer.m in Sources */ = {isa = PBXBuildFile; fileRef = C7759C0F12A97A3B003479EF /* KVDepthPlayer.m */; };
-		C775A0F812AC111E003479EF /* kinect_project.vs in Resources */ = {isa = PBXBuildFile; fileRef = C775A0D512AC0942003479EF /* kinect_project.vs */; };
 		C775A13312AC18EB003479EF /* main.fs in Resources */ = {isa = PBXBuildFile; fileRef = C775A11F12AC1749003479EF /* main.fs */; };
 		C775A1AE12AC1DF4003479EF /* main.vs in Resources */ = {isa = PBXBuildFile; fileRef = C775A1AD12AC1DF4003479EF /* main.vs */; };
 		C775A1CD12AC2022003479EF /* lighting.vs in Resources */ = {isa = PBXBuildFile; fileRef = C775A1CC12AC2022003479EF /* lighting.vs */; };
 		C7759C0612A97873003479EF /* KVDepthRecorder.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KVDepthRecorder.m; sourceTree = "<group>"; };
 		C7759C0E12A97A3B003479EF /* KVDepthPlayer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KVDepthPlayer.h; sourceTree = "<group>"; };
 		C7759C0F12A97A3B003479EF /* KVDepthPlayer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KVDepthPlayer.m; sourceTree = "<group>"; };
-		C775A0D512AC0942003479EF /* kinect_project.vs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = kinect_project.vs; sourceTree = "<group>"; };
 		C775A11F12AC1749003479EF /* main.fs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = main.fs; sourceTree = "<group>"; };
 		C775A1AD12AC1DF4003479EF /* main.vs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = main.vs; sourceTree = "<group>"; };
 		C775A1CC12AC2022003479EF /* lighting.vs */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = lighting.vs; sourceTree = "<group>"; };
 		C775A0D312AC08D4003479EF /* Shaders */ = {
 			isa = PBXGroup;
 			children = (
-				C775A0D512AC0942003479EF /* kinect_project.vs */,
 				C775A1CC12AC2022003479EF /* lighting.vs */,
 				C775A1AD12AC1DF4003479EF /* main.vs */,
 				C775A11F12AC1749003479EF /* main.fs */,
 				C775A1AE12AC1DF4003479EF /* main.vs in Resources */,
 				C775A13312AC18EB003479EF /* main.fs in Resources */,
 				C775A1CD12AC2022003479EF /* lighting.vs in Resources */,
-				C775A0F812AC111E003479EF /* kinect_project.vs in Resources */,
 				8D11072B0486CEB800E47090 /* InfoPlist.strings in Resources */,
 				1DDD58160DA1D0A300B32029 /* MainMenu.xib in Resources */,
 			);

File kinect_project.vs

-/*
- * Copyright 2010 Cliff L. Biffle.  All Rights Reserved.
- * Use of this source code is governed by the Apache License 2.0,
- * which can be found in the LICENSE file.
- */
- 
-/*
- * Kinect Projection Routines
- *
- * Since the Kinect is, fundamentally, a camera, its data has already
- * undergone perspective projection.  Closer objects seem relatively
- * larger, and distant objects seem small.  If we left the POV fixed,
- * we could simply display the Kinect's raw data using an orthographic
- * projection -- but what's the fun in that?
- *
- * This shader maps from one perspective space (the Kinect's) into
- * another (GL's modelview/projection space).  We receive unprocessed
- * vertices from the CPU, which are structured like this:
- *   X and Y: -1.0 .. 1.0, normalized pixel location in the input.
- *   Z: the Kinect's raw 11-bit non-linear depth sample.
- *
- * (Note: you might expect the Kinect's raw depth samples to measure
- * distance from the Kinect -- i.e. the length of the ray cast from
- * the lens to the object.  Not so!  The Kinect pre-processes depths,
- * and by the time we receive them, they are the distance from the
- * object to the nearest point on the camera's XY plane.)
- *
- * Our job is to:
- *  1. Linearize and scale the depth.  In this case, we convert it
- *     such that 1 GL Unit = 1 meter.  Because depths are already
- *     measured from the XY plane, we don't have to do any raycasting.
- *  2. Reverse the perspective warping of the XY data, using what we
- *     know about the Kinect's optics.
- */
-
-// The Kinect's lens appears to have a 57-degree horizontal FOV.
-const float kinectFovXDegrees = 57.0;
-
-// Multiplying this by our normalized XY gives us ray angles.
-const float halfFov = (kinectFovXDegrees / 180.0 * 3.14159265358) / 2.0;
-
-/*
- * Reverses the Kinect's perspective projection.  The input point must
- * be in our "raw" format, that is:
- *   X and Y: -1.0 .. 1.0, normalized pixel location in the input.
- *   Z: the Kinect's raw 11-bit non-linear depth sample.
- *
- * The result is the point's original location in orthonormal space,
- * with 1 GL unit per meter.
- */
-vec3 kinect_unproject(vec3 point) {
-  // Linearization equation derived by the ROS folks at CCNY.
-  float linearZ = -325.616 / (point.z + -1084.61);
-  
-  vec2 unprojected = linearZ * tan(halfFov) * point.xy;
-  
-  return vec3(unprojected, linearZ);
-}
-
-/*
- * In the incoming data, normals have been computed in Kinect-space.
- * This function converts them into GL space.  I am not spectacularly
- * happy with the approach and may try again later.
- *
- * Note that the result may not, itself, be normalized.
- */
-vec3 kinect_unproject_normal(vec3 normal, vec3 cameraOrigin, vec3 glOrigin) {
-  return kinect_unproject(normal + cameraOrigin) - glOrigin;
-}
 uniform bool useTextureFromCamera;
 
 /*
+ * Kinect Projection Correction
+ *
+ * Since the Kinect is, fundamentally, a camera, its data has already
+ * undergone perspective projection.  Closer objects seem relatively
+ * larger, and distant objects seem small.  If we left the POV fixed,
+ * we could simply display the Kinect's raw data using an orthographic
+ * projection -- but what's the fun in that?
+ *
+ * The Kinect's depth output looks a lot like a 16-bit grayscale image.
+ * Usefully, the depth samples are a function of 1/Z, and we need 1/Z
+ * during the perspective un-projection.
+ *
+ * The CPU provides the Kinect's depth data in a 16-bit luminance
+ * texture, 640x480 pixels in size.  To help us perform the texture
+ * lookup here, it also provides a canned array of 640x480 vertices,
+ * each of which has...
+ *  0 ≤ X, Y ≤ 1.0 - each vertex in the XY plane maps to a single pixel,
+ *     in texel coordinates.
+ *  Z is garbage.
+ *
+ * Finally, the CPU uses the texture unit's matrix to pass the
+ * un-projection matrix.
+ */
+vec4 kinect_unproject(sampler2D texture, vec4 point) {
+  vec4 sample = texture2D(texture, point.xy);
+  vec4 kinectPoint = vec4(gl_Vertex.xy, sample.g, 1.);
+  return gl_TextureMatrix[0] * kinectPoint;
+}
+
+/*
  * Vertex Shader entry point.
  */
 void main() {
-  // Load our depth from the camera's depth texture.
-  vec2 imageCoord = gl_Vertex.xy * vec2(1., 4./3.) * -0.5 + 0.5;
-  vec4 sample = texture2D(depthTex, imageCoord);
-  vec3 kinectPoint = vec3(gl_Vertex.xy, sample.g * 65536.);
-  
-  // Undo the Kinect's perspective projection, leaving the homogeneous term.
-  vec4 vertex = vec4(kinect_unproject(kinectPoint), gl_Vertex.w);
+  vec4 vertex = kinect_unproject(depthTex, gl_Vertex);
   
   // Project the point using GL's perspective settings.
   gl_Position = gl_ModelViewProjectionMatrix * vertex;
 
-  // The normal was computed in Kinect-space.  Unproject it too.
-  vec3 normal = kinect_unproject_normal(normalize(gl_Normal), kinectPoint, vertex.xyz);
-  
   if (vertex.z <= 0.) {
     gl_FrontColor = vec4(0,0,0,0);
   } else {
-    gl_FrontColor = light_vertex(normal);
+    gl_FrontColor = vec4(1,1,1,1);
   }
 }