Commits

Cliff Biffle committed 339dea4

Pixel color data can now be mapped onto the depth scene. The defaults are reasonably accurate for my Kinect, but keyboard shortcuts are available for tweaking the matrix.

  • Participants
  • Parent commits bf50f78

Comments (0)

Files changed (13)

File English.lproj/MainMenu.xib

 		</object>
 		<object class="NSMutableArray" key="IBDocument.EditedObjectIDs">
 			<bool key="EncodedWithXMLCoder">YES</bool>
+			<integer value="533"/>
 			<integer value="296"/>
-			<integer value="533"/>
 		</object>
 		<object class="NSArray" key="IBDocument.PluginDependencies">
 			<bool key="EncodedWithXMLCoder">YES</bool>
 									<reference key="NSOnImage" ref="35465992"/>
 									<reference key="NSMixedImage" ref="502551668"/>
 								</object>
+								<object class="NSMenuItem" id="1032511002">
+									<reference key="NSMenu" ref="466310130"/>
+									<string key="NSTitle">Texture From Camera</string>
+									<string key="NSKeyEquiv">t</string>
+									<int key="NSKeyEquivModMask">1048576</int>
+									<int key="NSMnemonicLoc">2147483647</int>
+									<reference key="NSOnImage" ref="35465992"/>
+									<reference key="NSMixedImage" ref="502551668"/>
+								</object>
 								<object class="NSMenuItem" id="851891320">
 									<reference key="NSMenu" ref="466310130"/>
 									<bool key="NSIsDisabled">YES</bool>
 					</object>
 					<int key="connectionID">566</int>
 				</object>
+				<object class="IBConnectionRecord">
+					<object class="IBActionConnection" key="connection">
+						<string key="label">toggleTextures:</string>
+						<reference key="source" ref="1014"/>
+						<reference key="destination" ref="1032511002"/>
+					</object>
+					<int key="connectionID">568</int>
+				</object>
 			</object>
 			<object class="IBMutableOrderedSet" key="objectRecords">
 				<object class="NSArray" key="orderedObjects">
 							<reference ref="264846174"/>
 							<reference ref="138643579"/>
 							<reference ref="63974516"/>
+							<reference ref="1032511002"/>
 						</object>
 						<reference key="parent" ref="586577488"/>
 					</object>
 						<reference key="object" ref="63974516"/>
 						<reference key="parent" ref="466310130"/>
 					</object>
+					<object class="IBObjectRecord">
+						<int key="objectID">567</int>
+						<reference key="object" ref="1032511002"/>
+						<reference key="parent" ref="466310130"/>
+					</object>
 				</object>
 			</object>
 			<object class="NSMutableDictionary" key="flattenedProperties">
 					<string>562.IBPluginDependency</string>
 					<string>563.IBPluginDependency</string>
 					<string>564.IBPluginDependency</string>
+					<string>567.IBPluginDependency</string>
 					<string>57.IBEditorWindowLastContentRect</string>
 					<string>57.IBPluginDependency</string>
 					<string>57.ImportedFromIB2</string>
 					<string>{74, 862}</string>
 					<string>{{6, 978}, {478, 20}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
-					<string>{{547, 613}, {264, 223}}</string>
+					<string>{{547, 593}, {264, 243}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>{{475, 832}, {234, 43}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>{{392, 653}, {223, 183}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<integer value="1"/>
 				</object>
 			</object>
 			<nil key="sourceID"/>
-			<int key="maxID">566</int>
+			<int key="maxID">568</int>
 		</object>
 		<object class="IBClassDescriber" key="IBDocument.Classes">
 			<object class="NSMutableArray" key="referencedPartialClassDescriptions">
 							<string>toggleDepthFieldUpdates:</string>
 							<string>toggleLighting:</string>
 							<string>toggleRecording:</string>
+							<string>toggleTextures:</string>
 						</object>
 						<object class="NSMutableArray" key="dict.values">
 							<bool key="EncodedWithXMLCoder">YES</bool>
 							<string>id</string>
 							<string>id</string>
 							<string>id</string>
+							<string>id</string>
 						</object>
 					</object>
 					<object class="NSMutableDictionary" key="actionInfosByName">
 							<string>toggleDepthFieldUpdates:</string>
 							<string>toggleLighting:</string>
 							<string>toggleRecording:</string>
+							<string>toggleTextures:</string>
 						</object>
 						<object class="NSMutableArray" key="dict.values">
 							<bool key="EncodedWithXMLCoder">YES</bool>
 								<string key="name">toggleRecording:</string>
 								<string key="candidateClassName">id</string>
 							</object>
+							<object class="IBActionInfo">
+								<string key="name">toggleTextures:</string>
+								<string key="candidateClassName">id</string>
+							</object>
 						</object>
 					</object>
 					<object class="IBClassDescriptionSource" key="sourceIdentifier">

File KVAbstractRenderer.h

 // These properties are synthesized for the benefit of
 // subclasses.
 @property(assign, nonatomic) BOOL useLighting;
+@property(assign, nonatomic) BOOL textured;
 @property(retain, nonatomic) NSColor *color;
+@property(assign, nonatomic) projective_transform_t colorTransform;
 
 // Sets the properties to sensible defaults.
 - init;

File KVAbstractRenderer.m

 
 @synthesize useLighting;
 @synthesize color;
+@synthesize colorTransform;
+@synthesize textured;
 
 - (void) drawInCurrentOpenGLContext: (const linear_depth_t *)zs colors: (const colors_t *)colors {
   [self doesNotRecognizeSelector: _cmd];  // subclass responsibility
   // We assume square pixels, and derive the Y FOV from the X.
   static const float fovXDegrees = 57.F;  // Pretty close to actual?
   
-  static const float fovX = fovXDegrees / 180.F * (float) M_PI;
+  const float fovX = fovXDegrees / 180.F * (float) M_PI;
   for (int y = 0; y < 240; y++) {
     float theta = (y / 320.F) * fovX/2.F;
     for (int x = 0; x < 320; x++) {
   }
 }
 
-@end
+@end

File KVAveragingDepthMap.h

   GLfloat linearizationTable[2048];
   linear_depth_t zs;
   uint32_t sampleCount[480][640];
+  
+  colors_t colors;
 }
 
 @end

File KVAveragingDepthMap.m

 }
 
 - (void) updateColor: (const colors_t *)pixels {
-  // Nothing to do yet.
+  memcpy(&colors, pixels, sizeof(colors_t));
 }
 
 - (void) drawInCurrentOpenGLContextWithRenderer: (id <KVRenderer>) renderer {
-  [renderer drawInCurrentOpenGLContext: &zs colors: nil];
+  [renderer drawInCurrentOpenGLContext: &zs colors: &colors];
 }
 
 #pragma mark --- Internals

File KVDepthView.h

 @property(assign, nonatomic) BOOL anaglyph;
 @property(assign, nonatomic) BOOL lightsOn;
 @property(nonatomic) BOOL recording;
+@property(assign, nonatomic) projective_transform_t colorTransform;
 
 - (void) depthFieldUpdated: (const depth_t *)buffer from: (KFKinect *)kinect;
 - (void) colorFieldUpdated: (const colors_t *)buffer from: (KFKinect *)kinect;
 - (IBAction) toggleLighting: sender;
 - (IBAction) toggleRecording: sender;
 - (IBAction) toggleAnaglyph: sender;
+- (IBAction) toggleTextures: sender;
 @end

File KVDepthView.m

 - (void) resizeGL;
 - (void) updateProjection;
 - (void) updateModelView;
+- (void) updateColorTransform;
 @end
 
 @implementation KVDepthView
 
 @synthesize lightsOn;
 @synthesize depthMap;
+@synthesize colorTransform;
+@synthesize renderer;
 
-@synthesize renderer;
+- (void) setColorTransform: (projective_transform_t)newTransform {
+  colorTransform = newTransform;
+  [self updateColorTransform];
+}
+
 - (void) setRenderer: (NSObject <KVRenderer> *)newRenderer {
   NSObject <KVRenderer> *currentRenderer = renderer;
   renderer = [newRenderer retain];
   [currentRenderer release];
   
   renderer.useLighting = lightsOn;
+  renderer.colorTransform = colorTransform;
   [self setNeedsDisplay: YES];
 }
 
 - (void) awakeFromNib {
   deviceWhite = [[NSColor colorWithDeviceRed: 1.F green: 1.F blue: 1.F alpha: 1.F] retain];
   devicePurplish = [[NSColor colorWithDeviceRed: 1.F green: 0.5F blue: 1.F alpha: 1.F] retain];
+  
+  colorTransform = (projective_transform_t) {
+    .offset = { 14, 40, 0 },
+    .scale = { 0.92F, 0.92F },
+  };
+  
   [self selectPointCloudRenderer: nil];
   [self selectStaticDepthMap: nil];
 }
 				spinX -= 10;
 				[self setNeedsDisplay: YES];
 				break;
+        
+      // Camera origin control
+      case 'i':
+        colorTransform.offset.y -= 1;
+        [self updateColorTransform];
+        break;
+        
+      case 'k':
+        colorTransform.offset.y += 1;
+        [self updateColorTransform];
+        break;
+
+      case 'j':
+        colorTransform.offset.x -= 1;
+        [self updateColorTransform];
+        break;
+        
+      case 'l':
+        colorTransform.offset.x += 1;
+        [self updateColorTransform];
+        break;
+        
+      case 'h':
+        colorTransform.offset.z -= 1.F/100.F;
+        [self updateColorTransform];
+        break;
+
+      case 'y':
+        colorTransform.offset.z += 1.F/100.F;
+        [self updateColorTransform];
+        break;
+        
+      // Scale control
+      case 'n':
+        colorTransform.scale.x -= 0.01F;
+        [self updateColorTransform];
+        break;
+        
+      case 'm':
+        colorTransform.scale.x += 0.01F;
+        [self updateColorTransform];
+        break;
+        
+      case ',':
+        colorTransform.scale.y -= 0.01F;
+        [self updateColorTransform];
+        break;
+        
+      case '.':
+        colorTransform.scale.y += 0.01F;
+        [self updateColorTransform];
+        break;
 		}
 	}
 }
 }
 
 - (void) colorFieldUpdated: (const colors_t *)buffer from: (KFKinect *)kinect {
-  // Nothing to do yet.
+  if (!self.frozen) {
+    [depthMap updateColor: buffer];
+    [self setNeedsDisplay: YES];
+  }
 }
 
 #pragma mark --- Actions ---
   [self setNeedsDisplay: YES];
 }
 
+- (IBAction) toggleTextures: sender {
+  renderer.textured = !renderer.textured;
+  [self setNeedsDisplay: YES];
+  [sender setState: renderer.textured? NSOnState : NSOffState];
+}
+
 @end
 
 @implementation KVDepthView (Private)
   glLightfv(GL_LIGHT0, GL_POSITION, lightPos);
 }
 
+- (void) updateColorTransform {
+  renderer.colorTransform = colorTransform;
+  [self setNeedsDisplay: YES];
+}
 @end

File KVPointCloudRenderer.m

                              colors: (const colors_t *)colors {
   // Avoid the property message send for a significant perf win.
   // (Sending a message doubles our render time.)
-  BOOL lights = self.useLighting;
+  BOOL textured = self.textured;
+  const projective_transform_t xform = self.colorTransform;
   KVAbstractRendererProjector projector =
       (KVAbstractRendererProjector) [self methodForSelector: @selector(projectInto:x:y:z:)];
   
       GLfloat depth = zs->z[y][x];
       if (depth == 0.F) continue;
       
-      projector(self, @selector(projectInto:x:y:z:), &coords, x, y, depth);
-      
-      if (!lights) {
-        GLfloat c = 1.F - (coords.z / 9.F);
-        vec3f_t falseColor = vector_scale(baseColor, c);
+      if (textured) {
+        int colorX = (int) (x * xform.scale.x + xform.offset.x);
+        int colorY = (int) (y * xform.scale.y + xform.offset.y);
+        const rgb_t *pixelColor = &colors->pixels[colorY][colorX];
+        
+        glColor3ubv(&pixelColor->r);
+      } else {
+        vec3f_t falseColor = vector_scale(baseColor, 1.F - depth/10.F);
         glColor3fv(&falseColor.x);
       }
       
+      projector(self, @selector(projectInto:x:y:z:), &coords, x, y, depth);
+      
       glVertex3fv(&coords.x);
     }
   }

File KVQuadRenderer.m

 @implementation KVQuadRenderer
 
 - (void) drawInCurrentOpenGLContext: (const linear_depth_t *)zs
-                             colors: (const colors_t *)colors_unused {
+                             colors: (const colors_t *)colors {
   // Avoid message send for large performance win.
-  BOOL lightsOn = self.useLighting;
+  BOOL textured = self.textured;
+  projective_transform_t xform = self.colorTransform;
   KVAbstractRendererProjector projector =
       (KVAbstractRendererProjector) [self methodForSelector: @selector(projectInto:x:y:z:)];
 
       
       BOOL skip = NO;
       vec3f_t coords[4];
-      GLfloat colors[4];
+      GLfloat vcolors[4];
       GLfloat minDepth = 1000.F, maxDepth = -1000.F;
       for (int i = 0; i < 4; i++) {
         GLfloat d = zs->z[ys[i]][xs[i]];
         maxDepth = fmaxf(maxDepth, d);
         
         projector(self, @selector(projectInto:x:y:z:), &coords[i], xs[i], ys[i], d);
-        colors[i] = 1.F - (d / 9.F);
+        
+        vcolors[i] = 1.F - (d / 9.F);
       }
       
       if (fabsf(minDepth - maxDepth) > 0.4F) skip = YES;
       glNormal3fv(&normal.x);
       
       for (int i = 0; i < 4; i++) {
-        if (!lightsOn) {
-          vec3f_t falseColor = vector_scale(baseColor, colors[i]);
+        if (textured) {
+          int colorX = (int) (xs[i] * xform.scale.x + xform.offset.x);
+          int colorY = (int) (ys[i] * xform.scale.y + xform.offset.y);
+          const rgb_t *pixelColor = &colors->pixels[colorY][colorX];
+          
+          glColor3ubv(&pixelColor->r);
+        } else {
+          vec3f_t falseColor = vector_scale(baseColor, vcolors[i]);
           glColor3fv(&falseColor.x);
         }
         glVertex3fv(&coords[i].x);

File KVRenderer.h

 // -[drawInCurrentOpenGLContext].
 @property(assign, nonatomic) BOOL useLighting;
 
+// Whether the renderer maps color data from the camera
+// onto the 3D scene.
+@property(assign, nonatomic) BOOL textured;
+
 // Selects the base color for the rendering.  Note that
 // if lighting is disabled, the renderer may transform
 // or even ignore this color.
 @property(retain, nonatomic) NSColor *color;
 
+// Sets the projection for color data onto depth data.
+@property(assign, nonatomic) projective_transform_t colorTransform;
+
 // Applies the renderer's internal state to the currently
 // active OpenGL context.  This method may be called many
 // times for a single call to -[update:].
 - (void) drawInCurrentOpenGLContext: (const linear_depth_t *)zs colors: (const colors_t *)colors;
 
-@end
+@end

File KVSinglePlaneDepthMap.h

  @private
   GLfloat linearizationTable[2048];
   linear_depth_t zs;
+  colors_t colors;
 }
 
 @end

File KVSinglePlaneDepthMap.m

 }
 
 - (void) updateColor: (const colors_t *)pixels {
-  // Nothing to do yet.
+  memcpy(&colors, pixels, sizeof(colors_t));
 }
 
 - (void) drawInCurrentOpenGLContextWithRenderer: (id <KVRenderer>) renderer {
-  [renderer drawInCurrentOpenGLContext: &zs colors: nil];
+  [renderer drawInCurrentOpenGLContext: &zs colors: &colors];
 }
 
 #pragma mark --- Internals

File KVTriStripRenderer.m

   // 1 3 5   ...and so on.
   
   // Avoid message send for a large performance win.
-  BOOL lightsOn = self.useLighting;
+  BOOL textured = self.textured;
+  const projective_transform_t xform = self.colorTransform;
+
   KVAbstractRendererProjector projector =
       (KVAbstractRendererProjector) [self methodForSelector: @selector(projectInto:x:y:z:)];
 
         compute_normal(&prev[0], &prev[1], &coord, &normal);
         glNormal3fv(&normal.x);
 
-        if (!lightsOn) {
-          GLfloat color = 1.F - (d / 9.F);
-          vec3f_t falseColor = vector_scale(baseColor, color);
+        if (textured) {
+          int colorX = (int) (x * xform.scale.x + xform.offset.x);
+          int colorY = (int) (y * xform.scale.y + xform.offset.y);
+          const rgb_t *pixelColor = &colors->pixels[colorY][colorX];
+          
+          glColor3ubv(&pixelColor->r);
+        } else {
+          vec3f_t falseColor = vector_scale(baseColor, 1.F - d/10.F);
           glColor3fv(&falseColor.x);
         }
         glVertex3fv(&coord.x);