Commits

Cliff Biffle  committed c737ef6

KVMultiDepthMap can separate foreground from background, using a slightly cleverer algorithm than I used before. Originally I had hoped to scale this beyond two occlusion planes, and it almost works at 8 planes...but (1) rendering is bad because the renderer sees only one plane at a time, even if they intersect, and (2) it's slow.

  • Participants
  • Parent commits bc68584

Comments (0)

Files changed (6)

File English.lproj/MainMenu.xib

 									<reference key="NSOnImage" ref="35465992"/>
 									<reference key="NSMixedImage" ref="502551668"/>
 								</object>
-								<object class="NSMenuItem" id="655022095">
+								<object class="NSMenuItem" id="858242009">
 									<reference key="NSMenu" ref="466310130"/>
 									<string key="NSTitle">Overlay Wireframe</string>
 									<string key="NSKeyEquiv">f</string>
 									<reference key="NSOnImage" ref="35465992"/>
 									<reference key="NSMixedImage" ref="502551668"/>
 								</object>
+								<object class="NSMenuItem" id="1038079737">
+									<reference key="NSMenu" ref="466310130"/>
+									<string key="NSTitle">Occlusion-Aware Depth</string>
+									<string key="NSKeyEquiv">3</string>
+									<int key="NSKeyEquivModMask">1572864</int>
+									<int key="NSMnemonicLoc">2147483647</int>
+									<reference key="NSOnImage" ref="35465992"/>
+									<reference key="NSMixedImage" ref="502551668"/>
+								</object>
 								<object class="NSMenuItem" id="264846174">
 									<reference key="NSMenu" ref="466310130"/>
 									<bool key="NSIsDisabled">YES</bool>
 					<object class="IBActionConnection" key="connection">
 						<string key="label">toggleWireframe:</string>
 						<reference key="source" ref="1014"/>
-						<reference key="destination" ref="655022095"/>
+						<reference key="destination" ref="858242009"/>
 					</object>
 					<int key="connectionID">570</int>
 				</object>
+				<object class="IBConnectionRecord">
+					<object class="IBActionConnection" key="connection">
+						<string key="label">selectMultiDepthMap:</string>
+						<reference key="source" ref="1014"/>
+						<reference key="destination" ref="1038079737"/>
+					</object>
+					<int key="connectionID">572</int>
+				</object>
 			</object>
 			<object class="IBMutableOrderedSet" key="objectRecords">
 				<object class="NSArray" key="orderedObjects">
 							<reference ref="138643579"/>
 							<reference ref="63974516"/>
 							<reference ref="1032511002"/>
-							<reference ref="655022095"/>
+							<reference ref="858242009"/>
+							<reference ref="1038079737"/>
 						</object>
 						<reference key="parent" ref="586577488"/>
 					</object>
 					</object>
 					<object class="IBObjectRecord">
 						<int key="objectID">569</int>
-						<reference key="object" ref="655022095"/>
+						<reference key="object" ref="858242009"/>
+						<reference key="parent" ref="466310130"/>
+					</object>
+					<object class="IBObjectRecord">
+						<int key="objectID">571</int>
+						<reference key="object" ref="1038079737"/>
 						<reference key="parent" ref="466310130"/>
 					</object>
 				</object>
 					<string>57.IBPluginDependency</string>
 					<string>57.ImportedFromIB2</string>
 					<string>57.editorWindowContentRectSynchronizationRect</string>
+					<string>571.IBPluginDependency</string>
 					<string>58.IBPluginDependency</string>
 					<string>58.ImportedFromIB2</string>
 					<string>72.IBPluginDependency</string>
 					<string>{74, 862}</string>
 					<string>{{6, 978}, {478, 20}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
-					<string>{{547, 573}, {264, 263}}</string>
+					<string>{{547, 553}, {264, 283}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<string>{{475, 832}, {234, 43}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<integer value="1"/>
 					<string>{{23, 794}, {245, 183}}</string>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
+					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<integer value="1"/>
 					<string>com.apple.InterfaceBuilder.CocoaPlugin</string>
 					<integer value="1"/>
 				</object>
 			</object>
 			<nil key="sourceID"/>
-			<int key="maxID">570</int>
+			<int key="maxID">572</int>
 		</object>
 		<object class="IBClassDescriber" key="IBDocument.Classes">
 			<object class="NSMutableArray" key="referencedPartialClassDescriptions">
 							<string>selectAveragingDepthMap:</string>
 							<string>selectFilledQuadRenderer:</string>
 							<string>selectFilledTriRenderer:</string>
+							<string>selectMultiDepthMap:</string>
 							<string>selectPointCloudRenderer:</string>
 							<string>selectStaticDepthMap:</string>
 							<string>toggleAnaglyph:</string>
 							<string>id</string>
 							<string>id</string>
 							<string>id</string>
+							<string>id</string>
 						</object>
 					</object>
 					<object class="NSMutableDictionary" key="actionInfosByName">
 							<string>selectAveragingDepthMap:</string>
 							<string>selectFilledQuadRenderer:</string>
 							<string>selectFilledTriRenderer:</string>
+							<string>selectMultiDepthMap:</string>
 							<string>selectPointCloudRenderer:</string>
 							<string>selectStaticDepthMap:</string>
 							<string>toggleAnaglyph:</string>
 								<string key="candidateClassName">id</string>
 							</object>
 							<object class="IBActionInfo">
+								<string key="name">selectMultiDepthMap:</string>
+								<string key="candidateClassName">id</string>
+							</object>
+							<object class="IBActionInfo">
 								<string key="name">selectPointCloudRenderer:</string>
 								<string key="candidateClassName">id</string>
 							</object>

File KVDepthView.h

 
 - (IBAction) selectStaticDepthMap: sender;
 - (IBAction) selectAveragingDepthMap: sender;
+- (IBAction) selectMultiDepthMap: sender;
 
 - (IBAction) toggleDepthFieldUpdates: sender;
 - (IBAction) toggleLighting: sender;

File KVDepthView.m

 #import "KVTriStripRenderer.h"
 #import "KVDoubleBufferDepthMap.h"
 #import "KVAveragingDepthMap.h"
+#import "KVMultiDepthMap.h"
 #import "trackball.h"
 #import "utility.h"
 
   self.depthMap = [[[KVAveragingDepthMap alloc] init] autorelease];
 }
 
+- (IBAction) selectMultiDepthMap: sender {
+  self.depthMap = [[[KVMultiDepthMap alloc] init] autorelease];
+}
+
 - (IBAction) toggleDepthFieldUpdates: sender {
   self.frozen = !self.frozen;
   [sender setState: self.frozen? NSOnState : NSOffState];

File KVMultiDepthMap.h

+/*
+ * Copyright 2010 Cliff L. Biffle.  All Rights Reserved.
+ * Use of this source code is governed by the Apache License 2.0,
+ * which can be found in the LICENSE file.
+ */
+
+#import <Cocoa/Cocoa.h>
+#import "types.h"
+#import "KVDepthMap.h"
+
+// Maintains two planes of depth map, so that foreground and
+// background can be separated.
+@interface KVMultiDepthMap : NSObject <KVDepthMap> {
+ @private
+  colors_t colors;
+  depth_t depthCopy;
+  BOOL blinky;
+}
+
+- init;
+
+@end

File KVMultiDepthMap.m

+/*
+ * Copyright 2010 Cliff L. Biffle.  All Rights Reserved.
+ * Use of this source code is governed by the Apache License 2.0,
+ * which can be found in the LICENSE file.
+ */
+
+#import "KVMultiDepthMap.h"
+#import "KVRenderer.h"
+
+@interface KVDepthPlane : NSObject {
+  GLfloat linearizationTable[2048];
+  depth_t cachedDepth;
+  linear_depth_t zs;
+}
+
+@property(retain, nonatomic) NSColor *color;
+
+- (void) updateAndTransformDepth: (depth_t *) depth finalPlane: (BOOL)finalPlane;
+- (void) drawInCurrentOpenGLContextWithRenderer: (id <KVRenderer>)renderer colors: (const colors_t *)colors;
+- (void) buildLinearizationTable;
+@end
+
+@interface KVMultiDepthMap ()
+
+@property(retain, nonatomic) NSMutableArray *planes;
+
+@end
+
+@implementation KVMultiDepthMap
+
+# pragma mark --- Initialization and properties
+
+@synthesize planes;
+
+- init {
+  if ((self = [super init])) {
+    static const int colorCount = 2;
+    NSColor *color[2] = {
+      [NSColor colorWithDeviceRed: 1. green: 1. blue: 1. alpha: 1.],
+      [NSColor colorWithDeviceRed: 1. green: 0. blue: 0. alpha: 1.],
+    };
+    self.planes = [NSMutableArray array];
+    for (NSUInteger i = 0; i < 2; i++) {
+      [self.planes addObject: [[[KVDepthPlane alloc] initWithColor: color[i % colorCount]] autorelease]];
+    }
+  }
+  return self;
+}
+
+#pragma mark --- KVDepthMap implementation
+
+- (void) updateDepth: (const depth_t *)rawDepthSamples {
+  memcpy(&depthCopy, rawDepthSamples, sizeof(depth_t));
+  
+  NSUInteger planeIndex = 0;
+  for (KVDepthPlane *plane in planes) {
+    [plane updateAndTransformDepth: &depthCopy finalPlane: (planeIndex == 1)];
+    planeIndex++;
+  }
+}
+
+- (void) updateColor: (const colors_t *)pixels {
+  memcpy(&colors, pixels, sizeof(colors_t));
+}
+
+- (void) drawInCurrentOpenGLContextWithRenderer: (id <KVRenderer>) renderer {
+  NSUInteger x = 0;
+  for (KVDepthPlane *plane in planes) {
+    [plane drawInCurrentOpenGLContextWithRenderer: renderer colors: &colors];
+    x++;
+  }
+}
+
+@end
+
+@implementation KVDepthPlane
+
+@synthesize color;
+
+#define NO_DEPTH (2047)
+#define ALL_CLEAR (2045)
+#define JUMP (3)
+
+- initWithColor: (NSColor *)newColor {
+  if ((self = [super init])) {
+    self.color = newColor;
+    [self buildLinearizationTable];
+    for (int y = 0; y < 480; y++) {
+      for (int x = 0; x < 640; x++) {
+        cachedDepth.samples[y][x] = 0;
+      }
+    }
+  }
+  return self;
+}
+
+- (void) drawInCurrentOpenGLContextWithRenderer: (id <KVRenderer>)renderer colors: (const colors_t *)colors {
+  renderer.color = color;
+  [renderer drawInCurrentOpenGLContext: &zs colors: colors];
+}
+
+- (void) updateAndTransformDepth: (depth_t *)depth finalPlane: (BOOL) finalPlane {
+  // Remember: we're working with raw camera depth samples here.  Smaller is
+  // closer.
+  for (int y = 0; y < 480; y++) {
+    for (int x = 0; x < 640; x++) {
+      // Ignore all holes in the depth data.
+      if (depth->samples[y][x] == NO_DEPTH) continue;
+      
+      if (depth->samples[y][x] == ALL_CLEAR) {
+        // This ray is not occluded!  I CAN SEE FOREVER
+        cachedDepth.samples[y][x] = 0;
+      } else {
+        if (!finalPlane && depth->samples[y][x] < cachedDepth.samples[y][x] - JUMP) {
+          // We are occluded.  Do not change the cache.
+          // Leave it for a closer plane.
+        } else {
+          // Update us.
+          cachedDepth.samples[y][x] = depth->samples[y][x];
+          // Knock a hole for later planes.
+          depth->samples[y][x] = ALL_CLEAR;
+        }
+      }
+      zs.z[y][x] = linearizationTable[cachedDepth.samples[y][x]];
+    }
+  }
+}
+
+// The Kinect produces 11-bit depth samples.  Z-resolution decreases
+// with distance, following a curve that looks suspiciously like the
+// one used for perspective projection.  The ROS folks at CCNY derived
+// the equation below, for converting Kinect depth samples to meters.
+// This method memoizes the function over the entire 2048-element
+// domain.  This implies replacing a couple of floating-point constant
+// loads, an addition, and a division with an indirection to very hot
+// memory.  On my machine this eliminates 60% of time spent calculating
+// linear depths.
+- (void) buildLinearizationTable {
+  for (uint16_t depth = 0; depth < 2048; depth++) {
+    linearizationTable[depth] = -325.616F / ((GLfloat) depth + -1084.61F);
+    
+    // Simplification: when the Kinect reports "no data" for a point
+    // (typically because the IR projector is occluded) the result is
+    // a "very close" reading of nearly zero meters.  All my renderers
+    // filter out such readings, so we coerce it to exactly 0.F here
+    // to simplify that.
+    if (linearizationTable[depth] < 0.1F) {
+      linearizationTable[depth] = 0.F;
+    }
+  }
+  linearizationTable[0] = 0.F;
+}
+
+@end

File KinectViewer.xcodeproj/project.pbxproj

 		C70E66BB129C906C004A44B3 /* libdriver.a in Frameworks */ = {isa = PBXBuildFile; fileRef = C70E669F129C9005004A44B3 /* libdriver.a */; };
 		C70E66CA129C9121004A44B3 /* libusb-1.0.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = C70E66C9129C9121004A44B3 /* libusb-1.0.dylib */; };
 		C758017712A3522700391C33 /* KVWireRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = C758017612A3522700391C33 /* KVWireRenderer.m */; };
+		C75801FB12A3590000391C33 /* KVMultiDepthMap.m in Sources */ = {isa = PBXBuildFile; fileRef = C75801FA12A3590000391C33 /* KVMultiDepthMap.m */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
 		C70E66C9129C9121004A44B3 /* libusb-1.0.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = "libusb-1.0.dylib"; path = "usr/local/lib/libusb-1.0.dylib"; sourceTree = SDKROOT; };
 		C758017512A3522700391C33 /* KVWireRenderer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KVWireRenderer.h; sourceTree = "<group>"; };
 		C758017612A3522700391C33 /* KVWireRenderer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KVWireRenderer.m; sourceTree = "<group>"; };
+		C75801F912A3590000391C33 /* KVMultiDepthMap.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KVMultiDepthMap.h; sourceTree = "<group>"; };
+		C75801FA12A3590000391C33 /* KVMultiDepthMap.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KVMultiDepthMap.m; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
 				C708F55812A0728800656901 /* KVDoubleBufferDepthMap.m */,
 				C708F5E212A0779500656901 /* KVAveragingDepthMap.h */,
 				C708F5E312A0779500656901 /* KVAveragingDepthMap.m */,
+				C75801F912A3590000391C33 /* KVMultiDepthMap.h */,
+				C75801FA12A3590000391C33 /* KVMultiDepthMap.m */,
 			);
 			name = Model;
 			sourceTree = "<group>";
 				C708F55912A0728800656901 /* KVDoubleBufferDepthMap.m in Sources */,
 				C708F5E412A0779500656901 /* KVAveragingDepthMap.m in Sources */,
 				C758017712A3522700391C33 /* KVWireRenderer.m in Sources */,
+				C75801FB12A3590000391C33 /* KVMultiDepthMap.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};