Commits

Evgeny Podjachev  committed 0f1d587

Intermediate commit.

  • Participants
  • Parent commits 2e31702

Comments (0)

Files changed (2)

File tests/benchmark.py

 else:
 	caseMod = lambda x: x
 
-def scan_dir(path, ignoreList, cache):
-	path = os.abspath(path)
+def in_ignore_list(f):
 	ignoreList = map(caseMod, ignoreList)
-	def in_ignore_list(f):
-		for i in ignoreList:
-			if fnmatch(caseMod(f), i):
-				return True
-		return False
+	for i in ignoreList:
+		if fnmatch(caseMod(f), i):
+			return True
+	return False
 
-	cachedPaths = cache.getCachedPaths(path)
+def scan_dir(path, checkOnIgnores, cache):
+	path = os.abspath(path)
+	pathLen = len(path)
 
 	fileList = []
+	cachedPaths = cache.getCachedPaths(path)
+	if len(cachedPaths):
+		if cachedPaths[0][0][1] == getModTime(fp):
+			for i in cachedPaths:
+				for j in i[1]:
+					fullPath.append(join(path, i[0][0], j))
+			return [x]	
+
 	addToCache = []
 	for root, dirs, files in walk(path):
-		fileList += [join(root, f) for f in files if not in_ignore_list(f)]
-		
-		toRemove = filter(in_ignore_list, dirs)
-		for j in toRemove:
-			dirs.remove(j)
-
 		toRemove = []
 		for i in dirs:
-			fp = join(root, dirs)
+			fp = join(root, i)
 			fpmt = getModTime(fp)
-			
+			fp = fp.encode("utf-8")
+			fpl = len(fp)
+		
+			inCache = False
 			for j in cachedPaths:
 				if fpmt == j[1] and fp == j[0]:
 					toRemove.append(i)
-					fileList += [join(root, f) for f in files if not in_ignore_list(f)]
+
+					# add cached paths
+					for k in cachedPaths:
+						if k[0].find(fp) == fpl:
+							fileList.append((k[0], k[3]))
+
+					inCache = True
+
 					break
-			if (fp, getModTime(fp)) in cachedPaths:
-				toRemove.append(i)
-				fileList += [join(root, f) for f in files if not in_ignore_list(f)]
+
+			if not inCache:
+				
+				fileList.append((root[pathLen : ].encode("utf-8"), [x.encode("utf-8") for x in files]))
+					fileList += [join(root, f) for f in files if not in_ignore_list(f)]
+
+		for j in toRemove:
+			dirs.remove(j)
 
 
 	n = len(path.encode("utf-8"))
 
-	fileList = map(lambda x: x.encode("utf-8"), fileList)
 	fileList = map(lambda x: (caseMod(x[n:]), x), fileList)
 
 	return fileList

File tests/paths_cache.py

 import os
 from os.path import getctime, getmtime
+from collections import defaultdict
 
 import sqlite3
 
 		self.caseSensitivePaths = caseSensitivePaths
 
 		if self.caseSensitivePaths:
-			self.caseSelectAddon = ''
+			self.caseSelectAddon = u''
 		else:
-			self.caseSelectAddon = 'COLLATE NOCASE'
+			self.caseSelectAddon = u'COLLATE NOCASE'
 
 		# Check if database file exists and creates all needed paths.
 		createTable = False
 			self.conn = conn
 			if createTable:
 				c = conn.cursor()
-				c.executescript("""
+				c.executescript(u"""
 					CREATE TABLE roots (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT, modtime INTEGER);
 					CREATE TABLE paths (id INTEGER PRIMARY KEY AUTOINCREMENT, root_id INTEGER SECONDARY KEY, path TEXT, modtime INTEGER);
 					CREATE TABLE files (path_id INTEGER PRIMARY KEY, file_name TEXT);
 				conn.commit()
 				c.close()
 
+	# Returns unicoded list of files
+	# ((path, mod_time), (files))
 	def getCachedPaths(self, root):
 		if self.conn == None:
 			return []
 
 		c = self.conn.cursor()
 
-		# get parent id
-		c.execute('SELECT id FROM roots WHERE path = "%s" %s' % (path, self.caseSelectAddon))
-		res = c.fetchone()
-		if res == None:
-			c.close()
-			return []
-		
-		root_id = res[0]
-
 		# get cached paths
-		res = []
-		c.execute('SELECT id, path, modtime FROM paths WHERE root_id = %s' % (root_id))
+		c.execute(u'''
+			SELECT paths.path, paths.modtime, files.file_name FROM roots 
+			JOIN paths ON (roots.id == paths.root_id)
+			JOIN files ON (paths.id == files.path_id)
+			WHERE path = "%s" %s
+		''' % (path, self.caseSelectAddon))
+
+		res = defaultdict(list)
+
 		paths = c.fetchmany()
 		while len(paths):
+			for i in paths
 			res += paths
 			paths = c.fetchmany()
 
 		files = []
 		for i in res:
-			c.execute('SELECT file_name FROM files WHERE path_id = %s' % (i[0]))
+			c.execute(u'SELECT file_name FROM files WHERE path_id = %s' % (i[0]))
 			fs = c.fetchmany()
 			while True:
 				r = c.fetchmany()
 					break
 
 				fs += r
-			files += fs
+			files.append(fs)
 
 		c.close()
 		
 		res = zip([(x[1], x[2]) for x in res], files)
+		res.sort(key = lambda x: x[0][0])
 
 		return res
 
+	# root, paths mustbe in unicode
 	def updateCachedPaths(self, root, paths):
 		if self.conn == None:
 			return
 			root += os.pathsep
 
 		# get root id
-		c.execute('SELECT id FROM roots WHERE path = "%s" %s' % (root, self.caseSelectAddon))
+		c.execute(u'SELECT id FROM roots WHERE path = "%s" %s' % (root, self.caseSelectAddon))
 
 		res = c.fetchone()
 		if res == None:
-			c.execute("INSERT OR REPLACE INTO roots VALUES(NULL, '%s', %s)" % (root, getModTime(root)))
-			c.execute('SELECT id FROM roots WHERE path = "%s" %s' % (root, self.caseSelectAddon))
+			c.execute(u"INSERT OR REPLACE INTO roots VALUES(NULL, '%s', %s)" % (root, getModTime(root)))
+			c.execute(u'SELECT id FROM roots WHERE path = "%s" %s' % (root, self.caseSelectAddon))
 			res = c.fetchone()
 			if res == None:
 				return
 		# insert paths
 		# TODO: Batch insert
 		for i in paths:
-			c.execute("INSERT OR REPLACE INTO paths VALUES(NULL, %s, '%s', %s)" % (root_id, i[0], i[1]))
+			c.execute(u"INSERT OR REPLACE INTO paths VALUES(NULL, %s, '%s', %s)" % (root_id, i[0], i[1]))
 		self.conn.commit()
 
 		# insert files
 		# TODO: Batch insert
 		for i in paths:
-			c.execute('SELECT id FROM paths WHERE root_id = %s, path = "%s" %s' % (root_id, i[0], self.caseSelectAddon))
+			c.execute(u'SELECT id FROM paths WHERE root_id = %s, path = "%s" %s' % (root_id, i[0], self.caseSelectAddon))
 			path_id = c.fetchone()
 			if path_id == None:
 				continue
 			
+			# delete old files
+			c.execute(u'DELETE FROM files WHERE path_id)
 			for j in i[2]:
 				c.execute("INSERT OR REPLACE INTO files VALUES(%s, '%s')" % (path_id[0], j))