Etienne Perot  committed 4b7b60c

Added concurrency max value and waiting time ebtween edits to not overload some servers

  • Participants
  • Parent commits bc6bdf7

Comments (0)

Files changed (2)

 		return self.func(*(self.pending + args), **kw)
 class BatchScheduler:
 	def __init__(self, concurrency=16):
-		self.concurrency = 16
+		global config
+		self.concurrency = concurrency
+		if 'maxConcurrency' in config:
+			self.concurrency = min(config['maxConcurrency'], self.concurrency)
 		self.tasks = []
 	def schedule(self, target, *args, **kwargs):
 		self.tasks.append((target, args, kwargs))
 	return summary
 def editPage(p, content, summary=u'', minor=True, bot=True, nocreate=True):
 	global config
+	if 'editWaitTime' in config and type(config['editWaitTime']) is type(()) and len(config['editWaitTime']) == 2:
+		time.sleep(random.uniform(*config['editWaitTime']))
 	summary = getSummary(summary)
 	p = page(p)


 	'pagePasses': 8, # Maximum number of parsing/filtering passes
 	'filterPasses': 64, # Maximum number of times to run a fitler on a filtering pass
 	'tempPrefix': 'MrGusty', # Prefix for temporary filenames
+	'maxConcurrency': 8, # Max number of requests going on concurrently
+	'editWaitTime': (0.1, 0.5), # Before every edit, wait a random number of seconds between the two provided numbers. Set to None or remove the line to ignore.
 	'pages': {
 		'filters': 'User:MrGusty/Filters', # Filters page
 		'blacklist': 'User:MrGusty/Blacklist', # Blacklist