Commits

Lars Yencken committed 03f426a

Made paralellism and memory usage configurable.

Comments (0)

Files changed (1)

 #
 
 COLORSET = inputs/colorset
+N_CORES = 8
+MEMORY = 1G
 
 default: \
 	outputs/pairs.normed \
 	outputs/color_monthly_popularity.js \
 
 outputs/dates.sorted: inputs/dates
-	sort -n --parallel=8 -S 500M $< >$@.tmp
+	sort -n --parallel=$(N_CORES) -S $(MEMORY) $< >$@.tmp
 	mv -f $@.tmp $@
 
 outputs/colors.sorted: inputs/colors
-	sort -n --parallel=8 -S 500M $< >$@.tmp
+	sort -n --parallel=$(N_CORES) -S $(MEMORY) $< >$@.tmp
 	mv -f $@.tmp $@
 
 outputs/colors.dated: outputs/colors.sorted outputs/dates.sorted src/join_dates_colors.py
 	python src/join_dates_colors.py outputs/colors.sorted outputs/dates.sorted outputs/
 
 outputs/colors.dated.sorted: outputs/colors.dated
-	sort -n --parallel=8 -S 500M $< >$@.tmp
+	sort -n --parallel=$(N_CORES) -S $(MEMORY) $< >$@.tmp
 	mv -f $@.tmp $@
 
 outputs/colors.dated.normed: outputs/colors.dated.sorted src/normalize_colors.py $(COLORSET)
 	mv -f $@.tmp $@
 
 outputs/colors.monthly: outputs/colors.dated.normed src/groupbymonth_mapper.py src/groupbymonth_reducer.py
-	pv -a --progress --eta $< | python src/groupbymonth_mapper.py | sort --parallel=8 -S 2G | python src/groupbymonth_reducer.py >$@.tmp
+	pv -a --progress --eta $< | python src/groupbymonth_mapper.py | sort --parallel=$(N_CORES) -S $(MEMORY) | python src/groupbymonth_reducer.py >$@.tmp
 	mv -f $@.tmp $@
 
 outputs/colors.weekly: outputs/colors.dated.normed src/groupbyweek_mapper.py src/groupbymonth_reducer.py
-	pv -a --progress --eta $< | python src/groupbyweek_mapper.py | sort --parallel=8 -S 2G | python src/groupbymonth_reducer.py >$@.tmp
+	pv -a --progress --eta $< | python src/groupbyweek_mapper.py | sort --parallel=$(N_CORES) -S $(MEMORY) | python src/groupbymonth_reducer.py >$@.tmp
 	mv -f $@.tmp $@
 
 outputs/color_weekly_hue.js: outputs/colors.weekly src/stream_viz.py