Miki Tebeka avatar Miki Tebeka committed 32611e7

Simpler and faster

Comments (0)

Files changed (1)

 (ns clj2010
   (:require [apricot-soup :as soup])
-  (:use [clj-time.core :only (hours minutes date-time plus year month day)])
-  (:use [clojure.contrib.string :only (trim lower-case split)])
+  (:use [clj-time.core :only (hours minutes date-time plus year month day)]
+        [clojure.contrib.string :only (trim lower-case split)])
   (:import java.io.File))
 ; "chouser: " -> "chouser"
 (defn load-data [root]
   (flatten (map process-logfile (log-files root))))
-(defn map-fn [mapper prev record]
-  (let [[key value] (mapper record)]
-    (assoc prev key (cons value (prev key)))))
+(defn flatten1 
+  "Flatten one level"
+  [lst]
+  (mapcat identity lst))
+; mapper returns [[k1 v1] [k1 v2] [k2 v3] ...], we aggregate it to
+; {k1 [v1 v2] k2 [v3] ...}
 (defn map-stage [mapper records]
-  (reduce (partial map-fn mapper) {} records))
+  (let [results (flatten1 (pmap mapper records))]
+    (reduce (fn [prev [k v]] (assoc prev k (cons v (prev k)))) {} results)))
 (defn reduce-stage [reducer map-result]
   (let [ks (keys map-result)]
-    (zipmap ks (map #(reducer % (map-result %)) ks))))
+    (zipmap ks (pmap #(reducer % (map-result %)) ks))))
 (defn map-reduce [job records]
   (reduce-stage (:reduce job) (map-stage (:map job) records)))
   (date-time (year time) (month time) (day time)))
 (def numlogs {
-    :map (fn [record] [(month-only (:time record)) 1])
+    :map (fn [record] [[(month-only (:time record)) 1]])
     :reduce (fn [key values] (reduce + values))})
 (defn -main []
   (let [records (load-data "logs")]
-    (map-reduce numlogs-map numlogs-reduce records)))
+    (map-reduce numlogs records)))
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.