Commits

Miki Tebeka committed 804994d

Minimal map/reduce working

Comments (0)

Files changed (1)

 (ns clj2010
   (:require [apricot-soup :as soup])
-  (:use [clj-time.core :only (hours minutes date-time plus)])
+  (:use [clj-time.core :only (hours minutes date-time plus year month day)])
   (:use [clojure.contrib.string :only (trim lower-case split)])
   (:import java.io.File))
 
 (defn load-data [root]
   (flatten (map process-logfile (log-files root))))
 
+(defn map-fn [mapper prev record]
+  (let [[key value] (mapper record)]
+    (assoc prev key (cons value (prev key)))))
+
+(defn map-stage [mapper records]
+  (reduce (partial map-fn mapper) {} records))
+
+(defn reduce-stage [reducer map-result]
+  (let [ks (keys map-result)]
+    (zipmap ks (map #(reducer % (map-result %)) ks))))
+
+(defn map-reduce [mapper reducer records]
+  (reduce-stage reducer (map-stage mapper records)))
+
+(defn month-only [time]
+  (date-time (year time) (month time)))
+
+(defn day-only [time]
+  (date-time (year time) (month time) (day time)))
+
+(defn numlogs-map [record]
+  [(month-only (:time record)) 1])
+
+(defn numlogs-reduce [key values]
+  (reduce + values))
+
 (defn -main []
-  (dorun (map println (take 10 (load-data "logs")))))
+  (let [records (load-data "logs")]
+    (map-reduce numlogs-map numlogs-reduce records)))