Miki Tebeka committed b89ffb0

Better parsing, words and users

Comments (0)

Files changed (1)

 (defn parse-text
   "\"21:38 chouser: great, thanks!\" -> 
-      [\"21:38\" \"chouser:\" \"great, thanks!\"]"
+      [\"21\" \"38\" \"chouser: great, thanks!\"]"
-  (rest (re-find #"^(\d+):(\d+)([^:]+:)?(.*)" text)))
+  (rest (re-find #"^(\d+):(\d+)(.*)?" text)))
+(defn find-user [p]
+  (let [elems (into [] (soup/search "b" p))]
+    (if (empty? elems)
+      nil
+      (soup/text (elems 0)))))
 (defn process-p
-  "Process a <p>...</p> log recrod, return {:time ... :tokens ... :user ..}"
+  "Process a <p>...</p> log record, return {:time ... :tokens ... :user ..}"
   [day previous-log p]
-  (let [[hour minute user text] (parse-text (soup/text p))]
+  (let [user (find-user p)
+        [hour minute text] (parse-text (soup/text p))
+        tokens (tokenize text)]
     { :time (add-time day (str->long hour) (str->long minute))
-      :tokens (tokenize text)
+      :tokens (if user (rest tokens) tokens)
       :user (if user (fix-user user) (:user previous-log))}))
 (defn process-logfile 
 (defn day-only [time]
   (date-time (year time) (month time) (day time)))
-(defn gen-chart [job records]
+(defn gen-chart [records job]
   (let [result (map-reduce (:map job) (:reduce job) records)
         xs (sort (keys result))
         ys (map #(result %) xs)
-        fxs (map (:x-format job) xs)]
+        fxs (map (:x-format job identity) xs)]
     (bar-chart fxs ys :title (:title job) :x-label (:x-label job) 
                :y-label (:y-label job))))
-(defn run-job [job records]
-  (let [chart (gen-chart job records)]
+(defn run-job [records job]
+  (let [chart (gen-chart records job)]
     (save chart (str "charts/" (:filename job) ".png"))))
 (defn time-fmt [fmt]
   (fn [dt]
     (unparse (formatter fmt) dt)))
+(defn sum [values]
+  (reduce + values))
 (def numlines {
     :map (fn [record] [[(month-only (:time record)) 1]])
-    :reduce (fn [key values] (reduce + values))
+    :reduce (fn [key values] (sum values))
     :title "Lines/Month"
     :x-label "Month"
     :y-label "Lines"
     :filename "users"
     :x-format (time-fmt "MMM")})
+(def active {
+    :map (fn [record] (if-let [u (:user record)] [[u 1]] []))
+    :reduce (fn [key values] (sum values))
+    :title "Users/Month"
+    :x-label "User"
+    :y-label "Lines"
+    :filename "active"})
+(def words {
+    :map (fn [record] (map (fn [tok] [tok 1]) (:tokens record)))
+    :reduce (fn [key values] (reduce + values))
+    :title "Words"
+    :x-label "Word"
+    :y-label "Count"
+    :filename "words"})
 (defn -main []
   (let [records (load-data "logs")]
-    (map-reduce numlogs records)))
+    (run-job records active)))
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.