Commits

Kelvin Jin committed f64a338

Page_rank mapper implemented.

  • Participants
  • Parent commits a26305b

Comments (0)

Files changed (2)

apps/page_rank/mapper.ml

 let (key, value) = Program.get_input() in
-Program.set_output (List.fold_left (fun acc k -> (k, "1")::acc) []
-                   (List.map String.lowercase (Util.split_words value)))
+let ht = Util.unmarshal (Program.get_shared_data()) in
+let website : Util.website = Util.unmarshal key in
+let num_links = float_of_int (List.length website.Util.links) in
+let rank = float_of_string value in
+let weight = if num_links = 0. then 0. else rank /. num_links in
+Program.set_output (List.fold_left (fun acc k -> (Hashtbl.find ht k, string_of_float weight)::acc) [] website.Util.links)

apps/page_rank/page_rank.ml

 let page_rank_map_reduce filename iterations =
   let websites = load_websites filename in
   let num_websites = float_of_int (List.length websites) in
+  let ht = Hashtbl.create 16 in
   let init_kv_pairs =
-    List.map (fun w -> (marshal w, string_of_float (1. /. num_websites))) websites in
+    let add w =
+      Hashtbl.add ht w.pageid w;
+      (marshal w, string_of_float (1. /. num_websites)) in
+    List.map add websites in
+  let marshalled_hash = marshal ht in
   let rec loop kv_pairs n =
     if n = 0 then kv_pairs else begin
-      let mapped = Map_reduce.map kv_pairs "" "apps/page_rank/mapper.ml" in
+      let mapped =
+        Map_reduce.map kv_pairs marshalled_hash "apps/page_rank/mapper.ml" in
       let combined = Map_reduce.combine mapped in
-      let reduced = Map_reduce.reduce combined "" "apps/page_rank/mapper.ml" in
+      let reduced = Map_reduce.reduce combined "" "apps/page_rank/reducer.ml" in
       let filter (k, vs) =
         match vs with
         | [] -> failwith "some kind of error came up"