OCaml Map Reduce Project / apps / page_rank /

Full commit
open Util

let page_rank_map_reduce filename iterations =
  let websites = load_websites filename in
  let num_websites = float_of_int (List.length websites) in
  let ht = Hashtbl.create 16 in
  let init_kv_pairs =
    let add w =
      Hashtbl.add ht w.pageid w;
      (marshal w, string_of_float (1. /. num_websites)) in add websites in
  let marshalled_hash = marshal ht in
  let rec loop kv_pairs n =
    if n = 0 then kv_pairs else begin
      let mapped = kv_pairs marshalled_hash "apps/page_rank/" in
      let combined = Map_reduce.combine mapped in
      let reduced = Map_reduce.reduce combined "" "apps/page_rank/" in
      let filter (k, vs) =
        match vs with
        | [] -> failwith "some kind of error came up"
        | h::_ -> (k, h) in
      let new_kv_pairs = filter reduced in
      loop new_kv_pairs (n - 1) end in (fun (k, v) -> ((unmarshal k).pagetitle, [v])) (loop init_kv_pairs iterations)

let main (args : string array) : unit = 
  if Array.length args < 3 then
    Printf.printf "Usage: page_rank <num_iterations> <filename>"
  else begin
    let filename = args.(3) in
    let results =
      page_rank_map_reduce filename (int_of_string args.(2)) in
    print_reduce_results results end