Source

OCaml Map Reduce Project / apps / page_rank / page_rank.ml

open Util

let page_rank_map_reduce filename iterations =
  let websites = load_websites filename in
  let num_websites = float_of_int (List.length websites) in
  let init_kv_pairs =
    List.map (fun w -> (marshal w, string_of_float (1. /. num_websites))) websites in
  let rec loop kv_pairs n =
    if n = 0 then kv_pairs else begin
      let mapped = Map_reduce.map kv_pairs "" "apps/page_rank/mapper.ml" in
      let combined = Map_reduce.combine mapped in
      let reduced = Map_reduce.reduce combined "" "apps/page_rank/mapper.ml" in
      let filter (k, vs) =
        match vs with
        | [] -> failwith "some kind of error came up"
        | h::_ -> (k, h) in
      let new_kv_pairs = List.map filter reduced in
      loop new_kv_pairs (n - 1) end in
  List.map (fun (k, v) -> ((unmarshal k).pagetitle, [v])) (loop init_kv_pairs iterations)

let main (args : string array) : unit = 
  if Array.length args < 3 then
    Printf.printf "Usage: page_rank <num_iterations> <filename>"
  else begin
    let filename = args.(3) in
    let results =
      page_rank_map_reduce filename (int_of_string args.(2)) in
    print_reduce_results results end