OCaml Map Reduce Project / apps / page_rank /

open Util

let page_rank_map_reduce filename iterations =
  let websites = load_websites filename in
  let num_websites = float_of_int (List.length websites) in
  let init_kv_pairs = (fun w -> (marshal w, string_of_float (1. /. num_websites))) websites in
  let rec loop kv_pairs n =
    if n = 0 then kv_pairs else begin
      let mapped = kv_pairs "" "apps/page_rank/" in
      let combined = Map_reduce.combine mapped in
      let reduced = Map_reduce.reduce combined "" "apps/page_rank/" in
      let filter (k, vs) =
        match vs with
        | [] -> failwith "some kind of error came up"
        | h::_ -> (k, h) in
      let new_kv_pairs = filter reduced in
      loop new_kv_pairs (n - 1) end in (fun (k, v) -> ((unmarshal k).pagetitle, [v])) (loop init_kv_pairs iterations)

let main (args : string array) : unit = 
  if Array.length args < 3 then
    Printf.printf "Usage: page_rank <num_iterations> <filename>"
  else begin
    let filename = args.(3) in
    let results =
      page_rank_map_reduce filename (int_of_string args.(2)) in
    print_reduce_results results end
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.