Commits

Nick Wade committed 8ad373c

First work on mapping/reducing - comments before office hours

  • Participants
  • Parent commits ad0a15b

Comments (0)

Files changed (1)

File controller/map_reduce.ml

 open Util
 open Worker_manager
 
-(* TODO implement these *)
+
+let do_map kv_pair pool manager
+  let worker = Worker_manager.pop_worker manager in
+    (** do work *)
+    match kv_pair with
+      [(k,v)] -> 
+  Worker_manager.push_worker manager worker
+
 let map kv_pairs shared_data map_filename : (string * string) list = 
-  failwith "Go back whence you came! Trouble the soul of my Mother no more!"
+  let manager = Worker_manager.initialize_mappers map_filename shared_data in
+    let pool = Thread_pool.create 200 in
+
+
+(* Idea of execution of map *)
+(* initialize worker manager
+initialize thread pool
+
+for each (id,body) pair in kv_pair, add it to QUEUE1/LIST1
+for each (id,body) pair in QUEUE1/LIST1
+  pop a worker, give it the pair to work on in a new thread
+  if it fails, add (id, body) back to QUEUE1/LIST1
+  if it suceeds, add returned data to QUEUE2/LIST2 and push worker back to manager *)
+  
+  
+  (* For combine, use the hash table *)
+
+
 let combine kv_pairs : (string * string list) list = 
   failwith "You have been doomed ever since you lost the ability to love."
 let reduce kvs_pairs shared_data reduce_filename : (string * string list) list =