Commits

Yit Phang Khoo  committed fa1bb76

Log every measurement in benchmark.

Oddly, the performance of filter improved slightly across all modules, due to code motion?

  • Participants
  • Parent commits 3e01316

Comments (0)

Files changed (3)

File Benchmarks/BenchmarkAdapton/runbenchmarkadapton.ml

 let opt_task = ref "filter"
 let opt_input_size = ref 1
 let opt_repeat_count = ref 1
+let opt_take_count = ref 1
 let opt_edit_count = ref 1
-let opt_take_count = ref 1
+let opt_monotonic = ref false
 let opt_random_seed = ref 1
-let opt_monotonic = ref false
 
 let header ff = Printf.fprintf ff "%24s %24s %8d %8d %20d" !opt_a !opt_task !opt_take_count !opt_input_size !opt_random_seed
+let config ff =
+    Printf.fprintf ff "\"module\": \"%s\", \"task\": \"%s\", \"size\": %d, \"repeat\": %d, \"take\": %d, \"edit\": %d, \"monotonic\": %b, \"seed\": %d"
+        !opt_a !opt_task !opt_input_size !opt_repeat_count !opt_take_count !opt_edit_count !opt_monotonic !opt_random_seed
 let stats ff s =
-    Printf.fprintf ff "\"time\": %.17g, \"heap\": %.17g, \"stack\": %.17g, \"update\": %.17g, \"evaluate\": %.17g, \"dirty\": %.17g, \"clean\": %.17g"
+    Printf.fprintf ff "\"time\": %.17g, \"heap\": %d, \"stack\": %d, \"update\": %d, \"evaluate\": %d, \"dirty\": %d, \"clean\": %d"
         s.time s.heap s.stack s.update s.evaluate s.dirty s.clean
-let top_heap_stack ff ( heap, stack ) = Printf.fprintf ff "\"max-heap\": %.17g, \"max-stack\": %.17g" (word_bytes heap) (word_bytes stack)
+let top_heap_stack ff ( heap, stack ) = Printf.fprintf ff "\"max-heap\": %d, \"max-stack\": %d" (word_bytes heap) (word_bytes stack)
 let units =
     "\"units\": { \"time\": \"seconds\", \"heap\": \"bytes\", \"stack\": \"bytes\", "
     ^ "\"update\": null, \"evaluate\": null, \"dirty\": null, \"clean\": null, "
             take ();
             take
         end in
-        let setup_stats = finish setup_stats 1 in
         let setup_top_heap_stack = get_top_heap_stack () in
 
         if A.is_incremental then begin
-            let rec do_edits past n update_stats take_stats edit_count =
-                if n == 0 then
-                    ( update_stats, take_stats, edit_count )
-                else
-                    let past =
-                        let now = get_time () in
-                        if now -. past < 20. then
-                            past
-                        else begin
-                            let heap, stack = get_top_heap_stack () in
-                            Printf.eprintf "%t edit %10d %9.2fMB %9.2fMB\n%!"
-                                header n (word_megabytes heap) (word_megabytes stack);
-                            now
-                        end
-                    in
-                    let update_stats', take_stats', edit_count' = do_edit input (fun () -> A.refresh (); take ()) in
-                    do_edits past (pred n)
-                        (add update_stats update_stats')
-                        (add take_stats take_stats')
-                        (edit_count + edit_count')
+            let stats_file, stats_out = Filename.open_temp_file ~mode:[ Open_append; Open_binary ] "runbenchmarkadapton." ".stats" in
+            let stats_in = open_in_bin stats_file in
+            Unix.unlink stats_file;
+            let rec do_edits past n = if n == 0 then () else
+                let past =
+                    let now = get_time () in
+                    if now -. past < 20. then
+                        past
+                    else begin
+                        let heap, stack = get_top_heap_stack () in
+                        Printf.eprintf "%t edit %10d %9.2fMB %9.2fMB\n%!"
+                            header n (word_megabytes heap) (word_megabytes stack);
+                        now
+                    end
+                in
+                let update_stats, take_stats, edit_count = do_edit input (fun () -> A.refresh (); take ()) in
+
+                let edit_heap, edit_stack = get_top_heap_stack () in
+                output_value stats_out ( update_stats, take_stats, edit_count, edit_heap, edit_stack );
+
+                do_edits past (pred n)
             in
-            let update_stats, take_stats, edit_count = do_edits 0. !opt_edit_count zero zero 0 in
-            let edit_top_heap_stack = get_top_heap_stack () in
-            let update_stats = finish update_stats edit_count in
-            let take_stats = finish take_stats edit_count in
-            Printf.printf "{ \"setup\": { %a, %a }, \"edits\": { \"update\": { %a }, \"take\": { %a }, %a }, %s }\n%!"
-                stats setup_stats top_heap_stack setup_top_heap_stack stats update_stats stats take_stats top_heap_stack edit_top_heap_stack units;
+            do_edits 0. !opt_edit_count;
+            close_out stats_out;
+            Printf.printf "{ %t, \"setup\": { %a, %a }, \"edits\": [ "
+                config stats setup_stats top_heap_stack setup_top_heap_stack;
+            let edit_time = ref 0. in
+            let first = ref true in
+            begin try
+                while true do
+                    let update_stats, take_stats, edit_count, edit_heap, edit_stack = input_value stats_in in
+
+                    if not !first then begin
+                        print_string ", ";
+                    end;
+                    first := false;
+                    Printf.printf "{ \"update\": { %a }, \"take\": { %a }, \"edit-count\": %d, %a }"
+                         stats update_stats
+                         stats take_stats
+                         edit_count
+                         top_heap_stack ( edit_heap, edit_stack );
+                    edit_time := !edit_time +. update_stats.time +. take_stats.time;
+                done
+            with End_of_file ->
+                ()
+            end;
+            Printf.printf " ], %s }\n%!" units;
+            close_in stats_in;
             Printf.eprintf "%t ... done (%9.2fs) %9.3gs edit %9.3gs\n%!"
-                header (get_time () -. start_time) setup_stats.time (update_stats.time +. take_stats.time)
+                header (get_time () -. start_time) setup_stats.time !edit_time
         end else begin
-            Printf.printf "{ \"setup\": { %a, %a }, %s }\n%!"
-                stats setup_stats top_heap_stack setup_top_heap_stack units;
+            Printf.printf "{ %t, \"setup\": { %a, %a }, %s }\n%!"
+                config stats setup_stats top_heap_stack setup_top_heap_stack units;
             Printf.eprintf "%t ... done (%9.2fs) %9.3gs\n%!" header (get_time () -. start_time) setup_stats.time
         end
     with e ->
         let err = Printexc.to_string e in
-        Printf.printf ("{ \"error\": %S }\n%!") err;
+        Printf.printf ("{ %t, \"error\": %S }\n%!") config err;
         Printf.eprintf "%s\n%!" err;
         Printf.eprintf "%t ... done (%9.2fs)\n%!" header (get_time () -. start_time)
 
         ( "-R", Arg.Set_int opt_repeat_count, "count repeat count" );
         ( "-T", Arg.Set_int opt_take_count, "count take count" );
         ( "-E", Arg.Set_int opt_edit_count, "count edit count" );
+        ( "-M", Arg.Set opt_monotonic, "monotonic edits" );
         ( "-S", Arg.Set_int opt_random_seed, "seed random seed" );
-        ( "-M", Arg.Set opt_monotonic, "monotonic edits" );
     ]) (fun s -> raise (Arg.Bad ("extraneous argument " ^ s))) (Sys.argv.(0) ^ " [options]");
 
     let rng = Random.State.make [| !opt_random_seed |] in

File Benchmarks/BenchmarkAdapton/runbenchmarkadapton.py

 def driver(( module, task, size, repeat, take, edit, monotonic, seed )):
     driver_start_time = time.time()
     rng = random.Random(seed)
-    results = OrderedDict((
-        ( "module", module ), ( "task", task ),
-        ( "size", size ), ( "repeat", repeat), ( "take", take ), ( "edit", edit ), ( "monotonic", monotonic ),
-        ( "seed", seed ) ))
     try:
-        cmd = [ runbenchmarkadapton_native, "-m", "%s" % ( module, ), "-t", str(task), "-I", str(size), "-R", str(repeat), "-T", str(take), "-E", str(edit) ]
+        cmd = [ runbenchmarkadapton_native, "-m", str(module), "-t", str(task), "-I", str(size), "-R", str(repeat), "-T", str(take), "-E", str(edit) ]
         if monotonic:
             cmd.append("-M")
         cmd.extend(( "-S", str(seed) ))
         native = subprocess.Popen(cmd, stdout=subprocess.PIPE, env={ "BENCHMARK_SALIST_ENV": " " * rng.randrange(4096) })
-        results.update(json.load(native.stdout, object_pairs_hook=OrderedDict))
-        return results
+        return native.stdout.read()
 
     except Exception, KeyboardInterrupt:
         error = traceback.format_exc()
-        results["error"] = error
         print>>sys.stderr, error
         print>>sys.stderr, "%32s %24s %4d %10d %20d ... error (%9.2fs)" % ( module, task, take, size, seed, time.time() - driver_start_time )
-        return results
+        return json.dumps(OrderedDict((
+            ( "module", module ), ( "task", task ),
+            ( "size", size ), ( "repeat", repeat), ( "take", take ), ( "edit", edit ), ( "monotonic", monotonic ),
+            ( "seed", seed ), ( "error", error ) )))
     finally:
         try:
             native.kill()
                 finally:
                     if len(args.input_sizes) > 1:
                         assert len(args.take_counts) == 1
-                        results = OrderedDict((
-                            ( "label", "take count = %d" % ( args.take_counts[0], ) ),
-                            ( "x-axis", "size" ),
-                            ( "x-label", "input size" ),
-                            ( "data", results )
-                        ))
+                        label = "take count = %d" % ( args.take_counts[0], )
+                        x_axis = "size"
+                        x_label = "input size"
                     elif len(args.take_counts) > 1:
                         assert len(args.input_sizes) == 1
-                        results = OrderedDict((
-                            ( "label", "input size = %d" % ( args.input_sizes[0], ) ),
-                            ( "x-axis", "take" ),
-                            ( "x-label", "take count" ),
-                            ( "data", results )
-                        ))
+                        label = "input size = %d" % ( args.input_sizes[0], )
+                        x_axis = "take"
+                        x_label = "take count"
                     else:
                         assert len(args.take_counts) == 1 and len(args.input_sizes) == 1
-                        results = OrderedDict((
-                            ( "label", "input size = %d; count = %d" % ( args.input_sizes[0], args.take_counts[0] ) ),
-                            ( "x-axis", "size" ),
-                            ( "x-label", "" ),
-                            ( "data", results )
-                        ))
+                        label = "input size = %d; count = %d" % ( args.input_sizes[0], args.take_counts[0] )
+                        x_axis = "size"
+                        x_label = ""
                     with gzip.open(os.path.join(output, "%s-%04d.json.gz" % ( task, len(results) )), "w") as jsonfile:
-                        json.dump(results, jsonfile, indent=4, separators=( ",", ":" ))
+                        print>>jsonfile, "{ \"label\": \"%s\", \"x-axis\": \"%s\", \"x-label\": \"%s\", \"data\": [ %s ] }" \
+                            % ( label, x_axis, x_label, ", ".join(results) )
 
     if not args.summaries:
         sys.exit()
                         if "edits" in record:
                             editables.add(record["module"])
                             try:
+                                edit_count = float(sum( edit["edit-count"] for edit in record["edits"] ))
                                 for key in table.iterkeys():
                                     if key.startswith("max-"):
                                         table[key].setdefault("propagate", {}).setdefault(record["module"], {}) \
-                                            .setdefault(record[results["x-axis"]], []).append(record["edits"][key])
+                                            .setdefault(record[results["x-axis"]], []).append(record["edits"][-1][key])
                                     else:
+                                        update = sum( edit["update"][key] for edit in record["edits"] ) / edit_count
+                                        take = sum( edit["take"][key] for edit in record["edits"] ) / edit_count
                                         table[key].setdefault("propagate", {}).setdefault(record["module"], {}) \
-                                            .setdefault(record[results["x-axis"]], []).append(record["edits"]["update"][key] + record["edits"]["take"][key])
+                                            .setdefault(record[results["x-axis"]], []).append(update + take)
                                         table[key].setdefault("update", {}).setdefault(record["module"], {}) \
-                                            .setdefault(record[results["x-axis"]], []).append(record["edits"]["update"][key])
+                                            .setdefault(record[results["x-axis"]], []).append(update)
                             except Exception:
                                 traceback.print_exc()
                                 if "error" in record:

File Source/AdaptonUtil/Statistics.ml

 
 let word_size = Sys.word_size / 8
 
-let word_bytes words = float_of_int (word_size * words)
+let word_bytes words = word_size * words
 
-let word_megabytes words = word_bytes words /. 1048576.
+let word_megabytes words = float_of_int (word_bytes words) /. 1048576.
 
 let get_time = Unix.gettimeofday
 
 
 type t = {
     time : float; (** Elapsed time in seconds. *)
-    heap : float; (** Max heap delta in bytes. *)
-    stack : float; (** Max stack delta in bytes. *)
-    update : float; (** Thunks updated. *)
-    evaluate : float; (** Thunks re-evaluated. *)
-    dirty : float; (** For {!Adapton}, dependencies to be checked; for {!EagerTotalOrder}, thunks scheduled for re-evaluation. *)
-    clean : float; (** For {!Adapton}, dependencies checked clean; for {!EagerTotalOrder}, thunks unscheduled for re-evaluation (due to invalidation). *)
+    heap : int; (** Max heap delta in bytes. *)
+    stack : int; (** Max stack delta in bytes. *)
+    update : int; (** Thunks updated. *)
+    evaluate : int; (** Thunks re-evaluated. *)
+    dirty : int; (** For {!Adapton}, dependencies to be checked; for {!EagerTotalOrder}, thunks scheduled for re-evaluation. *)
+    clean : int; (** For {!Adapton}, dependencies checked clean; for {!EagerTotalOrder}, thunks unscheduled for re-evaluation (due to invalidation). *)
 }
 
-type u = {
-    time' : float;
-    heap' : int;
-    stack' : int;
-    update' : int;
-    evaluate' : int;
-    dirty' : int;
-    clean' : int;
-}
-
-let zero = { time'=0.; heap'=0; stack'=0; update'=0; evaluate'=0; dirty'=0; clean'=0 }
-
 let add s s' = {
-    time'=s.time' +. s'.time';
-    heap'=s.heap' + s'.heap';
-    stack'=s.stack' + s'.stack';
-    update'=s.update' + s'.update';
-    evaluate'=s.evaluate' + s'.evaluate';
-    dirty'=s.dirty' + s'.dirty';
-    clean'=s.clean' +s'.clean';
+    time=s.time +. s'.time;
+    heap=s.heap + s'.heap;
+    stack=s.stack + s'.stack;
+    update=s.update + s'.update;
+    evaluate=s.evaluate + s'.evaluate;
+    dirty=s.dirty + s'.dirty;
+    clean=s.clean +s'.clean;
 }
 
 let measure f =
     stack' := max stack !stack';
     let time = get_time () in
     let x = f () in
-    let time' = get_time () -. time in
+    let time = get_time () -. time in
     heap_stacks := old_heap_stacks;
     let measurement = {
-        time';
-        heap'=(!heap' - heap);
-        stack'=(!stack' - stack);
-        update'=(!Counts.update - update);
-        evaluate'=(!Counts.evaluate - evaluate);
-        dirty'=(!Counts.dirty - dirty);
-        clean'=(!Counts.clean - clean);
+        time;
+        heap=word_bytes (!heap' - heap);
+        stack=word_bytes (!stack' - stack);
+        update=(!Counts.update - update);
+        evaluate=(!Counts.evaluate - evaluate);
+        dirty=(!Counts.dirty - dirty);
+        clean=(!Counts.clean - clean);
     } in
     ( x, measurement )
-
-let finish s k =
-    let k = float_of_int k in {
-        time=s.time' /. k;
-        heap=word_bytes s.heap' /. k;
-        stack=word_bytes s.stack' /. k;
-        update=float_of_int s.update' /. k;
-        evaluate=float_of_int s.evaluate' /. k;
-        dirty=float_of_int s.dirty' /. k;
-        clean=float_of_int s.clean' /. k;
-    }