Commits

Robert MacLean committed a168303

Fixed too many people issue & added rate limit error checking

Added the stuff to break the too many people into batches of 100 (found
some documentation saying that is the limit) which required a boat load of
promise work to get it all nicely tied up.

Added some general error & edge case handling scenarios (like if you end
up with 0 matches or > 13000 matches).

Trying to play nicer with the rate limiting - limiting the calls and also
implemented the error checking.

Known issues are
+ rate limit error check is untested
+ only get the first 5000 people for each tweeter and then compare
those, need to get a better way of handling it - this will not hit
normal tweeters but ruins the Bieber coalesce Mother Monster scenario.

  • Participants
  • Parent commits f674a66

Comments (0)

Files changed (2)

File TwitterCoalesce/HtmlPage1.html

     <script type="text/javascript" src="Magic.js"></script>
     <script type="text/javascript">
         $(document).ready(function () {
-            $("#go").click(function () {
+            $("#go").click(function () {                
+                $("#results").html("<li>loading....</li>");
+
                 twitterCoalesce.findFollowers($("#firstTweeter").val(), $("#secondTweeter").val()).then(function (data) {
-                    var result = twitterCoalesce.makeListItems(twitterCoalesce.sortResults(data));
+                    var result = "";
+
+                    if (data.success) {
+                        if (data.filtered) {
+                            result = result + "<li>Filtered to the first 13000 responses - note you will likely be rate limited by Twitter soon</li>";
+                        }
+
+                        result = result + twitterCoalesce.makeListItems(twitterCoalesce.sortResults(data.names));
+                        console.log("item count: " + data.names.length);
+                    } else {
+                        result = "<li>An error occured: " + data.errorMessage + "</li>";
+                    }
+
                     $("#results").html(result);
-                    console.log("item count" + data.length);
                 });
             });
         });

File TwitterCoalesce/Magic.js

-(function (twitterCoalesce, $, undefined) {
-    twitterCoalesce.findFollowers = function (firstUsername, secondUsername) {
-        var def = $.Deferred();
-        getFollowerIdList(firstUsername).then(function (firstFollowers) {
-            getFollowerIdList(secondUsername).then(function (secondFollowers) {
-                var ids = [];
-
-                $.each(firstFollowers, function (index, item) {
-                    if (secondFollowers.indexOf(item) !== -1) {
-                        ids.push(item);
+/*jslint browser: true, devel: true, windows: false, white: true */
+/*globals jQuery: true*/
+(function (twitterCoalesce, $, undefined) {
+    "use strict";
+
+    function getTwitterNames(ids) {
+        var def = $.Deferred(),
+            result = {
+                success: false,
+                names: [],
+                errorMessage: ""
+            };
+
+        $.getJSON("https://api.twitter.com/1/users/lookup.json?user_id=" + ids.join(",") + " &callback=?", function (data) {
+            result.success = true;
+            $.each(data, function (index, item) {
+                result.names.push({
+                    name: item.name,
+                    screen_name: item.screen_name,
+                    toString: function () {
+                        return this.screen_name;
                     }
                 });
+            });
 
-                getAllTwitterNames(ids).then(function (names) {
-                    def.resolve(names);
-                });
-            })
+            def.resolve(result);
         })
+        .error(function (jqXHR) {
+            result.success = false;
+            switch (jqXHR.status) {
+                case 400:
+                    result.errorMessage = "Twitter rate limite hit.";
+                    break;
+            }
 
-        return def;
-    }
-
-    twitterCoalesce.makeListItems = function (results) {
-        var result = "";
-        $.each(results, function (index, item) {
-            result += "<li><a href='http://twitter.com/" + item.screen_name + "' target='_blank'>" + item.name + "</a></li>";
+            def.resolve(result);
         });
 
-        return result;
+        return def;
     }
 
-    twitterCoalesce.sortResults = function (results) {
-        return results.sort(function (a, b) {
-            if (a.name === b.name) {
-                return 0;
-            }
-
-            if (a.name < b.name) {
-                return -1;
-            }
+    function buildNameResults(nameResult, currentList) {
+        var result = {
+            results: [],
+            success: false,
+            errorMessage: ""
+        };
+
+        if (nameResult.success) {
+            result.success = true;
+            result.results = currentList.concat(nameResult.names);
+        } else {
+            result.errorMessage = nameResult.errorMessage;
+        }
 
-            return 1;
-        })
+        return result;
     }
 
     function getAllTwitterNames(ids) {
         var def = $.Deferred(),
-                  results = [],
-                  batch,
-                  more = true,
-                  counter,
-                  batchCounter = 0,
-                  batchStart = 0,
-                  step = 50;
+            result = {
+                results: [],
+                success: true,
+                errorMessage: ""
+            },
+            batch,
+            more = true,
+            counter,
+            batchCounter = 0,
+            batchStart = 0,
+            batches = [],
+            step = 100;
 
         while (more) {
             batch = [];
             for (counter = batchStart; counter < batchStart + step; counter++) {
-                //console.log(counter);
                 if (ids[counter] === undefined) {
                     more = false;
                     break;
             }
 
             if (batch.length > 0) {
-                getTwitterNames(batch).then(function (names) {
-                    results = results.concat(names);
-                });
+                batches.push(function (b) {
+                    var x = $.Deferred();
+                    getTwitterNames(b).then(function (nameResult) {
+                        var buildResult = buildNameResults(nameResult, result.results);
+                        if (buildResult.success) {
+                            result.results = buildResult.results;
+                        } else {
+                            result.success = false;
+                            result.errorMessage = buildResult.errorMessage;
+                        }
+
+                        x.resolve();
+                    });
+
+                    return x;
+                }(batch));
             }
 
             batchCounter = batchCounter + 1;
             batchStart = batchCounter * step;
         }
 
-        def.resolve(results);
-
+        $.when.apply(this, batches).then(function () {
+            def.resolve(result);
+        });
+           
         return def;
     }
 
-    function getTwitterNames(ids) {
+    function getFollowerIdList(username) {
         var def = $.Deferred(),
-            results = [];
-
-        $.getJSON("https://api.twitter.com/1/users/lookup.json?user_id=" + ids.join(",") + " &callback=?", function (data) {
-            $.each(data, function (index, item) {
-                results.push({
-                    name: item.name,
-                    screen_name: item.screen_name,
-                    toString: function () {
-                        return this.screen_name;
-                    }
-                });
-            });
+            result = {
+                errorMessage: "",
+                success: false,
+                ids: []
+            };
+
+        $.getJSON("https://api.twitter.com/1/followers/ids.json?cursor=-1&screen_name=" + username + "&callback=?", function (data) {
+            result.success = true;
+            result.ids = data.ids;
+            def.resolve(result);
+        })
+        .error(function (jqXHR) {
+            result.success = false;
+            switch (jqXHR.status) {
+                case 400:
+                    result.errorMessage = "Twitter rate limite hit.";
+                    break;
+                default:
+                    result.errorMessage = jqXHR.responseText;
+                    break;
+            }
 
-            def.resolve(results);
+            def.resolve(result);
         });
 
         return def;
     }
 
-    function getFollowerIdList(username) {
-        var def = $.Deferred();
-
-        $.getJSON("https://api.twitter.com/1/followers/ids.json?screen_name=" + username + "&callback=?", function (data) {
-            def.resolve(data.ids);
+    twitterCoalesce.findFollowers = function (firstUsername, secondUsername) {
+        var def = $.Deferred(),
+            nameResult,
+            result = {
+                names: [],
+                filtered: false,
+                errorMessage: "",
+                success: false
+            };
+
+        getFollowerIdList(firstUsername).then(function (firstFollowerResult) {
+            if (!firstFollowerResult.success) {
+                result.errorMessage = firstFollowerResult.errorMessage;
+                def.resolve(result);
+            } else {
+                getFollowerIdList(secondUsername).then(function (secondFollowerResult) {
+                    if (!secondFollowerResult.success) {
+                        result.errorMessage = secondFollowerResult.errorMessage;
+                        def.resolve(result);
+                    } else {
+                        if (firstFollowerResult.ids.length === 0 ||
+                            secondFollowerResult.ids === 0) {
+                            result.success = true;
+                            def.resolve(result);
+                        } else {
+                            var ids = [];
+                            $.each(firstFollowerResult.ids, function (index, item) {
+                                if (secondFollowerResult.ids.indexOf(item) !== -1) {
+                                    ids.push(item);
+                                }
+                            });
+
+                            if (ids.length > 13000) {
+                                result.filtered = true;
+                                ids.splice(13000, ids.length - 13000);
+                            }
+
+                            getAllTwitterNames(ids).then(function (nameResult) {
+                                if (!nameResult.success) {
+                                    def.resolve(result);
+                                } else {
+                                    result.names = nameResult.results;
+                                    result.success = true;
+                                    def.resolve(result);
+                                }
+                            });
+                        }
+                    }
+                });
+            }
         });
 
         return def;
-    }
+    };
+
+    twitterCoalesce.makeListItems = function (results) {
+        var result = "";
+        $.each(results, function (index, item) {
+            result += "<li><a href='http://twitter.com/" + item.screen_name + "' target='_blank'>" + item.name + "</a></li>";
+        });
+
+        return result;
+    };
+
+    twitterCoalesce.sortResults = function (results) {
+        return results.sort(function (a, b) {
+            if (a.name === b.name) {
+                return 0;
+            }
+
+            if (a.name < b.name) {
+                return -1;
+            }
+
+            return 1;
+        });
+    };
 
 })(window.twitterCoalesce = window.twitterCoalesce || {}, jQuery);