1. A S
  2. pypy-site

Commits

A S  committed d901b76 Draft

Fix for data scraping with unmatched binaries/sandbox releases.

  • Participants
  • Parent commits 5024268
  • Branches default

Comments (0)

Files changed (3)

File pypysite/data/downloads.json

View file
             "", 
             "", 
             "32Bit", 
-            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-linux.tar.bz2", 
-            "pypy-1.8-linux.tar.bz2", 
-            "12.0 MB", 
-            "2012-02-09", 
-            "2666"
+            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.9-linux.tar.bz2", 
+            "pypy-1.9-linux.tar.bz2", 
+            "11.2 MB", 
+            "2012-06-08", 
+            "708"
         ], 
         "linux64": [
             "Linux", 
             "", 
             "", 
             "64Bit", 
-            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-linux64.tar.bz2", 
-            "pypy-1.8-linux64.tar.bz2", 
-            "13.5 MB", 
-            "2012-02-09", 
-            "5427"
+            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.9-linux64.tar.bz2", 
+            "pypy-1.9-linux64.tar.bz2", 
+            "12.8 MB", 
+            "2012-06-08", 
+            "1563"
         ], 
         "osx64": [
             "Mac", 
             "OS X 10.6+", 
             "", 
             "64Bit", 
-            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-osx64.tar.bz2", 
-            "pypy-1.8-osx64.tar.bz2", 
-            "13.6 MB", 
-            "2012-02-10", 
-            "4016"
+            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.9-osx64.tar.bz2", 
+            "pypy-1.9-osx64.tar.bz2", 
+            "12.5 MB", 
+            "2012-06-08", 
+            "1507"
         ], 
         "win32": [
             "Windows", 
             "", 
             "BETA!", 
             "32Bit", 
-            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-win32.zip", 
-            "pypy-1.8-win32.zip", 
-            "16.8 MB", 
-            "2012-02-09", 
-            "14235"
+            "https://bitbucket.org/pypy/pypy/downloads/pypy-1.9-win32.zip", 
+            "pypy-1.9-win32.zip", 
+            "15.0 MB", 
+            "2012-06-08", 
+            "2627"
         ]
     }, 
     "sandbox": {
             "pypy-1.8-sandbox-linux.tar.bz2", 
             "3.7 MB", 
             "2012-02-15", 
-            "265"
+            "332"
         ], 
         "sandbox-linux64": [
             "Linux", 
             "pypy-1.8-sandbox-linux64.tar.bz2", 
             "4.7 MB", 
             "2012-02-15", 
-            "449"
+            "558"
         ]
     }, 
     "source": [
         [
-            "pypy-1.8-src.tar.bz2", 
+            "pypy-1.9-src.tar.bz2", 
             "Unix line endings", 
-            "https://bitbucket.org/pypy/pypy/get/release-1.8.tar.bz2"
+            "https://bitbucket.org/pypy/pypy/get/release-1.9.tar.bz2"
         ], 
         [
-            "pypy-1.8-src.zip", 
+            "pypy-1.9-src.zip", 
             "Unix line endings", 
-            "https://bitbucket.org/pypy/pypy/get/release-1.8.zip"
+            "https://bitbucket.org/pypy/pypy/get/release-1.9.zip"
         ]
     ], 
-    "version": "1.8"
+    "version": "1.9"
 }

File pypysite/data/twitter_pypyproject.json

View file
     {
         "contributors": null, 
         "coordinates": null, 
-        "created_at": "Fri Apr 27 16:26:01 +0000 2012", 
+        "created_at": "Fri Jun 08 09:52:23 +0000 2012", 
         "favorited": false, 
         "geo": null, 
-        "id": 195911692008423424, 
-        "id_str": "195911692008423424", 
+        "id": 211032921484697602, 
+        "id_str": "211032921484697602", 
         "in_reply_to_screen_name": null, 
         "in_reply_to_status_id": null, 
         "in_reply_to_status_id_str": null, 
         "in_reply_to_user_id_str": null, 
         "place": null, 
         "possibly_sensitive": false, 
-        "retweet_count": 23, 
+        "retweet_count": 53, 
         "retweeted": false, 
-        "source": "<a href=\"https://wiki.ubuntu.com/Gwibber\" rel=\"nofollow\">Ubuntu</a>", 
-        "text": "STM status update from Armin Rigo: http://t.co/sONbrKT1", 
+        "source": "web", 
+        "text": "PyPy 1.9 - Yard Wolf released! http://t.co/NbrLmzD4", 
         "truncated": false, 
         "user": {
             "contributors_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 1077, 
+            "followers_count": 1139, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "id_str": "277141637", 
             "is_translator": false, 
             "lang": "en", 
-            "listed_count": 52, 
+            "listed_count": 55, 
             "location": "", 
             "name": "The PyPy Project", 
             "notifications": null, 
             "protected": false, 
             "screen_name": "pypyproject", 
             "show_all_inline_media": false, 
-            "statuses_count": 63, 
+            "statuses_count": 65, 
+            "time_zone": null, 
+            "url": "http://pypy.org", 
+            "utc_offset": null, 
+            "verified": false
+        }
+    }, 
+    {
+        "contributors": null, 
+        "coordinates": null, 
+        "created_at": "Fri Apr 27 16:26:01 +0000 2012", 
+        "favorited": false, 
+        "geo": null, 
+        "id": 195911692008423424, 
+        "id_str": "195911692008423424", 
+        "in_reply_to_screen_name": null, 
+        "in_reply_to_status_id": null, 
+        "in_reply_to_status_id_str": null, 
+        "in_reply_to_user_id": null, 
+        "in_reply_to_user_id_str": null, 
+        "place": null, 
+        "possibly_sensitive": false, 
+        "retweet_count": 22, 
+        "retweeted": false, 
+        "source": "<a href=\"https://wiki.ubuntu.com/Gwibber\" rel=\"nofollow\">Ubuntu</a>", 
+        "text": "STM status update from Armin Rigo: http://t.co/sONbrKT1", 
+        "truncated": false, 
+        "user": {
+            "contributors_enabled": false, 
+            "created_at": "Mon Apr 04 19:44:41 +0000 2011", 
+            "default_profile": true, 
+            "default_profile_image": false, 
+            "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
+            "favourites_count": 0, 
+            "follow_request_sent": null, 
+            "followers_count": 1139, 
+            "following": null, 
+            "friends_count": 7, 
+            "geo_enabled": false, 
+            "id": 277141637, 
+            "id_str": "277141637", 
+            "is_translator": false, 
+            "lang": "en", 
+            "listed_count": 55, 
+            "location": "", 
+            "name": "The PyPy Project", 
+            "notifications": null, 
+            "profile_background_color": "C0DEED", 
+            "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", 
+            "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", 
+            "profile_background_tile": false, 
+            "profile_image_url": "http://a0.twimg.com/profile_images/1331828944/pypy_small64_normal.png", 
+            "profile_image_url_https": "https://si0.twimg.com/profile_images/1331828944/pypy_small64_normal.png", 
+            "profile_link_color": "0084B4", 
+            "profile_sidebar_border_color": "C0DEED", 
+            "profile_sidebar_fill_color": "DDEEF6", 
+            "profile_text_color": "333333", 
+            "profile_use_background_image": true, 
+            "protected": false, 
+            "screen_name": "pypyproject", 
+            "show_all_inline_media": false, 
+            "statuses_count": 65, 
             "time_zone": null, 
             "url": "http://pypy.org", 
             "utc_offset": null, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 1077, 
+            "followers_count": 1139, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "id_str": "277141637", 
             "is_translator": false, 
             "lang": "en", 
-            "listed_count": 52, 
+            "listed_count": 55, 
             "location": "", 
             "name": "The PyPy Project", 
             "notifications": null, 
             "protected": false, 
             "screen_name": "pypyproject", 
             "show_all_inline_media": false, 
-            "statuses_count": 63, 
+            "statuses_count": 65, 
             "time_zone": null, 
             "url": "http://pypy.org", 
             "utc_offset": null, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 1077, 
+            "followers_count": 1139, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "id_str": "277141637", 
             "is_translator": false, 
             "lang": "en", 
-            "listed_count": 52, 
+            "listed_count": 55, 
             "location": "", 
             "name": "The PyPy Project", 
             "notifications": null, 
             "protected": false, 
             "screen_name": "pypyproject", 
             "show_all_inline_media": false, 
-            "statuses_count": 63, 
+            "statuses_count": 65, 
             "time_zone": null, 
             "url": "http://pypy.org", 
             "utc_offset": null, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 1077, 
+            "followers_count": 1139, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "id_str": "277141637", 
             "is_translator": false, 
             "lang": "en", 
-            "listed_count": 52, 
+            "listed_count": 55, 
             "location": "", 
             "name": "The PyPy Project", 
             "notifications": null, 
             "protected": false, 
             "screen_name": "pypyproject", 
             "show_all_inline_media": false, 
-            "statuses_count": 63, 
+            "statuses_count": 65, 
             "time_zone": null, 
             "url": "http://pypy.org", 
             "utc_offset": null, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 1077, 
+            "followers_count": 1139, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "id_str": "277141637", 
             "is_translator": false, 
             "lang": "en", 
-            "listed_count": 52, 
+            "listed_count": 55, 
             "location": "", 
             "name": "The PyPy Project", 
             "notifications": null, 
             "protected": false, 
             "screen_name": "pypyproject", 
             "show_all_inline_media": false, 
-            "statuses_count": 63, 
-            "time_zone": null, 
-            "url": "http://pypy.org", 
-            "utc_offset": null, 
-            "verified": false
-        }
-    }, 
-    {
-        "contributors": null, 
-        "coordinates": null, 
-        "created_at": "Thu Jan 26 19:31:32 +0000 2012", 
-        "favorited": false, 
-        "geo": null, 
-        "id": 162618691857559552, 
-        "id_str": "162618691857559552", 
-        "in_reply_to_screen_name": "paidhi", 
-        "in_reply_to_status_id": 162605188014817280, 
-        "in_reply_to_status_id_str": "162605188014817280", 
-        "in_reply_to_user_id": 14812261, 
-        "in_reply_to_user_id_str": "14812261", 
-        "place": null, 
-        "retweet_count": 0, 
-        "retweeted": false, 
-        "source": "<a href=\"http://itunes.apple.com/us/app/twitter/id409789998?mt=12\" rel=\"nofollow\">Twitter for Mac</a>", 
-        "text": "@paidhi We do!  Take a look at the bottom of each blog post.", 
-        "truncated": false, 
-        "user": {
-            "contributors_enabled": false, 
-            "created_at": "Mon Apr 04 19:44:41 +0000 2011", 
-            "default_profile": true, 
-            "default_profile_image": false, 
-            "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
-            "favourites_count": 0, 
-            "follow_request_sent": null, 
-            "followers_count": 1077, 
-            "following": null, 
-            "friends_count": 7, 
-            "geo_enabled": false, 
-            "id": 277141637, 
-            "id_str": "277141637", 
-            "is_translator": false, 
-            "lang": "en", 
-            "listed_count": 52, 
-            "location": "", 
-            "name": "The PyPy Project", 
-            "notifications": null, 
-            "profile_background_color": "C0DEED", 
-            "profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png", 
-            "profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png", 
-            "profile_background_tile": false, 
-            "profile_image_url": "http://a0.twimg.com/profile_images/1331828944/pypy_small64_normal.png", 
-            "profile_image_url_https": "https://si0.twimg.com/profile_images/1331828944/pypy_small64_normal.png", 
-            "profile_link_color": "0084B4", 
-            "profile_sidebar_border_color": "C0DEED", 
-            "profile_sidebar_fill_color": "DDEEF6", 
-            "profile_text_color": "333333", 
-            "profile_use_background_image": true, 
-            "protected": false, 
-            "screen_name": "pypyproject", 
-            "show_all_inline_media": false, 
-            "statuses_count": 63, 
+            "statuses_count": 65, 
             "time_zone": null, 
             "url": "http://pypy.org", 
             "utc_offset": null, 

File pypysite/update.py

View file
     def get(self):
         downloads = self.process_html()
 
-        # set latest version, by using max
+        # set latest version
         # self.curver = '1.8'
         self.curver = max(downloads['binaries'].keys())
+        self.curversandbox = max(downloads['sandbox'].keys())
 
         # prepare json output dict
         downloads_json = {}
         downloads_json['version'] = self.curver
         downloads_json['binaries'] = downloads['binaries'][self.curver]
-        downloads_json['sandbox'] = downloads['sandbox'][self.curver]
+        downloads_json['sandbox'] = downloads['sandbox'][self.curversandbox]
         downloads_json['source'] = self.get_source_files()
 
         self.save(downloads_json)
+        print('Downloads done')
 
     def save(self, json_obj):
         # save downloads to json
             ])
 
         self.save_json(compatibility_json)
+        print('Compatibility done')
 
     def refresh(self):
         self.r = requests.get(self.url)