Commits

A S committed 9bd8d30

Rough version of Update classes, needs to be abstracted further

  • Participants
  • Parent commits 76259c6

Comments (0)

Files changed (5)

pypysite/__init__.py

 
 import config
 import data
+import update
 import twitter
 import utils
 
 # Admin pages
 
 @app.route('/update')
-def update():
-    import update
-    return make_response("Downloads: " + str(update.dr.status_code) + " / Compatiblity Wiki: " + str(update.cr.status_code))
+def updateall():
+    update.downloads.refresh()
+    update.compatibility.refresh()
+    return make_response("Downloads: " + str(update.downloads.r.status_code) + " / Compatiblity Wiki: " + str(update.compatibility.r.status_code))
 
 
 #JSON
 import config
+import update
 from flask import json
 
 # load downloads data
         downloads_json = f.read()
         downloads = json.loads(downloads_json)
 except IOError:
-    import update
+    update.downloads.refresh()
 
 # load downloads data
 try:
         compatibility_json = f.read()
         compatibility = json.loads(compatibility_json)
 except IOError:
-    import update
+    update.compatibility.refresh()

pypysite/data/downloads.json

             "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-linux.tar.bz2", 
             "pypy-1.8-linux.tar.bz2", 
             "12.0 MB", 
-            "16 days ago", 
-            "507"
+            "18 days ago", 
+            "553"
         ], 
         "linux64": [
             "Linux", 
             "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-linux64.tar.bz2", 
             "pypy-1.8-linux64.tar.bz2", 
             "13.5 MB", 
-            "16 days ago", 
-            "980"
+            "18 days ago", 
+            "1028"
         ], 
         "osx64": [
             "Mac", 
             "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-osx64.tar.bz2", 
             "pypy-1.8-osx64.tar.bz2", 
             "13.6 MB", 
-            "15 days ago", 
-            "1083"
+            "17 days ago", 
+            "1143"
         ], 
         "win32": [
             "Windows", 
             "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-win32.zip", 
             "pypy-1.8-win32.zip", 
             "16.8 MB", 
-            "16 days ago", 
-            "2152"
+            "18 days ago", 
+            "2331"
         ]
     }, 
     "sandbox": {
             "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-sandbox-linux.tar.bz2", 
             "pypy-1.8-sandbox-linux.tar.bz2", 
             "3.7 MB", 
-            "10 days ago", 
-            "23"
+            "12 days ago", 
+            "26"
         ], 
         "sandbox-linux64": [
             "Linux", 
             "https://bitbucket.org/pypy/pypy/downloads/pypy-1.8-sandbox-linux64.tar.bz2", 
             "pypy-1.8-sandbox-linux64.tar.bz2", 
             "4.7 MB", 
-            "10 days ago", 
-            "52"
+            "13 days ago", 
+            "58"
         ]
     }, 
     "source": [

pypysite/data/twitter_pypyproject.json

             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 
             "description": "The developers of the PyPy project: a fast, compliant Python interpreter.", 
             "favourites_count": 0, 
             "follow_request_sent": null, 
-            "followers_count": 881, 
+            "followers_count": 883, 
             "following": null, 
             "friends_count": 7, 
             "geo_enabled": false, 

pypysite/update.py

 
 from bs4 import BeautifulSoup, SoupStrainer
 
+
+class DownloadsUpdate:
+    """Update"""
+
+    def __init__(self, domain, repo, systems):
+        self.domain = domain
+        self.repo = repo
+        self.systems = systems
+        self.refresh()
+
+    def get_group(self, s):
+        if 'sandbox' in s:
+            return 'sandbox'
+        else:
+            return 'binaries'
+
+    def get_sys_name(self, s):
+        return s[9:s.find('.', 8)]
+
+    def get_sys_info(self, key, s):
+        for sys, properties in systems.items():
+            if sys in s:
+                return properties[key]
+
+    def get_sys_bit(self, s):
+        if 'src' not in s:
+            if '64' in s:
+                return '64Bit'
+            else:
+                return '32Bit'
+
+    def get_source_files(self):
+        return (
+            (
+                'pypy-' + self.curver + '-src.tar.bz2',
+                'Unix line endings',
+                self.domain + '/pypy/pypy/get/release-' + self.curver + '.tar.bz2'
+            ), (
+                'pypy-' + self.curver + '-src.zip',
+                'Unix line endings',
+                self.domain + '/pypy/pypy/get/release-' + self.curver + '.zip'
+            )
+        )
+
+    def process_html(self):
+        # Soupify
+        downloads_rows = BeautifulSoup(self.r.content,
+            parse_only=SoupStrainer(id='uploaded-files'))
+
+        # Dict build vars
+        downloads = {'binaries': {}}
+        group = 'binaries'
+        ver = False
+        fname = False
+
+        for x in downloads_rows.find_all('td'):
+            if 'name' in x['class']:
+                group = self.get_group(x.string)
+                ver = x.string[5:8]
+                fname = self.get_sys_name(x.string)
+
+                if group not in downloads:
+                    downloads[group] = {}
+
+                # pypy version group
+                if ver not in downloads[group]:
+                    downloads[group][ver] = {}
+
+                # pypy system
+                if fname not in downloads[group][ver]:
+                    downloads[group][ver][fname] = []
+
+                # sys pretty name
+                downloads[group][ver][fname].append(self.get_sys_info('name', fname))
+                # sys version
+                downloads[group][ver][fname].append(self.get_sys_info('version', fname))
+                # sys notes
+                downloads[group][ver][fname].append(self.get_sys_info('notes', fname))
+                # sys bits
+                downloads[group][ver][fname].append(self.get_sys_bit(fname))
+                # download url
+                downloads[group][ver][fname].append(self.domain + x.a['href'])
+
+            if x.string != "" and x.string[0].isalnum():
+                downloads[group][ver][fname].append(x.string)
+
+        return downloads
+
+    def get(self):
+        downloads = self.process_html()
+
+        # set latest version, by using max
+        # self.curver = '1.8'
+        self.curver = max(downloads['binaries'].keys())
+
+        # prepare json output dict
+        downloads_json = {}
+        downloads_json['version'] = self.curver
+        downloads_json['binaries'] = downloads['binaries'][self.curver]
+        downloads_json['sandbox'] = downloads['sandbox'][self.curver]
+        downloads_json['source'] = self.get_source_files()
+
+        self.save(downloads_json)
+
+    def save(self, json_obj):
+        # save downloads to json
+        j = open(config.DATA_DIR + '/downloads.json', 'w')
+        # j.write(json.dumps(downloads, sort_keys=True, indent=4))
+        j.write(json.dumps(json_obj, sort_keys=True, indent=4))
+        j.close()
+
+    def refresh(self):
+        self.r = requests.get(self.domain + self.repo)
+        print('Downloads response code: ' + str(self.r.status_code))
+
+        if self.r.status_code == 200:
+            print('Processing...')
+            self.get()
+        else:
+            print('Failed to get Downloads.')
+            return False
+
 # Naming
 systems = {
     'linux': {
     }
 }
 
-# Get page
-bb = 'https://bitbucket.org'
-dr = requests.get(bb + '/pypy/pypy/downloads')
-print('Downloads response code: ' + str(dr.status_code))
-
-# Soupify
-downloads_rows = BeautifulSoup(dr.content,
-    parse_only=SoupStrainer(id='uploaded-files'))
-
-
-# Helper functions
-def get_group(s):
-    if 'sandbox' in s:
-        return 'sandbox'
-    else:
-        return 'binaries'
-
-
-def get_sys_name(s):
-    return s[9:s.find('.', 8)]
-
-
-def get_sys_info(key, s):
-    for sys, properties in systems.items():
-        if sys in s:
-            return properties[key]
-
-
-def get_sys_bit(s):
-    if 'src' not in s:
-        if '64' in s:
-            return '64Bit'
-        else:
-            return '32Bit'
-
-
-# Dict build vars
-downloads = {'binaries': {}}
-group = 'binaries'
-ver = False
-fname = False
-
-for x in downloads_rows.find_all('td'):
-    if 'name' in x['class']:
-        group = get_group(x.string)
-        ver = x.string[5:8]
-        fname = get_sys_name(x.string)
-
-        if group not in downloads:
-            downloads[group] = {}
-
-        # pypy version group
-        if ver not in downloads[group]:
-            downloads[group][ver] = {}
-
-        # pypy system
-        if fname not in downloads[group][ver]:
-            downloads[group][ver][fname] = []
-
-        # sys pretty name
-        downloads[group][ver][fname].append(get_sys_info('name', fname))
-        # sys version
-        downloads[group][ver][fname].append(get_sys_info('version', fname))
-        # sys notes
-        downloads[group][ver][fname].append(get_sys_info('notes', fname))
-        # sys bits
-        downloads[group][ver][fname].append(get_sys_bit(fname))
-        # download url
-        downloads[group][ver][fname].append(bb + x.a['href'])
-
-    if x.string != "" and x.string[0].isalnum():
-        downloads[group][ver][fname].append(x.string)
-
-# set latest version, by using max
-# current_version = '1.8'
-current_version = max(downloads['binaries'].keys())
-
-# set the source files
-source_files = (
-    (
-        'pypy-' + current_version + '-src.tar.bz2',
-        'Unix line endings',
-        bb + '/pypy/pypy/get/release-' + current_version + '.tar.bz2'
-    ), (
-        'pypy-' + current_version + '-src.zip',
-        'Unix line endings',
-        bb + '/pypy/pypy/get/release-' + current_version + '.zip'
-    )
-)
-
-# prepare json output dict
-downloads_json = {}
-downloads_json['version'] = current_version
-downloads_json['binaries'] = downloads['binaries'][current_version]
-downloads_json['sandbox'] = downloads['sandbox'][current_version]
-downloads_json['source'] = source_files
-
-
-# save downloads to json
-j = open(config.DATA_DIR + '/downloads.json', 'w')
-# j.write(json.dumps(downloads, sort_keys=True, indent=4))
-j.write(json.dumps(downloads_json, sort_keys=True, indent=4))
-j.close()
+downloads = DownloadsUpdate('https://bitbucket.org',
+    '/pypy/pypy/downloads',
+    systems)
 
 
 # Compatibility
 # ~~~~~~~~~~~~~
 
-compatibility_url = 'https://api.bitbucket.org/1.0/repositories/pypy/compatibility/wiki/Home'
+class CompatibilityUpdate:
 
-cr = requests.get(compatibility_url)
-print("Compatibility response code: " + str(cr.status_code))
+    def __init__(self):
+        self.url = 'https://api.bitbucket.org/1.0/repositories/pypy/compatibility/wiki/Home'
 
-raw = cr.content
-data = json.loads(raw)
+        self.refresh()
 
-# save file to disk.
-f = open(config.DATA_DIR + '/compatibility.creole', 'w')
-f.write(data['data'].encode('utf-8'))
-f.close()
+    def save(self):
+        # save file to disk.
+        f = open(config.DATA_DIR + '/compatibility.creole', 'w')
+        f.write(self.data['data'].encode('utf-8'))
+        f.close()
 
-# process raw file
-supported = {}
-line_count = 0
-block_marker = -100
-name = None
+    def save_json(self, json_data):
+        # save compatibility to json
+        j = open(config.DATA_DIR + '/compatibility.json', 'w')
+        j.write(json.dumps(json_data, sort_keys=True, indent=4))
+        j.close()
 
-for line in data['data'].splitlines():
-    if line.startswith('== ') and not line.startswith('== **'):
-        category = line[3:]
+    def process(self):
+        # process raw file
+        supported = {}
+        line_count = 0
+        block_marker = -100
+        name = None
 
-    # detect name, set to block to be used as a ref for other details
-    if line.startswith('=== '):
-        sep = line.find('|')
-        block_marker = line_count
-        name = line[6:sep].strip()
+        for line in self.data['data'].splitlines():
+            if line.startswith('== ') and not line.startswith('== **'):
+                category = line[3:]
 
-        supported[name] = {}
-        supported[name]['prettyname'] = line[sep + 2:].strip(']]')
-        supported[name]['category'] = category
+            # detect name, set to block to be used as a ref for other details
+            if line.startswith('=== '):
+                sep = line.find('|')
+                block_marker = line_count
+                name = line[6:sep].strip()
 
-    # compatibility
-    if line_count == block_marker + 1:
-        sep = line.find('|')
-        end = line.find('}}')
-        supported[name]['compatibility'] = line[sep + 2:end]
+                supported[name] = {}
+                supported[name]['prettyname'] = line[sep + 2:].strip(']]')
+                supported[name]['category'] = category
 
-    # url
-    if line_count == block_marker + 2:
-        supported[name]['url'] = line[3:]
+            # compatibility
+            if line_count == block_marker + 1:
+                sep = line.find('|')
+                end = line.find('}}')
+                supported[name]['compatibility'] = line[sep + 2:end]
 
-    # desc
-    if line_count == block_marker + 3:
-        supported[name]['description'] = line[3:]
+            # url
+            if line_count == block_marker + 2:
+                supported[name]['url'] = line[3:]
 
-    if line_count >= block_marker + 4 and block_marker >= 0 and not line.startswith('----'):
-        supported[name]['description'] += " " + line
+            # desc
+            if line_count == block_marker + 3:
+                supported[name]['description'] = line[3:]
 
-    # end block
-    if line.startswith('----'):
-        block_marker = -100
+            if line_count >= block_marker + 4 and block_marker >= 0 and not line.startswith('----'):
+                supported[name]['description'] += " " + line
 
-    line_count += 1
+            # end block
+            if line.startswith('----'):
+                block_marker = -100
 
+            line_count += 1
 
-# convert to an list/array for faster JavaScript processing
-compatibility_json = []
+        return supported
 
-for key, value in sorted(supported.items()):
-    compatibility_json.append([
-        key,
-        value['prettyname'],
-        value['url'],
-        value['description'],
-        value['category'],
-        value['compatibility']
-    ])
+    def get(self):
+        self.data = json.loads(self.r.content)
 
+        supported = self.process()
 
-# save compatibility to json
-j = open(config.DATA_DIR + '/compatibility.json', 'w')
-j.write(json.dumps(compatibility_json, sort_keys=True, indent=4))
-j.close()
+        # convert to an list/array for faster JavaScript processing
+        compatibility_json = []
+
+        for key, value in sorted(supported.items()):
+            compatibility_json.append([
+                key,
+                value['prettyname'],
+                value['url'],
+                value['description'],
+                value['category'],
+                value['compatibility']
+            ])
+
+        self.save_json(compatibility_json)
+
+    def refresh(self):
+        self.r = requests.get(self.url)
+        print("Compatibility response code: " + str(self.r.status_code))
+
+        if self.r.status_code == 200:
+            print('Processing...')
+            self.get()
+            self.save()
+        else:
+            print('Failed to get Compatibility Wiki.')
+            return False
+
+compatibility = CompatibilityUpdate()