Commits

Dan Chudnov committed 8a42e1b Merge

merged latest

  • Participants
  • Parent commits 86ba78b, 5f95732

Comments (0)

Files changed (7)

 
 Install ubuntu packages (and dependencies) for:
 
-    python ipython python-setuptools python-simplejson
+    python ipython python-setuptools python-simplejson python-feedparser
     postgresql-8.3 postgresql-client-8.3 python-psycopg2
     sun-java6-jdk solr-tomcat5.5
     libapache2-mod-wsgi 
 
 Install non-.deb python dependencies:
 
-    from tarball:   Django 1.2.1
+    from tarball:   Django 1.2.4
     w/easy_install: solrpy 0.9 (and iso8601, if import from zodb install)
 
 Set up solr config.  Move the original /etc/solr/conf/schema.xml
     list_display = ['id', 'entry', 'tag', 'sequence_num']
     search_fields = ['tag']
 
-#class EntryWebhookAdmin(admin.ModelAdmin):
-#    list_display = ['id', 'entry', 'url', 'date_first_attempt',
-#        'num_attempts', 'date_latest_attempt']
-#    list_filter = ['date_first_attempt', 'date_latest_attempt',
-#        'num_attempts']
-
 admin.site.register(m.GroupProfile, GroupProfileAdmin)
 admin.site.register(m.UserProfile, UserProfileAdmin)
 admin.site.register(m.Filter, FilterAdmin)
 admin.site.register(m.Url, UrlAdmin)
 admin.site.register(m.Entry, EntryAdmin)
 admin.site.register(m.EntryTag, EntryTagAdmin)
-#admin.site.register(m.EntryWebhook, EntryWebhookAdmin)
     token = m.CharField(blank=True, max_length=32)
     tz = m.CharField(blank=True, max_length=6)
     group_invites = m.ManyToManyField(Group, related_name='invitees', blank=True)
-    webhook_url = m.URLField(blank=True, verify_exists=True)
     date_modified = m.DateTimeField(auto_now=True)
     
     def solr_reindex (self):
 class UserProfileForm (ModelForm):
     class Meta:
         model = UserProfile
-        fields = ['url', 'is_private', 'default_to_private_entry',
-            'webhook_url']
+        fields = ['url', 'is_private', 'default_to_private_entry']
 
 
 class Filter (m.Model):
         if solr_delete:
             self.solr_delete()
         super(Entry, self).delete()
-        
 
-#class EntryWebhook (m.Model):
-#    entry = m.ForeignKey(Entry, unique=True)
-#    url = m.URLField()
-#    num_attempts = m.SmallIntegerField(default=0)
-#    date_first_attempt = m.DateTimeField(auto_now_add=True)
-#    date_latest_attempt = m.DateTimeField(auto_now=True, db_index=True)
-#    latest_attempt_status = m.CharField(max_length=3, blank=True)
-#
-#    class Meta:
-#        verbose_name = "Webhook POST URL"
-
-        
-## This handler is wired up to Entry's post_save to create a new
-## EntryWebhook.  Done here with a signal instead of on Entry.save()
-## because we only want to do it when the Entry is first created.
-#def create_entry_webhook_handler(sender, **kwargs):
-#    if kwargs['created']:
-#        instance = kwargs['instance']
-#        webhook_url = instance.user.get_profile().webhook_url
-#        if webhook_url:
-#            entry_webhook = EntryWebhook(entry=kwargs['instance'],
-#                url=webhook_url)
-#            entry_webhook.save()
-#post_save.connect(create_entry_webhook_handler, sender=Entry)

base/templates/prefs.html

     by default -- even if your account isn't private.  
 </p>
 
-
 <table class='gentable'>
     <thead>
     </thead>
                 <td>{{ profile_form.default_to_private_entry }}</td>
             </tr>
             <tr>
+            <tr>
                 <td>&nbsp;</td>
                 <td>
                     <input type='submit' value='update'>
 
 
 {% endblock content %}
+
         client = Client()
         self.assertTrue(client.login(username='unalog', password='unalog'))
         response = client.post('/entry/new', self.test_entry)
+        self.assertEqual(response.status_code, 302)
         self.assertEqual(response['location'], 
                 'http://testserver/entry/1/edit/')
         entry = m.Entry.objects.get(id=1)
         entry_json = json.dumps(self.test_entry)
         response = client.post('/entry/new', entry_json, 
                 content_type='application/json')
-        self.assertEqual(response.status_code, 302)
-        self.assertEqual(response['location'], 'http://testserver/entry/1/edit/')
-        entry = m.Entry.objects.get(id=1)
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response['location'], 'http://testserver/entry/2/')
+        entry = m.Entry.objects.get(id=2)
         self.assertEqual(entry.url.value, 'http://example.com/')
         self.assertEqual(entry.title, 'hey example.com!')
         self.assertEqual(entry.is_private, False)
         entry_json = json.dumps(test_entry)
         response = client.post('/entry/new', entry_json, 
                 content_type='application/json')
-        self.assertEqual(response.status_code, 302)
-        self.assertEqual(response['location'], 'http://testserver/entry/1/edit/')
-        entry = m.Entry.objects.get(id=1)
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response['location'], 'http://testserver/entry/3/')
+        entry = m.Entry.objects.get(id=3)
         self.assertEqual(entry.date_created, 
             datetime.datetime(1985, 4, 12, 23, 20, 50))
         
             new_entry.save()
             new_entry.add_tags(tags_orig)
 
+            if payload == "application/json":
+                url = reverse('entry', args=[new_entry.id])
+                if was_created:
+                    return HttpResponseCreated(url)
+                else:
+                    return HttpResponseRedirect(url)
             request.user.message_set.create(message='Saved!')
+
             # If they had to confirm this, they don't need an edit screen again
             if submit == 'Save anyway':
                 return HttpResponseRedirect(reverse('index'))
         except:
             page = None
     return render_to_response('index.html', context)
+
+class HttpResponseCreated(HttpResponse):
+    status_code = 201
+
+    def __init__(self, location):
+        HttpResponse.__init__(self)
+        self["Location"] = location
 #!/usr/bin/env python
 
 """
-This is a delcious to unalog import tool. 
+This is a delicious to unalog import tool. You'll need to export your
+delicious links and save them as a file ... it should be html. Then
+you run this script with your unalog username and password with the 
+delicious export file:
 
   ./d2u.py --username me --password secret delicious-20101216.htm
 
 import json
 import optparse
 import os
+import socket
+import sys
 import time
-import urllib
+import urllib2
 
 from html5lib import HTMLParser, treebuilders
 from lxml import etree
 
+socket.setdefaulttimeout(10)
+
 opt_parser = optparse.OptionParser()
 opt_parser.add_option('-u', '--username', dest='username')
 opt_parser.add_option('-p', '--password', dest='password')
 opt_parser.add_option('-n', '--unalog', dest='unalog', 
                       default="http://unalog.com")
+opt_parser.add_option('-s', '--skip', type=int, dest='skip', default=0)
+opt_parser.add_option('-c', '--check', action='store_true', dest='check')
 
 opts, args = opt_parser.parse_args()
 
 h.add_credentials(opts.username, opts.password)
 unalog = opts.unalog.rstrip("/") + "/entry/new"
 
-status = {}
+summary = {}
 count = 0
 
 for dt in doc.findall(".//{%s}dt" % xhtml):
+    count += 1
+    if opts.skip > 1 and count < opts.skip:
+        continue
+
     # get the bookmark from the dt
     a = dt.find('{%s}a' % xhtml)
     b  = a.attrib
     t = time.localtime(int(b["add_date"]))
     t = time.strftime('%Y-%m-%dT%H:%M:%S%z', t)
 
-    # get the content at the url
+    # get the content at the url only if it looks like html or text
     url = b["href"]
-    resp, content = h.request(url, "GET")
-    status[resp.status] = status.get(resp.status, 0) + 1
+    try:
+        resp = urllib2.urlopen(url)
+        content = resp.read()
+        status = resp.code 
+        summary[status] = summary.get(status, 0) + 1
+        content_type = resp.headers['content-type']
+        if 'html' in content_type or 'text' in content_type:
+            content = content.decode('utf-8', 'replace')
+        else:
+            content = None
+    except urllib2.HTTPError, e:
+        content = None
+        status = e.code
+    except urllib2.URLError, e:
+        content = None
+        status = e.reason
+    except Exception, e:
+        content = None
+        status = str(type(e))
+
+    summary[status] = summary.get(status, 0) + 1
+    print "\t".join([url, str(status)])
+
+    # don't bother posting to unalog if we're just checking
+    if opts.check:
+        continue
 
     # build the bookmark entry
     entry = {
         "private": b["private"] == 1,
         "date_created": t,
         "comment": comment,
-        #"content": content
+        "content": content
     }
 
     # send the bookmark to unalog as json
-    print url
     resp, content = h.request(unalog, "POST", body=json.dumps(entry),
             headers={"content-type": "application/json"})
 
-    count += 1
-    if count % 30 == 0:
-        break
+    if resp.status not in [200, 201, 302]:
+        print "post to unalog failed! %s" % url
 
-print status
+
+
+print "imported %s bookmarks" % count
+print "response summary: %s" % summary