[cc-commits] [SCM] deedscraper: Retrieve metadata and deed referrers and format for use on the deeds (branch next) updated. 52f8d9c9359dd2c279976d899ecb9fa463b88a1d

Gitosis User git at gandi0.creativecommons.org
Thu Jan 26 15:26:19 EST 2012


The branch, next has been updated
       via  52f8d9c9359dd2c279976d899ecb9fa463b88a1d (commit)
       via  b633174e1086f9fc871e2e6a8bbd3bbe43468c7f (commit)
       via  1cab53ff3f7534783e9fdf3f365136ff3169af53 (commit)
       via  5f7422572c53c095dc09622942657e5e3d908f4b (commit)
       via  0757e505191259127c56be50baa628cf3ac76471 (commit)
       via  e185e7224c3e7ba0972935c7fafce60f6ffe5ab9 (commit)
       via  19f8d49a8d7cd88997f4ecd99a9e2010c8ec75d1 (commit)
       via  f67cce2a512f4016ca7e2e2b360e792d5c13e244 (commit)
       via  1e5b9b2540379203e68c4d2bf57e8415acd77b39 (commit)
       via  d242c638c7dd6a254130cb73c1882b98fad484f1 (commit)
       via  a0fa6cf8c6d2933f395631676b295c844bb0f042 (commit)
      from  23fb98cbb30dbf5d2a9d81665e40e843cbc56186 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
commit 52f8d9c9359dd2c279976d899ecb9fa463b88a1d
Merge: b633174 5f74225
Author: Jonathan Palecek <jonathan at creativecommons.org>
Date:   Thu Jan 26 14:26:39 2012 -0600

    Merge branch 'master' into next
    
    Conflicts:
    	setup.py

commit b633174e1086f9fc871e2e6a8bbd3bbe43468c7f
Author: Jonathan Palecek <jonathan at creativecommons.org>
Date:   Thu Jan 26 14:25:05 2012 -0600

    Bumped the version number.

commit 1cab53ff3f7534783e9fdf3f365136ff3169af53
Author: Jonathan Palecek <jonathan at creativecommons.org>
Date:   Thu Jan 26 14:23:14 2012 -0600

    Added 'python-gettext<2.0' as a dependency to resolve a version conflict.

-----------------------------------------------------------------------

Summary of changes (followed by patch):
 buildout.cfg                  |    1 +
 paste.ini                     |   10 ++++++++++
 setup.py                      |    6 ++++--
 src/cc/deedscraper/app.py     |   13 +++++++++----
 src/cc/deedscraper/scraper.py |    5 +++--
 src/cc/deedscraper/support.py |   19 +++++++++++++++++++
 6 files changed, 46 insertions(+), 8 deletions(-)
 create mode 100644 paste.ini

diff --git a/buildout.cfg b/buildout.cfg
index fcd0f46..a11d255 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -15,6 +15,7 @@ eggs =
 entry-points = 
     zdaemon=zdaemon.zdctl:main
     nosetests=nose:main
+dependent-scripts = true
 interpreter = python
 
 # include in parts only when need to extract strings
diff --git a/paste.ini b/paste.ini
new file mode 100644
index 0000000..2b9ba02
--- /dev/null
+++ b/paste.ini
@@ -0,0 +1,10 @@
+[DEFAULT]
+debug = true
+
+[app:main]
+use = egg:cc.deedscraper#deedscraper
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 6542
diff --git a/setup.py b/setup.py
index 8868443..796886e 100644
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@ from setuptools import setup, find_packages
 import sys
 
 requires = ['setuptools',
-            'rdfadict==0.7',
+            'rdfadict',
             'web.py',
             'Jinja2',
             'decorator',
@@ -32,6 +32,8 @@ requires = ['setuptools',
             'cc.license',
             'lxml',
             'rdflib<3.0',
+            'PasteScript',
+            'python-gettext<2.0',
             ]
 
 if sys.version_info < (2, 6):
@@ -39,7 +41,7 @@ if sys.version_info < (2, 6):
 
 setup(
     name = "cc.deedscraper",
-    version = "0.3",
+    version = "0.3.2",
 
     packages = ['cc.deedscraper'],
     namespace_packages = ['cc'],
diff --git a/src/cc/deedscraper/app.py b/src/cc/deedscraper/app.py
index 0458f74..f1dc795 100644
--- a/src/cc/deedscraper/app.py
+++ b/src/cc/deedscraper/app.py
@@ -205,12 +205,17 @@ class PublicDomainReferer(RefererHandler):
     def results(self):
         """ Process the scraped triples for the deed. """
         
-        # extra all license relations to check for dual-licensing
+        def cc0_filter(license_name):
+            return CC0_SELECTOR.has_license(str(license_name))
+
+        # look through all license relations to check for dual-licensing
         licenses = metadata.get_license_uri(self.subject, self.triples) or []
-        cc0 = filter(lambda l: CC0_SELECTOR.has_license(l), licenses) or None
-        if cc0: cc0 = cc0[0]
+        if type(licenses) in (str, unicode):
+            licenses = [licenses]
+        cc0 = (filter(cc0_filter, licenses) or [None])[0]
         
-        regist = metadata.registration(self.subject, self.triples, self.license_uri)
+        regist = metadata.registration(
+            self.subject, self.triples, self.license_uri)
 
         # empty values are represented by None
         results = {
diff --git a/src/cc/deedscraper/scraper.py b/src/cc/deedscraper/scraper.py
index 81fd91d..eb5cfcf 100644
--- a/src/cc/deedscraper/scraper.py
+++ b/src/cc/deedscraper/scraper.py
@@ -26,7 +26,7 @@ import logging
 
 from copy import deepcopy
 from decorator import decorator
-from support import LOG
+from support import LOG, barf_if_not_http
 
 from rdfadict.sink import DictSetTripleSink
 
@@ -71,6 +71,8 @@ class ScrapeRequestHandler(object):
 
     def _load_source(self, url, subjects=None, sink=None,
                      depth=2, redirects=None):
+        barf_if_not_http(url)
+
         # bail out if we've hit the parsing limit
         if depth < 0:
             return sink
@@ -121,7 +123,6 @@ class ScrapeRequestHandler(object):
         return triples
 
     def _first_pass(self, url, action='triples'):
-
         redirects = {}
         subjects = []
         sink = TripleDictSink(redirects)
diff --git a/src/cc/deedscraper/support.py b/src/cc/deedscraper/support.py
index 08ecf8a..2b6075c 100644
--- a/src/cc/deedscraper/support.py
+++ b/src/cc/deedscraper/support.py
@@ -24,6 +24,7 @@ import os
 import logging
 import logging.config
 import urllib2
+import urlparse
 import lxml.etree
 from StringIO import StringIO
 from urllib import urlencode
@@ -57,6 +58,8 @@ def get_document_locale(url):
     >>> print get_document_locale('http://code.creativecommons.org/tests/metadata_scraper/')
     None
     """ 
+    barf_if_not_http(url)
+
     try:
         doc =  urllib2.urlopen(url)
         data = StringIO(doc.read())
@@ -75,6 +78,7 @@ def add_qs_parameter(url, key, value):
     >>> add_qs_parameter('http://localhost/example?test=example', 'foo', 'bar')
     'http://localhost/example?test=example&foo=bar'
     """
+    barf_if_not_http(url)
 
     url = urlparse(url)
     query = parse_qs(url.query)
@@ -99,6 +103,8 @@ def get_permissions_link(url):
     >>> get_permissions_link('should return null string')
     'should return null string'
     """
+    barf_if_not_http(url)
+
     parsed = urlparse(url)
     if parsed.scheme in ['http', 'https']:
         return parsed.netloc
@@ -106,3 +112,16 @@ def get_permissions_link(url):
         return parsed.path
     else:
         return url
+
+
+class NotHttpError(Exception): pass
+
+def barf_if_not_http(url):
+    """
+    If a url is not http:// or https://, we shouldn't open it!
+
+    ie, prevent us from opening file:// links.
+    """
+    if urlparse(url)[0] not in ('http', 'https', ''):
+        raise NotHttpError(
+            "url not in form http:// or https://, refusing to open")


hooks/post-receive
-- 
deedscraper: Retrieve metadata and deed referrers and format for use on the deeds


More information about the cc-commits mailing list