[cc-commits] [SCM] deedscraper: Retrieve metadata and deed referrers and format for use on the deeds (branch next) updated. 52f8d9c9359dd2c279976d899ecb9fa463b88a1d
Gitosis User
git at gandi0.creativecommons.org
Thu Jan 26 15:26:19 EST 2012
The branch, next has been updated
via 52f8d9c9359dd2c279976d899ecb9fa463b88a1d (commit)
via b633174e1086f9fc871e2e6a8bbd3bbe43468c7f (commit)
via 1cab53ff3f7534783e9fdf3f365136ff3169af53 (commit)
via 5f7422572c53c095dc09622942657e5e3d908f4b (commit)
via 0757e505191259127c56be50baa628cf3ac76471 (commit)
via e185e7224c3e7ba0972935c7fafce60f6ffe5ab9 (commit)
via 19f8d49a8d7cd88997f4ecd99a9e2010c8ec75d1 (commit)
via f67cce2a512f4016ca7e2e2b360e792d5c13e244 (commit)
via 1e5b9b2540379203e68c4d2bf57e8415acd77b39 (commit)
via d242c638c7dd6a254130cb73c1882b98fad484f1 (commit)
via a0fa6cf8c6d2933f395631676b295c844bb0f042 (commit)
from 23fb98cbb30dbf5d2a9d81665e40e843cbc56186 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 52f8d9c9359dd2c279976d899ecb9fa463b88a1d
Merge: b633174 5f74225
Author: Jonathan Palecek <jonathan at creativecommons.org>
Date: Thu Jan 26 14:26:39 2012 -0600
Merge branch 'master' into next
Conflicts:
setup.py
commit b633174e1086f9fc871e2e6a8bbd3bbe43468c7f
Author: Jonathan Palecek <jonathan at creativecommons.org>
Date: Thu Jan 26 14:25:05 2012 -0600
Bumped the version number.
commit 1cab53ff3f7534783e9fdf3f365136ff3169af53
Author: Jonathan Palecek <jonathan at creativecommons.org>
Date: Thu Jan 26 14:23:14 2012 -0600
Added 'python-gettext<2.0' as a dependency to resolve a version conflict.
-----------------------------------------------------------------------
Summary of changes (followed by patch):
buildout.cfg | 1 +
paste.ini | 10 ++++++++++
setup.py | 6 ++++--
src/cc/deedscraper/app.py | 13 +++++++++----
src/cc/deedscraper/scraper.py | 5 +++--
src/cc/deedscraper/support.py | 19 +++++++++++++++++++
6 files changed, 46 insertions(+), 8 deletions(-)
create mode 100644 paste.ini
diff --git a/buildout.cfg b/buildout.cfg
index fcd0f46..a11d255 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -15,6 +15,7 @@ eggs =
entry-points =
zdaemon=zdaemon.zdctl:main
nosetests=nose:main
+dependent-scripts = true
interpreter = python
# include in parts only when need to extract strings
diff --git a/paste.ini b/paste.ini
new file mode 100644
index 0000000..2b9ba02
--- /dev/null
+++ b/paste.ini
@@ -0,0 +1,10 @@
+[DEFAULT]
+debug = true
+
+[app:main]
+use = egg:cc.deedscraper#deedscraper
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 6542
diff --git a/setup.py b/setup.py
index 8868443..796886e 100644
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@ from setuptools import setup, find_packages
import sys
requires = ['setuptools',
- 'rdfadict==0.7',
+ 'rdfadict',
'web.py',
'Jinja2',
'decorator',
@@ -32,6 +32,8 @@ requires = ['setuptools',
'cc.license',
'lxml',
'rdflib<3.0',
+ 'PasteScript',
+ 'python-gettext<2.0',
]
if sys.version_info < (2, 6):
@@ -39,7 +41,7 @@ if sys.version_info < (2, 6):
setup(
name = "cc.deedscraper",
- version = "0.3",
+ version = "0.3.2",
packages = ['cc.deedscraper'],
namespace_packages = ['cc'],
diff --git a/src/cc/deedscraper/app.py b/src/cc/deedscraper/app.py
index 0458f74..f1dc795 100644
--- a/src/cc/deedscraper/app.py
+++ b/src/cc/deedscraper/app.py
@@ -205,12 +205,17 @@ class PublicDomainReferer(RefererHandler):
def results(self):
""" Process the scraped triples for the deed. """
- # extra all license relations to check for dual-licensing
+ def cc0_filter(license_name):
+ return CC0_SELECTOR.has_license(str(license_name))
+
+ # look through all license relations to check for dual-licensing
licenses = metadata.get_license_uri(self.subject, self.triples) or []
- cc0 = filter(lambda l: CC0_SELECTOR.has_license(l), licenses) or None
- if cc0: cc0 = cc0[0]
+ if type(licenses) in (str, unicode):
+ licenses = [licenses]
+ cc0 = (filter(cc0_filter, licenses) or [None])[0]
- regist = metadata.registration(self.subject, self.triples, self.license_uri)
+ regist = metadata.registration(
+ self.subject, self.triples, self.license_uri)
# empty values are represented by None
results = {
diff --git a/src/cc/deedscraper/scraper.py b/src/cc/deedscraper/scraper.py
index 81fd91d..eb5cfcf 100644
--- a/src/cc/deedscraper/scraper.py
+++ b/src/cc/deedscraper/scraper.py
@@ -26,7 +26,7 @@ import logging
from copy import deepcopy
from decorator import decorator
-from support import LOG
+from support import LOG, barf_if_not_http
from rdfadict.sink import DictSetTripleSink
@@ -71,6 +71,8 @@ class ScrapeRequestHandler(object):
def _load_source(self, url, subjects=None, sink=None,
depth=2, redirects=None):
+ barf_if_not_http(url)
+
# bail out if we've hit the parsing limit
if depth < 0:
return sink
@@ -121,7 +123,6 @@ class ScrapeRequestHandler(object):
return triples
def _first_pass(self, url, action='triples'):
-
redirects = {}
subjects = []
sink = TripleDictSink(redirects)
diff --git a/src/cc/deedscraper/support.py b/src/cc/deedscraper/support.py
index 08ecf8a..2b6075c 100644
--- a/src/cc/deedscraper/support.py
+++ b/src/cc/deedscraper/support.py
@@ -24,6 +24,7 @@ import os
import logging
import logging.config
import urllib2
+import urlparse
import lxml.etree
from StringIO import StringIO
from urllib import urlencode
@@ -57,6 +58,8 @@ def get_document_locale(url):
>>> print get_document_locale('http://code.creativecommons.org/tests/metadata_scraper/')
None
"""
+ barf_if_not_http(url)
+
try:
doc = urllib2.urlopen(url)
data = StringIO(doc.read())
@@ -75,6 +78,7 @@ def add_qs_parameter(url, key, value):
>>> add_qs_parameter('http://localhost/example?test=example', 'foo', 'bar')
'http://localhost/example?test=example&foo=bar'
"""
+ barf_if_not_http(url)
url = urlparse(url)
query = parse_qs(url.query)
@@ -99,6 +103,8 @@ def get_permissions_link(url):
>>> get_permissions_link('should return null string')
'should return null string'
"""
+ barf_if_not_http(url)
+
parsed = urlparse(url)
if parsed.scheme in ['http', 'https']:
return parsed.netloc
@@ -106,3 +112,16 @@ def get_permissions_link(url):
return parsed.path
else:
return url
+
+
+class NotHttpError(Exception): pass
+
+def barf_if_not_http(url):
+ """
+ If a url is not http:// or https://, we shouldn't open it!
+
+ ie, prevent us from opening file:// links.
+ """
+ if urlparse(url)[0] not in ('http', 'https', ''):
+ raise NotHttpError(
+ "url not in form http:// or https://, refusing to open")
hooks/post-receive
--
deedscraper: Retrieve metadata and deed referrers and format for use on the deeds
More information about the cc-commits
mailing list