[cc-commits] [CC SVN] r15253 - in stats: . europeana
webmaster at creativecommons.org
webmaster at creativecommons.org
Fri Sep 30 12:16:14 EDT 2011
Author: cronuser at backup
Date: 2011-09-30 16:16:13 +0000 (Fri, 30 Sep 2011)
New Revision: 15253
Added:
stats/europeana/
stats/europeana/nightly_import_scrape.py
Log:
Added script to scrape europeana.eu for PDM counts.
Added: stats/europeana/nightly_import_scrape.py
===================================================================
--- stats/europeana/nightly_import_scrape.py (rev 0)
+++ stats/europeana/nightly_import_scrape.py 2011-09-30 16:16:13 UTC (rev 15253)
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+from BeautifulSoup import BeautifulSoup
+from sqlalchemy.ext.sqlsoup import SqlSoup
+import urllib2
+import re
+import datetime
+import sys
+sys.path.append('..')
+import dbconfig
+
+
+def scrape_europeana_pdm(query):
+ '''Scrape Europeana for PDM 1.0 count'''
+ page = urllib2.urlopen(query).read()
+ soup = BeautifulSoup(page)
+ regex = re.compile('Results\s+1\s+-\s+12\s+of\s+([0-9,]+)')
+ count_part = soup.firstText(regex)
+ if count_part:
+ pdm_count = regex.match(count_part).group(1)
+ return int(pdm_count.replace(',',''))
+ else:
+ raise Exception, "No PDM count found at Europeana!"
+
+
+def main(query):
+ # Extract a UTC datetime object
+ utc_datetime = datetime.datetime.utcnow()
+ # Connect to the DB; if we can't, this will fail anyway.
+ db = SqlSoup(dbconfig.dburl)
+ # Scrape for PDM count
+ count = scrape_europeana_pdm(query)
+
+ # Build a db row
+ row = {}
+ row['utc_time_stamp'] = utc_datetime
+ row['site'] = 'http://europeana.eu/'
+ row['license_uri'] = 'http://creativecommons.org/publicdomain/mark/1.0/'
+ row['count'] = count
+
+ # Insert count into site_specific table
+ db.site_specific.insert(**row)
+ db.flush()
+
+
+if __name__ == '__main__':
+ url = 'http://europeana.eu/portal/search.html?query=*:*&qf=RIGHTS:http://creativecommons.org/publicdomain/mark/1.0/&view=table&wpfv=N'
+ main(url)
More information about the cc-commits
mailing list