[cc-commits] [CC SVN] r15253 - in stats: . europeana

webmaster at creativecommons.org webmaster at creativecommons.org
Fri Sep 30 12:16:14 EDT 2011


Author: cronuser at backup
Date: 2011-09-30 16:16:13 +0000 (Fri, 30 Sep 2011)
New Revision: 15253

Added:
   stats/europeana/
   stats/europeana/nightly_import_scrape.py
Log:
Added script to scrape europeana.eu for PDM counts.

Added: stats/europeana/nightly_import_scrape.py
===================================================================
--- stats/europeana/nightly_import_scrape.py	                        (rev 0)
+++ stats/europeana/nightly_import_scrape.py	2011-09-30 16:16:13 UTC (rev 15253)
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+from BeautifulSoup import BeautifulSoup
+from sqlalchemy.ext.sqlsoup import SqlSoup
+import urllib2
+import re
+import datetime
+import sys
+sys.path.append('..')
+import dbconfig
+
+
+def scrape_europeana_pdm(query):
+    '''Scrape Europeana for PDM 1.0 count'''
+    page = urllib2.urlopen(query).read()
+    soup = BeautifulSoup(page)
+    regex = re.compile('Results\s+1\s+-\s+12\s+of\s+([0-9,]+)')
+    count_part = soup.firstText(regex)
+    if count_part:
+        pdm_count = regex.match(count_part).group(1)
+        return int(pdm_count.replace(',',''))
+    else:
+        raise Exception, "No PDM count found at Europeana!" 
+
+
+def main(query):
+    # Extract a UTC datetime object
+    utc_datetime = datetime.datetime.utcnow()
+    # Connect to the DB; if we can't, this will fail anyway.
+    db = SqlSoup(dbconfig.dburl)
+    # Scrape for PDM count
+    count = scrape_europeana_pdm(query)
+
+    # Build a db row
+    row = {}
+    row['utc_time_stamp'] = utc_datetime
+    row['site'] = 'http://europeana.eu/'
+    row['license_uri'] = 'http://creativecommons.org/publicdomain/mark/1.0/'
+    row['count'] = count
+
+    # Insert count into site_specific table
+    db.site_specific.insert(**row)
+    db.flush()
+
+
+if __name__ == '__main__':
+    url = 'http://europeana.eu/portal/search.html?query=*:*&qf=RIGHTS:http://creativecommons.org/publicdomain/mark/1.0/&view=table&wpfv=N'
+    main(url) 



More information about the cc-commits mailing list