[cc-commits] [CC SVN] r12524 - stats/wikibot

webmaster at creativecommons.org webmaster at creativecommons.org
Sun May 3 13:08:12 EDT 2009


Author: bhy
Date: 2009-05-03 17:08:12 +0000 (Sun, 03 May 2009)
New Revision: 12524

Added:
   stats/wikibot/grab_links.py
Modified:
   stats/wikibot/ccquery.py
   stats/wikibot/juris.csv
   stats/wikibot/utils.py
Log:
Done grab related links from various external resources

Modified: stats/wikibot/ccquery.py
===================================================================
--- stats/wikibot/ccquery.py	2009-05-03 15:47:02 UTC (rev 12523)
+++ stats/wikibot/ccquery.py	2009-05-03 17:08:12 UTC (rev 12524)
@@ -214,6 +214,12 @@
         """
         return self._juris_code2x('fips', code)
 
+    def juris_fips2code(self, fips):
+        fips = fips.upper()
+        c = self.conn.execute('select code from juris where fips=?', (fips,) )
+        result = c.fetchone()[0]
+        return result
+
     def region_code2name(self, code):
         """
         Get region name from code.
@@ -233,12 +239,16 @@
         result = [t[0] for t in c]
         return result
 
-    def all_juris(self):
+    def all_juris(self, ported_only = True):
         """
         All juris avaiable with ported license.
         """
-        c = self.conn.cursor()
-        c.execute('select distinct code from juris where is_ported<>0')
+        if ported_only:
+            where_clause = 'where is_ported<>0'
+        else:
+            where_clause = ''
+        c = self.conn.cursor()        
+        c.execute('select distinct code from juris '+where_clause)
         result = [t[0] for t in c]
         return result
 

Added: stats/wikibot/grab_links.py
===================================================================
--- stats/wikibot/grab_links.py	                        (rev 0)
+++ stats/wikibot/grab_links.py	2009-05-03 17:08:12 UTC (rev 12524)
@@ -0,0 +1,266 @@
+"""
+Grab related links for jurisdictions.
+"""
+import re
+import json
+import urllib2
+import hashlib
+import StringIO
+from os import path
+from xml.etree import ElementTree
+
+import ccquery
+import grab_flags
+from utils import log
+
+DB_FILE = 'ccdata.sqlite'
+PAGE_GLOBALVOICE = 'http://globalvoicesonline.org/'
+PAGE_OPENNET = 'http://map.opennet.net/filtering-pol.xml'
+
+class FuzzyDict(dict):
+    """
+    Provides a dictionary that performs fuzzy lookup.
+    Got from here with some modification:
+    http://code.activestate.com/recipes/475148/
+    """
+    def __init__(self, items = None, cutoff = .6):
+        """Construct a new FuzzyDict instance
+
+        items is an dictionary to copy items from (optional)
+        cutoff is the match ratio below which mathes should not be considered
+        cutoff needs to be a float between 0 and 1 (where zero is no match
+        and 1 is a perfect match)"""
+        super(FuzzyDict, self).__init__()
+
+        if items:
+            self.update(items)
+        self.cutoff =  cutoff
+
+        # short wrapper around some super (dict) methods
+        self._dict_contains = lambda x: dict.__contains__(self, x)
+        self._dict_getitem = lambda x: dict.__getitem__(self, x)
+        return
+
+    def _match(self, u, v):
+        """
+        Caculate match radio for two given object.
+        """
+        u = u.lower()
+        v = v.lower()
+        spacechars = '(),-'
+        for c in spacechars:
+            u = u.replace(c, ' ')
+            v = v.replace(c, ' ')
+        
+        u_words = set(u.split())
+        v_words = set(v.split())
+        null = set()
+
+        if null.issuperset(u_words - v_words) or \
+            null.issuperset(v_words - u_words):
+                return 1.0
+
+        diffsize = len(u_words.symmetric_difference(v_words))
+        total = len(u_words) + len(v_words)
+        radio = 1.0 - float(diffsize) / float(total)
+        return radio
+
+    def _search(self, lookfor, stop_on_first = False):
+        """Returns the value whose key best matches lookfor
+
+        if stop_on_first is True then the method returns as soon
+        as it finds the first item
+        """
+        # if the item is in the dictionary then just return it
+        if self._dict_contains(lookfor):
+            return True, lookfor, self._dict_getitem(lookfor), 1
+
+        # test each key in the dictionary
+        best_ratio = 0
+        best_match = None
+        best_key = None
+        for key in self:
+            try:
+            # calculate the match value
+                ratio = self._match(key, lookfor)
+            except TypeError:
+                break
+
+            # if this is the best ratio so far - save it and the value
+            if ratio > best_ratio:
+                best_ratio = ratio
+                best_key = key
+                best_match = self._dict_getitem(key)
+
+            if stop_on_first and ratio >= self.cutoff:
+                break
+
+        return (
+            best_ratio >= self.cutoff,
+            best_key,
+            best_match,
+            best_ratio)
+
+    def __contains__(self, item):
+        "Overides Dictionary __contains__ to use fuzzy matching"
+        if self._search(item, True)[0]:
+            return True
+        else:
+            return False
+
+    def __getitem__(self, lookfor):
+        "Overides Dictionary __getitem__ to use fuzzy matching"
+        matched, key, item, ratio = self._search(lookfor)
+
+        if not matched:
+            raise KeyError(
+                "'%s'. closest match: '%s' with ratio %.3f"%
+                    (str(lookfor), str(key), ratio))
+
+        return item
+
+def cached_urlread(url):
+    """
+    urlopen with local disk cache support, and retry 5 times if we get connection error.
+    """
+    log("Querying...")
+    page = None
+    cache_fn = path.join('/tmp/',  hashlib.md5(url).hexdigest())
+    try:
+        page = open(cache_fn).read()
+        log("Got from cache.\n")
+    except IOError:
+        #no cache
+        pass
+    
+    if page is None:
+        page = urllib2.urlopen(url).read()
+
+        open(cache_fn, 'w').write(page)
+        log("OK.\n")
+    return page
+
+class LinksGrabber(object):
+    def __init__(self):
+        self.query = ccquery.CCQuery(DB_FILE)
+
+        matching_dict = FuzzyDict()
+        code_dict = {}
+
+        for code, name in self.jurisiter():
+            links = list()
+            if code:
+                code_dict[code] = links
+            if name:
+                matching_dict[name] = links
+        # Some more alias
+        matching_dict['usa'] = code_dict['US']
+
+        self.matching_dict = matching_dict
+        self.code_dict = code_dict
+
+        return
+
+    def jurisiter(self):
+        q = self.query
+        for code in q.all_juris(ported_only=False):
+            name = q.juris_code2name(code)
+            # convert from unicode to string
+            code = str(code)
+            name = str(name)
+            yield (code, name)
+        return
+
+    def grab_globalvoices(self):
+        print "Grabing Global Voices links..."
+        page = cached_urlread(PAGE_GLOBALVOICE)
+        # Match HTML tags like:
+        #   <option value="/-/world/central-asia-caucasus/afghanistan/" >
+        matches = re.finditer(r'<option value="/-/world/(.*)/(.*)/" >', page)
+        for match in matches:
+            region, country = match.group(1,2)
+            url = "http://globalvoicesonline.org/-/world/%s/%s/"%(region, country)
+            try:
+                links = self.matching_dict[country]
+                links.append(('Global Voices', url))
+            except KeyError:
+                print "No juris for:", url
+
+        return
+
+    def grab_opennet(self):
+        print "Grabing OpenNet links..."
+        page = cached_urlread(PAGE_OPENNET)
+        t = ElementTree.parse(StringIO.StringIO(page))
+        for state in t.findall('state'):
+            urle = state.find('url')
+            if urle is None:
+                continue
+            fips = state.get('id')
+            url = urle.text
+            
+            code = self.query.juris_fips2code(fips)
+            code = str(code)
+            links = self.code_dict[code]
+            links.append(('OpenNet Profile', url))
+        return
+
+    def grab_wikipedia(self):        
+        # Just reuse the Wikipedia country page grabber in grab_flags.py
+        print "Grabing Wikipedia links..."
+        for code, page in grab_flags.country_pages():
+            links = self.code_dict[code]
+            page = page.replace(' ', '_')
+            url = 'http://en.wikipedia.org/wiki/' + page
+            links.append(('Wikipedia Article', url))
+        return
+
+    def gen_cci(self):
+        " Generate urls like http://creativecommons.org/international/cn/"
+        for code, links in self.code_dict.iteritems():
+            url = 'http://creativecommons.org/international/%s/'%(code.lower())
+            links.append(('CCI Page', url))
+        return
+
+    def gen_herdict(self):
+        " Generate urls like http://www.herdict.org/web/explore/country/CN"
+        for code, links in self.code_dict.iteritems():
+            url = 'http://www.herdict.org/web/explore/country/%s'%(code)
+            links.append(('Herdict Web', url))
+        return
+
+    def __call__(self):
+        self.gen_cci()
+        self.grab_wikipedia()
+        self.grab_globalvoices()
+        self.grab_opennet()
+        self.gen_herdict()
+        return
+
+    def check(self):
+        for code, links in self.code_dict.iteritems():
+            sites = [l[0] for l in links]
+            if len(set(sites)) < len(sites):
+                print 'Please check:', code
+        return
+
+    def dump(self, filename):
+        json.dump(self.code_dict, open(filename, 'w'), indent=4)
+        return
+
+def main():
+    g = LinksGrabber()
+    g()
+    g.check()
+    g.dump('related_links.txt')
+
+
+if __name__=='__main__':
+    main()
+
+
+    
+        
+
+
+

Modified: stats/wikibot/juris.csv
===================================================================
--- stats/wikibot/juris.csv	2009-05-03 15:47:02 UTC (rev 12523)
+++ stats/wikibot/juris.csv	2009-05-03 17:08:12 UTC (rev 12524)
@@ -156,7 +156,7 @@
 "Congo, Republic of the",CG,CF,AF,0
 Central African Republic,CF,CT,AF,0
 "Congo, Democratic Republic of the",CD,CG,AF,0
-Czech Republic,CZ,EZ,EU,0
+Czech Republic,CZ,EZ,EU,1
 Cyprus,CY,CY,AS,0
 Christmas Island,CX,KT,AS,0
 Costa Rica,CR,CS,SA,0
@@ -221,7 +221,7 @@
 Libya,LY,LY,AF,0
 Holy See (Vatican City),VA,VT,EU,0
 Saint Vincent and the Grenadines,VC,VC,SA,0
-United Arab Emirates,AE,AE,AS,0
+United Arab Emirates,AE,TC,AS,0
 Andorra,AD,AN,EU,0
 Antigua and Barbuda,AG,AC,SA,0
 British Virgin Islands,VG,VI,SA,0

Modified: stats/wikibot/utils.py
===================================================================
--- stats/wikibot/utils.py	2009-05-03 15:47:02 UTC (rev 12523)
+++ stats/wikibot/utils.py	2009-05-03 17:08:12 UTC (rev 12524)
@@ -2,7 +2,8 @@
 import time
 
 def log(s):
-    print >>sys.stderr, s
+    print >>sys.stderr, s,
+    sys.stderr.flush()
     return
 
 def tries(ntries, fn, *arglist, **argdict):




More information about the cc-commits mailing list