summaryrefslogtreecommitdiff
path: root/modules/url_titles.py
diff options
context:
space:
mode:
Diffstat (limited to 'modules/url_titles.py')
-rw-r--r--modules/url_titles.py25
1 files changed, 18 insertions, 7 deletions
diff --git a/modules/url_titles.py b/modules/url_titles.py
index fe6282b..b7d5e25 100644
--- a/modules/url_titles.py
+++ b/modules/url_titles.py
@@ -10,6 +10,7 @@ from PIL import ImageFile
class Module:
re_http = re.compile(r'(https?://[^\ #]+)')
re_title = re.compile(r'<title[^>]*?>(.*?)</title>', re.S | re.I)
+ re_meta = re.compile(r'<meta\s+http-equiv="content-type"\s+content="[^"]+charset=([^"]+)"', re.I)
def __init__(self, bot):
self.irc = bot
@@ -62,11 +63,6 @@ class Module:
u = urllib2.urlopen(url)
except:
return
- #enc = ct.split('encoding=')
- #if len(enc) == 2:
- #enc = enc[1]
- #else:
- #enc = None
if u.headers['content-type'].startswith('text/html'):
#s = u.read()
if 'content-encoding' in u.headers and u.headers['content-encoding'] == 'gzip':
@@ -74,6 +70,7 @@ class Module:
s.seek(0)
s = gzip.GzipFile(fileobj = s).read()
m = self.re_title.search(s)
+ enc = self.re_meta.search(s)
else:
s = ''
m = None
@@ -81,11 +78,25 @@ class Module:
while buf and time.time() - t < 5.0:
s += buf
m = self.re_title.search(s)
- if m:
+ enc = self.re_meta.search(s)
+ if m and (enc or '</head>' in s):
break
buf = u.read(1024)
+ if enc:
+ enc = enc.groups()[0]
+ else:
+ ct = u.headers['content-type']
+ enc = ct.lower().split('charset=')
+ if len(enc) == 2:
+ enc = enc[1]
+ else:
+ enc = None
+
if m:
- titles.append(m.groups()[0])
+ s = m.groups()[0]
+ if enc:
+ s = s.decode(enc).encode('utf8')
+ titles.append(s)
elif u.headers['content-type'] in ('image/gif', 'image/png', 'image/jpeg'):
def pretty_size(size):
suffixes = (('B', 2**10), ('KiB', 2**20), ('MiB', 2**30), ('GiB', 2**40), ('TiB', 2**50))