info = { 'author': 'Jon Bergli Heier', 'title': 'Posten.no tracking', 'description': 'Fetches tracking info from posten.no.', } cfg_section = 'module/tracking' import urllib2, datetime, re, pytz, json, urllib from xml.etree import ElementTree as ET from twisted.internet import reactor from twisted.internet.task import LoopingCall from sqlalchemy import create_engine, Column, Integer, String, DateTime, ForeignKey from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker, relation, backref from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.exc import IntegrityError from sqlalchemy.sql import or_, and_ engine = Session = None Base = declarative_base() class Consignment(Base): __tablename__ = 'consignment' id = Column(Integer, primary_key = True) server = Column(String, nullable = False) nick = Column(String, nullable = False) channel = Column(String) # NULL for private code = Column(String, nullable = False, unique = True) label = Column(String) added = Column(DateTime, nullable = False) last = Column(DateTime) type = Column(String, nullable = False) # Tracking module packages = relation('Package', backref = 'consignment', primaryjoin = 'Package.consignment_id == Consignment.id') def __init__(self, server, nick, channel, type, code, label, added): self.server = server self.nick = nick self.channel = channel self.type = type self.code = code self.label = label self.added = added def __str__(self): s = '\002%s\002' % self.code.encode('utf-8') if self.label: s += ' (%s)' % self.label.encode('utf-8') return s def __unicode__(self): return str(self).decode('utf-8') class Package(Base): __tablename__ = 'package' id = Column(Integer, primary_key = True) consignment_id = Column(Integer, ForeignKey('consignment.id')) code = Column(String, nullable = False, unique = True) last = Column(DateTime) # Last update status = Column(String) # Last status def __init__(self, consignment_id, code): self.consignment_id = consignment_id self.code = code def __str__(self): s = '\002%s\002' % self.code.encode('utf-8') if self.consignment.label: s += ' (%s)' % self.consignment.label.encode('utf-8') return s def __unicode__(self): return str(self).decode('utf-8') class PackageError(Exception): pass class NoPackageFound(PackageError): pass class TrackingResult: def __init__(self, code, date, desc, delivered = False, previous_code = None): self.code = code self.date = date self.desc = desc self.delivered = delivered # Set when a package changes tracking code. self.previous_code = previous_code def __str__(self): return '\002{code}\002 {date} - {desc}'.format(code = self.code, date = self.date, desc = self.desc) tracking_modules = {} class TrackingModuleMeta(type): def __init__(cls, name, bases, dict): if name != 'TrackingModule': tracking_modules[cls.name] = cls class TrackingModule(object): __metaclass__ = TrackingModuleMeta def splitcode(self, code): return code_split(code)[-1] class PostenModule(TrackingModule): name = 'posten' url = 'http://sporing.posten.no/sporing.xml?q=%s' def get_xml(self, url): try: u = urllib2.urlopen(url, timeout = config.getfloat(cfg_section, 'timeout')) except urllib2.HTTPError as e: if e.code == 404: raise NoPackageFound('No packages found') raise PackageError(str(e)) except Exception as e: raise PackageError(str(e)) if u.headers['content-type'].startswith('application/xml'): xml = ET.parse(u) else: xml = None u.close() return xml def get_url(self, code = None): url = 'http://sporing.posten.no/' if code: url += 'sporing.html?q=' + code return url def track(self, code): code = self.splitcode(code) xml = self.get_xml(self.url % code) if not xml: return ns = 'http://www.bring.no/sporing/1.0' packages = xml.find('.//{%s}PackageSet' % (ns,)) if packages is None: raise NoPackageFound('No packages found for \002%s\002.' % code) results = [] for package in packages: code = package.attrib['packageId'] if 'previousPackageId' in package.attrib: previous_code = package.attrib['previousPackageId'] else: previous_code = None eventset = package.find('{%s}EventSet' % ns) if not len(eventset): continue last = eventset[0] desc = last.find('{%s}Description' % ns).text.replace('<', '<').replace('
', ' ') desc = re.sub(r'<[^>]*?>', '', desc).encode('utf8').strip() isodate = last.find('{%s}OccuredAtIsoDateTime' % ns).text[:-10] isodate = datetime.datetime.strptime(isodate, '%Y-%m-%dT%H:%M:%S') city = last.find('{%s}City' % ns).text status = last.find('{%s}Status' % ns).text.encode('utf-8') if city: desc = '%s (%s)' % (desc, city.encode('utf8')) date = last.find('{%s}OccuredAtDisplayDate' % ns).text + ' ' + last.find('{%s}OccuredAtDisplayTime' % ns).text results.append(TrackingResult(code, isodate, desc, status == 'DELIVERED', previous_code = previous_code)) return results class FedexModule(TrackingModule): name = 'fedex' def get_url(self, code = None): url = 'https://www.fedex.com/fedextrack/index.html' if code: url += '?tracknumbers=' + code return url def track(self, code): code = self.splitcode(code) data = {'TrackPackagesRequest': {'appType': 'wtrk', 'uniqueKey': '', 'processingParameters': {'anonymousTransaction': True, 'clientId': 'WTRK', 'returnDetailedErrors': True, 'returnLocalizedDateTime': False}, 'trackingInfoList': [{'trackNumberInfo': {'trackingNumber': code, 'trackingQualifier': '', 'trackingCarrier': ''}}]}} url = 'https://www.fedex.com/trackingCal/track?' + urllib.urlencode({'data': json.dumps(data), 'action': 'trackpackages', 'locale': 'en_US', 'format': 'json', 'version': 99}) u = urllib2.urlopen(url) data = u.read() u.close() for m in re.findall(r'(\\x[a-fA-F\d]{2})', data): data = data.replace(m, chr(int(m[-2:], 16))) data = json.loads(data) response = data['TrackPackagesResponse'] packages = response['packageList'] results = [] for package in packages: lastevent = package['scanEventList'][0] # Ignore events with missing date or time, this seems to be a temporary thing. if not lastevent['date'] or not lastevent['time']: continue date = datetime.datetime.strptime('%s %s' % (lastevent['date'], lastevent['time']), '%Y-%m-%d %H:%M:%S') if lastevent['gmtOffset']: tz = pytz.timezone('Etc/GMT' + ('+' if lastevent['gmtOffset'][0] == '-' else '-') + str(int(lastevent['gmtOffset'][1:3]))) date = tz.localize(date).astimezone(pytz.timezone(config.get(cfg_section, 'local_timezone'))).replace(tzinfo = None) status = lastevent['status'].encode('utf-8') if lastevent['scanDetails']: status += ' - %s' % lastevent['scanDetails'].encode('utf-8') if lastevent['scanLocation']: status += ' (%s)' % lastevent['scanLocation'].encode('utf-8') results.append(TrackingResult(code, date, status, lastevent['isDelivered'])) return results class TollpostModule(TrackingModule): name = 'tollpost' def get_url(self, code = None): url = 'http://www.tollpost.no/minside/SOPS/' if code: url += 'consignment/' + code return code def track(self, code): code = self.splitcode(code) url = 'http://www.tollpost.no/XMLServer/rest/trackandtrace' data = urllib.urlencode({'q': code}) u = urllib2.urlopen(url, data) data = u.read() u.close() data = json.loads(data) response = data['TrackingInformationResponse'] results = [] for shipment in response['shipments']: for item in shipment['items']: code = item['itemId'] delivered = (item['status'] == 'DELIVERED') lastevent = item['events'][-1] date = datetime.datetime.strptime(lastevent['eventTime'], '%Y-%m-%dT%H:%M:%S') event = lastevent['eventDescription'] location = lastevent['location']['name'] status = '%s (%s)' % (event, location) results.append(TrackingResult(code, date, status.encode('utf-8'), delivered)) return results def code_split(code): if ':' in code: return code.split(':') else: return 'posten', code def get_tracking_module(arg): type, code = code_split(arg) return tracking_modules[type]() class Module: def __init__(self, bot): self.setup_db() self.irc = bot self.tracking = [] self.lc = LoopingCall(self.lc_callback) if bot: self.lc.start(config.getfloat(cfg_section, 'interval'), False) self.irc.register_keyword('!track', self) def stop(self): try: self.lc.stop() except: pass def setup_db(self): global engine, Session if engine and Session: return engine = create_engine(config.get(cfg_section, 'db_path')) Base.metadata.create_all(engine) Session = sessionmaker(bind = engine, autoflush = True, autocommit = False) def track_start(self, code, label, nick, channel): type, code = code_split(code) if not type in tracking_modules: return '\002%s\002 is an invalid type.' % type if not len(code): return 'Missing tracking number.' msg = None try: session = Session() consignment = Consignment(self.irc.factory.server, nick, channel, type, code, label.decode('utf8'), datetime.datetime.utcnow()) session.add(consignment) session.commit() msg = 'Now tracking \002%s\002.' % code reactor.callLater(1, self.track_update, code, True) except IntegrityError as e: msg = 'Already tracking \002%s\002.' % code finally: session.close() return msg def track_stop(self, code, nick, channel): type, code = code_split(code) if not len(code): return 'Missing tracking number or label.' msg = None try: session = Session() consignments = session.query(Consignment).outerjoin(Consignment.packages) \ .filter(and_(Consignment.server == self.irc.factory.server, or_(Consignment.code == code, Consignment.label == code, Package.code == code))) if nick: consignments = consignments.filter(Consignment.nick == nick) deleted = [] consignments = consignments.all() if not len(consignments): raise NoResultFound # we should have exactly one consignment, but need to handle cases when there are more for consignment in consignments: code = str(consignment) for p in consignment.packages: session.delete(p) deleted.append(code) session.delete(consignment) session.commit() msg = 'No longer tracking %s' % ', '.join(deleted) except NoResultFound: msg = '\002%s\002 is not being tracked.' % code.decode('utf8') finally: session.close() return msg def track_status(self, code, nick, channel): # don't code_split() here; this is done later if no results are returned msg = [] try: session = Session() consignments = session.query(Consignment).filter(and_(Consignment.server == self.irc.factory.server, or_(Consignment.code.like(code + '%'), Consignment.label.like((code + '%'))))) if nick: consignments = consignments.filter(Consignment.nick == nick) results = [] for row in consignments: try: self.track_update(row.code, propagate_error = True) except NoPackageFound: results.append('No packages found for %s' % unicode(row)) continue except PackageError as e: results.append('Failed to fetch data for %s.' % unicode(row)) continue i = 0 for package in row.packages: i += 1 date = package.last desc = package.status if date and desc: s = '%s %s - %s' % (unicode(package), date, desc) else: s = 'No tracking info found for %s' % unicode(package) results.append(s) if i == 0: results.append('No packages found for %s' % unicode(row)) if len(results): msg = results elif len(code): try: # type:code is handled by get_tracking_module() tm = get_tracking_module(code) data = tm.track(code) if data: for package in data: msg.append('\002%s\002 %s - %s' % (package.code, package.date, package.desc.decode('utf8'))) elif code: msg = 'Failed to fetch tracking data for \002%s\002.' % code.decode('utf8') except PackageError as e: msg = str(e) else: msg = 'No tracking number given or registered.' finally: session.close() return msg def track_label(self, code, label, nick, channel): type, code = code_split(code) if not len(code): return 'Missing tracking number or label.' msg = None try: session = Session() consignment = session.query(Consignment).filter(and_(Consignment.server == self.irc.factory.server, Consignment.code == code, Consignment.nick == nick)).one() consignment.label = label.decode('utf8') session.add(consignment) session.commit() msg = '\002%s\002 now labelled \002%s\002' % (code, label.decode('utf8')) finally: session.close() return msg def track_list(self, nick): msg = None try: session = Session() trackings = [] consignments = session.query(Consignment).filter(and_(Consignment.server == self.irc.factory.server, Consignment.nick == nick)) for row in consignments: trackings.append(str(row)) if len(trackings): msg = 'Trackings for %s: %s' % (nick, ', '.join(trackings)) else: msg = 'No trackings for %s' % nick except NoResultFound: msg = 'No trackings registered for %s.' finally: session.close() return msg def track_url(self, code, nick, channel): msg = [] try: session = Session() consignments = session.query(Consignment).filter(and_(Consignment.server == self.irc.factory.server, or_(Consignment.code.like(code + '%'), Consignment.label.like(code+ '%')))) if nick: consignments = consignments.filter(Consignment.nick == nick) results = [] for consignment in consignments: for package in consignment.packages: m = get_tracking_module('%s:%s' % (consignment.type, package.code)) msg.append(m.get_url(package.code)) if not len(msg): # Rreturn an error only if we got a code to filter with. if code: msg.append('No packages found.') else: m = get_tracking_module('') msg = [m.get_url()] finally: session.close() return msg def keyword(self, nick, channel, kw, msg): usage = 'Usage: !track (start TRACKINGNO [LABEL])|(stop|status TRACKINGNO|LABEL) - See !track help for full list' args = msg.split() if len(args) < 1: self.irc.msg(channel if not channel == self.irc.nickname else nick.split('!')[0], usage) return mode = args[0] if mode.lower() in ('start', 'add', 'stop', 'status'): if len(args) < 2 and mode.lower() != 'status': self.irc.msg(channel if not channel == self.irc.nickname else nick.split('!')[0], usage) return if mode.lower() in ('start', 'add'): code = args[1] label = ' '.join(args[2:]) if len(args) > 2 else '' else: code = ' '.join(args[1:]) elif mode.lower() == 'label': if len(args) < 3: self.irc.msg(channel if not channel == self.irc.nickname else nick.split('!')[0], usage) return code = args[1] label = ' '.join(args[2:]) elif mode.lower() == 'url': code = ' '.join(args[1:]) if len(args) > 1 else '' elif mode.lower() == 'help': self.irc.msg(nick.split('!')[0], ''' start TRACKINGNO [LABEL] - Start tracking package with optional label (alias: add) stop TRACKINGNO|LABEL - Stop tracking package given by either tracking number or label status TRACKINGNO|LABEL - Show status for registered matches, or look up tracking number if no matches (implicitly updates status) list - List all registered tracking numbers without fetching status label TRACKINGNO LABEL - Changes the label for the given tracking number url [TRACKINGNO] - Lists URLs for matches to the corresponding website''') return msg = None if mode.lower() in ('start', 'add'): msg = self.track_start(code, label, nick.split('!')[0], channel if not channel == self.irc.nickname else None) elif mode.lower() == 'stop': msg = self.track_stop(code, nick.split('!')[0], channel if not channel == self.irc.nickname else None) elif mode.lower() == 'status': msg = self.track_status(code, nick.split('!')[0], channel if not channel == self.irc.nickname else None) elif mode.lower() == 'list': msg = self.track_list(nick.split('!')[0]) elif mode.lower() == 'label': msg = self.track_label(code, label, nick.split('!')[0], channel if not channel == self.irc.nickname else None) elif mode.lower() == 'url': msg = self.track_url(code, nick.split('!')[0], channel if not channel == self.irc.nickname else None) else: msg = 'Invalid mode "%s".' % mode if msg: if type(msg) == list: for i in msg: if isinstance(i, unicode): i = i.encode('utf-8') self.irc.msg(channel if not channel == self.irc.nickname else nick.split('!')[0], i) else: if isinstance(msg, unicode): msg = msg.encode('utf-8') self.irc.msg(channel if not channel == self.irc.nickname else nick.split('!')[0], msg) else: self.irc.msg(channel if not channel == self.irc.nickname else nick.split('!')[0], 'No data returned (this is a bug).') # called by start and status def track_update(self, code, announce = False, propagate_error = False): try: session = Session() consignment = session.query(Consignment).filter(and_(Consignment.server == self.irc.factory.server, Consignment.code == code)).one() self.update_consignment(session, consignment, announce, propagate_error) session.commit() finally: session.close() def update_consignment(self, session, consignment, announce = True, propagate_error = False): now = datetime.datetime.utcnow() td = datetime.timedelta(seconds = config.getint(cfg_section, 'cache_time')) # return if consignment was cached within cache_time seconds if consignment.last and consignment.last > now - td: return consignment.last = now target = consignment.channel or consignment.nick target = target.encode('utf-8') removed = False tm = tracking_modules[consignment.type]() try: package_results = tm.track(consignment.code) or [] except PackageError as e: if propagate_error: raise e # ignore, set results to an empty list so that we can reach the stale check at the end package_results = [] for data in package_results: try: package = session.query(Package).filter(Package.consignment_id == consignment.id) if data.previous_code: package = package.filter(Package.code.in_((data.code, data.previous_code))) else: package = package.filter(Package.code == data.code) package = package.one() except NoResultFound: # don't add delivered packages if data.delivered: continue package = Package(consignment.id, data.code) session.add(package) try: session.commit() except IntegrityError: # assume several packets within the same consignment was added as a consignment session.rollback() removed = True self.irc.msg(target, '%s: \002%s\002 conflicts with another consignment' % (consignment.nick.encode('utf-8'), data.code)) break package = session.query(Package).filter(and_(Package.consignment_id == consignment.id, Package.code == data.code)).one() # This happens when the package changes code. if package.code != data.code: # Always announce. msg = '%s: %s has changed code to \002%s\002' % (consignment.nick.encode('utf-8'), package, data.code) self.irc.msg(target, msg) package.code = data.code session.add(package) if package.last == None or data.date > package.last: code = data.code last = data.date desc = data.desc msg = '%s: %s %s - %s' % (consignment.nick.encode('utf-8'), package, last, desc) if data.delivered: session.delete(package) msg += ' (Package delivered - tracking stopped)' removed = True else: package.last = last package.status = desc session.add(package) if announce or removed: self.irc.msg(target, msg) elif package.last != None and package.last < datetime.datetime.utcnow() - datetime.timedelta(hours = 24*30): msg = '%s: Removing stale package %s' % (consignment.nick.encode('utf-8'), consignment) self.irc.msg(target, msg) session.delete(package) removed = True # empty consignment and 30 days old if len(consignment.packages) == 0 and consignment.added < datetime.datetime.utcnow() - datetime.timedelta(hours = 24*30): msg = '%s: Removing stale consignment %s' % (consignment.nick.encode('utf-8'), consignment) self.irc.msg(target, msg) session.delete(consignment) if removed and len(consignment.packages) == 0: msg = '%s: %s is no longer being tracked' % (consignment.nick.encode('utf-8'), consignment) self.irc.msg(target, msg) session.delete(consignment) def lc_callback(self): try: session = Session() consignments = session.query(Consignment).filter(and_(Consignment.server == self.irc.factory.server, or_(Consignment.channel.in_(config.get(self.irc.factory.server, 'channels').split()), Consignment.channel == None))) for row in consignments: self.update_consignment(session, row) session.commit() except Exception: import traceback print 'Exception as %s' % datetime.datetime.now() traceback.print_exc() finally: session.close() if __name__ == '__main__': import sys, ConfigParser, os config = ConfigParser.ConfigParser() config.read([os.path.expanduser('~/.fot')]) m = Module(None) for code in sys.argv[1:]: tm = get_tracking_module(code) tracking = tm.track(code) if tracking: for track in tracking: if type(track) == list: print '\n'.join(track) else: print track