phone

#! /usr/bin/env python
# encoding:UTF-8

# Copyright (c) 2011 Victor Terron. All rights reserved.
# Institute of Astrophysics of Andalusia, IAA-CSIC
#
# This file is part of phone.
#
# phone is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import htmlentitydefs
import HTMLParser
import optparse
import os.path
import pickle
import re

# Set the default encoding to UTF-8. This solution is undoubtedly unclean, and
# would potentially break in larger scripts, but good enough to make our simple
# script work when used with pipes (avoiding the UnicodeEncodeError exception).

import sys
reload(sys) # restore sys.setdefaultencoding()
sys.setdefaultencoding('UTF-8')

import tempfile
import time
import unicodedata
import urllib

class Scraper(HTMLParser.HTMLParser):
    in_h4 = False
    in_link = False

    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        if tag == 'h4':
            self.in_h4 = True
        if tag == 'a' and 'href' in attrs:
            self.in_link = True
            self.chunks = []
            self.url = attrs['href']

    def handle_data(self, data):
        if self.in_link:
            self.chunks.append(data)
    def handle_endtag(self, tag):
        if tag == 'h4':
            self.in_h4 = False
        if tag == 'a':
            if self.in_h4 and self.in_link:
                print '%s (%s)' % (''.join(self.chunks), self.url)
            self.in_link = False


# http://stackoverflow.com/q/930303/
def remove_accents (unicrap):
    """This replaces UNICODE Latin-1 characters with
    something equivalent in 7-bit ASCII. All characters in the standard
    7-bit ASCII range are preserved. In the 8th bit range all the Latin-1
    accented letters are stripped of their accents. Most symbol characters
    are converted to something meaninful. Anything not converted is deleted.
    """
    xlate={0xc0:'A', 0xc1:'A', 0xc2:'A', 0xc3:'A', 0xc4:'A', 0xc5:'A',
    0xc6:'Ae', 0xc7:'C',
    0xc8:'E', 0xc9:'E', 0xca:'E', 0xcb:'E',
    0xcc:'I', 0xcd:'I', 0xce:'I', 0xcf:'I',
    0xd0:'Th', 0xd1:'N',
    0xd2:'O', 0xd3:'O', 0xd4:'O', 0xd5:'O', 0xd6:'O', 0xd8:'O',
    0xd9:'U', 0xda:'U', 0xdb:'U', 0xdc:'U',
    0xdd:'Y', 0xde:'th', 0xdf:'ss',
    0xe0:'a', 0xe1:'a', 0xe2:'a', 0xe3:'a', 0xe4:'a', 0xe5:'a',
    0xe6:'ae', 0xe7:'c',
    0xe8:'e', 0xe9:'e', 0xea:'e', 0xeb:'e',
    0xec:'i', 0xed:'i', 0xee:'i', 0xef:'i',
    0xf0:'th', 0xf1:'n',
    0xf2:'o', 0xf3:'o', 0xf4:'o', 0xf5:'o', 0xf6:'o', 0xf8:'o',
    0xf9:'u', 0xfa:'u', 0xfb:'u', 0xfc:'u',
    0xfd:'y', 0xfe:'th', 0xff:'y',
    0xa1:'!', 0xa2:'{cent}', 0xa3:'{pound}', 0xa4:'{currency}',
    0xa5:'{yen}', 0xa6:'|', 0xa7:'{section}', 0xa8:'{umlaut}',
    0xa9:'{C}', 0xaa:'{^a}', 0xab:'<<', 0xac:'{not}',
    0xad:'-', 0xae:'{R}', 0xaf:'_', 0xb0:'{degrees}',
    0xb1:'{+/-}', 0xb2:'{^2}', 0xb3:'{^3}', 0xb4:"'",
    0xb5:'{micro}', 0xb6:'{paragraph}', 0xb7:'*', 0xb8:'{cedilla}',
    0xb9:'{^1}', 0xba:'{^o}', 0xbb:'>>',
    0xbc:'{1/4}', 0xbd:'{1/2}', 0xbe:'{3/4}', 0xbf:'?',
    0xd7:'*', 0xf7:'/'
    }

    r = ''
    for i in unicrap:
        if xlate.has_key(ord(i)):
            r += xlate[ord(i)]
        elif ord(i) >= 0x80:
            pass
        else:
            r += i
    return r

# http://farmdev.com/talks/unicode/
def to_unicode_or_bust(obj, encoding='utf-8'):
    if isinstance(obj, basestring):
        if not isinstance(obj, unicode):
            obj = unicode(obj, encoding)
    return obj


def unescape(s):
    "unescape HTML code refs; c.f. http://wiki.python.org/moin/EscapingHtml"
    return re.sub('&(%s);' % '|'.join(htmlentitydefs.name2codepoint),
              lambda m: unichr(htmlentitydefs.name2codepoint[m.group(1)]), s)


class Person(object):
    def __init__(self, name, department, position, phone, email, encoding = 'utf-8'):
        self.name = unescape(name.decode(encoding))
        self.department = unescape(department.decode(encoding))
        self.position = unescape(position.decode(encoding))
        self.phone = phone.decode(encoding)
        self.email = unescape(email.decode(encoding))

    @property
    def name_wo_accents(self):
        return remove_accents(self.name)

    def __hash__(self):
        return hash((self.name, self.department, self.position,
                     self.phone, self.email))

    def __eq__(self, other):
        return hash(self) == hash(other)

    def __ne__(self, other):
        return not self == other


def query_website():
    url = "http://www.iaa.es/content/general-alphabetical-list.php"
    html = urllib.urlopen(url).read()
    return html

def parse_alphabetical_list(html):

    # A row in the table will look something like this:
    # <tr><td>605</td><td><a href='http://www.python.org/~guido/'>Guido van Rossum</a></td>
    #<td>Python Department</td><td>Benevolent Dictator for Life</td>
    #<td>guido@python.org</td></tr>

    regexp = re.compile("<tr><td>([0-9]*)</td><td>(<a .*?>)?(.*?)(</a>)?</td>"
                        "<td>(.*?)</td><td>(.*?)</td><td>(.*?)</td></tr>",
                        re.IGNORECASE)

    found_persons = []
    for matched_line in regexp.findall(html):
        phone = matched_line[0].strip()
        name  = matched_line[2].strip()
        department = matched_line[4].strip()
        position = matched_line[5].strip()
        email = matched_line[6].strip()
        found_persons.append(Person(name, department, position, phone, email))
    return found_persons

def search(target, encoding ='utf-8', cache_path = None, cache_max = 7):

    """ The IAA website is taking too long to load (~4.5 secs), so we can cache
    the list of people so that a connection to the database is not needed every
    time we run the script. If 'cache_path' exists and it is less than
    'cache_max' days old, the cache will be unpickled and used instead. """

    if not cache_path or not os.path.exists(cache_path) or \
       (time.time() - os.path.getmtime(cache_path)) > (cache_max * 24 * 3600):
          html = query_website()
          people = parse_alphabetical_list(html)

          def add_place(name, phone):
              """ Append a new Person object to 'people'. Except for the name
              and telephone number, all the fields are set to empty strings"""
              people.append(Person(name, '', '', phone, ''))

          # The rooms and laboratories telephone numbers have to be hard-coded
          # as they are only available to employees through the IAA intranet.

          for name, phone in \
              [('Almacén Material Fungible', '562'),
               ('Almacén Papelería', '500'),
               ('Biblioteca', '583'),
               ('Cafetería IAA', '565'),
               ('Cafetería UDIT', '571'),
               ('Centralita', '500'),
               ('Dirección', '535'),
               ('Fax Administración', '545'),
               ('Gerencia', '539'),
               ('Laboratorio Electrónica P1', '582'),
               ('Laboratorio Informática', '563'),
               ('Laboratorio Óptica', '572'),
               ('Laboratorio Revelado', '586'),
               ('Laboratorio Scattering', '587'),
               ('Polycom IMaX', '597'),
               ('Recepción', '555'),
               ('Sala Blanca Electrónica', '573'),
               ('Sala Blanca Óptica', '572'),
               ('Sala Juntas Edificio A', '554'),
               ('Sala Juntas Edificio C', '625'),
               ('Sala Reuniones Edificio B', '571'),
               ('Sala Reuniones Edificio C', '636'),
               ('Salón Actos', '559'),
               ('Seguridad Edificio A', '500'),
               ('Seguridad Edificio B', '501'),
               ('Seguridad Edificio C', '502'),
               ('Taller Mantenimiento', '558'),
               ('Taller Mecánica', '570'),
               ('Biblioteca OSN', '2003'),
               ('Consola Telescopio 0.9m OSN', '2004'),
               ('Consola Telescopio 1.5 m OSN', '2002'),
               ('Consola Telescopio IR OSN', '2005'),
               ('Laboratorio Electrónica OSN', '2001'),
               ('Recepción OSN', '2000')]:
                  add_place(name, phone)

          # Include in the search the restaurant at which, as part of a
          # time-honored and almost sacred tradition, we have lunch every
          # Friday: http://www.restauranteraices.es/
          add_place("Restaurante Raíces", '958-120-103')

          if cache_path:  # cache (pickle) the list if a path was given
              pickle.dump(people, open(cache_path, 'wb'))
    else:
        people = pickle.load(open(cache_path, 'rb'))

    target = remove_accents(target.decode(encoding)).lower()
    matches = filter(lambda x: target in x.name_wo_accents.lower() or \
                               target in str(x.phone) or \
                               target in x.email, people)

    return sorted(matches, key = lambda x: x.name)

if __name__ == "__main__":

    if sys.version_info < (2, 5):
        print "Error: phone requires Python 2.5 os later :-("
        sys.exit(3)

    description = \
    "phone is intended to serve as the white pages of the Institute of "       \
    "Astrophysics of Andalusia: a command-line application that queries the "  \
    "website in order to find the telephone number given a name or surname. "  \
    "Note that visiting and associate researchers are included in the "        \
    "search. The reverse telephone directory (grey pages) is also available, " \
    "as searches may also be done by the phone number or e-mail address. All " \
    "searches are case- and accent-insensitive."

    parser = optparse.OptionParser(description = description)
    parser.usage = "%prog [NAME...] [PHONE] [EMAIL]"

    parser.add_option('-c', action = 'store_true', dest = 'clear',
                      help = "due to the slow speed of the IAA website; we "
                         "use a local cache of the alphabetical list, so that "
                         "that the website is only accessed *once a week*. "
                         "This option clears the local cache, forcing the "
                         "connection to the IAA website -- useful in you know "
                         "the website was recently updated!")

    (options, args) = parser.parse_args(args = sys.argv)
    if len(args) < 2:
        parser.print_help()
        sys.exit(2)

    # If the user wants to clear the local cache, we use a maximum number of
    # days equal to a negative value (although zero would also work); in this
    # manner, the cache will always considered too old and discarded.
    cache_path = os.path.join(tempfile.gettempdir(), 'iaa-phone-cache.pkl')
    if options.clear:
        cache_max = -1
    else:
        cache_max = 7 # days

    target = args[1]
    kwargs = dict(cache_path = cache_path, cache_max = cache_max)
    matches = set(search(target, **kwargs))

    # If more than one search term is given, perform the search for all of them
    # and compute the intersection of the results. For example: if "John" and
    # "Doe" are the arguments passed to the script, we first search the records
    # that contain "John", then those that contain "Doe" and finally return the
    # intersection of both sets

    for target in args[2:]:
        matches &=  set(search(target, **kwargs))

    if not matches:
        sys.exit(0)

    # Determine the max length of each column
    name_column_length       = max(len(x.name) for x in matches)
    department_column_length = max(len(x.department) for x in matches)
    phone_column_length      = max(len(x.phone) for x in matches)
    email_column_length      = max(len(x.email) for x in matches)

    for person in matches:
        print person.name.ljust(name_column_length), '|', \
              person.department.ljust(department_column_length), '|', \
              person.phone.ljust(phone_column_length), '|', \
              person.email.ljust(email_column_length)

    sys.exit(0)