Adding validation for uniqueness of service_id X date in calendar_dates

python/transitfeed.py

 #!/usr/bin/python2.5
 
 # Copyright (C) 2007 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
@@ skipping 51 matching lines @@
 import warnings
 # Objects in a schedule (Route, Trip, etc) should not keep a strong reference
 # to the Schedule object to avoid a reference cycle. Schedule needs to use
 # __del__ to cleanup its temporary file. The garbage collector can't handle
 # reference cycles containing objects with custom cleanup code.
 import weakref
 import zipfile
 
 OUTPUT_ENCODING = 'utf-8'
 MAX_DISTANCE_FROM_STOP_TO_SHAPE = 1000
-
-
-__version__ = '1.2.1'
+MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_WARNING = 100.0
+MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_ERROR = 1000.0
+
+__version__ = '1.2.2'
 
 
 def EncodeUnicode(text):
   """
   Optionally encode text and return it. The result should be safe to print.
   """
   if type(text) == type(u''):
     return text.encode(OUTPUT_ENCODING)
   else:
     return text
@@ skipping 79 matching lines @@
                      context2=self._context)
     self._Report(e)
 
   def InvalidValue(self, column_name, value, reason=None, context=None,
                    type=TYPE_ERROR):
     e = InvalidValue(column_name=column_name, value=value, reason=reason,
                      context=context, context2=self._context, type=type)
     self._Report(e)
 
   def DuplicateID(self, column_names, values, context=None, type=TYPE_ERROR):
-    if not isinstance(column_names, tuple):
-      column_names = (column_names,)
-    if not isinstance(values, tuple):
-      values = (values,)
-    e = DuplicateID(column_names=column_names, values=values,
+    if isinstance(column_names, tuple):
+      column_names = '(' + ', '.join(column_names) + ')'
+    if isinstance(values, tuple):
+      values = '(' + ', '.join(values) + ')'
+    e = DuplicateID(column_name=column_names, value=values,
                     context=context, context2=self._context, type=type)
     self._Report(e)
 
   def UnusedStop(self, stop_id, stop_name, context=None):
     e = UnusedStop(stop_id=stop_id, stop_name=stop_name,
                    context=context, context2=self._context, type=TYPE_WARNING)
     self._Report(e)
 
   def UsedStation(self, stop_id, stop_name, context=None):
     e = UsedStation(stop_id=stop_id, stop_name=stop_name,
                     context=context, context2=self._context, type=TYPE_ERROR)
     self._Report(e)
 
+  def StopTooFarFromParentStation(self, stop_id, stop_name, parent_stop_id,
+                                  parent_stop_name, distance,
+                                  type=TYPE_WARNING, context=None):
+    e = StopTooFarFromParentStation(
+        stop_id=stop_id, stop_name=stop_name,
+        parent_stop_id=parent_stop_id,
+        parent_stop_name=parent_stop_name, distance=distance,
+        context=context, context2=self._context, type=type)
+    self._Report(e)
+
   def ExpirationDate(self, expiration, context=None):
     e = ExpirationDate(expiration=expiration, context=context,
                        context2=self._context, type=TYPE_WARNING)
+    self._Report(e)
+
+  def FutureService(self, start_date, context=None):
+    e = FutureService(start_date=start_date, context=context,
+                      context2=self._context, type=TYPE_WARNING)
     self._Report(e)
 
   def InvalidLineEnd(self, bad_line_end, context=None):
     """bad_line_end is a human readable string."""
     e = InvalidLineEnd(bad_line_end=bad_line_end, context=context,
                        context2=self._context, type=TYPE_WARNING)
     self._Report(e)
 
+  def TooFastTravel(self, trip_id, prev_stop, next_stop, dist, time, speed,
+                    type=TYPE_ERROR):
+    e = TooFastTravel(trip_id=trip_id, prev_stop=prev_stop,
+                      next_stop=next_stop, time=time, dist=dist, speed=speed,
+                      context=None, context2=self._context, type=type)
+    self._Report(e)
+
+  def StopWithMultipleRouteTypes(self, stop_name, stop_id, route_id1, route_id2,
+                                 context=None):
+    e = StopWithMultipleRouteTypes(stop_name=stop_name, stop_id=stop_id, 
+                                   route_id1=route_id1, route_id2=route_id2, 
+                                   context=context, context2=self._context, 
+                                   type=TYPE_WARNING)
+    self._Report(e)
+
+  def DuplicateTrip(self, trip_id1, route_id1, trip_id2, route_id2, 
+                    context=None):
+    e = DuplicateTrip(trip_id1=trip_id1, route_id1=route_id1, trip_id2=trip_id2,
+                      route_id2=route_id2, context=context,
+                      context2=self._context, type=TYPE_WARNING)
+    self._Report(e) 
+
   def OtherProblem(self, description, context=None, type=TYPE_ERROR):
     e = OtherProblem(description=description,
                     context=context, context2=self._context, type=type)
     self._Report(e)
 
 class ProblemReporter(ProblemReporterBase):
   """This is a basic problem reporter that just prints to console."""
   def _Report(self, e):
-    print EncodeUnicode(self._LineWrap(e.FormatProblem(), 78))
     context = e.FormatContext()
     if context:
       print context
+    print EncodeUnicode(self._LineWrap(e.FormatProblem(), 78))
 
   @staticmethod
   def _LineWrap(text, width):
     """
     A word-wrap function that preserves existing line breaks
     and most spaces in the text. Expects that existing line
     breaks are posix newlines (\n).
 
     Taken from:
     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/148061
@@ skipping 131 matching lines @@
 class CsvSyntax(ExceptionWithContext):
   ERROR_TEXT = '%(description)s'
 
 class MissingValue(ExceptionWithContext):
   ERROR_TEXT = 'Missing value for column %(column_name)s'
 
 class InvalidValue(ExceptionWithContext):
   ERROR_TEXT = 'Invalid value %(value)s in field %(column_name)s'
 
 class DuplicateID(ExceptionWithContext):
-  ERROR_TEXT = 'Duplicate ID %(values)s in column %(column_names)s'
+  ERROR_TEXT = 'Duplicate ID %(value)s in column %(column_name)s'
 
 class UnusedStop(ExceptionWithContext):
   ERROR_TEXT = "%(stop_name)s (ID %(stop_id)s) isn't used in any trips"
 
 class UsedStation(ExceptionWithContext):
   ERROR_TEXT = "%(stop_name)s (ID %(stop_id)s) has location_type=1 " \
                "(station) so it should not appear in stop_times"
+
+class StopTooFarFromParentStation(ExceptionWithContext):
+  ERROR_TEXT = "%(stop_name)s (ID %(stop_id)s) is too far from its" \
+               " parent station %(parent_stop_name)s (ID %(parent_stop_id)s)" \
+               " : %(distance)s meters."
 
 class ExpirationDate(ExceptionWithContext):
   def FormatProblem(self, d=None):
     if not d:
       d = self.GetDictToFormat()
     expiration = d['expiration']
     formatted_date = time.strftime("%B %d, %Y",
                                    time.localtime(expiration))
     if (expiration < time.mktime(time.localtime())):
       return "This feed expired on %s" % formatted_date
     else:
       return "This feed will soon expire, on %s" % formatted_date
 
+class FutureService(ExceptionWithContext):
+  def FormatProblem(self, d=None):
+    if not d:
+      d = self.GetDictToFormat()
+    formatted_date = time.strftime("%B %d, %Y", time.localtime(d['start_date']))
+    return ("The earliest service date in this feed is in the future, on %s. "
+            "Published feeds must always include the current date." %
+            formatted_date)
+
+
 class InvalidLineEnd(ExceptionWithContext):
   ERROR_TEXT = "Each line must end with CR LF or LF except for the last line " \
                "of the file. This line ends with \"%(bad_line_end)s\"."
+
+class StopWithMultipleRouteTypes(ExceptionWithContext):
+  ERROR_TEXT = "Stop %(stop_name)s (ID=%(stop_id)s) belongs to both " \
+               "subway (ID=%(route_id1)s) and bus line (ID=%(route_id2)s)."
+
+class TooFastTravel(ExceptionWithContext):
+  def FormatProblem(self, d=None):
+    if not d:
+      d = self.GetDictToFormat()
+    if not d['speed']:
+      return "High speed travel detected in trip %(trip_id)s: %(prev_stop)s" \
+                " to %(next_stop)s. %(dist)d meters in %(time)d seconds." % d
+    else:
+      return "High speed travel detected in trip %(trip_id)s: %(prev_stop)s" \
+                " to %(next_stop)s. %(dist)d meters in %(time)d seconds." \
+                " (%(speed)f km/h)." % d
+
+class DuplicateTrip(ExceptionWithContext):
+  ERROR_TEXT = "Trip %(trip_id1)s of route %(route_id1)s might be duplicated " \
+               "with trip %(trip_id2)s of route %(route_id2)s. They go " \
+               "through the same stops with same service."
 
 class OtherProblem(ExceptionWithContext):
   ERROR_TEXT = '%(description)s'
 
 
 class ExceptionProblemReporter(ProblemReporter):
   def __init__(self, raise_warnings=False):
     ProblemReporterBase.__init__(self)
     self.raise_warnings = raise_warnings
 
@@ skipping 46 matching lines @@
 
 def FindUniqueId(dic):
   """Return a string not used as a key in the dictionary dic"""
   name = str(len(dic))
   while name in dic:
     name = str(random.randint(1, 999999999))
   return name
 
 
 def TimeToSecondsSinceMidnight(time_string):
-  """Convert HH:MM:SS into seconds since midnight.
+  """Convert HHH:MM:SS into seconds since midnight.
 
   For example "01:02:03" returns 3723. The leading zero of the hours may be
   omitted. HH may be more than 23 if the time is on the following day."""
-  m = re.match(r'(\d{1,2}):([0-5]\d):([0-5]\d)$', time_string)
+  m = re.match(r'(\d{1,3}):([0-5]\d):([0-5]\d)$', time_string)
   # ignored: matching for leap seconds
   if not m:
     raise Error, 'Bad HH:MM:SS "%s"' % time_string
   return int(m.group(1)) * 3600 + int(m.group(2)) * 60 + int(m.group(3))
 
 
 def FormatSecondsSinceMidnight(s):
   """Formats an int number of seconds past midnight into a string
   as "HH:MM:SS"."""
   return "%02d:%02d:%02d" % (s / 3600, (s / 60) % 60, s % 60)
@@ skipping 335 matching lines @@
 
 class Route(GenericGTFSObject):
   """Represents a single route."""
 
   _REQUIRED_FIELD_NAMES = [
     'route_id', 'route_short_name', 'route_long_name', 'route_type'
     ]
   _FIELD_NAMES = _REQUIRED_FIELD_NAMES + [
     'agency_id', 'route_desc', 'route_url', 'route_color', 'route_text_color'
     ]
-  _ROUTE_TYPE_NAMES = {
-    'Tram': 0,
-    'Subway': 1,
-    'Rail': 2,
-    'Bus': 3,
-    'Ferry': 4,
-    'Cable Car': 5,
-    'Gondola': 6,
-    'Funicular': 7
+  _ROUTE_TYPES = {
+    0: {'name':'Tram', 'max_speed':50},
+    1: {'name':'Subway', 'max_speed':150},
+    2: {'name':'Rail', 'max_speed':300},
+    3: {'name':'Bus', 'max_speed':100},
+    4: {'name':'Ferry', 'max_speed':80},
+    5: {'name':'Cable Car', 'max_speed':50},
+    6: {'name':'Gondola', 'max_speed':50},
+    7: {'name':'Funicular', 'max_speed':50},
     }
-  _ROUTE_TYPE_IDS = set(_ROUTE_TYPE_NAMES.values())
+  # Create a reverse lookup dict of route type names to route types.
+  _ROUTE_TYPE_IDS = set(_ROUTE_TYPES.keys())
+  _ROUTE_TYPE_NAMES = dict((v['name'], k) for k, v in _ROUTE_TYPES.items())
   _TABLE_NAME = 'routes'
 
   def __init__(self, short_name=None, long_name=None, route_type=None,
                route_id=None, agency_id=None, field_dict=None):
     self._schedule = None
     self._trips = []
 
     if not field_dict:
       field_dict = {}
       if short_name is not None:
@@ skipping 818 matching lines @@
         problems.OtherProblem(
           'No time for start of trip_id "%s""' % (self.trip_id))
       if stoptimes[-1].arrival_time is None and stoptimes[-1].departure_time is None:
         problems.OtherProblem(
           'No time for end of trip_id "%s""' % (self.trip_id))
 
       # Sorts the stoptimes by sequence and then checks that the arrival time
       # for each time point is after the departure time of the previous.
       stoptimes.sort(key=lambda x: x.stop_sequence)
       prev_departure = 0
+      prev_stop = None
+      try:
+        route_type = self._schedule.GetRoute(self.route_id).route_type
+        max_speed = Route._ROUTE_TYPES[route_type]['max_speed']
+      except KeyError, e:
+        # If route_type cannot be found, assume it is 0 (Tram) for checking
+        # speeds between stops.
+        route_type = 0
+        max_speed = 30
       for timepoint in stoptimes:
         if timepoint.arrival_secs is not None:
+          self._CheckSpeed(prev_stop, timepoint.stop, prev_departure,
+                           timepoint.arrival_secs, max_speed, problems)
+
           if timepoint.arrival_secs >= prev_departure:
             prev_departure = timepoint.departure_secs
+            prev_stop = timepoint.stop
           else:
             problems.OtherProblem('Timetravel detected! Arrival time '
                                   'is before previous departure '
                                   'at sequence number %s in trip %s' %
                                   (timepoint.stop_sequence, self.trip_id))
 
       if self.shape_id and self.shape_id in self._schedule._shapes:
         shape = self._schedule.GetShape(self.shape_id)
         max_shape_dist = shape.max_distance
         st = stoptimes[-1]
@@ skipping 28 matching lines @@
                     type=TYPE_WARNING)
 
     # O(n^2), but we don't anticipate many headway periods per trip
     for headway_index, headway in enumerate(self._headways[0:-1]):
       for other in self._headways[headway_index + 1:]:
         if (other[0] < headway[1]) and (other[1] > headway[0]):
           problems.OtherProblem('Trip contains overlapping headway periods '
                                 '%s and %s' %
                                 (self._HeadwayOutputTuple(headway),
                                  self._HeadwayOutputTuple(other)))
+
+  def _CheckSpeed(self, prev_stop, next_stop, depart_time,
+                  arrive_time, max_speed, problems):
+    # Checks that the speed between two stops is not faster than max_speed
+    if prev_stop != None:
+      try:
+        time_between_stops = arrive_time - depart_time
+      except TypeError:
+        return
+
+      try:
+        dist_between_stops = \
+          ApproximateDistanceBetweenStops(next_stop, prev_stop)
+      except TypeError, e:
+          return
+
+      if time_between_stops == 0:
+        problems.TooFastTravel(self.trip_id,
+                               prev_stop.stop_name,
+                               next_stop.stop_name,
+                               dist_between_stops,
+                               time_between_stops,
+                               speed=None,
+                               type=TYPE_WARNING)
+        return
+      # This needs floating point division for precision.
+      speed_between_stops = ((float(dist_between_stops) / 1000) /
+                                (float(time_between_stops) / 3600))
+      if speed_between_stops > max_speed:
+        problems.TooFastTravel(self.trip_id,
+                               prev_stop.stop_name,
+                               next_stop.stop_name,
+                               dist_between_stops,
+                               time_between_stops,
+                               speed_between_stops,
+                               type=TYPE_WARNING)
 
 # TODO: move these into a separate file
 class ISO4217(object):
   """Represents the set of currencies recognized by the ISO-4217 spec."""
   codes = {  # map of alpha code to numerical code
     'AED': 784, 'AFN': 971, 'ALL':   8, 'AMD':  51, 'ANG': 532, 'AOA': 973,
     'ARS':  32, 'AUD':  36, 'AWG': 533, 'AZN': 944, 'BAM': 977, 'BBD':  52,
     'BDT':  50, 'BGN': 975, 'BHD':  48, 'BIF': 108, 'BMD':  60, 'BND':  96,
     'BOB':  68, 'BOV': 984, 'BRL': 986, 'BSD':  44, 'BTN':  64, 'BWP':  72,
     'BYR': 974, 'BZD':  84, 'CAD': 124, 'CDF': 976, 'CHE': 947, 'CHF': 756,
@@ skipping 593 matching lines @@
       yield (self.service_id, date, unicode(exception_type))
 
   def GetCalendarDatesFieldValuesTuples(self):
     """Return a list of date execeptions"""
     result = []
     for date_tuple in self.GenerateCalendarDatesFieldValuesTuples():
       result.append(date_tuple)
     result.sort()  # helps with __eq__
     return result
 
-  def SetDateHasService(self, date, has_service=True):
+  def SetDateHasService(self, date, has_service=True, problems=None):
+    if date in self.date_exceptions and problems:
+      problems.DuplicateID(('service_id', 'date'),
+                           (self.service_id, date),
+                           type=TYPE_WARNING)
     self.date_exceptions[date] = has_service and 1 or 2
 
   def ResetDateToNormalService(self, date):
     if date in self.date_exceptions:
       del self.date_exceptions[date]
 
   def SetStartDate(self, start_date):
     """Set the first day of service as a string in YYYYMMDD format"""
     self.start_date = start_date
 
@@ skipping 32 matching lines @@
 
   def IsActiveOn(self, date):
     """Test if this service period is active on a date.
 
     Args:
       date: a string of form "YYYYMMDD"
 
     Returns:
       True iff this service is active on date.
     """
-    date_obj = DateStringToDateObject(date)
     if date in self.date_exceptions:
       if self.date_exceptions[date] == 1:
         return True
       else:
         return False
     if (self.start_date and self.end_date and self.start_date <= date and
         date <= self.end_date):
+      date_obj = DateStringToDateObject(date)
       return self.day_of_week[date_obj.weekday()]
     return False
 
   def ActiveDates(self):
     """Return dates this service period is active as a list of "YYYYMMDD"."""
     (earliest, latest) = self.GetDateRange()
     if earliest is None:
       return []
     dates = []
     date_it = DateStringToDateObject(earliest)
@@ skipping 316 matching lines @@
 
     ranges = [period.GetDateRange() for period in self.GetServicePeriodList()]
     starts = filter(lambda x: x, [item[0] for item in ranges])
     ends = filter(lambda x: x, [item[1] for item in ranges])
 
     if not starts or not ends:
       return (None, None)
 
     return (min(starts), max(ends))
 
+  def GetServicePeriodsActiveEachDate(self, date_start, date_end):
+    """Return a list of tuples (date, [period1, period2, ...]).
+
+    For each date in the range [date_start, date_end) make list of each
+    ServicePeriod object which is active.
+
+    Args:
+      date_start: The first date in the list, a date object
+      date_end: The first date after the list, a date object
+
+    Returns:
+      A list of tuples. Each tuple contains a date object and a list of zero or
+      more ServicePeriod objects.
+    """
+    date_it = date_start
+    one_day = datetime.timedelta(days=1)
+    date_service_period_list = []
+    while date_it < date_end:
+      periods_today = []
+      for service in self.GetServicePeriodList():
+        if service.IsActiveOn(date_it.strftime("%Y%m%d")):
+          periods_today.append(service)
+      date_service_period_list.append((date_it, periods_today))
+      date_it += one_day
+    return date_service_period_list
+
+
   def AddStop(self, lat, lng, name):
     """Add a stop to this schedule.
 
     A new stop_id is created for this stop. Do not use this method unless all
     stops in this Schedule are created with it. See source for details.
 
     Args:
       lat: Latitude of the stop as a float or string
       lng: Longitude of the stop as a float or string
       name: Name of the stop, which will appear in the feed
@@ skipping 357 matching lines @@
 
     archive.close()
 
   def Validate(self, problems=None, validate_children=True):
     """Validates various holistic aspects of the schedule
        (mostly interrelationships between the various data sets)."""
     if not problems:
       problems = self.problem_reporter
 
     (start_date, end_date) = self.GetDateRange()
-    if not end_date:
+    if not end_date or not start_date:
       problems.OtherProblem('This feed has no effective service dates!',
                             type=TYPE_WARNING)
     else:
       try:
         expiration = time.mktime(time.strptime(end_date, "%Y%m%d"))
+        start_date_time = time.mktime(time.strptime(start_date, "%Y%m%d"))
         now = time.mktime(time.localtime())
         warning_cutoff = now + 60 * 60 * 24 * 30  # one month from expiration
         if expiration < warning_cutoff:
           problems.ExpirationDate(expiration)
+        if start_date_time > now:
+          problems.FutureService(start_date_time)
       except ValueError:
-        problems.InvalidValue('end_date', end_date)
+        # Format of start_date and end_date checked in class ServicePeriod
+        pass
 
     # TODO: Check Trip fields against valid values
 
     # Check for stops that aren't referenced by any trips and broken
-    # parent_station references.
+    # parent_station references. Also check that the parent station isn't too
+    # far from its child stops.
     for stop in self.stops.values():
       if validate_children:
         stop.Validate(problems)
       cursor = self._connection.cursor()
       cursor.execute("SELECT count(*) FROM stop_times WHERE stop_id=? LIMIT 1",
                      (stop.stop_id,))
       count = cursor.fetchone()[0]
       if stop.location_type == 0 and count == 0:
           problems.UnusedStop(stop.stop_id, stop.stop_name)
       elif stop.location_type == 1 and count != 0:
           problems.UsedStation(stop.stop_id, stop.stop_name)
 
       if stop.location_type != 1 and stop.parent_station:
         if stop.parent_station not in self.stops:
           problems.InvalidValue("parent_station",
                                 EncodeUnicode(stop.parent_station),
                                 "parent_station '%s' not found for stop_id "
                                 "'%s' in stops.txt" %
                                 (EncodeUnicode(stop.parent_station),
                                  EncodeUnicode(stop.stop_id)))
         elif self.stops[stop.parent_station].location_type != 1:
           problems.InvalidValue("parent_station",
                                 EncodeUnicode(stop.parent_station),
                                 "parent_station '%s' of stop_id '%s' must "
                                 "have location_type=1 in stops.txt" %
                                 (EncodeUnicode(stop.parent_station),
                                  EncodeUnicode(stop.stop_id)))
-
-    #TODO: check that every station is used and within 1km of stops that are
-    # part of it. Then uncomment testStationWithoutReference.
+        else:
+          parent_station = self.stops[stop.parent_station]
+          distance = ApproximateDistanceBetweenStops(stop, parent_station)
+          if distance > MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_ERROR:
+            problems.StopTooFarFromParentStation(
+                stop.stop_id, stop.stop_name, parent_station.stop_id,
+                parent_station.stop_name, distance, TYPE_ERROR)
+          elif distance > MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_WARNING:
+            problems.StopTooFarFromParentStation(
+                stop.stop_id, stop.stop_name, parent_station.stop_id,
+                parent_station.stop_name, distance, TYPE_WARNING)
+
+    #TODO: check that every station is used.
+    # Then uncomment testStationWithoutReference.
 
     # Check for stops that might represent the same location (specifically,
     # stops that are less that 2 meters apart) First filter out stops without a
     # valid lat and lon. Then sort by latitude, then find the distance between
     # each pair of stations within 2 meters latitude of each other. This avoids
     # doing n^2 comparisons in the average case and doesn't need a spatial
     # index.
     sorted_stops = filter(lambda s: s.stop_lat and s.stop_lon,
                           self.GetStopList())
     sorted_stops.sort(key=(lambda x: x.stop_lat))
@@ skipping 29 matching lines @@
             elif stop.location_type == 1 and other_stop.location_type == 0:
               this_stop = other_stop
               this_station = stop
             if this_stop.parent_station != this_station.stop_id:
               problems.OtherProblem(
                   'The parent_station of stop "%s" (ID "%s") is not '
                   'station "%s" (ID "%s") but they are only %0.2fm apart.' %
                   (EncodeUnicode(this_stop.stop_name),
                    EncodeUnicode(this_stop.stop_id),
                    EncodeUnicode(this_station.stop_name),
-                   EncodeUnicode(this_station.stop_id), distance))
+                   EncodeUnicode(this_station.stop_id), distance),
+                  type=TYPE_WARNING)
         index += 1
 
     # Check for multiple routes using same short + long name
     route_names = {}
     for route in self.routes.values():
       if validate_children:
         route.Validate(problems)
       short_name = ''
       if not IsEmpty(route.route_short_name):
         short_name = route.route_short_name.lower().strip()
       long_name = ''
       if not IsEmpty(route.route_long_name):
         long_name = route.route_long_name.lower().strip()
       name = (short_name, long_name)
       if name in route_names:
         problems.InvalidValue('route_long_name',
                               long_name,
                               'The same combination of '
                               'route_short_name and route_long_name '
                               'shouldn\'t be used for more than one '
                               'route, as it is for the for the two routes '
                               'with IDs "%s" and "%s".' %
                               (route.route_id, route_names[name].route_id),
                               type=TYPE_WARNING)
       else:
         route_names[name] = route
 
-    # Check duplicate trips which go through the same stops with same
-    # service and start times.
-
-    if self._check_duplicate_trips:
-      trips = {}
-      for trip in self.trips.values():
-        stop_ids = []
-        stop_times = []
-        for index, st in enumerate(trip.GetStopTimes(problems)):
-          stop_times.append(st.arrival_time)
-          stop_ids.append(st.stop.stop_id)
-        if not stop_ids or not stop_times:
+    stop_types = {} # a dict mapping stop_id to [route_id, route_type, is_match]
+    trips = {} # a dict mapping tuple to (route_id, trip_id)
+    for trip in sorted(self.trips.values()):
+      if trip.route_id not in self.routes:
+        continue
+      route_type = self.GetRoute(trip.route_id).route_type
+      arrival_times = []
+      stop_ids = []      
+      for index, st in enumerate(trip.GetStopTimes(problems)):
+        stop_id = st.stop.stop_id
+        arrival_times.append(st.arrival_time)
+        stop_ids.append(stop_id)
+        # Check a stop if which belongs to both subway and bus.
+        if (route_type == Route._ROUTE_TYPE_NAMES['Subway'] or 
+            route_type == Route._ROUTE_TYPE_NAMES['Bus']):  
+          if stop_id not in stop_types:
+            stop_types[stop_id] = [trip.route_id, route_type, 0]
+          elif (stop_types[stop_id][1] != route_type and 
+                stop_types[stop_id][2] == 0):
+            stop_types[stop_id][2] = 1
+            if stop_types[stop_id][1] == Route._ROUTE_TYPE_NAMES['Subway']:
+              subway_route_id = stop_types[stop_id][0]
+              bus_route_id = trip.route_id
+            else:
+              subway_route_id = trip.route_id
+              bus_route_id = stop_types[stop_id][0]
+            problems.StopWithMultipleRouteTypes(st.stop.stop_name, stop_id, 
+                                                subway_route_id, bus_route_id) 
+
+      # Check duplicate trips which go through the same stops with same
+      # service and start times.
+      if self._check_duplicate_trips:
+        if not stop_ids or not arrival_times:
           continue
-        key = (trip.service_id, min(stop_times), str(stop_ids))
+        key = (trip.service_id, min(arrival_times), str(stop_ids))
         if key not in trips:
-          trips[key] = [trip.route_id, trip.trip_id]
+          trips[key] = (trip.route_id, trip.trip_id)
         else:
-          problems.OtherProblem('Trip %s of route %s might be duplicated '
-                                'with trip %s of route %s. They go through '
-                                'the same stops with same service. '
-                                % (trips[key][1], trips[key][0],
-                                trip.trip_id, trip.route_id), 
-                                type=TYPE_WARNING)
+          problems.DuplicateTrip(trips[key][1], trips[key][0], trip.trip_id, 
+                                 trip.route_id)
 
     # Check that routes' agency IDs are valid, if set
     for route in self.routes.values():
       if (not IsEmpty(route.agency_id) and
           not route.agency_id in self._agencies):
         problems.InvalidValue('agency_id',
                               route.agency_id,
                               'The route with ID "%s" specifies agency_id '
                               '"%s", which doesn\'t exist.' %
                               (route.route_id, route.agency_id))
@@ skipping 463 matching lines @@
   def _FileContents(self, file_name):
     results = None
     if self._zip:
       try:
         results = self._zip.read(file_name)
       except KeyError:  # file not found in archve
         self._problems.MissingFile(file_name)
         return None
     else:
       try:
-        data_file = open(os.path.join(self._path, file_name), 'r')
+        data_file = open(os.path.join(self._path, file_name), 'rb')
         results = data_file.read()
       except IOError:  # file not found
         self._problems.MissingFile(file_name)
         return None
 
     if not results:
       self._problems.EmptyFile(file_name)
     return results
 
   def _LoadAgencies(self):
@@ skipping 41 matching lines @@
     periods = {}
 
     # process calendar.txt
     if self._HasFile(file_name):
       has_useful_contents = False
       for (row, row_num, cols) in \
               self._ReadCSV(file_name,
                             ServicePeriod._FIELD_NAMES,
                             ServicePeriod._FIELD_NAMES_REQUIRED):
         context = (file_name, row_num, row, cols)
+        self._problems.SetFileContext(*context)
 
         period = ServicePeriod(field_list=row)
 
         if period.service_id in periods:
           self._problems.DuplicateID('service_id', period.service_id)
-          continue
-
-        periods[period.service_id] = (period, context)
+        else:
+          periods[period.service_id] = (period, context)
+        self._problems.ClearContext()
 
     # process calendar_dates.txt
     if self._HasFile(file_name_dates):
-      service_id_date_entries = {}
       # ['service_id', 'date', 'exception_type']
       fields = ServicePeriod._FIELD_NAMES_CALENDAR_DATES
       for (row, row_num, cols) in self._ReadCSV(file_name_dates,
                                                 fields, fields):
         context = (file_name_dates, row_num, row, cols)
+        self._problems.SetFileContext(*context)
 
         service_id = row[0]
 
         period = None
         if service_id in periods:
           period = periods[service_id][0]
         else:
           period = ServicePeriod(service_id)
           periods[period.service_id] = (period, context)
-          
-        # Check uniqueness of service_id x date
-        if (service_id, row[1]) in service_id_date_entries:
-          self._problems.DuplicateID(('service_id', 'date'),
-                                     (service_id, row[1]),
-                                     type=TYPE_WARNING)
-        service_id_date_entries[service_id, row[1]] = 1
 
         exception_type = row[2]
         if exception_type == u'1':
-          period.SetDateHasService(row[1], True)
+          period.SetDateHasService(row[1], True, self._problems)
         elif exception_type == u'2':
-          period.SetDateHasService(row[1], False)
+          period.SetDateHasService(row[1], False, self._problems)
         else:
           self._problems.InvalidValue('exception_type', exception_type)
+        self._problems.ClearContext()
 
     # Now insert the periods into the schedule object, so that they're
     # validated with both calendar and calendar_dates info present
     for period, context in periods.values():
       self._problems.SetFileContext(*context)
       self._schedule.AddServicePeriodObject(period, self._problems)
       self._problems.ClearContext()
 
   def _LoadShapes(self):
     if not self._HasFile('shapes.txt'):
@@ skipping 201 matching lines @@
   def __init__(self, *args, **kwargs):
     """Initialize a new ShapeLoader object.
 
     See Loader.__init__ for argument documentation.
     """
     Loader.__init__(self, *args, **kwargs)
 
   def Load(self):
     self._LoadShapes()
     return self._schedule