Adding validation for uniqueness of service_id X date in calendar_dates

python/transitfeed.py

 #!/usr/bin/python2.5
 
 # Copyright (C) 2007 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
@@ skipping 51 matching lines @@
 import warnings
 # Objects in a schedule (Route, Trip, etc) should not keep a strong reference
 # to the Schedule object to avoid a reference cycle. Schedule needs to use
 # __del__ to cleanup its temporary file. The garbage collector can't handle
 # reference cycles containing objects with custom cleanup code.
 import weakref
 import zipfile
 
 OUTPUT_ENCODING = 'utf-8'
 MAX_DISTANCE_FROM_STOP_TO_SHAPE = 1000
-
-
-__version__ = '1.2.1'
+MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_WARNING = 100.0
+MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_ERROR = 1000.0
+
+__version__ = '1.2.2'
 
 
 def EncodeUnicode(text):
   """
   Optionally encode text and return it. The result should be safe to print.
   """
   if type(text) == type(u''):
     return text.encode(OUTPUT_ENCODING)
   else:
     return text
@@ skipping 78 matching lines @@
     e = MissingValue(column_name=column_name, reason=reason, context=context,
                      context2=self._context)
     self._Report(e)
 
   def InvalidValue(self, column_name, value, reason=None, context=None,
                    type=TYPE_ERROR):
     e = InvalidValue(column_name=column_name, value=value, reason=reason,
                      context=context, context2=self._context, type=type)
     self._Report(e)
 
-  def DuplicateID(self, column_name, value, context=None, type=TYPE_ERROR):
-    e = DuplicateID(column_name=column_name, value=value,
+  def DuplicateID(self, column_names, values, context=None, type=TYPE_ERROR):
+    if isinstance(column_names, tuple):
+      column_names = '(' + ', '.join(column_names) + ')'
+    if isinstance(values, tuple):
+      values = '(' + ', '.join(values) + ')'
+    e = DuplicateID(column_name=column_names, value=values,
                     context=context, context2=self._context, type=type)
     self._Report(e)
 
   def UnusedStop(self, stop_id, stop_name, context=None):
     e = UnusedStop(stop_id=stop_id, stop_name=stop_name,
                    context=context, context2=self._context, type=TYPE_WARNING)
     self._Report(e)
 
   def UsedStation(self, stop_id, stop_name, context=None):
     e = UsedStation(stop_id=stop_id, stop_name=stop_name,
                     context=context, context2=self._context, type=TYPE_ERROR)
     self._Report(e)
 
+  def StopTooFarFromParentStation(self, stop_id, stop_name, parent_stop_id,
+                                  parent_stop_name, distance,
+                                  type=TYPE_WARNING, context=None):
+    e = StopTooFarFromParentStation(
+        stop_id=stop_id, stop_name=stop_name,
+        parent_stop_id=parent_stop_id,
+        parent_stop_name=parent_stop_name, distance=distance,
+        context=context, context2=self._context, type=type)
+    self._Report(e)
+
   def ExpirationDate(self, expiration, context=None):
     e = ExpirationDate(expiration=expiration, context=context,
                        context2=self._context, type=TYPE_WARNING)
+    self._Report(e)
+
+  def FutureService(self, start_date, context=None):
+    e = FutureService(start_date=start_date, context=context,
+                      context2=self._context, type=TYPE_WARNING)
     self._Report(e)
 
   def InvalidLineEnd(self, bad_line_end, context=None):
     """bad_line_end is a human readable string."""
     e = InvalidLineEnd(bad_line_end=bad_line_end, context=context,
                        context2=self._context, type=TYPE_WARNING)
     self._Report(e)
 
+  def TooFastTravel(self, trip_id, prev_stop, next_stop, dist, time, speed,
+                    type=TYPE_ERROR):
+    e = TooFastTravel(trip_id=trip_id, prev_stop=prev_stop,
+                      next_stop=next_stop, time=time, dist=dist, speed=speed,
+                      context=None, context2=self._context, type=type)
+    self._Report(e)
+
+  def StopWithMultipleRouteTypes(self, stop_name, stop_id, route_id1, route_id2,
+                                 context=None):
+    e = StopWithMultipleRouteTypes(stop_name=stop_name, stop_id=stop_id, 
+                                   route_id1=route_id1, route_id2=route_id2, 
+                                   context=context, context2=self._context, 
+                                   type=TYPE_WARNING)
+    self._Report(e)
+
+  def DuplicateTrip(self, trip_id1, route_id1, trip_id2, route_id2, 
+                    context=None):
+    e = DuplicateTrip(trip_id1=trip_id1, route_id1=route_id1, trip_id2=trip_id2,
+                      route_id2=route_id2, context=context,
+                      context2=self._context, type=TYPE_WARNING)
+    self._Report(e) 
+
   def OtherProblem(self, description, context=None, type=TYPE_ERROR):
     e = OtherProblem(description=description,
                     context=context, context2=self._context, type=type)
     self._Report(e)
 
 class ProblemReporter(ProblemReporterBase):
   """This is a basic problem reporter that just prints to console."""
   def _Report(self, e):
-    print EncodeUnicode(self._LineWrap(e.FormatProblem(), 78))
     context = e.FormatContext()
     if context:
       print context
+    print EncodeUnicode(self._LineWrap(e.FormatProblem(), 78))
 
   @staticmethod
   def _LineWrap(text, width):
     """
     A word-wrap function that preserves existing line breaks
     and most spaces in the text. Expects that existing line
     breaks are posix newlines (\n).
 
     Taken from:
     http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/148061
@@ skipping 140 matching lines @@
 class DuplicateID(ExceptionWithContext):
   ERROR_TEXT = 'Duplicate ID %(value)s in column %(column_name)s'
 
 class UnusedStop(ExceptionWithContext):
   ERROR_TEXT = "%(stop_name)s (ID %(stop_id)s) isn't used in any trips"
 
 class UsedStation(ExceptionWithContext):
   ERROR_TEXT = "%(stop_name)s (ID %(stop_id)s) has location_type=1 " \
                "(station) so it should not appear in stop_times"
 
+class StopTooFarFromParentStation(ExceptionWithContext):
+  ERROR_TEXT = "%(stop_name)s (ID %(stop_id)s) is too far from its" \
+               " parent station %(parent_stop_name)s (ID %(parent_stop_id)s)" \
+               " : %(distance)s meters."
+
 class ExpirationDate(ExceptionWithContext):
   def FormatProblem(self, d=None):
     if not d:
       d = self.GetDictToFormat()
     expiration = d['expiration']
     formatted_date = time.strftime("%B %d, %Y",
                                    time.localtime(expiration))
     if (expiration < time.mktime(time.localtime())):
       return "This feed expired on %s" % formatted_date
     else:
       return "This feed will soon expire, on %s" % formatted_date
 
+class FutureService(ExceptionWithContext):
+  def FormatProblem(self, d=None):
+    if not d:
+      d = self.GetDictToFormat()
+    formatted_date = time.strftime("%B %d, %Y", time.localtime(d['start_date']))
+    return ("The earliest service date in this feed is in the future, on %s. "
+            "Published feeds must always include the current date." %
+            formatted_date)
+
+
 class InvalidLineEnd(ExceptionWithContext):
   ERROR_TEXT = "Each line must end with CR LF or LF except for the last line " \
                "of the file. This line ends with \"%(bad_line_end)s\"."
+
+class StopWithMultipleRouteTypes(ExceptionWithContext):
+  ERROR_TEXT = "Stop %(stop_name)s (ID=%(stop_id)s) belongs to both " \
+               "subway (ID=%(route_id1)s) and bus line (ID=%(route_id2)s)."
+
+class TooFastTravel(ExceptionWithContext):
+  def FormatProblem(self, d=None):
+    if not d:
+      d = self.GetDictToFormat()
+    if not d['speed']:
+      return "High speed travel detected in trip %(trip_id)s: %(prev_stop)s" \
+                " to %(next_stop)s. %(dist)d meters in %(time)d seconds." % d
+    else:
+      return "High speed travel detected in trip %(trip_id)s: %(prev_stop)s" \
+                " to %(next_stop)s. %(dist)d meters in %(time)d seconds." \
+                " (%(speed)f km/h)." % d
+
+class DuplicateTrip(ExceptionWithContext):
+  ERROR_TEXT = "Trip %(trip_id1)s of route %(route_id1)s might be duplicated " \
+               "with trip %(trip_id2)s of route %(route_id2)s. They go " \
+               "through the same stops with same service."
 
 class OtherProblem(ExceptionWithContext):
   ERROR_TEXT = '%(description)s'
 
 
 class ExceptionProblemReporter(ProblemReporter):
   def __init__(self, raise_warnings=False):
     ProblemReporterBase.__init__(self)
     self.raise_warnings = raise_warnings
 
@@ skipping 46 matching lines @@
 
 def FindUniqueId(dic):
   """Return a string not used as a key in the dictionary dic"""
   name = str(len(dic))
   while name in dic:
     name = str(random.randint(1, 999999999))
   return name
 
 
 def TimeToSecondsSinceMidnight(time_string):
-  """Convert HH:MM:SS into seconds since midnight.
+  """Convert HHH:MM:SS into seconds since midnight.
 
   For example "01:02:03" returns 3723. The leading zero of the hours may be
   omitted. HH may be more than 23 if the time is on the following day."""
-  m = re.match(r'(\d{1,2}):([0-5]\d):([0-5]\d)$', time_string)
+  m = re.match(r'(\d{1,3}):([0-5]\d):([0-5]\d)$', time_string)
   # ignored: matching for leap seconds
   if not m:
     raise Error, 'Bad HH:MM:SS "%s"' % time_string
   return int(m.group(1)) * 3600 + int(m.group(2)) * 60 + int(m.group(3))
 
 
 def FormatSecondsSinceMidnight(s):
   """Formats an int number of seconds past midnight into a string
   as "HH:MM:SS"."""
   return "%02d:%02d:%02d" % (s / 3600, (s / 60) % 60, s % 60)
@@ skipping 335 matching lines @@
 
 class Route(GenericGTFSObject):
   """Represents a single route."""
 
   _REQUIRED_FIELD_NAMES = [
     'route_id', 'route_short_name', 'route_long_name', 'route_type'
     ]
   _FIELD_NAMES = _REQUIRED_FIELD_NAMES + [
     'agency_id', 'route_desc', 'route_url', 'route_color', 'route_text_color'
     ]
-  _ROUTE_TYPE_NAMES = {
-    'Tram': 0,
-    'Subway': 1,
-    'Rail': 2,
-    'Bus': 3,
-    'Ferry': 4,
-    'Cable Car': 5,
-    'Gondola': 6,
-    'Funicular': 7
+  _ROUTE_TYPES = {
+    0: {'name':'Tram', 'max_speed':50},
+    1: {'name':'Subway', 'max_speed':150},
+    2: {'name':'Rail', 'max_speed':300},
+    3: {'name':'Bus', 'max_speed':100},
+    4: {'name':'Ferry', 'max_speed':80},
+    5: {'name':'Cable Car', 'max_speed':50},
+    6: {'name':'Gondola', 'max_speed':50},
+    7: {'name':'Funicular', 'max_speed':50},
     }
-  _ROUTE_TYPE_IDS = set(_ROUTE_TYPE_NAMES.values())
+  # Create a reverse lookup dict of route type names to route types.
+  _ROUTE_TYPE_IDS = set(_ROUTE_TYPES.keys())
+  _ROUTE_TYPE_NAMES = dict((v['name'], k) for k, v in _ROUTE_TYPES.items())
   _TABLE_NAME = 'routes'
 
   def __init__(self, short_name=None, long_name=None, route_type=None,
                route_id=None, agency_id=None, field_dict=None):
     self._schedule = None
     self._trips = []
 
     if not field_dict:
       field_dict = {}
       if short_name is not None:
@@ skipping 818 matching lines @@
         problems.OtherProblem(
           'No time for start of trip_id "%s""' % (self.trip_id))
       if stoptimes[-1].arrival_time is None and stoptimes[-1].departure_time is None:
         problems.OtherProblem(
           'No time for end of trip_id "%s""' % (self.trip_id))
 
       # Sorts the stoptimes by sequence and then checks that the arrival time
       # for each time point is after the departure time of the previous.
       stoptimes.sort(key=lambda x: x.stop_sequence)
       prev_departure = 0
+      prev_stop = None
+      try:
+        route_type = self._schedule.GetRoute(self.route_id).route_type
+        max_speed = Route._ROUTE_TYPES[route_type]['max_speed']
+      except KeyError, e:
+        # If route_type cannot be found, assume it is 0 (Tram) for checking
+        # speeds between stops.
+        route_type = 0
+        max_speed = 30
       for timepoint in stoptimes:
         if timepoint.arrival_secs is not None:
+          self._CheckSpeed(prev_stop, timepoint.stop, prev_departure,
+                           timepoint.arrival_secs, max_speed, problems)
+
           if timepoint.arrival_secs >= prev_departure:
             prev_departure = timepoint.departure_secs
+            prev_stop = timepoint.stop
           else:
             problems.OtherProblem('Timetravel detected! Arrival time '
                                   'is before previous departure '
                                   'at sequence number %s in trip %s' %
                                   (timepoint.stop_sequence, self.trip_id))
 
       if self.shape_id and self.shape_id in self._schedule._shapes:
         shape = self._schedule.GetShape(self.shape_id)
         max_shape_dist = shape.max_distance
         st = stoptimes[-1]
@@ skipping 28 matching lines @@
                     type=TYPE_WARNING)
 
     # O(n^2), but we don't anticipate many headway periods per trip
     for headway_index, headway in enumerate(self._headways[0:-1]):
       for other in self._headways[headway_index + 1:]:
         if (other[0] < headway[1]) and (other[1] > headway[0]):
           problems.OtherProblem('Trip contains overlapping headway periods '
                                 '%s and %s' %
                                 (self._HeadwayOutputTuple(headway),
                                  self._HeadwayOutputTuple(other)))
+
+  def _CheckSpeed(self, prev_stop, next_stop, depart_time,
+                  arrive_time, max_speed, problems):
+    # Checks that the speed between two stops is not faster than max_speed
+    if prev_stop != None:
+      try:
+        time_between_stops = arrive_time - depart_time
+      except TypeError:
+        return
+
+      try:
+        dist_between_stops = \
+          ApproximateDistanceBetweenStops(next_stop, prev_stop)
+      except TypeError, e:
+          return
+
+      if time_between_stops == 0:
+        problems.TooFastTravel(self.trip_id,
+                               prev_stop.stop_name,
+                               next_stop.stop_name,
+                               dist_between_stops,
+                               time_between_stops,
+                               speed=None,
+                               type=TYPE_WARNING)
+        return
+      # This needs floating point division for precision.
+      speed_between_stops = ((float(dist_between_stops) / 1000) /
+                                (float(time_between_stops) / 3600))
+      if speed_between_stops > max_speed:
+        problems.TooFastTravel(self.trip_id,
+                               prev_stop.stop_name,
+                               next_stop.stop_name,
+                               dist_between_stops,
+                               time_between_stops,
+                               speed_between_stops,
+                               type=TYPE_WARNING)
 
 # TODO: move these into a separate file
 class ISO4217(object):
   """Represents the set of currencies recognized by the ISO-4217 spec."""
   codes = {  # map of alpha code to numerical code
     'AED': 784, 'AFN': 971, 'ALL':   8, 'AMD':  51, 'ANG': 532, 'AOA': 973,
     'ARS':  32, 'AUD':  36, 'AWG': 533, 'AZN': 944, 'BAM': 977, 'BBD':  52,
     'BDT':  50, 'BGN': 975, 'BHD':  48, 'BIF': 108, 'BMD':  60, 'BND':  96,
     'BOB':  68, 'BOV': 984, 'BRL': 986, 'BSD':  44, 'BTN':  64, 'BWP':  72,
     'BYR': 974, 'BZD':  84, 'CAD': 124, 'CDF': 976, 'CHE': 947, 'CHF': 756,
@@ skipping 593 matching lines @@
       yield (self.service_id, date, unicode(exception_type))
 
   def GetCalendarDatesFieldValuesTuples(self):
     """Return a list of date execeptions"""
     result = []
     for date_tuple in self.GenerateCalendarDatesFieldValuesTuples():
       result.append(date_tuple)
     result.sort()  # helps with __eq__
     return result
 
-  def SetDateHasService(self, date, has_service=True):
+  def SetDateHasService(self, date, has_service=True, problems=None):
+    if date in self.date_exceptions and problems:
+      problems.DuplicateID(('service_id', 'date'),
+                           (self.service_id, date),
+                           type=TYPE_WARNING)
     self.date_exceptions[date] = has_service and 1 or 2
 
   def ResetDateToNormalService(self, date):
     if date in self.date_exceptions:
       del self.date_exceptions[date]
 
   def SetStartDate(self, start_date):
     """Set the first day of service as a string in YYYYMMDD format"""
     self.start_date = start_date
 
@@ skipping 32 matching lines @@
 
   def IsActiveOn(self, date):
     """Test if this service period is active on a date.
 
     Args:
       date: a string of form "YYYYMMDD"
 
     Returns:
       True iff this service is active on date.
     """
-    date_obj = DateStringToDateObject(date)
     if date in self.date_exceptions:
       if self.date_exceptions[date] == 1:
         return True
       else:
         return False
     if (self.start_date and self.end_date and self.start_date <= date and
         date <= self.end_date):
+      date_obj = DateStringToDateObject(date)
       return self.day_of_week[date_obj.weekday()]
     return False
 
   def ActiveDates(self):
     """Return dates this service period is active as a list of "YYYYMMDD"."""
     (earliest, latest) = self.GetDateRange()
     if earliest is None:
       return []
     dates = []
     date_it = DateStringToDateObject(earliest)
@@ skipping 316 matching lines @@
 
     ranges = [period.GetDateRange() for period in self.GetServicePeriodList()]
     starts = filter(lambda x: x, [item[0] for item in ranges])
     ends = filter(lambda x: x, [item[1] for item in ranges])
 
     if not starts or not ends:
       return (None, None)
 
     return (min(starts), max(ends))
 
+  def GetServicePeriodsActiveEachDate(self, date_start, date_end):
+    """Return a list of tuples (date, [period1, period2, ...]).
+
+    For each date in the range [date_start, date_end) make list of each
+    ServicePeriod object which is active.
+
+    Args:
+      date_start: The first date in the list, a date object
+      date_end: The first date after the list, a date object
+
+    Returns:
+      A list of tuples. Each tuple contains a date object and a list of zero or
+      more ServicePeriod objects.
+    """
+    date_it = date_start
+    one_day = datetime.timedelta(days=1)
+    date_service_period_list = []
+    while date_it < date_end:
+      periods_today = []
+      for service in self.GetServicePeriodList():
+        if service.IsActiveOn(date_it.strftime("%Y%m%d")):
+          periods_today.append(service)
+      date_service_period_list.append((date_it, periods_today))
+      date_it += one_day
+    return date_service_period_list
+
+
   def AddStop(self, lat, lng, name):
     """Add a stop to this schedule.
 
     A new stop_id is created for this stop. Do not use this method unless all
     stops in this Schedule are created with it. See source for details.
 
     Args:
       lat: Latitude of the stop as a float or string
       lng: Longitude of the stop as a float or string
       name: Name of the stop, which will appear in the feed
@@ skipping 357 matching lines @@
 
     archive.close()
 
   def Validate(self, problems=None, validate_children=True):
     """Validates various holistic aspects of the schedule
        (mostly interrelationships between the various data sets)."""
     if not problems:
       problems = self.problem_reporter
 
     (start_date, end_date) = self.GetDateRange()
-    if not end_date:
+    if not end_date or not start_date:
       problems.OtherProblem('This feed has no effective service dates!',
                             type=TYPE_WARNING)
     else:
       try:
         expiration = time.mktime(time.strptime(end_date, "%Y%m%d"))
+        start_date_time = time.mktime(time.strptime(start_date, "%Y%m%d"))
         now = time.mktime(time.localtime())
         warning_cutoff = now + 60 * 60 * 24 * 30  # one month from expiration
         if expiration < warning_cutoff:
           problems.ExpirationDate(expiration)
+        if start_date_time > now:
+          problems.FutureService(start_date_time)
       except ValueError:
-        problems.InvalidValue('end_date', end_date)
+        # Format of start_date and end_date checked in class ServicePeriod
+        pass
 
     # TODO: Check Trip fields against valid values
 
     # Check for stops that aren't referenced by any trips and broken
-    # parent_station references.
+    # parent_station references. Also check that the parent station isn't too
+    # far from its child stops.
     for stop in self.stops.values():
       if validate_children:
         stop.Validate(problems)
       cursor = self._connection.cursor()
       cursor.execute("SELECT count(*) FROM stop_times WHERE stop_id=? LIMIT 1",
                      (stop.stop_id,))
       count = cursor.fetchone()[0]
       if stop.location_type == 0 and count == 0:
           problems.UnusedStop(stop.stop_id, stop.stop_name)
       elif stop.location_type == 1 and count != 0:
           problems.UsedStation(stop.stop_id, stop.stop_name)
 
       if stop.location_type != 1 and stop.parent_station:
         if stop.parent_station not in self.stops:
           problems.InvalidValue("parent_station",
                                 EncodeUnicode(stop.parent_station),
                                 "parent_station '%s' not found for stop_id "
                                 "'%s' in stops.txt" %
                                 (EncodeUnicode(stop.parent_station),
                                  EncodeUnicode(stop.stop_id)))
         elif self.stops[stop.parent_station].location_type != 1:
           problems.InvalidValue("parent_station",
                                 EncodeUnicode(stop.parent_station),
                                 "parent_station '%s' of stop_id '%s' must "
                                 "have location_type=1 in stops.txt" %
                                 (EncodeUnicode(stop.parent_station),
                                  EncodeUnicode(stop.stop_id)))
-
-    #TODO: check that every station is used and within 1km of stops that are
-    # part of it. Then uncomment testStationWithoutReference.
+        else:
+          parent_station = self.stops[stop.parent_station]
+          distance = ApproximateDistanceBetweenStops(stop, parent_station)
+          if distance > MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_ERROR:
+            problems.StopTooFarFromParentStation(
+                stop.stop_id, stop.stop_name, parent_station.stop_id,
+                parent_station.stop_name, distance, TYPE_ERROR)
+          elif distance > MAX_DISTANCE_BETWEEN_STOP_AND_PARENT_STATION_WARNING:
+            problems.StopTooFarFromParentStation(
+                stop.stop_id, stop.stop_name, parent_station.stop_id,
+                parent_station.stop_name, distance, TYPE_WARNING)
+
+    #TODO: check that every station is used.
+    # Then uncomment testStationWithoutReference.
 
     # Check for stops that might represent the same location (specifically,
     # stops that are less that 2 meters apart) First filter out stops without a
     # valid lat and lon. Then sort by latitude, then find the distance between
     # each pair of stations within 2 meters latitude of each other. This avoids
     # doing n^2 comparisons in the average case and doesn't need a spatial
     # index.
     sorted_stops = filter(lambda s: s.stop_lat and s.stop_lon,
                           self.GetStopList())
     sorted_stops.sort(key=(lambda x: x.stop_lat))
@@ skipping 29 matching lines @@
             elif stop.location_type == 1 and other_stop.location_type == 0:
               this_stop = other_stop
               this_station = stop
             if this_stop.parent_station != this_station.stop_id:
               problems.OtherProblem(
                   'The parent_station of stop "%s" (ID "%s") is not '
                   'station "%s" (ID "%s") but they are only %0.2fm apart.' %
                   (EncodeUnicode(this_stop.stop_name),
                    EncodeUnicode(this_stop.stop_id),
                    EncodeUnicode(this_station.stop_name),
-                   EncodeUnicode(this_station.stop_id), distance))
+                   EncodeUnicode(this_station.stop_id), distance),
+                  type=TYPE_WARNING)
         index += 1
 
     # Check for multiple routes using same short + long name
     route_names = {}
     for route in self.routes.values():
       if validate_children:
         route.Validate(problems)
       short_name = ''
       if not IsEmpty(route.route_short_name):
         short_name = route.route_short_name.lower().strip()
       long_name = ''
       if not IsEmpty(route.route_long_name):
         long_name = route.route_long_name.lower().strip()
       name = (short_name, long_name)
       if name in route_names:
         problems.InvalidValue('route_long_name',
                               long_name,
                               'The same combination of '
                               'route_short_name and route_long_name '
                               'shouldn\'t be used for more than one '
                               'route, as it is for the for the two routes '
                               'with IDs "%s" and "%s".' %
                               (route.route_id, route_names[name].route_id),
                               type=TYPE_WARNING)
       else:
         route_names[name] = route
 
-    # Check duplicate trips which go through the same stops with same
-    # service and start times.
-
-    if self._check_duplicate_trips:
-      trips = {}
-      for trip in self.trips.values():
-        stop_ids = []
-        stop_times = []
-        for index, st in enumerate(trip.GetStopTimes(problems)):
-          stop_times.append(st.arrival_time)
-          stop_ids.append(st.stop.stop_id)
-        if not stop_ids or not stop_times:
+    stop_types = {} # a dict mapping stop_id to [route_id, route_type, is_match]
+    trips = {} # a dict mapping tuple to (route_id, trip_id)
+    for trip in sorted(self.trips.values()):
+      if trip.route_id not in self.routes:
+        continue
+      route_type = self.GetRoute(trip.route_id).route_type
+      arrival_times = []
+      stop_ids = []      
+      for index, st in enumerate(trip.GetStopTimes(problems)):
+        stop_id = st.stop.stop_id
+        arrival_times.append(st.arrival_time)
+        stop_ids.append(stop_id)
+        # Check a stop if which belongs to both subway and bus.
+        if (route_type == Route._ROUTE_TYPE_NAMES['Subway'] or 
+            route_type == Route._ROUTE_TYPE_NAMES['Bus']):  
+          if stop_id not in stop_types:
+            stop_types[stop_id] = [trip.route_id, route_type, 0]
+          elif (stop_types[stop_id][1] != route_type and 
+                stop_types[stop_id][2] == 0):
+            stop_types[stop_id][2] = 1
+            if stop_types[stop_id][1] == Route._ROUTE_TYPE_NAMES['Subway']:
+              subway_route_id = stop_types[stop_id][0]
+              bus_route_id = trip.route_id
+            else:
+              subway_route_id = trip.route_id
+              bus_route_id = stop_types[stop_id][0]
+            problems.StopWithMultipleRouteTypes(st.stop.stop_name, stop_id, 
+                                                subway_route_id, bus_route_id) 
+
+      # Check duplicate trips which go through the same stops with same
+      # service and start times.
+      if self._check_duplicate_trips:
+        if not stop_ids or not arrival_times:
           continue
-        key = (trip.service_id, min(stop_times), str(stop_ids))
+        key = (trip.service_id, min(arrival_times), str(stop_ids))
         if key not in trips:
-          trips[key] = [trip.route_id, trip.trip_id]
+          trips[key] = (trip.route_id, trip.trip_id)
         else:
-          problems.OtherProblem('Trip %s of route %s might be duplicated '
-                                'with trip %s of route %s. They go through '
-                                'the same stops with same service. '
-                                % (trips[key][1], trips[key][0],
-                                trip.trip_id, trip.route_id), 
-                                type=TYPE_WARNING)
+          problems.DuplicateTrip(trips[key][1], trips[key][0], trip.trip_id, 
+                                 trip.route_id)
 
     # Check that routes' agency IDs are valid, if set
     for route in self.routes.values():
       if (not IsEmpty(route.agency_id) and
           not route.agency_id in self._agencies):
         problems.InvalidValue('agency_id',
                               route.agency_id,
                               'The route with ID "%s" specifies agency_id '
                               '"%s", which doesn\'t exist.' %
                               (route.route_id, route.agency_id))
@@ skipping 463 matching lines @@
   def _FileContents(self, file_name):
     results = None
     if self._zip:
       try:
         results = self._zip.read(file_name)
       except KeyError:  # file not found in archve
         self._problems.MissingFile(file_name)
         return None
     else:
       try:
-        data_file = open(os.path.join(self._path, file_name), 'r')
+        data_file = open(os.path.join(self._path, file_name), 'rb')
         results = data_file.read()
       except IOError:  # file not found
         self._problems.MissingFile(file_name)
         return None
 
     if not results:
       self._problems.EmptyFile(file_name)
     return results
 
   def _LoadAgencies(self):
@@ skipping 41 matching lines @@
     periods = {}
 
     # process calendar.txt
     if self._HasFile(file_name):
       has_useful_contents = False
       for (row, row_num, cols) in \
               self._ReadCSV(file_name,
                             ServicePeriod._FIELD_NAMES,
                             ServicePeriod._FIELD_NAMES_REQUIRED):
         context = (file_name, row_num, row, cols)
+        self._problems.SetFileContext(*context)
 
         period = ServicePeriod(field_list=row)
 
         if period.service_id in periods:
           self._problems.DuplicateID('service_id', period.service_id)
-          continue
-
-        periods[period.service_id] = (period, context)
+        else:
+          periods[period.service_id] = (period, context)
+        self._problems.ClearContext()
 
     # process calendar_dates.txt
     if self._HasFile(file_name_dates):
-      service_id_date_entries = {}
       # ['service_id', 'date', 'exception_type']
       fields = ServicePeriod._FIELD_NAMES_CALENDAR_DATES
       for (row, row_num, cols) in self._ReadCSV(file_name_dates,
                                                 fields, fields):
         context = (file_name_dates, row_num, row, cols)
+        self._problems.SetFileContext(*context)
 
         service_id = row[0]
 
         period = None
         if service_id in periods:
           period = periods[service_id][0]
         else:
           period = ServicePeriod(service_id)
           periods[period.service_id] = (period, context)
-          
-        # Check uniqueness of service_id x date
-        if (service_id, row[1]) in service_id_date_entries:
-          self._problems.DuplicateID('service_id x date',
-                                     service_id+' X '+row[1],

[Tom, 2009/06/18 22:17:33]

I don't think this should use DuplicateID as is because column_name should be a
literal column. Instead you need to generalize it so "ID" can mean a tuple of
values or make a new type of error.
It would also be good to check existing feeds for this problem.
I've been trying to keep validation logic in the individual GenericGTFSObject
subclasses but there doesn't seem to be an easy way to do it here. Maybe add
more parameters to SetDateHasService that enable checking for existing dates?

[jirka, 2009/10/22 22:25:58]

On 2009/06/18 22:17:33, Tom wrote:
> I don't think this should use DuplicateID as is because column_name should be
a
> literal column. Instead you need to generalize it so "ID" can mean a tuple of
> values or make a new type of error.
> It would also be good to check existing feeds for this problem.
> I've been trying to keep validation logic in the individual GenericGTFSObject
> subclasses but there doesn't seem to be an easy way to do it here. Maybe add
> more parameters to SetDateHasService that enable checking for existing dates?

Done.

-                                     type=TYPE_WARNING)
-        service_id_date_entries[service_id, row[1]] = 1
 
         exception_type = row[2]
         if exception_type == u'1':
-          period.SetDateHasService(row[1], True)
+          period.SetDateHasService(row[1], True, self._problems)
         elif exception_type == u'2':
-          period.SetDateHasService(row[1], False)
+          period.SetDateHasService(row[1], False, self._problems)
         else:
           self._problems.InvalidValue('exception_type', exception_type)
+        self._problems.ClearContext()
 
     # Now insert the periods into the schedule object, so that they're
     # validated with both calendar and calendar_dates info present
     for period, context in periods.values():
       self._problems.SetFileContext(*context)
       self._schedule.AddServicePeriodObject(period, self._problems)
       self._problems.ClearContext()
 
   def _LoadShapes(self):
     if not self._HasFile('shapes.txt'):
@@ skipping 201 matching lines @@
   def __init__(self, *args, **kwargs):
     """Initialize a new ShapeLoader object.
 
     See Loader.__init__ for argument documentation.
     """
     Loader.__init__(self, *args, **kwargs)
 
   def Load(self):
     self._LoadShapes()
     return self._schedule