Squashed 'src/cpp/protobuf/' content from commit 29edac6

git-subtree-dir: src/cpp/protobuf
git-subtree-split: 29edac6e7d78bdb050ed47adc79c9ac1ba22071e

python/google/protobuf/internal/api_implementation.py

@@ -0,0 +1,87 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+This module is the central entity that determines which implementation of the
+API is used.
+"""
+
+__author__ = 'petar@google.com (Petar Petrov)'
+
+import os
+# This environment variable can be used to switch to a certain implementation
+# of the Python API. Right now only 'python' and 'cpp' are valid values. Any
+# other value will be ignored.
+_implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION',
+                                 'python')
+
+
+if _implementation_type != 'python':
+  # For now, by default use the pure-Python implementation.
+  # The code below checks if the C extension is available and
+  # uses it if it is available.
+  _implementation_type = 'cpp'
+  ## Determine automatically which implementation to use.
+  #try:
+  #  from google.protobuf.internal import cpp_message
+  #  _implementation_type = 'cpp'
+  #except ImportError, e:
+  #  _implementation_type = 'python'
+
+
+# This environment variable can be used to switch between the two
+# 'cpp' implementations. Right now only 1 and 2 are valid values. Any
+# other value will be ignored.
+_implementation_version_str = os.getenv(
+    'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION',
+    '1')
+
+
+if _implementation_version_str not in ('1', '2'):
+  raise ValueError(
+      "unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: '" +
+      _implementation_version_str + "' (supported versions: 1, 2)"
+      )
+
+
+_implementation_version = int(_implementation_version_str)
+
+
+
+# Usage of this function is discouraged. Clients shouldn't care which
+# implementation of the API is in use. Note that there is no guarantee
+# that differences between APIs will be maintained.
+# Please don't use this function if possible.
+def Type():
+  return _implementation_type
+
+# See comment on 'Type' above.
+def Version():
+  return _implementation_version

python/google/protobuf/internal/containers.py

@@ -0,0 +1,269 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Contains container classes to represent different protocol buffer types.
+
+This file defines container classes which represent categories of protocol
+buffer field types which need extra maintenance. Currently these categories
+are:
+  - Repeated scalar fields - These are all repeated fields which aren't
+    composite (e.g. they are of simple types like int32, string, etc).
+  - Repeated composite fields - Repeated fields which are composite. This
+    includes groups and nested messages.
+"""
+
+__author__ = 'petar@google.com (Petar Petrov)'
+
+
+class BaseContainer(object):
+
+  """Base container class."""
+
+  # Minimizes memory usage and disallows assignment to other attributes.
+  __slots__ = ['_message_listener', '_values']
+
+  def __init__(self, message_listener):
+    """
+    Args:
+      message_listener: A MessageListener implementation.
+        The RepeatedScalarFieldContainer will call this object's
+        Modified() method when it is modified.
+    """
+    self._message_listener = message_listener
+    self._values = []
+
+  def __getitem__(self, key):
+    """Retrieves item by the specified key."""
+    return self._values[key]
+
+  def __len__(self):
+    """Returns the number of elements in the container."""
+    return len(self._values)
+
+  def __ne__(self, other):
+    """Checks if another instance isn't equal to this one."""
+    # The concrete classes should define __eq__.
+    return not self == other
+
+  def __hash__(self):
+    raise TypeError('unhashable object')
+
+  def __repr__(self):
+    return repr(self._values)
+
+  def sort(self, *args, **kwargs):
+    # Continue to support the old sort_function keyword argument.
+    # This is expected to be a rare occurrence, so use LBYL to avoid
+    # the overhead of actually catching KeyError.
+    if 'sort_function' in kwargs:
+      kwargs['cmp'] = kwargs.pop('sort_function')
+    self._values.sort(*args, **kwargs)
+
+
+class RepeatedScalarFieldContainer(BaseContainer):
+
+  """Simple, type-checked, list-like container for holding repeated scalars."""
+
+  # Disallows assignment to other attributes.
+  __slots__ = ['_type_checker']
+
+  def __init__(self, message_listener, type_checker):
+    """
+    Args:
+      message_listener: A MessageListener implementation.
+        The RepeatedScalarFieldContainer will call this object's
+        Modified() method when it is modified.
+      type_checker: A type_checkers.ValueChecker instance to run on elements
+        inserted into this container.
+    """
+    super(RepeatedScalarFieldContainer, self).__init__(message_listener)
+    self._type_checker = type_checker
+
+  def append(self, value):
+    """Appends an item to the list. Similar to list.append()."""
+    self._type_checker.CheckValue(value)
+    self._values.append(value)
+    if not self._message_listener.dirty:
+      self._message_listener.Modified()
+
+  def insert(self, key, value):
+    """Inserts the item at the specified position. Similar to list.insert()."""
+    self._type_checker.CheckValue(value)
+    self._values.insert(key, value)
+    if not self._message_listener.dirty:
+      self._message_listener.Modified()
+
+  def extend(self, elem_seq):
+    """Extends by appending the given sequence. Similar to list.extend()."""
+    if not elem_seq:
+      return
+
+    new_values = []
+    for elem in elem_seq:
+      self._type_checker.CheckValue(elem)
+      new_values.append(elem)
+    self._values.extend(new_values)
+    self._message_listener.Modified()
+
+  def MergeFrom(self, other):
+    """Appends the contents of another repeated field of the same type to this
+    one. We do not check the types of the individual fields.
+    """
+    self._values.extend(other._values)
+    self._message_listener.Modified()
+
+  def remove(self, elem):
+    """Removes an item from the list. Similar to list.remove()."""
+    self._values.remove(elem)
+    self._message_listener.Modified()
+
+  def __setitem__(self, key, value):
+    """Sets the item on the specified position."""
+    self._type_checker.CheckValue(value)
+    self._values[key] = value
+    self._message_listener.Modified()
+
+  def __getslice__(self, start, stop):
+    """Retrieves the subset of items from between the specified indices."""
+    return self._values[start:stop]
+
+  def __setslice__(self, start, stop, values):
+    """Sets the subset of items from between the specified indices."""
+    new_values = []
+    for value in values:
+      self._type_checker.CheckValue(value)
+      new_values.append(value)
+    self._values[start:stop] = new_values
+    self._message_listener.Modified()
+
+  def __delitem__(self, key):
+    """Deletes the item at the specified position."""
+    del self._values[key]
+    self._message_listener.Modified()
+
+  def __delslice__(self, start, stop):
+    """Deletes the subset of items from between the specified indices."""
+    del self._values[start:stop]
+    self._message_listener.Modified()
+
+  def __eq__(self, other):
+    """Compares the current instance with another one."""
+    if self is other:
+      return True
+    # Special case for the same type which should be common and fast.
+    if isinstance(other, self.__class__):
+      return other._values == self._values
+    # We are presumably comparing against some other sequence type.
+    return other == self._values
+
+
+class RepeatedCompositeFieldContainer(BaseContainer):
+
+  """Simple, list-like container for holding repeated composite fields."""
+
+  # Disallows assignment to other attributes.
+  __slots__ = ['_message_descriptor']
+
+  def __init__(self, message_listener, message_descriptor):
+    """
+    Note that we pass in a descriptor instead of the generated directly,
+    since at the time we construct a _RepeatedCompositeFieldContainer we
+    haven't yet necessarily initialized the type that will be contained in the
+    container.
+
+    Args:
+      message_listener: A MessageListener implementation.
+        The RepeatedCompositeFieldContainer will call this object's
+        Modified() method when it is modified.
+      message_descriptor: A Descriptor instance describing the protocol type
+        that should be present in this container.  We'll use the
+        _concrete_class field of this descriptor when the client calls add().
+    """
+    super(RepeatedCompositeFieldContainer, self).__init__(message_listener)
+    self._message_descriptor = message_descriptor
+
+  def add(self, **kwargs):
+    """Adds a new element at the end of the list and returns it. Keyword
+    arguments may be used to initialize the element.
+    """
+    new_element = self._message_descriptor._concrete_class(**kwargs)
+    new_element._SetListener(self._message_listener)
+    self._values.append(new_element)
+    if not self._message_listener.dirty:
+      self._message_listener.Modified()
+    return new_element
+
+  def extend(self, elem_seq):
+    """Extends by appending the given sequence of elements of the same type
+    as this one, copying each individual message.
+    """
+    message_class = self._message_descriptor._concrete_class
+    listener = self._message_listener
+    values = self._values
+    for message in elem_seq:
+      new_element = message_class()
+      new_element._SetListener(listener)
+      new_element.MergeFrom(message)
+      values.append(new_element)
+    listener.Modified()
+
+  def MergeFrom(self, other):
+    """Appends the contents of another repeated field of the same type to this
+    one, copying each individual message.
+    """
+    self.extend(other._values)
+
+  def remove(self, elem):
+    """Removes an item from the list. Similar to list.remove()."""
+    self._values.remove(elem)
+    self._message_listener.Modified()
+
+  def __getslice__(self, start, stop):
+    """Retrieves the subset of items from between the specified indices."""
+    return self._values[start:stop]
+
+  def __delitem__(self, key):
+    """Deletes the item at the specified position."""
+    del self._values[key]
+    self._message_listener.Modified()
+
+  def __delslice__(self, start, stop):
+    """Deletes the subset of items from between the specified indices."""
+    del self._values[start:stop]
+    self._message_listener.Modified()
+
+  def __eq__(self, other):
+    """Compares the current instance with another one."""
+    if self is other:
+      return True
+    if not isinstance(other, self.__class__):
+      raise TypeError('Can only compare repeated composite fields against '
+                      'other repeated composite fields.')
+    return self._values == other._values

python/google/protobuf/internal/cpp_message.py

@@ -0,0 +1,663 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Contains helper functions used to create protocol message classes from
+Descriptor objects at runtime backed by the protocol buffer C++ API.
+"""
+
+__author__ = 'petar@google.com (Petar Petrov)'
+
+import copy_reg
+import operator
+from google.protobuf.internal import _net_proto2___python
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import message
+
+
+_LABEL_REPEATED = _net_proto2___python.LABEL_REPEATED
+_LABEL_OPTIONAL = _net_proto2___python.LABEL_OPTIONAL
+_CPPTYPE_MESSAGE = _net_proto2___python.CPPTYPE_MESSAGE
+_TYPE_MESSAGE = _net_proto2___python.TYPE_MESSAGE
+
+
+def GetDescriptorPool():
+  """Creates a new DescriptorPool C++ object."""
+  return _net_proto2___python.NewCDescriptorPool()
+
+
+_pool = GetDescriptorPool()
+
+
+def GetFieldDescriptor(full_field_name):
+  """Searches for a field descriptor given a full field name."""
+  return _pool.FindFieldByName(full_field_name)
+
+
+def BuildFile(content):
+  """Registers a new proto file in the underlying C++ descriptor pool."""
+  _net_proto2___python.BuildFile(content)
+
+
+def GetExtensionDescriptor(full_extension_name):
+  """Searches for extension descriptor given a full field name."""
+  return _pool.FindExtensionByName(full_extension_name)
+
+
+def NewCMessage(full_message_name):
+  """Creates a new C++ protocol message by its name."""
+  return _net_proto2___python.NewCMessage(full_message_name)
+
+
+def ScalarProperty(cdescriptor):
+  """Returns a scalar property for the given descriptor."""
+
+  def Getter(self):
+    return self._cmsg.GetScalar(cdescriptor)
+
+  def Setter(self, value):
+    self._cmsg.SetScalar(cdescriptor, value)
+
+  return property(Getter, Setter)
+
+
+def CompositeProperty(cdescriptor, message_type):
+  """Returns a Python property the given composite field."""
+
+  def Getter(self):
+    sub_message = self._composite_fields.get(cdescriptor.name, None)
+    if sub_message is None:
+      cmessage = self._cmsg.NewSubMessage(cdescriptor)
+      sub_message = message_type._concrete_class(__cmessage=cmessage)
+      self._composite_fields[cdescriptor.name] = sub_message
+    return sub_message
+
+  return property(Getter)
+
+
+class RepeatedScalarContainer(object):
+  """Container for repeated scalar fields."""
+
+  __slots__ = ['_message', '_cfield_descriptor', '_cmsg']
+
+  def __init__(self, msg, cfield_descriptor):
+    self._message = msg
+    self._cmsg = msg._cmsg
+    self._cfield_descriptor = cfield_descriptor
+
+  def append(self, value):
+    self._cmsg.AddRepeatedScalar(
+        self._cfield_descriptor, value)
+
+  def extend(self, sequence):
+    for element in sequence:
+      self.append(element)
+
+  def insert(self, key, value):
+    values = self[slice(None, None, None)]
+    values.insert(key, value)
+    self._cmsg.AssignRepeatedScalar(self._cfield_descriptor, values)
+
+  def remove(self, value):
+    values = self[slice(None, None, None)]
+    values.remove(value)
+    self._cmsg.AssignRepeatedScalar(self._cfield_descriptor, values)
+
+  def __setitem__(self, key, value):
+    values = self[slice(None, None, None)]
+    values[key] = value
+    self._cmsg.AssignRepeatedScalar(self._cfield_descriptor, values)
+
+  def __getitem__(self, key):
+    return self._cmsg.GetRepeatedScalar(self._cfield_descriptor, key)
+
+  def __delitem__(self, key):
+    self._cmsg.DeleteRepeatedField(self._cfield_descriptor, key)
+
+  def __len__(self):
+    return len(self[slice(None, None, None)])
+
+  def __eq__(self, other):
+    if self is other:
+      return True
+    if not operator.isSequenceType(other):
+      raise TypeError(
+          'Can only compare repeated scalar fields against sequences.')
+    # We are presumably comparing against some other sequence type.
+    return other == self[slice(None, None, None)]
+
+  def __ne__(self, other):
+    return not self == other
+
+  def __hash__(self):
+    raise TypeError('unhashable object')
+
+  def sort(self, *args, **kwargs):
+    # Maintain compatibility with the previous interface.
+    if 'sort_function' in kwargs:
+      kwargs['cmp'] = kwargs.pop('sort_function')
+    self._cmsg.AssignRepeatedScalar(self._cfield_descriptor,
+                                    sorted(self, *args, **kwargs))
+
+
+def RepeatedScalarProperty(cdescriptor):
+  """Returns a Python property the given repeated scalar field."""
+
+  def Getter(self):
+    container = self._composite_fields.get(cdescriptor.name, None)
+    if container is None:
+      container = RepeatedScalarContainer(self, cdescriptor)
+      self._composite_fields[cdescriptor.name] = container
+    return container
+
+  def Setter(self, new_value):
+    raise AttributeError('Assignment not allowed to repeated field '
+                         '"%s" in protocol message object.' % cdescriptor.name)
+
+  doc = 'Magic attribute generated for "%s" proto field.' % cdescriptor.name
+  return property(Getter, Setter, doc=doc)
+
+
+class RepeatedCompositeContainer(object):
+  """Container for repeated composite fields."""
+
+  __slots__ = ['_message', '_subclass', '_cfield_descriptor', '_cmsg']
+
+  def __init__(self, msg, cfield_descriptor, subclass):
+    self._message = msg
+    self._cmsg = msg._cmsg
+    self._subclass = subclass
+    self._cfield_descriptor = cfield_descriptor
+
+  def add(self, **kwargs):
+    cmessage = self._cmsg.AddMessage(self._cfield_descriptor)
+    return self._subclass(__cmessage=cmessage, __owner=self._message, **kwargs)
+
+  def extend(self, elem_seq):
+    """Extends by appending the given sequence of elements of the same type
+    as this one, copying each individual message.
+    """
+    for message in elem_seq:
+      self.add().MergeFrom(message)
+
+  def remove(self, value):
+    # TODO(protocol-devel): This is inefficient as it needs to generate a
+    # message pointer for each message only to do index().  Move this to a C++
+    # extension function.
+    self.__delitem__(self[slice(None, None, None)].index(value))
+
+  def MergeFrom(self, other):
+    for message in other[:]:
+      self.add().MergeFrom(message)
+
+  def __getitem__(self, key):
+    cmessages = self._cmsg.GetRepeatedMessage(
+        self._cfield_descriptor, key)
+    subclass = self._subclass
+    if not isinstance(cmessages, list):
+      return subclass(__cmessage=cmessages, __owner=self._message)
+
+    return [subclass(__cmessage=m, __owner=self._message) for m in cmessages]
+
+  def __delitem__(self, key):
+    self._cmsg.DeleteRepeatedField(
+        self._cfield_descriptor, key)
+
+  def __len__(self):
+    return self._cmsg.FieldLength(self._cfield_descriptor)
+
+  def __eq__(self, other):
+    """Compares the current instance with another one."""
+    if self is other:
+      return True
+    if not isinstance(other, self.__class__):
+      raise TypeError('Can only compare repeated composite fields against '
+                      'other repeated composite fields.')
+    messages = self[slice(None, None, None)]
+    other_messages = other[slice(None, None, None)]
+    return messages == other_messages
+
+  def __hash__(self):
+    raise TypeError('unhashable object')
+
+  def sort(self, cmp=None, key=None, reverse=False, **kwargs):
+    # Maintain compatibility with the old interface.
+    if cmp is None and 'sort_function' in kwargs:
+      cmp = kwargs.pop('sort_function')
+
+    # The cmp function, if provided, is passed the results of the key function,
+    # so we only need to wrap one of them.
+    if key is None:
+      index_key = self.__getitem__
+    else:
+      index_key = lambda i: key(self[i])
+
+    # Sort the list of current indexes by the underlying object.
+    indexes = range(len(self))
+    indexes.sort(cmp=cmp, key=index_key, reverse=reverse)
+
+    # Apply the transposition.
+    for dest, src in enumerate(indexes):
+      if dest == src:
+        continue
+      self._cmsg.SwapRepeatedFieldElements(self._cfield_descriptor, dest, src)
+      # Don't swap the same value twice.
+      indexes[src] = src
+
+
+def RepeatedCompositeProperty(cdescriptor, message_type):
+  """Returns a Python property for the given repeated composite field."""
+
+  def Getter(self):
+    container = self._composite_fields.get(cdescriptor.name, None)
+    if container is None:
+      container = RepeatedCompositeContainer(
+          self, cdescriptor, message_type._concrete_class)
+      self._composite_fields[cdescriptor.name] = container
+    return container
+
+  def Setter(self, new_value):
+    raise AttributeError('Assignment not allowed to repeated field '
+                         '"%s" in protocol message object.' % cdescriptor.name)
+
+  doc = 'Magic attribute generated for "%s" proto field.' % cdescriptor.name
+  return property(Getter, Setter, doc=doc)
+
+
+class ExtensionDict(object):
+  """Extension dictionary added to each protocol message."""
+
+  def __init__(self, msg):
+    self._message = msg
+    self._cmsg = msg._cmsg
+    self._values = {}
+
+  def __setitem__(self, extension, value):
+    from google.protobuf import descriptor
+    if not isinstance(extension, descriptor.FieldDescriptor):
+      raise KeyError('Bad extension %r.' % (extension,))
+    cdescriptor = extension._cdescriptor
+    if (cdescriptor.label != _LABEL_OPTIONAL or
+        cdescriptor.cpp_type == _CPPTYPE_MESSAGE):
+      raise TypeError('Extension %r is repeated and/or a composite type.' % (
+          extension.full_name,))
+    self._cmsg.SetScalar(cdescriptor, value)
+    self._values[extension] = value
+
+  def __getitem__(self, extension):
+    from google.protobuf import descriptor
+    if not isinstance(extension, descriptor.FieldDescriptor):
+      raise KeyError('Bad extension %r.' % (extension,))
+
+    cdescriptor = extension._cdescriptor
+    if (cdescriptor.label != _LABEL_REPEATED and
+        cdescriptor.cpp_type != _CPPTYPE_MESSAGE):
+      return self._cmsg.GetScalar(cdescriptor)
+
+    ext = self._values.get(extension, None)
+    if ext is not None:
+      return ext
+
+    ext = self._CreateNewHandle(extension)
+    self._values[extension] = ext
+    return ext
+
+  def ClearExtension(self, extension):
+    from google.protobuf import descriptor
+    if not isinstance(extension, descriptor.FieldDescriptor):
+      raise KeyError('Bad extension %r.' % (extension,))
+    self._cmsg.ClearFieldByDescriptor(extension._cdescriptor)
+    if extension in self._values:
+      del self._values[extension]
+
+  def HasExtension(self, extension):
+    from google.protobuf import descriptor
+    if not isinstance(extension, descriptor.FieldDescriptor):
+      raise KeyError('Bad extension %r.' % (extension,))
+    return self._cmsg.HasFieldByDescriptor(extension._cdescriptor)
+
+  def _FindExtensionByName(self, name):
+    """Tries to find a known extension with the specified name.
+
+    Args:
+      name: Extension full name.
+
+    Returns:
+      Extension field descriptor.
+    """
+    return self._message._extensions_by_name.get(name, None)
+
+  def _CreateNewHandle(self, extension):
+    cdescriptor = extension._cdescriptor
+    if (cdescriptor.label != _LABEL_REPEATED and
+        cdescriptor.cpp_type == _CPPTYPE_MESSAGE):
+      cmessage = self._cmsg.NewSubMessage(cdescriptor)
+      return extension.message_type._concrete_class(__cmessage=cmessage)
+
+    if cdescriptor.label == _LABEL_REPEATED:
+      if cdescriptor.cpp_type == _CPPTYPE_MESSAGE:
+        return RepeatedCompositeContainer(
+            self._message, cdescriptor, extension.message_type._concrete_class)
+      else:
+        return RepeatedScalarContainer(self._message, cdescriptor)
+    # This shouldn't happen!
+    assert False
+    return None
+
+
+def NewMessage(bases, message_descriptor, dictionary):
+  """Creates a new protocol message *class*."""
+  _AddClassAttributesForNestedExtensions(message_descriptor, dictionary)
+  _AddEnumValues(message_descriptor, dictionary)
+  _AddDescriptors(message_descriptor, dictionary)
+  return bases
+
+
+def InitMessage(message_descriptor, cls):
+  """Constructs a new message instance (called before instance's __init__)."""
+  cls._extensions_by_name = {}
+  _AddInitMethod(message_descriptor, cls)
+  _AddMessageMethods(message_descriptor, cls)
+  _AddPropertiesForExtensions(message_descriptor, cls)
+  copy_reg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
+
+
+def _AddDescriptors(message_descriptor, dictionary):
+  """Sets up a new protocol message class dictionary.
+
+  Args:
+    message_descriptor: A Descriptor instance describing this message type.
+    dictionary: Class dictionary to which we'll add a '__slots__' entry.
+  """
+  dictionary['__descriptors'] = {}
+  for field in message_descriptor.fields:
+    dictionary['__descriptors'][field.name] = GetFieldDescriptor(
+        field.full_name)
+
+  dictionary['__slots__'] = list(dictionary['__descriptors'].iterkeys()) + [
+      '_cmsg', '_owner', '_composite_fields', 'Extensions', '_HACK_REFCOUNTS']
+
+
+def _AddEnumValues(message_descriptor, dictionary):
+  """Sets class-level attributes for all enum fields defined in this message.
+
+  Args:
+    message_descriptor: Descriptor object for this message type.
+    dictionary: Class dictionary that should be populated.
+  """
+  for enum_type in message_descriptor.enum_types:
+    dictionary[enum_type.name] = enum_type_wrapper.EnumTypeWrapper(enum_type)
+    for enum_value in enum_type.values:
+      dictionary[enum_value.name] = enum_value.number
+
+
+def _AddClassAttributesForNestedExtensions(message_descriptor, dictionary):
+  """Adds class attributes for the nested extensions."""
+  extension_dict = message_descriptor.extensions_by_name
+  for extension_name, extension_field in extension_dict.iteritems():
+    assert extension_name not in dictionary
+    dictionary[extension_name] = extension_field
+
+
+def _AddInitMethod(message_descriptor, cls):
+  """Adds an __init__ method to cls."""
+
+  # Create and attach message field properties to the message class.
+  # This can be done just once per message class, since property setters and
+  # getters are passed the message instance.
+  # This makes message instantiation extremely fast, and at the same time it
+  # doesn't require the creation of property objects for each message instance,
+  # which saves a lot of memory.
+  for field in message_descriptor.fields:
+    field_cdescriptor = cls.__descriptors[field.name]
+    if field.label == _LABEL_REPEATED:
+      if field.cpp_type == _CPPTYPE_MESSAGE:
+        value = RepeatedCompositeProperty(field_cdescriptor, field.message_type)
+      else:
+        value = RepeatedScalarProperty(field_cdescriptor)
+    elif field.cpp_type == _CPPTYPE_MESSAGE:
+      value = CompositeProperty(field_cdescriptor, field.message_type)
+    else:
+      value = ScalarProperty(field_cdescriptor)
+    setattr(cls, field.name, value)
+
+    # Attach a constant with the field number.
+    constant_name = field.name.upper() + '_FIELD_NUMBER'
+    setattr(cls, constant_name, field.number)
+
+  def Init(self, **kwargs):
+    """Message constructor."""
+    cmessage = kwargs.pop('__cmessage', None)
+    if cmessage:
+      self._cmsg = cmessage
+    else:
+      self._cmsg = NewCMessage(message_descriptor.full_name)
+
+    # Keep a reference to the owner, as the owner keeps a reference to the
+    # underlying protocol buffer message.
+    owner = kwargs.pop('__owner', None)
+    if owner:
+      self._owner = owner
+
+    if message_descriptor.is_extendable:
+      self.Extensions = ExtensionDict(self)
+    else:
+      # Reference counting in the C++ code is broken and depends on
+      # the Extensions reference to keep this object alive during unit
+      # tests (see b/4856052).  Remove this once b/4945904 is fixed.
+      self._HACK_REFCOUNTS = self
+    self._composite_fields = {}
+
+    for field_name, field_value in kwargs.iteritems():
+      field_cdescriptor = self.__descriptors.get(field_name, None)
+      if not field_cdescriptor:
+        raise ValueError('Protocol message has no "%s" field.' % field_name)
+      if field_cdescriptor.label == _LABEL_REPEATED:
+        if field_cdescriptor.cpp_type == _CPPTYPE_MESSAGE:
+          field_name = getattr(self, field_name)
+          for val in field_value:
+            field_name.add().MergeFrom(val)
+        else:
+          getattr(self, field_name).extend(field_value)
+      elif field_cdescriptor.cpp_type == _CPPTYPE_MESSAGE:
+        getattr(self, field_name).MergeFrom(field_value)
+      else:
+        setattr(self, field_name, field_value)
+
+  Init.__module__ = None
+  Init.__doc__ = None
+  cls.__init__ = Init
+
+
+def _IsMessageSetExtension(field):
+  """Checks if a field is a message set extension."""
+  return (field.is_extension and
+          field.containing_type.has_options and
+          field.containing_type.GetOptions().message_set_wire_format and
+          field.type == _TYPE_MESSAGE and
+          field.message_type == field.extension_scope and
+          field.label == _LABEL_OPTIONAL)
+
+
+def _AddMessageMethods(message_descriptor, cls):
+  """Adds the methods to a protocol message class."""
+  if message_descriptor.is_extendable:
+
+    def ClearExtension(self, extension):
+      self.Extensions.ClearExtension(extension)
+
+    def HasExtension(self, extension):
+      return self.Extensions.HasExtension(extension)
+
+  def HasField(self, field_name):
+    return self._cmsg.HasField(field_name)
+
+  def ClearField(self, field_name):
+    child_cmessage = None
+    if field_name in self._composite_fields:
+      child_field = self._composite_fields[field_name]
+      del self._composite_fields[field_name]
+
+      child_cdescriptor = self.__descriptors[field_name]
+      # TODO(anuraag): Support clearing repeated message fields as well.
+      if (child_cdescriptor.label != _LABEL_REPEATED and
+          child_cdescriptor.cpp_type == _CPPTYPE_MESSAGE):
+        child_field._owner = None
+        child_cmessage = child_field._cmsg
+
+    if child_cmessage is not None:
+      self._cmsg.ClearField(field_name, child_cmessage)
+    else:
+      self._cmsg.ClearField(field_name)
+
+  def Clear(self):
+    cmessages_to_release = []
+    for field_name, child_field in self._composite_fields.iteritems():
+      child_cdescriptor = self.__descriptors[field_name]
+      # TODO(anuraag): Support clearing repeated message fields as well.
+      if (child_cdescriptor.label != _LABEL_REPEATED and
+          child_cdescriptor.cpp_type == _CPPTYPE_MESSAGE):
+        child_field._owner = None
+        cmessages_to_release.append((child_cdescriptor, child_field._cmsg))
+    self._composite_fields.clear()
+    self._cmsg.Clear(cmessages_to_release)
+
+  def IsInitialized(self, errors=None):
+    if self._cmsg.IsInitialized():
+      return True
+    if errors is not None:
+      errors.extend(self.FindInitializationErrors());
+    return False
+
+  def SerializeToString(self):
+    if not self.IsInitialized():
+      raise message.EncodeError(
+          'Message %s is missing required fields: %s' % (
+          self._cmsg.full_name, ','.join(self.FindInitializationErrors())))
+    return self._cmsg.SerializeToString()
+
+  def SerializePartialToString(self):
+    return self._cmsg.SerializePartialToString()
+
+  def ParseFromString(self, serialized):
+    self.Clear()
+    self.MergeFromString(serialized)
+
+  def MergeFromString(self, serialized):
+    byte_size = self._cmsg.MergeFromString(serialized)
+    if byte_size < 0:
+      raise message.DecodeError('Unable to merge from string.')
+    return byte_size
+
+  def MergeFrom(self, msg):
+    if not isinstance(msg, cls):
+      raise TypeError(
+          "Parameter to MergeFrom() must be instance of same class: "
+          "expected %s got %s." % (cls.__name__, type(msg).__name__))
+    self._cmsg.MergeFrom(msg._cmsg)
+
+  def CopyFrom(self, msg):
+    self._cmsg.CopyFrom(msg._cmsg)
+
+  def ByteSize(self):
+    return self._cmsg.ByteSize()
+
+  def SetInParent(self):
+    return self._cmsg.SetInParent()
+
+  def ListFields(self):
+    all_fields = []
+    field_list = self._cmsg.ListFields()
+    fields_by_name = cls.DESCRIPTOR.fields_by_name
+    for is_extension, field_name in field_list:
+      if is_extension:
+        extension = cls._extensions_by_name[field_name]
+        all_fields.append((extension, self.Extensions[extension]))
+      else:
+        field_descriptor = fields_by_name[field_name]
+        all_fields.append(
+            (field_descriptor, getattr(self, field_name)))
+    all_fields.sort(key=lambda item: item[0].number)
+    return all_fields
+
+  def FindInitializationErrors(self):
+    return self._cmsg.FindInitializationErrors()
+
+  def __str__(self):
+    return self._cmsg.DebugString()
+
+  def __eq__(self, other):
+    if self is other:
+      return True
+    if not isinstance(other, self.__class__):
+      return False
+    return self.ListFields() == other.ListFields()
+
+  def __ne__(self, other):
+    return not self == other
+
+  def __hash__(self):
+    raise TypeError('unhashable object')
+
+  def __unicode__(self):
+    # Lazy import to prevent circular import when text_format imports this file.
+    from google.protobuf import text_format
+    return text_format.MessageToString(self, as_utf8=True).decode('utf-8')
+
+  # Attach the local methods to the message class.
+  for key, value in locals().copy().iteritems():
+    if key not in ('key', 'value', '__builtins__', '__name__', '__doc__'):
+      setattr(cls, key, value)
+
+  # Static methods:
+
+  def RegisterExtension(extension_handle):
+    extension_handle.containing_type = cls.DESCRIPTOR
+    cls._extensions_by_name[extension_handle.full_name] = extension_handle
+
+    if _IsMessageSetExtension(extension_handle):
+      # MessageSet extension.  Also register under type name.
+      cls._extensions_by_name[
+          extension_handle.message_type.full_name] = extension_handle
+  cls.RegisterExtension = staticmethod(RegisterExtension)
+
+  def FromString(string):
+    msg = cls()
+    msg.MergeFromString(string)
+    return msg
+  cls.FromString = staticmethod(FromString)
+
+
+
+def _AddPropertiesForExtensions(message_descriptor, cls):
+  """Adds properties for all fields in this protocol message type."""
+  extension_dict = message_descriptor.extensions_by_name
+  for extension_name, extension_field in extension_dict.iteritems():
+    constant_name = extension_name.upper() + '_FIELD_NUMBER'
+    setattr(cls, constant_name, extension_field.number)

python/google/protobuf/internal/decoder.py

@@ -0,0 +1,720 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Code for decoding protocol buffer primitives.
+
+This code is very similar to encoder.py -- read the docs for that module first.
+
+A "decoder" is a function with the signature:
+  Decode(buffer, pos, end, message, field_dict)
+The arguments are:
+  buffer:     The string containing the encoded message.
+  pos:        The current position in the string.
+  end:        The position in the string where the current message ends.  May be
+              less than len(buffer) if we're reading a sub-message.
+  message:    The message object into which we're parsing.
+  field_dict: message._fields (avoids a hashtable lookup).
+The decoder reads the field and stores it into field_dict, returning the new
+buffer position.  A decoder for a repeated field may proactively decode all of
+the elements of that field, if they appear consecutively.
+
+Note that decoders may throw any of the following:
+  IndexError:  Indicates a truncated message.
+  struct.error:  Unpacking of a fixed-width field failed.
+  message.DecodeError:  Other errors.
+
+Decoders are expected to raise an exception if they are called with pos > end.
+This allows callers to be lax about bounds checking:  it's fineto read past
+"end" as long as you are sure that someone else will notice and throw an
+exception later on.
+
+Something up the call stack is expected to catch IndexError and struct.error
+and convert them to message.DecodeError.
+
+Decoders are constructed using decoder constructors with the signature:
+  MakeDecoder(field_number, is_repeated, is_packed, key, new_default)
+The arguments are:
+  field_number:  The field number of the field we want to decode.
+  is_repeated:   Is the field a repeated field? (bool)
+  is_packed:     Is the field a packed field? (bool)
+  key:           The key to use when looking up the field within field_dict.
+                 (This is actually the FieldDescriptor but nothing in this
+                 file should depend on that.)
+  new_default:   A function which takes a message object as a parameter and
+                 returns a new instance of the default value for this field.
+                 (This is called for repeated fields and sub-messages, when an
+                 instance does not already exist.)
+
+As with encoders, we define a decoder constructor for every type of field.
+Then, for every field of every message class we construct an actual decoder.
+That decoder goes into a dict indexed by tag, so when we decode a message
+we repeatedly read a tag, look up the corresponding decoder, and invoke it.
+"""
+
+__author__ = 'kenton@google.com (Kenton Varda)'
+
+import struct
+from google.protobuf.internal import encoder
+from google.protobuf.internal import wire_format
+from google.protobuf import message
+
+
+# This will overflow and thus become IEEE-754 "infinity".  We would use
+# "float('inf')" but it doesn't work on Windows pre-Python-2.6.
+_POS_INF = 1e10000
+_NEG_INF = -_POS_INF
+_NAN = _POS_INF * 0
+
+
+# This is not for optimization, but rather to avoid conflicts with local
+# variables named "message".
+_DecodeError = message.DecodeError
+
+
+def _VarintDecoder(mask):
+  """Return an encoder for a basic varint value (does not include tag).
+
+  Decoded values will be bitwise-anded with the given mask before being
+  returned, e.g. to limit them to 32 bits.  The returned decoder does not
+  take the usual "end" parameter -- the caller is expected to do bounds checking
+  after the fact (often the caller can defer such checking until later).  The
+  decoder returns a (value, new_pos) pair.
+  """
+
+  local_ord = ord
+  def DecodeVarint(buffer, pos):
+    result = 0
+    shift = 0
+    while 1:
+      b = local_ord(buffer[pos])
+      result |= ((b & 0x7f) << shift)
+      pos += 1
+      if not (b & 0x80):
+        result &= mask
+        return (result, pos)
+      shift += 7
+      if shift >= 64:
+        raise _DecodeError('Too many bytes when decoding varint.')
+  return DecodeVarint
+
+
+def _SignedVarintDecoder(mask):
+  """Like _VarintDecoder() but decodes signed values."""
+
+  local_ord = ord
+  def DecodeVarint(buffer, pos):
+    result = 0
+    shift = 0
+    while 1:
+      b = local_ord(buffer[pos])
+      result |= ((b & 0x7f) << shift)
+      pos += 1
+      if not (b & 0x80):
+        if result > 0x7fffffffffffffff:
+          result -= (1 << 64)
+          result |= ~mask
+        else:
+          result &= mask
+        return (result, pos)
+      shift += 7
+      if shift >= 64:
+        raise _DecodeError('Too many bytes when decoding varint.')
+  return DecodeVarint
+
+
+_DecodeVarint = _VarintDecoder((1 << 64) - 1)
+_DecodeSignedVarint = _SignedVarintDecoder((1 << 64) - 1)
+
+# Use these versions for values which must be limited to 32 bits.
+_DecodeVarint32 = _VarintDecoder((1 << 32) - 1)
+_DecodeSignedVarint32 = _SignedVarintDecoder((1 << 32) - 1)
+
+
+def ReadTag(buffer, pos):
+  """Read a tag from the buffer, and return a (tag_bytes, new_pos) tuple.
+
+  We return the raw bytes of the tag rather than decoding them.  The raw
+  bytes can then be used to look up the proper decoder.  This effectively allows
+  us to trade some work that would be done in pure-python (decoding a varint)
+  for work that is done in C (searching for a byte string in a hash table).
+  In a low-level language it would be much cheaper to decode the varint and
+  use that, but not in Python.
+  """
+
+  start = pos
+  while ord(buffer[pos]) & 0x80:
+    pos += 1
+  pos += 1
+  return (buffer[start:pos], pos)
+
+
+# --------------------------------------------------------------------
+
+
+def _SimpleDecoder(wire_type, decode_value):
+  """Return a constructor for a decoder for fields of a particular type.
+
+  Args:
+      wire_type:  The field's wire type.
+      decode_value:  A function which decodes an individual value, e.g.
+        _DecodeVarint()
+  """
+
+  def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default):
+    if is_packed:
+      local_DecodeVarint = _DecodeVarint
+      def DecodePackedField(buffer, pos, end, message, field_dict):
+        value = field_dict.get(key)
+        if value is None:
+          value = field_dict.setdefault(key, new_default(message))
+        (endpoint, pos) = local_DecodeVarint(buffer, pos)
+        endpoint += pos
+        if endpoint > end:
+          raise _DecodeError('Truncated message.')
+        while pos < endpoint:
+          (element, pos) = decode_value(buffer, pos)
+          value.append(element)
+        if pos > endpoint:
+          del value[-1]   # Discard corrupt value.
+          raise _DecodeError('Packed element was truncated.')
+        return pos
+      return DecodePackedField
+    elif is_repeated:
+      tag_bytes = encoder.TagBytes(field_number, wire_type)
+      tag_len = len(tag_bytes)
+      def DecodeRepeatedField(buffer, pos, end, message, field_dict):
+        value = field_dict.get(key)
+        if value is None:
+          value = field_dict.setdefault(key, new_default(message))
+        while 1:
+          (element, new_pos) = decode_value(buffer, pos)
+          value.append(element)
+          # Predict that the next tag is another copy of the same repeated
+          # field.
+          pos = new_pos + tag_len
+          if buffer[new_pos:pos] != tag_bytes or new_pos >= end:
+            # Prediction failed.  Return.
+            if new_pos > end:
+              raise _DecodeError('Truncated message.')
+            return new_pos
+      return DecodeRepeatedField
+    else:
+      def DecodeField(buffer, pos, end, message, field_dict):
+        (field_dict[key], pos) = decode_value(buffer, pos)
+        if pos > end:
+          del field_dict[key]  # Discard corrupt value.
+          raise _DecodeError('Truncated message.')
+        return pos
+      return DecodeField
+
+  return SpecificDecoder
+
+
+def _ModifiedDecoder(wire_type, decode_value, modify_value):
+  """Like SimpleDecoder but additionally invokes modify_value on every value
+  before storing it.  Usually modify_value is ZigZagDecode.
+  """
+
+  # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but
+  # not enough to make a significant difference.
+
+  def InnerDecode(buffer, pos):
+    (result, new_pos) = decode_value(buffer, pos)
+    return (modify_value(result), new_pos)
+  return _SimpleDecoder(wire_type, InnerDecode)
+
+
+def _StructPackDecoder(wire_type, format):
+  """Return a constructor for a decoder for a fixed-width field.
+
+  Args:
+      wire_type:  The field's wire type.
+      format:  The format string to pass to struct.unpack().
+  """
+
+  value_size = struct.calcsize(format)
+  local_unpack = struct.unpack
+
+  # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but
+  # not enough to make a significant difference.
+
+  # Note that we expect someone up-stack to catch struct.error and convert
+  # it to _DecodeError -- this way we don't have to set up exception-
+  # handling blocks every time we parse one value.
+
+  def InnerDecode(buffer, pos):
+    new_pos = pos + value_size
+    result = local_unpack(format, buffer[pos:new_pos])[0]
+    return (result, new_pos)
+  return _SimpleDecoder(wire_type, InnerDecode)
+
+
+def _FloatDecoder():
+  """Returns a decoder for a float field.
+
+  This code works around a bug in struct.unpack for non-finite 32-bit
+  floating-point values.
+  """
+
+  local_unpack = struct.unpack
+
+  def InnerDecode(buffer, pos):
+    # We expect a 32-bit value in little-endian byte order.  Bit 1 is the sign
+    # bit, bits 2-9 represent the exponent, and bits 10-32 are the significand.
+    new_pos = pos + 4
+    float_bytes = buffer[pos:new_pos]
+
+    # If this value has all its exponent bits set, then it's non-finite.
+    # In Python 2.4, struct.unpack will convert it to a finite 64-bit value.
+    # To avoid that, we parse it specially.
+    if ((float_bytes[3] in '\x7F\xFF')
+        and (float_bytes[2] >= '\x80')):
+      # If at least one significand bit is set...
+      if float_bytes[0:3] != '\x00\x00\x80':
+        return (_NAN, new_pos)
+      # If sign bit is set...
+      if float_bytes[3] == '\xFF':
+        return (_NEG_INF, new_pos)
+      return (_POS_INF, new_pos)
+
+    # Note that we expect someone up-stack to catch struct.error and convert
+    # it to _DecodeError -- this way we don't have to set up exception-
+    # handling blocks every time we parse one value.
+    result = local_unpack('<f', float_bytes)[0]
+    return (result, new_pos)
+  return _SimpleDecoder(wire_format.WIRETYPE_FIXED32, InnerDecode)
+
+
+def _DoubleDecoder():
+  """Returns a decoder for a double field.
+
+  This code works around a bug in struct.unpack for not-a-number.
+  """
+
+  local_unpack = struct.unpack
+
+  def InnerDecode(buffer, pos):
+    # We expect a 64-bit value in little-endian byte order.  Bit 1 is the sign
+    # bit, bits 2-12 represent the exponent, and bits 13-64 are the significand.
+    new_pos = pos + 8
+    double_bytes = buffer[pos:new_pos]
+
+    # If this value has all its exponent bits set and at least one significand
+    # bit set, it's not a number.  In Python 2.4, struct.unpack will treat it
+    # as inf or -inf.  To avoid that, we treat it specially.
+    if ((double_bytes[7] in '\x7F\xFF')
+        and (double_bytes[6] >= '\xF0')
+        and (double_bytes[0:7] != '\x00\x00\x00\x00\x00\x00\xF0')):
+      return (_NAN, new_pos)
+
+    # Note that we expect someone up-stack to catch struct.error and convert
+    # it to _DecodeError -- this way we don't have to set up exception-
+    # handling blocks every time we parse one value.
+    result = local_unpack('<d', double_bytes)[0]
+    return (result, new_pos)
+  return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode)
+
+
+# --------------------------------------------------------------------
+
+
+Int32Decoder = EnumDecoder = _SimpleDecoder(
+    wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32)
+
+Int64Decoder = _SimpleDecoder(
+    wire_format.WIRETYPE_VARINT, _DecodeSignedVarint)
+
+UInt32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32)
+UInt64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint)
+
+SInt32Decoder = _ModifiedDecoder(
+    wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode)
+SInt64Decoder = _ModifiedDecoder(
+    wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode)
+
+# Note that Python conveniently guarantees that when using the '<' prefix on
+# formats, they will also have the same size across all platforms (as opposed
+# to without the prefix, where their sizes depend on the C compiler's basic
+# type sizes).
+Fixed32Decoder  = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<I')
+Fixed64Decoder  = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<Q')
+SFixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<i')
+SFixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<q')
+FloatDecoder = _FloatDecoder()
+DoubleDecoder = _DoubleDecoder()
+
+BoolDecoder = _ModifiedDecoder(
+    wire_format.WIRETYPE_VARINT, _DecodeVarint, bool)
+
+
+def StringDecoder(field_number, is_repeated, is_packed, key, new_default):
+  """Returns a decoder for a string field."""
+
+  local_DecodeVarint = _DecodeVarint
+  local_unicode = unicode
+
+  assert not is_packed
+  if is_repeated:
+    tag_bytes = encoder.TagBytes(field_number,
+                                 wire_format.WIRETYPE_LENGTH_DELIMITED)
+    tag_len = len(tag_bytes)
+    def DecodeRepeatedField(buffer, pos, end, message, field_dict):
+      value = field_dict.get(key)
+      if value is None:
+        value = field_dict.setdefault(key, new_default(message))
+      while 1:
+        (size, pos) = local_DecodeVarint(buffer, pos)
+        new_pos = pos + size
+        if new_pos > end:
+          raise _DecodeError('Truncated string.')
+        value.append(local_unicode(buffer[pos:new_pos], 'utf-8'))
+        # Predict that the next tag is another copy of the same repeated field.
+        pos = new_pos + tag_len
+        if buffer[new_pos:pos] != tag_bytes or new_pos == end:
+          # Prediction failed.  Return.
+          return new_pos
+    return DecodeRepeatedField
+  else:
+    def DecodeField(buffer, pos, end, message, field_dict):
+      (size, pos) = local_DecodeVarint(buffer, pos)
+      new_pos = pos + size
+      if new_pos > end:
+        raise _DecodeError('Truncated string.')
+      field_dict[key] = local_unicode(buffer[pos:new_pos], 'utf-8')
+      return new_pos
+    return DecodeField
+
+
+def BytesDecoder(field_number, is_repeated, is_packed, key, new_default):
+  """Returns a decoder for a bytes field."""
+
+  local_DecodeVarint = _DecodeVarint
+
+  assert not is_packed
+  if is_repeated:
+    tag_bytes = encoder.TagBytes(field_number,
+                                 wire_format.WIRETYPE_LENGTH_DELIMITED)
+    tag_len = len(tag_bytes)
+    def DecodeRepeatedField(buffer, pos, end, message, field_dict):
+      value = field_dict.get(key)
+      if value is None:
+        value = field_dict.setdefault(key, new_default(message))
+      while 1:
+        (size, pos) = local_DecodeVarint(buffer, pos)
+        new_pos = pos + size
+        if new_pos > end:
+          raise _DecodeError('Truncated string.')
+        value.append(buffer[pos:new_pos])
+        # Predict that the next tag is another copy of the same repeated field.
+        pos = new_pos + tag_len
+        if buffer[new_pos:pos] != tag_bytes or new_pos == end:
+          # Prediction failed.  Return.
+          return new_pos
+    return DecodeRepeatedField
+  else:
+    def DecodeField(buffer, pos, end, message, field_dict):
+      (size, pos) = local_DecodeVarint(buffer, pos)
+      new_pos = pos + size
+      if new_pos > end:
+        raise _DecodeError('Truncated string.')
+      field_dict[key] = buffer[pos:new_pos]
+      return new_pos
+    return DecodeField
+
+
+def GroupDecoder(field_number, is_repeated, is_packed, key, new_default):
+  """Returns a decoder for a group field."""
+
+  end_tag_bytes = encoder.TagBytes(field_number,
+                                   wire_format.WIRETYPE_END_GROUP)
+  end_tag_len = len(end_tag_bytes)
+
+  assert not is_packed
+  if is_repeated:
+    tag_bytes = encoder.TagBytes(field_number,
+                                 wire_format.WIRETYPE_START_GROUP)
+    tag_len = len(tag_bytes)
+    def DecodeRepeatedField(buffer, pos, end, message, field_dict):
+      value = field_dict.get(key)
+      if value is None:
+        value = field_dict.setdefault(key, new_default(message))
+      while 1:
+        value = field_dict.get(key)
+        if value is None:
+          value = field_dict.setdefault(key, new_default(message))
+        # Read sub-message.
+        pos = value.add()._InternalParse(buffer, pos, end)
+        # Read end tag.
+        new_pos = pos+end_tag_len
+        if buffer[pos:new_pos] != end_tag_bytes or new_pos > end:
+          raise _DecodeError('Missing group end tag.')
+        # Predict that the next tag is another copy of the same repeated field.
+        pos = new_pos + tag_len
+        if buffer[new_pos:pos] != tag_bytes or new_pos == end:
+          # Prediction failed.  Return.
+          return new_pos
+    return DecodeRepeatedField
+  else:
+    def DecodeField(buffer, pos, end, message, field_dict):
+      value = field_dict.get(key)
+      if value is None:
+        value = field_dict.setdefault(key, new_default(message))
+      # Read sub-message.
+      pos = value._InternalParse(buffer, pos, end)
+      # Read end tag.
+      new_pos = pos+end_tag_len
+      if buffer[pos:new_pos] != end_tag_bytes or new_pos > end:
+        raise _DecodeError('Missing group end tag.')
+      return new_pos
+    return DecodeField
+
+
+def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
+  """Returns a decoder for a message field."""
+
+  local_DecodeVarint = _DecodeVarint
+
+  assert not is_packed
+  if is_repeated:
+    tag_bytes = encoder.TagBytes(field_number,
+                                 wire_format.WIRETYPE_LENGTH_DELIMITED)
+    tag_len = len(tag_bytes)
+    def DecodeRepeatedField(buffer, pos, end, message, field_dict):
+      value = field_dict.get(key)
+      if value is None:
+        value = field_dict.setdefault(key, new_default(message))
+      while 1:
+        value = field_dict.get(key)
+        if value is None:
+          value = field_dict.setdefault(key, new_default(message))
+        # Read length.
+        (size, pos) = local_DecodeVarint(buffer, pos)
+        new_pos = pos + size
+        if new_pos > end:
+          raise _DecodeError('Truncated message.')
+        # Read sub-message.
+        if value.add()._InternalParse(buffer, pos, new_pos) != new_pos:
+          # The only reason _InternalParse would return early is if it
+          # encountered an end-group tag.
+          raise _DecodeError('Unexpected end-group tag.')
+        # Predict that the next tag is another copy of the same repeated field.
+        pos = new_pos + tag_len
+        if buffer[new_pos:pos] != tag_bytes or new_pos == end:
+          # Prediction failed.  Return.
+          return new_pos
+    return DecodeRepeatedField
+  else:
+    def DecodeField(buffer, pos, end, message, field_dict):
+      value = field_dict.get(key)
+      if value is None:
+        value = field_dict.setdefault(key, new_default(message))
+      # Read length.
+      (size, pos) = local_DecodeVarint(buffer, pos)
+      new_pos = pos + size
+      if new_pos > end:
+        raise _DecodeError('Truncated message.')
+      # Read sub-message.
+      if value._InternalParse(buffer, pos, new_pos) != new_pos:
+        # The only reason _InternalParse would return early is if it encountered
+        # an end-group tag.
+        raise _DecodeError('Unexpected end-group tag.')
+      return new_pos
+    return DecodeField
+
+
+# --------------------------------------------------------------------
+
+MESSAGE_SET_ITEM_TAG = encoder.TagBytes(1, wire_format.WIRETYPE_START_GROUP)
+
+def MessageSetItemDecoder(extensions_by_number):
+  """Returns a decoder for a MessageSet item.
+
+  The parameter is the _extensions_by_number map for the message class.
+
+  The message set message looks like this:
+    message MessageSet {
+      repeated group Item = 1 {
+        required int32 type_id = 2;
+        required string message = 3;
+      }
+    }
+  """
+
+  type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT)
+  message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)
+  item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP)
+
+  local_ReadTag = ReadTag
+  local_DecodeVarint = _DecodeVarint
+  local_SkipField = SkipField
+
+  def DecodeItem(buffer, pos, end, message, field_dict):
+    message_set_item_start = pos
+    type_id = -1
+    message_start = -1
+    message_end = -1
+
+    # Technically, type_id and message can appear in any order, so we need
+    # a little loop here.
+    while 1:
+      (tag_bytes, pos) = local_ReadTag(buffer, pos)
+      if tag_bytes == type_id_tag_bytes:
+        (type_id, pos) = local_DecodeVarint(buffer, pos)
+      elif tag_bytes == message_tag_bytes:
+        (size, message_start) = local_DecodeVarint(buffer, pos)
+        pos = message_end = message_start + size
+      elif tag_bytes == item_end_tag_bytes:
+        break
+      else:
+        pos = SkipField(buffer, pos, end, tag_bytes)
+        if pos == -1:
+          raise _DecodeError('Missing group end tag.')
+
+    if pos > end:
+      raise _DecodeError('Truncated message.')
+
+    if type_id == -1:
+      raise _DecodeError('MessageSet item missing type_id.')
+    if message_start == -1:
+      raise _DecodeError('MessageSet item missing message.')
+
+    extension = extensions_by_number.get(type_id)
+    if extension is not None:
+      value = field_dict.get(extension)
+      if value is None:
+        value = field_dict.setdefault(
+            extension, extension.message_type._concrete_class())
+      if value._InternalParse(buffer, message_start,message_end) != message_end:
+        # The only reason _InternalParse would return early is if it encountered
+        # an end-group tag.
+        raise _DecodeError('Unexpected end-group tag.')
+    else:
+      if not message._unknown_fields:
+        message._unknown_fields = []
+      message._unknown_fields.append((MESSAGE_SET_ITEM_TAG,
+                                      buffer[message_set_item_start:pos]))
+
+    return pos
+
+  return DecodeItem
+
+# --------------------------------------------------------------------
+# Optimization is not as heavy here because calls to SkipField() are rare,
+# except for handling end-group tags.
+
+def _SkipVarint(buffer, pos, end):
+  """Skip a varint value.  Returns the new position."""
+
+  while ord(buffer[pos]) & 0x80:
+    pos += 1
+  pos += 1
+  if pos > end:
+    raise _DecodeError('Truncated message.')
+  return pos
+
+def _SkipFixed64(buffer, pos, end):
+  """Skip a fixed64 value.  Returns the new position."""
+
+  pos += 8
+  if pos > end:
+    raise _DecodeError('Truncated message.')
+  return pos
+
+def _SkipLengthDelimited(buffer, pos, end):
+  """Skip a length-delimited value.  Returns the new position."""
+
+  (size, pos) = _DecodeVarint(buffer, pos)
+  pos += size
+  if pos > end:
+    raise _DecodeError('Truncated message.')
+  return pos
+
+def _SkipGroup(buffer, pos, end):
+  """Skip sub-group.  Returns the new position."""
+
+  while 1:
+    (tag_bytes, pos) = ReadTag(buffer, pos)
+    new_pos = SkipField(buffer, pos, end, tag_bytes)
+    if new_pos == -1:
+      return pos
+    pos = new_pos
+
+def _EndGroup(buffer, pos, end):
+  """Skipping an END_GROUP tag returns -1 to tell the parent loop to break."""
+
+  return -1
+
+def _SkipFixed32(buffer, pos, end):
+  """Skip a fixed32 value.  Returns the new position."""
+
+  pos += 4
+  if pos > end:
+    raise _DecodeError('Truncated message.')
+  return pos
+
+def _RaiseInvalidWireType(buffer, pos, end):
+  """Skip function for unknown wire types.  Raises an exception."""
+
+  raise _DecodeError('Tag had invalid wire type.')
+
+def _FieldSkipper():
+  """Constructs the SkipField function."""
+
+  WIRETYPE_TO_SKIPPER = [
+      _SkipVarint,
+      _SkipFixed64,
+      _SkipLengthDelimited,
+      _SkipGroup,
+      _EndGroup,
+      _SkipFixed32,
+      _RaiseInvalidWireType,
+      _RaiseInvalidWireType,
+      ]
+
+  wiretype_mask = wire_format.TAG_TYPE_MASK
+  local_ord = ord
+
+  def SkipField(buffer, pos, end, tag_bytes):
+    """Skips a field with the specified tag.
+
+    |pos| should point to the byte immediately after the tag.
+
+    Returns:
+        The new position (after the tag value), or -1 if the tag is an end-group
+        tag (in which case the calling loop should break).
+    """
+
+    # The wire type is always in the first byte since varints are little-endian.
+    wire_type = local_ord(tag_bytes[0]) & wiretype_mask
+    return WIRETYPE_TO_SKIPPER[wire_type](buffer, pos, end)
+
+  return SkipField
+
+SkipField = _FieldSkipper()

python/google/protobuf/internal/descriptor_database_test.py

@@ -0,0 +1,63 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests for google.protobuf.descriptor_database."""
+
+__author__ = 'matthewtoia@google.com (Matt Toia)'
+
+import unittest
+from google.protobuf import descriptor_pb2
+from google.protobuf.internal import factory_test2_pb2
+from google.protobuf import descriptor_database
+
+
+class DescriptorDatabaseTest(unittest.TestCase):
+
+  def testAdd(self):
+    db = descriptor_database.DescriptorDatabase()
+    file_desc_proto = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test2_pb2.DESCRIPTOR.serialized_pb)
+    db.Add(file_desc_proto)
+
+    self.assertEquals(file_desc_proto, db.FindFileByName(
+        'net/proto2/python/internal/factory_test2.proto'))
+    self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
+        'net.proto2.python.internal.Factory2Message'))
+    self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
+        'net.proto2.python.internal.Factory2Message.NestedFactory2Message'))
+    self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
+        'net.proto2.python.internal.Factory2Enum'))
+    self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
+        'net.proto2.python.internal.Factory2Message.NestedFactory2Enum'))
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/descriptor_pool_test.py

@@ -0,0 +1,220 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests for google.protobuf.descriptor_pool."""
+
+__author__ = 'matthewtoia@google.com (Matt Toia)'
+
+import unittest
+from google.protobuf import descriptor_pb2
+from google.protobuf.internal import factory_test1_pb2
+from google.protobuf.internal import factory_test2_pb2
+from google.protobuf import descriptor
+from google.protobuf import descriptor_database
+from google.protobuf import descriptor_pool
+
+
+class DescriptorPoolTest(unittest.TestCase):
+
+  def setUp(self):
+    self.pool = descriptor_pool.DescriptorPool()
+    self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test1_pb2.DESCRIPTOR.serialized_pb)
+    self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test2_pb2.DESCRIPTOR.serialized_pb)
+    self.pool.Add(self.factory_test1_fd)
+    self.pool.Add(self.factory_test2_fd)
+
+  def testFindFileByName(self):
+    name1 = 'net/proto2/python/internal/factory_test1.proto'
+    file_desc1 = self.pool.FindFileByName(name1)
+    self.assertIsInstance(file_desc1, descriptor.FileDescriptor)
+    self.assertEquals(name1, file_desc1.name)
+    self.assertEquals('net.proto2.python.internal', file_desc1.package)
+    self.assertIn('Factory1Message', file_desc1.message_types_by_name)
+
+    name2 = 'net/proto2/python/internal/factory_test2.proto'
+    file_desc2 = self.pool.FindFileByName(name2)
+    self.assertIsInstance(file_desc2, descriptor.FileDescriptor)
+    self.assertEquals(name2, file_desc2.name)
+    self.assertEquals('net.proto2.python.internal', file_desc2.package)
+    self.assertIn('Factory2Message', file_desc2.message_types_by_name)
+
+  def testFindFileByNameFailure(self):
+    try:
+      self.pool.FindFileByName('Does not exist')
+      self.fail('Expected KeyError')
+    except KeyError:
+      pass
+
+  def testFindFileContainingSymbol(self):
+    file_desc1 = self.pool.FindFileContainingSymbol(
+        'net.proto2.python.internal.Factory1Message')
+    self.assertIsInstance(file_desc1, descriptor.FileDescriptor)
+    self.assertEquals('net/proto2/python/internal/factory_test1.proto',
+                      file_desc1.name)
+    self.assertEquals('net.proto2.python.internal', file_desc1.package)
+    self.assertIn('Factory1Message', file_desc1.message_types_by_name)
+
+    file_desc2 = self.pool.FindFileContainingSymbol(
+        'net.proto2.python.internal.Factory2Message')
+    self.assertIsInstance(file_desc2, descriptor.FileDescriptor)
+    self.assertEquals('net/proto2/python/internal/factory_test2.proto',
+                      file_desc2.name)
+    self.assertEquals('net.proto2.python.internal', file_desc2.package)
+    self.assertIn('Factory2Message', file_desc2.message_types_by_name)
+
+  def testFindFileContainingSymbolFailure(self):
+    try:
+      self.pool.FindFileContainingSymbol('Does not exist')
+      self.fail('Expected KeyError')
+    except KeyError:
+      pass
+
+  def testFindMessageTypeByName(self):
+    msg1 = self.pool.FindMessageTypeByName(
+        'net.proto2.python.internal.Factory1Message')
+    self.assertIsInstance(msg1, descriptor.Descriptor)
+    self.assertEquals('Factory1Message', msg1.name)
+    self.assertEquals('net.proto2.python.internal.Factory1Message',
+                      msg1.full_name)
+    self.assertEquals(None, msg1.containing_type)
+
+    nested_msg1 = msg1.nested_types[0]
+    self.assertEquals('NestedFactory1Message', nested_msg1.name)
+    self.assertEquals(msg1, nested_msg1.containing_type)
+
+    nested_enum1 = msg1.enum_types[0]
+    self.assertEquals('NestedFactory1Enum', nested_enum1.name)
+    self.assertEquals(msg1, nested_enum1.containing_type)
+
+    self.assertEquals(nested_msg1, msg1.fields_by_name[
+        'nested_factory_1_message'].message_type)
+    self.assertEquals(nested_enum1, msg1.fields_by_name[
+        'nested_factory_1_enum'].enum_type)
+
+    msg2 = self.pool.FindMessageTypeByName(
+        'net.proto2.python.internal.Factory2Message')
+    self.assertIsInstance(msg2, descriptor.Descriptor)
+    self.assertEquals('Factory2Message', msg2.name)
+    self.assertEquals('net.proto2.python.internal.Factory2Message',
+                      msg2.full_name)
+    self.assertIsNone(msg2.containing_type)
+
+    nested_msg2 = msg2.nested_types[0]
+    self.assertEquals('NestedFactory2Message', nested_msg2.name)
+    self.assertEquals(msg2, nested_msg2.containing_type)
+
+    nested_enum2 = msg2.enum_types[0]
+    self.assertEquals('NestedFactory2Enum', nested_enum2.name)
+    self.assertEquals(msg2, nested_enum2.containing_type)
+
+    self.assertEquals(nested_msg2, msg2.fields_by_name[
+        'nested_factory_2_message'].message_type)
+    self.assertEquals(nested_enum2, msg2.fields_by_name[
+        'nested_factory_2_enum'].enum_type)
+
+    self.assertTrue(msg2.fields_by_name['int_with_default'].has_default)
+    self.assertEquals(
+        1776, msg2.fields_by_name['int_with_default'].default_value)
+
+    self.assertTrue(msg2.fields_by_name['double_with_default'].has_default)
+    self.assertEquals(
+        9.99, msg2.fields_by_name['double_with_default'].default_value)
+
+    self.assertTrue(msg2.fields_by_name['string_with_default'].has_default)
+    self.assertEquals(
+        'hello world', msg2.fields_by_name['string_with_default'].default_value)
+
+    self.assertTrue(msg2.fields_by_name['bool_with_default'].has_default)
+    self.assertFalse(msg2.fields_by_name['bool_with_default'].default_value)
+
+    self.assertTrue(msg2.fields_by_name['enum_with_default'].has_default)
+    self.assertEquals(
+        1, msg2.fields_by_name['enum_with_default'].default_value)
+
+    msg3 = self.pool.FindMessageTypeByName(
+        'net.proto2.python.internal.Factory2Message.NestedFactory2Message')
+    self.assertEquals(nested_msg2, msg3)
+
+  def testFindMessageTypeByNameFailure(self):
+    try:
+      self.pool.FindMessageTypeByName('Does not exist')
+      self.fail('Expected KeyError')
+    except KeyError:
+      pass
+
+  def testFindEnumTypeByName(self):
+    enum1 = self.pool.FindEnumTypeByName(
+        'net.proto2.python.internal.Factory1Enum')
+    self.assertIsInstance(enum1, descriptor.EnumDescriptor)
+    self.assertEquals(0, enum1.values_by_name['FACTORY_1_VALUE_0'].number)
+    self.assertEquals(1, enum1.values_by_name['FACTORY_1_VALUE_1'].number)
+
+    nested_enum1 = self.pool.FindEnumTypeByName(
+        'net.proto2.python.internal.Factory1Message.NestedFactory1Enum')
+    self.assertIsInstance(nested_enum1, descriptor.EnumDescriptor)
+    self.assertEquals(
+        0, nested_enum1.values_by_name['NESTED_FACTORY_1_VALUE_0'].number)
+    self.assertEquals(
+        1, nested_enum1.values_by_name['NESTED_FACTORY_1_VALUE_1'].number)
+
+    enum2 = self.pool.FindEnumTypeByName(
+        'net.proto2.python.internal.Factory2Enum')
+    self.assertIsInstance(enum2, descriptor.EnumDescriptor)
+    self.assertEquals(0, enum2.values_by_name['FACTORY_2_VALUE_0'].number)
+    self.assertEquals(1, enum2.values_by_name['FACTORY_2_VALUE_1'].number)
+
+    nested_enum2 = self.pool.FindEnumTypeByName(
+        'net.proto2.python.internal.Factory2Message.NestedFactory2Enum')
+    self.assertIsInstance(nested_enum2, descriptor.EnumDescriptor)
+    self.assertEquals(
+        0, nested_enum2.values_by_name['NESTED_FACTORY_2_VALUE_0'].number)
+    self.assertEquals(
+        1, nested_enum2.values_by_name['NESTED_FACTORY_2_VALUE_1'].number)
+
+  def testFindEnumTypeByNameFailure(self):
+    try:
+      self.pool.FindEnumTypeByName('Does not exist')
+      self.fail('Expected KeyError')
+    except KeyError:
+      pass
+
+  def testUserDefinedDB(self):
+    db = descriptor_database.DescriptorDatabase()
+    self.pool = descriptor_pool.DescriptorPool(db)
+    db.Add(self.factory_test1_fd)
+    db.Add(self.factory_test2_fd)
+    self.testFindMessageTypeByName()
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/descriptor_test.py

@@ -0,0 +1,613 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unittest for google.protobuf.internal.descriptor."""
+
+__author__ = 'robinson@google.com (Will Robinson)'
+
+import unittest
+from google.protobuf import unittest_custom_options_pb2
+from google.protobuf import unittest_import_pb2
+from google.protobuf import unittest_pb2
+from google.protobuf import descriptor_pb2
+from google.protobuf import descriptor
+from google.protobuf import text_format
+
+
+TEST_EMPTY_MESSAGE_DESCRIPTOR_ASCII = """
+name: 'TestEmptyMessage'
+"""
+
+
+class DescriptorTest(unittest.TestCase):
+
+  def setUp(self):
+    self.my_file = descriptor.FileDescriptor(
+        name='some/filename/some.proto',
+        package='protobuf_unittest'
+        )
+    self.my_enum = descriptor.EnumDescriptor(
+        name='ForeignEnum',
+        full_name='protobuf_unittest.ForeignEnum',
+        filename=None,
+        file=self.my_file,
+        values=[
+          descriptor.EnumValueDescriptor(name='FOREIGN_FOO', index=0, number=4),
+          descriptor.EnumValueDescriptor(name='FOREIGN_BAR', index=1, number=5),
+          descriptor.EnumValueDescriptor(name='FOREIGN_BAZ', index=2, number=6),
+        ])
+    self.my_message = descriptor.Descriptor(
+        name='NestedMessage',
+        full_name='protobuf_unittest.TestAllTypes.NestedMessage',
+        filename=None,
+        file=self.my_file,
+        containing_type=None,
+        fields=[
+          descriptor.FieldDescriptor(
+            name='bb',
+            full_name='protobuf_unittest.TestAllTypes.NestedMessage.bb',
+            index=0, number=1,
+            type=5, cpp_type=1, label=1,
+            has_default_value=False, default_value=0,
+            message_type=None, enum_type=None, containing_type=None,
+            is_extension=False, extension_scope=None),
+        ],
+        nested_types=[],
+        enum_types=[
+          self.my_enum,
+        ],
+        extensions=[])
+    self.my_method = descriptor.MethodDescriptor(
+        name='Bar',
+        full_name='protobuf_unittest.TestService.Bar',
+        index=0,
+        containing_service=None,
+        input_type=None,
+        output_type=None)
+    self.my_service = descriptor.ServiceDescriptor(
+        name='TestServiceWithOptions',
+        full_name='protobuf_unittest.TestServiceWithOptions',
+        file=self.my_file,
+        index=0,
+        methods=[
+            self.my_method
+        ])
+
+  def testEnumValueName(self):
+    self.assertEqual(self.my_message.EnumValueName('ForeignEnum', 4),
+                     'FOREIGN_FOO')
+
+    self.assertEqual(
+        self.my_message.enum_types_by_name[
+            'ForeignEnum'].values_by_number[4].name,
+        self.my_message.EnumValueName('ForeignEnum', 4))
+
+  def testEnumFixups(self):
+    self.assertEqual(self.my_enum, self.my_enum.values[0].type)
+
+  def testContainingTypeFixups(self):
+    self.assertEqual(self.my_message, self.my_message.fields[0].containing_type)
+    self.assertEqual(self.my_message, self.my_enum.containing_type)
+
+  def testContainingServiceFixups(self):
+    self.assertEqual(self.my_service, self.my_method.containing_service)
+
+  def testGetOptions(self):
+    self.assertEqual(self.my_enum.GetOptions(),
+                     descriptor_pb2.EnumOptions())
+    self.assertEqual(self.my_enum.values[0].GetOptions(),
+                     descriptor_pb2.EnumValueOptions())
+    self.assertEqual(self.my_message.GetOptions(),
+                     descriptor_pb2.MessageOptions())
+    self.assertEqual(self.my_message.fields[0].GetOptions(),
+                     descriptor_pb2.FieldOptions())
+    self.assertEqual(self.my_method.GetOptions(),
+                     descriptor_pb2.MethodOptions())
+    self.assertEqual(self.my_service.GetOptions(),
+                     descriptor_pb2.ServiceOptions())
+
+  def testSimpleCustomOptions(self):
+    file_descriptor = unittest_custom_options_pb2.DESCRIPTOR
+    message_descriptor =\
+        unittest_custom_options_pb2.TestMessageWithCustomOptions.DESCRIPTOR
+    field_descriptor = message_descriptor.fields_by_name["field1"]
+    enum_descriptor = message_descriptor.enum_types_by_name["AnEnum"]
+    enum_value_descriptor =\
+        message_descriptor.enum_values_by_name["ANENUM_VAL2"]
+    service_descriptor =\
+        unittest_custom_options_pb2.TestServiceWithCustomOptions.DESCRIPTOR
+    method_descriptor = service_descriptor.FindMethodByName("Foo")
+
+    file_options = file_descriptor.GetOptions()
+    file_opt1 = unittest_custom_options_pb2.file_opt1
+    self.assertEqual(9876543210, file_options.Extensions[file_opt1])
+    message_options = message_descriptor.GetOptions()
+    message_opt1 = unittest_custom_options_pb2.message_opt1
+    self.assertEqual(-56, message_options.Extensions[message_opt1])
+    field_options = field_descriptor.GetOptions()
+    field_opt1 = unittest_custom_options_pb2.field_opt1
+    self.assertEqual(8765432109, field_options.Extensions[field_opt1])
+    field_opt2 = unittest_custom_options_pb2.field_opt2
+    self.assertEqual(42, field_options.Extensions[field_opt2])
+    enum_options = enum_descriptor.GetOptions()
+    enum_opt1 = unittest_custom_options_pb2.enum_opt1
+    self.assertEqual(-789, enum_options.Extensions[enum_opt1])
+    enum_value_options = enum_value_descriptor.GetOptions()
+    enum_value_opt1 = unittest_custom_options_pb2.enum_value_opt1
+    self.assertEqual(123, enum_value_options.Extensions[enum_value_opt1])
+
+    service_options = service_descriptor.GetOptions()
+    service_opt1 = unittest_custom_options_pb2.service_opt1
+    self.assertEqual(-9876543210, service_options.Extensions[service_opt1])
+    method_options = method_descriptor.GetOptions()
+    method_opt1 = unittest_custom_options_pb2.method_opt1
+    self.assertEqual(unittest_custom_options_pb2.METHODOPT1_VAL2,
+                     method_options.Extensions[method_opt1])
+
+  def testDifferentCustomOptionTypes(self):
+    kint32min = -2**31
+    kint64min = -2**63
+    kint32max = 2**31 - 1
+    kint64max = 2**63 - 1
+    kuint32max = 2**32 - 1
+    kuint64max = 2**64 - 1
+
+    message_descriptor =\
+        unittest_custom_options_pb2.CustomOptionMinIntegerValues.DESCRIPTOR
+    message_options = message_descriptor.GetOptions()
+    self.assertEqual(False, message_options.Extensions[
+        unittest_custom_options_pb2.bool_opt])
+    self.assertEqual(kint32min, message_options.Extensions[
+        unittest_custom_options_pb2.int32_opt])
+    self.assertEqual(kint64min, message_options.Extensions[
+        unittest_custom_options_pb2.int64_opt])
+    self.assertEqual(0, message_options.Extensions[
+        unittest_custom_options_pb2.uint32_opt])
+    self.assertEqual(0, message_options.Extensions[
+        unittest_custom_options_pb2.uint64_opt])
+    self.assertEqual(kint32min, message_options.Extensions[
+        unittest_custom_options_pb2.sint32_opt])
+    self.assertEqual(kint64min, message_options.Extensions[
+        unittest_custom_options_pb2.sint64_opt])
+    self.assertEqual(0, message_options.Extensions[
+        unittest_custom_options_pb2.fixed32_opt])
+    self.assertEqual(0, message_options.Extensions[
+        unittest_custom_options_pb2.fixed64_opt])
+    self.assertEqual(kint32min, message_options.Extensions[
+        unittest_custom_options_pb2.sfixed32_opt])
+    self.assertEqual(kint64min, message_options.Extensions[
+        unittest_custom_options_pb2.sfixed64_opt])
+
+    message_descriptor =\
+        unittest_custom_options_pb2.CustomOptionMaxIntegerValues.DESCRIPTOR
+    message_options = message_descriptor.GetOptions()
+    self.assertEqual(True, message_options.Extensions[
+        unittest_custom_options_pb2.bool_opt])
+    self.assertEqual(kint32max, message_options.Extensions[
+        unittest_custom_options_pb2.int32_opt])
+    self.assertEqual(kint64max, message_options.Extensions[
+        unittest_custom_options_pb2.int64_opt])
+    self.assertEqual(kuint32max, message_options.Extensions[
+        unittest_custom_options_pb2.uint32_opt])
+    self.assertEqual(kuint64max, message_options.Extensions[
+        unittest_custom_options_pb2.uint64_opt])
+    self.assertEqual(kint32max, message_options.Extensions[
+        unittest_custom_options_pb2.sint32_opt])
+    self.assertEqual(kint64max, message_options.Extensions[
+        unittest_custom_options_pb2.sint64_opt])
+    self.assertEqual(kuint32max, message_options.Extensions[
+        unittest_custom_options_pb2.fixed32_opt])
+    self.assertEqual(kuint64max, message_options.Extensions[
+        unittest_custom_options_pb2.fixed64_opt])
+    self.assertEqual(kint32max, message_options.Extensions[
+        unittest_custom_options_pb2.sfixed32_opt])
+    self.assertEqual(kint64max, message_options.Extensions[
+        unittest_custom_options_pb2.sfixed64_opt])
+
+    message_descriptor =\
+        unittest_custom_options_pb2.CustomOptionOtherValues.DESCRIPTOR
+    message_options = message_descriptor.GetOptions()
+    self.assertEqual(-100, message_options.Extensions[
+        unittest_custom_options_pb2.int32_opt])
+    self.assertAlmostEqual(12.3456789, message_options.Extensions[
+        unittest_custom_options_pb2.float_opt], 6)
+    self.assertAlmostEqual(1.234567890123456789, message_options.Extensions[
+        unittest_custom_options_pb2.double_opt])
+    self.assertEqual("Hello, \"World\"", message_options.Extensions[
+        unittest_custom_options_pb2.string_opt])
+    self.assertEqual("Hello\0World", message_options.Extensions[
+        unittest_custom_options_pb2.bytes_opt])
+    dummy_enum = unittest_custom_options_pb2.DummyMessageContainingEnum
+    self.assertEqual(
+        dummy_enum.TEST_OPTION_ENUM_TYPE2,
+        message_options.Extensions[unittest_custom_options_pb2.enum_opt])
+
+    message_descriptor =\
+        unittest_custom_options_pb2.SettingRealsFromPositiveInts.DESCRIPTOR
+    message_options = message_descriptor.GetOptions()
+    self.assertAlmostEqual(12, message_options.Extensions[
+        unittest_custom_options_pb2.float_opt], 6)
+    self.assertAlmostEqual(154, message_options.Extensions[
+        unittest_custom_options_pb2.double_opt])
+
+    message_descriptor =\
+        unittest_custom_options_pb2.SettingRealsFromNegativeInts.DESCRIPTOR
+    message_options = message_descriptor.GetOptions()
+    self.assertAlmostEqual(-12, message_options.Extensions[
+        unittest_custom_options_pb2.float_opt], 6)
+    self.assertAlmostEqual(-154, message_options.Extensions[
+        unittest_custom_options_pb2.double_opt])
+
+  def testComplexExtensionOptions(self):
+    descriptor =\
+        unittest_custom_options_pb2.VariousComplexOptions.DESCRIPTOR
+    options = descriptor.GetOptions()
+    self.assertEqual(42, options.Extensions[
+        unittest_custom_options_pb2.complex_opt1].foo)
+    self.assertEqual(324, options.Extensions[
+        unittest_custom_options_pb2.complex_opt1].Extensions[
+            unittest_custom_options_pb2.quux])
+    self.assertEqual(876, options.Extensions[
+        unittest_custom_options_pb2.complex_opt1].Extensions[
+            unittest_custom_options_pb2.corge].qux)
+    self.assertEqual(987, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].baz)
+    self.assertEqual(654, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].Extensions[
+            unittest_custom_options_pb2.grault])
+    self.assertEqual(743, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].bar.foo)
+    self.assertEqual(1999, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].bar.Extensions[
+            unittest_custom_options_pb2.quux])
+    self.assertEqual(2008, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].bar.Extensions[
+            unittest_custom_options_pb2.corge].qux)
+    self.assertEqual(741, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].Extensions[
+            unittest_custom_options_pb2.garply].foo)
+    self.assertEqual(1998, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].Extensions[
+            unittest_custom_options_pb2.garply].Extensions[
+                unittest_custom_options_pb2.quux])
+    self.assertEqual(2121, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].Extensions[
+            unittest_custom_options_pb2.garply].Extensions[
+                unittest_custom_options_pb2.corge].qux)
+    self.assertEqual(1971, options.Extensions[
+        unittest_custom_options_pb2.ComplexOptionType2
+        .ComplexOptionType4.complex_opt4].waldo)
+    self.assertEqual(321, options.Extensions[
+        unittest_custom_options_pb2.complex_opt2].fred.waldo)
+    self.assertEqual(9, options.Extensions[
+        unittest_custom_options_pb2.complex_opt3].qux)
+    self.assertEqual(22, options.Extensions[
+        unittest_custom_options_pb2.complex_opt3].complexoptiontype5.plugh)
+    self.assertEqual(24, options.Extensions[
+        unittest_custom_options_pb2.complexopt6].xyzzy)
+
+  # Check that aggregate options were parsed and saved correctly in
+  # the appropriate descriptors.
+  def testAggregateOptions(self):
+    file_descriptor = unittest_custom_options_pb2.DESCRIPTOR
+    message_descriptor =\
+        unittest_custom_options_pb2.AggregateMessage.DESCRIPTOR
+    field_descriptor = message_descriptor.fields_by_name["fieldname"]
+    enum_descriptor = unittest_custom_options_pb2.AggregateEnum.DESCRIPTOR
+    enum_value_descriptor = enum_descriptor.values_by_name["VALUE"]
+    service_descriptor =\
+        unittest_custom_options_pb2.AggregateService.DESCRIPTOR
+    method_descriptor = service_descriptor.FindMethodByName("Method")
+
+    # Tests for the different types of data embedded in fileopt
+    file_options = file_descriptor.GetOptions().Extensions[
+        unittest_custom_options_pb2.fileopt]
+    self.assertEqual(100, file_options.i)
+    self.assertEqual("FileAnnotation", file_options.s)
+    self.assertEqual("NestedFileAnnotation", file_options.sub.s)
+    self.assertEqual("FileExtensionAnnotation", file_options.file.Extensions[
+        unittest_custom_options_pb2.fileopt].s)
+    self.assertEqual("EmbeddedMessageSetElement", file_options.mset.Extensions[
+        unittest_custom_options_pb2.AggregateMessageSetElement
+        .message_set_extension].s)
+
+    # Simple tests for all the other types of annotations
+    self.assertEqual(
+        "MessageAnnotation",
+        message_descriptor.GetOptions().Extensions[
+            unittest_custom_options_pb2.msgopt].s)
+    self.assertEqual(
+        "FieldAnnotation",
+        field_descriptor.GetOptions().Extensions[
+            unittest_custom_options_pb2.fieldopt].s)
+    self.assertEqual(
+        "EnumAnnotation",
+        enum_descriptor.GetOptions().Extensions[
+            unittest_custom_options_pb2.enumopt].s)
+    self.assertEqual(
+        "EnumValueAnnotation",
+        enum_value_descriptor.GetOptions().Extensions[
+            unittest_custom_options_pb2.enumvalopt].s)
+    self.assertEqual(
+        "ServiceAnnotation",
+        service_descriptor.GetOptions().Extensions[
+            unittest_custom_options_pb2.serviceopt].s)
+    self.assertEqual(
+        "MethodAnnotation",
+        method_descriptor.GetOptions().Extensions[
+            unittest_custom_options_pb2.methodopt].s)
+
+  def testNestedOptions(self):
+    nested_message =\
+        unittest_custom_options_pb2.NestedOptionType.NestedMessage.DESCRIPTOR
+    self.assertEqual(1001, nested_message.GetOptions().Extensions[
+        unittest_custom_options_pb2.message_opt1])
+    nested_field = nested_message.fields_by_name["nested_field"]
+    self.assertEqual(1002, nested_field.GetOptions().Extensions[
+        unittest_custom_options_pb2.field_opt1])
+    outer_message =\
+        unittest_custom_options_pb2.NestedOptionType.DESCRIPTOR
+    nested_enum = outer_message.enum_types_by_name["NestedEnum"]
+    self.assertEqual(1003, nested_enum.GetOptions().Extensions[
+        unittest_custom_options_pb2.enum_opt1])
+    nested_enum_value = outer_message.enum_values_by_name["NESTED_ENUM_VALUE"]
+    self.assertEqual(1004, nested_enum_value.GetOptions().Extensions[
+        unittest_custom_options_pb2.enum_value_opt1])
+    nested_extension = outer_message.extensions_by_name["nested_extension"]
+    self.assertEqual(1005, nested_extension.GetOptions().Extensions[
+        unittest_custom_options_pb2.field_opt2])
+
+  def testFileDescriptorReferences(self):
+    self.assertEqual(self.my_enum.file, self.my_file)
+    self.assertEqual(self.my_message.file, self.my_file)
+
+  def testFileDescriptor(self):
+    self.assertEqual(self.my_file.name, 'some/filename/some.proto')
+    self.assertEqual(self.my_file.package, 'protobuf_unittest')
+
+
+class DescriptorCopyToProtoTest(unittest.TestCase):
+  """Tests for CopyTo functions of Descriptor."""
+
+  def _AssertProtoEqual(self, actual_proto, expected_class, expected_ascii):
+    expected_proto = expected_class()
+    text_format.Merge(expected_ascii, expected_proto)
+
+    self.assertEqual(
+        actual_proto, expected_proto,
+        'Not equal,\nActual:\n%s\nExpected:\n%s\n'
+        % (str(actual_proto), str(expected_proto)))
+
+  def _InternalTestCopyToProto(self, desc, expected_proto_class,
+                               expected_proto_ascii):
+    actual = expected_proto_class()
+    desc.CopyToProto(actual)
+    self._AssertProtoEqual(
+        actual, expected_proto_class, expected_proto_ascii)
+
+  def testCopyToProto_EmptyMessage(self):
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestEmptyMessage.DESCRIPTOR,
+        descriptor_pb2.DescriptorProto,
+        TEST_EMPTY_MESSAGE_DESCRIPTOR_ASCII)
+
+  def testCopyToProto_NestedMessage(self):
+    TEST_NESTED_MESSAGE_ASCII = """
+      name: 'NestedMessage'
+      field: <
+        name: 'bb'
+        number: 1
+        label: 1  # Optional
+        type: 5  # TYPE_INT32
+      >
+      """
+
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR,
+        descriptor_pb2.DescriptorProto,
+        TEST_NESTED_MESSAGE_ASCII)
+
+  def testCopyToProto_ForeignNestedMessage(self):
+    TEST_FOREIGN_NESTED_ASCII = """
+      name: 'TestForeignNested'
+      field: <
+        name: 'foreign_nested'
+        number: 1
+        label: 1  # Optional
+        type: 11  # TYPE_MESSAGE
+        type_name: '.protobuf_unittest.TestAllTypes.NestedMessage'
+      >
+      """
+
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestForeignNested.DESCRIPTOR,
+        descriptor_pb2.DescriptorProto,
+        TEST_FOREIGN_NESTED_ASCII)
+
+  def testCopyToProto_ForeignEnum(self):
+    TEST_FOREIGN_ENUM_ASCII = """
+      name: 'ForeignEnum'
+      value: <
+        name: 'FOREIGN_FOO'
+        number: 4
+      >
+      value: <
+        name: 'FOREIGN_BAR'
+        number: 5
+      >
+      value: <
+        name: 'FOREIGN_BAZ'
+        number: 6
+      >
+      """
+
+    self._InternalTestCopyToProto(
+        unittest_pb2._FOREIGNENUM,
+        descriptor_pb2.EnumDescriptorProto,
+        TEST_FOREIGN_ENUM_ASCII)
+
+  def testCopyToProto_Options(self):
+    TEST_DEPRECATED_FIELDS_ASCII = """
+      name: 'TestDeprecatedFields'
+      field: <
+        name: 'deprecated_int32'
+        number: 1
+        label: 1  # Optional
+        type: 5  # TYPE_INT32
+        options: <
+          deprecated: true
+        >
+      >
+      """
+
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestDeprecatedFields.DESCRIPTOR,
+        descriptor_pb2.DescriptorProto,
+        TEST_DEPRECATED_FIELDS_ASCII)
+
+  def testCopyToProto_AllExtensions(self):
+    TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII = """
+      name: 'TestEmptyMessageWithExtensions'
+      extension_range: <
+        start: 1
+        end: 536870912
+      >
+      """
+
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestEmptyMessageWithExtensions.DESCRIPTOR,
+        descriptor_pb2.DescriptorProto,
+        TEST_EMPTY_MESSAGE_WITH_EXTENSIONS_ASCII)
+
+  def testCopyToProto_SeveralExtensions(self):
+    TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII = """
+      name: 'TestMultipleExtensionRanges'
+      extension_range: <
+        start: 42
+        end: 43
+      >
+      extension_range: <
+        start: 4143
+        end: 4244
+      >
+      extension_range: <
+        start: 65536
+        end: 536870912
+      >
+      """
+
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR,
+        descriptor_pb2.DescriptorProto,
+        TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII)
+
+  def testCopyToProto_FileDescriptor(self):
+    UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = ("""
+      name: 'google/protobuf/unittest_import.proto'
+      package: 'protobuf_unittest_import'
+      dependency: 'google/protobuf/unittest_import_public.proto'
+      message_type: <
+        name: 'ImportMessage'
+        field: <
+          name: 'd'
+          number: 1
+          label: 1  # Optional
+          type: 5  # TYPE_INT32
+        >
+      >
+      """ +
+      """enum_type: <
+        name: 'ImportEnum'
+        value: <
+          name: 'IMPORT_FOO'
+          number: 7
+        >
+        value: <
+          name: 'IMPORT_BAR'
+          number: 8
+        >
+        value: <
+          name: 'IMPORT_BAZ'
+          number: 9
+        >
+      >
+      options: <
+        java_package: 'com.google.protobuf.test'
+        optimize_for: 1  # SPEED
+      >
+      public_dependency: 0
+      """)
+
+    self._InternalTestCopyToProto(
+        unittest_import_pb2.DESCRIPTOR,
+        descriptor_pb2.FileDescriptorProto,
+        UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII)
+
+  def testCopyToProto_ServiceDescriptor(self):
+    TEST_SERVICE_ASCII = """
+      name: 'TestService'
+      method: <
+        name: 'Foo'
+        input_type: '.protobuf_unittest.FooRequest'
+        output_type: '.protobuf_unittest.FooResponse'
+      >
+      method: <
+        name: 'Bar'
+        input_type: '.protobuf_unittest.BarRequest'
+        output_type: '.protobuf_unittest.BarResponse'
+      >
+      """
+
+    self._InternalTestCopyToProto(
+        unittest_pb2.TestService.DESCRIPTOR,
+        descriptor_pb2.ServiceDescriptorProto,
+        TEST_SERVICE_ASCII)
+
+
+class MakeDescriptorTest(unittest.TestCase):
+  def testMakeDescriptorWithUnsignedIntField(self):
+    file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
+    file_descriptor_proto.name = 'Foo'
+    message_type = file_descriptor_proto.message_type.add()
+    message_type.name = file_descriptor_proto.name
+    field = message_type.field.add()
+    field.number = 1
+    field.name = 'uint64_field'
+    field.label = descriptor.FieldDescriptor.LABEL_REQUIRED
+    field.type = descriptor.FieldDescriptor.TYPE_UINT64
+    result = descriptor.MakeDescriptor(message_type)
+    self.assertEqual(result.fields[0].cpp_type,
+                     descriptor.FieldDescriptor.CPPTYPE_UINT64)
+
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/encoder.py

@@ -0,0 +1,769 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Code for encoding protocol message primitives.
+
+Contains the logic for encoding every logical protocol field type
+into one of the 5 physical wire types.
+
+This code is designed to push the Python interpreter's performance to the
+limits.
+
+The basic idea is that at startup time, for every field (i.e. every
+FieldDescriptor) we construct two functions:  a "sizer" and an "encoder".  The
+sizer takes a value of this field's type and computes its byte size.  The
+encoder takes a writer function and a value.  It encodes the value into byte
+strings and invokes the writer function to write those strings.  Typically the
+writer function is the write() method of a cStringIO.
+
+We try to do as much work as possible when constructing the writer and the
+sizer rather than when calling them.  In particular:
+* We copy any needed global functions to local variables, so that we do not need
+  to do costly global table lookups at runtime.
+* Similarly, we try to do any attribute lookups at startup time if possible.
+* Every field's tag is encoded to bytes at startup, since it can't change at
+  runtime.
+* Whatever component of the field size we can compute at startup, we do.
+* We *avoid* sharing code if doing so would make the code slower and not sharing
+  does not burden us too much.  For example, encoders for repeated fields do
+  not just call the encoders for singular fields in a loop because this would
+  add an extra function call overhead for every loop iteration; instead, we
+  manually inline the single-value encoder into the loop.
+* If a Python function lacks a return statement, Python actually generates
+  instructions to pop the result of the last statement off the stack, push
+  None onto the stack, and then return that.  If we really don't care what
+  value is returned, then we can save two instructions by returning the
+  result of the last statement.  It looks funny but it helps.
+* We assume that type and bounds checking has happened at a higher level.
+"""
+
+__author__ = 'kenton@google.com (Kenton Varda)'
+
+import struct
+from google.protobuf.internal import wire_format
+
+
+# This will overflow and thus become IEEE-754 "infinity".  We would use
+# "float('inf')" but it doesn't work on Windows pre-Python-2.6.
+_POS_INF = 1e10000
+_NEG_INF = -_POS_INF
+
+
+def _VarintSize(value):
+  """Compute the size of a varint value."""
+  if value <= 0x7f: return 1
+  if value <= 0x3fff: return 2
+  if value <= 0x1fffff: return 3
+  if value <= 0xfffffff: return 4
+  if value <= 0x7ffffffff: return 5
+  if value <= 0x3ffffffffff: return 6
+  if value <= 0x1ffffffffffff: return 7
+  if value <= 0xffffffffffffff: return 8
+  if value <= 0x7fffffffffffffff: return 9
+  return 10
+
+
+def _SignedVarintSize(value):
+  """Compute the size of a signed varint value."""
+  if value < 0: return 10
+  if value <= 0x7f: return 1
+  if value <= 0x3fff: return 2
+  if value <= 0x1fffff: return 3
+  if value <= 0xfffffff: return 4
+  if value <= 0x7ffffffff: return 5
+  if value <= 0x3ffffffffff: return 6
+  if value <= 0x1ffffffffffff: return 7
+  if value <= 0xffffffffffffff: return 8
+  if value <= 0x7fffffffffffffff: return 9
+  return 10
+
+
+def _TagSize(field_number):
+  """Returns the number of bytes required to serialize a tag with this field
+  number."""
+  # Just pass in type 0, since the type won't affect the tag+type size.
+  return _VarintSize(wire_format.PackTag(field_number, 0))
+
+
+# --------------------------------------------------------------------
+# In this section we define some generic sizers.  Each of these functions
+# takes parameters specific to a particular field type, e.g. int32 or fixed64.
+# It returns another function which in turn takes parameters specific to a
+# particular field, e.g. the field number and whether it is repeated or packed.
+# Look at the next section to see how these are used.
+
+
+def _SimpleSizer(compute_value_size):
+  """A sizer which uses the function compute_value_size to compute the size of
+  each value.  Typically compute_value_size is _VarintSize."""
+
+  def SpecificSizer(field_number, is_repeated, is_packed):
+    tag_size = _TagSize(field_number)
+    if is_packed:
+      local_VarintSize = _VarintSize
+      def PackedFieldSize(value):
+        result = 0
+        for element in value:
+          result += compute_value_size(element)
+        return result + local_VarintSize(result) + tag_size
+      return PackedFieldSize
+    elif is_repeated:
+      def RepeatedFieldSize(value):
+        result = tag_size * len(value)
+        for element in value:
+          result += compute_value_size(element)
+        return result
+      return RepeatedFieldSize
+    else:
+      def FieldSize(value):
+        return tag_size + compute_value_size(value)
+      return FieldSize
+
+  return SpecificSizer
+
+
+def _ModifiedSizer(compute_value_size, modify_value):
+  """Like SimpleSizer, but modify_value is invoked on each value before it is
+  passed to compute_value_size.  modify_value is typically ZigZagEncode."""
+
+  def SpecificSizer(field_number, is_repeated, is_packed):
+    tag_size = _TagSize(field_number)
+    if is_packed:
+      local_VarintSize = _VarintSize
+      def PackedFieldSize(value):
+        result = 0
+        for element in value:
+          result += compute_value_size(modify_value(element))
+        return result + local_VarintSize(result) + tag_size
+      return PackedFieldSize
+    elif is_repeated:
+      def RepeatedFieldSize(value):
+        result = tag_size * len(value)
+        for element in value:
+          result += compute_value_size(modify_value(element))
+        return result
+      return RepeatedFieldSize
+    else:
+      def FieldSize(value):
+        return tag_size + compute_value_size(modify_value(value))
+      return FieldSize
+
+  return SpecificSizer
+
+
+def _FixedSizer(value_size):
+  """Like _SimpleSizer except for a fixed-size field.  The input is the size
+  of one value."""
+
+  def SpecificSizer(field_number, is_repeated, is_packed):
+    tag_size = _TagSize(field_number)
+    if is_packed:
+      local_VarintSize = _VarintSize
+      def PackedFieldSize(value):
+        result = len(value) * value_size
+        return result + local_VarintSize(result) + tag_size
+      return PackedFieldSize
+    elif is_repeated:
+      element_size = value_size + tag_size
+      def RepeatedFieldSize(value):
+        return len(value) * element_size
+      return RepeatedFieldSize
+    else:
+      field_size = value_size + tag_size
+      def FieldSize(value):
+        return field_size
+      return FieldSize
+
+  return SpecificSizer
+
+
+# ====================================================================
+# Here we declare a sizer constructor for each field type.  Each "sizer
+# constructor" is a function that takes (field_number, is_repeated, is_packed)
+# as parameters and returns a sizer, which in turn takes a field value as
+# a parameter and returns its encoded size.
+
+
+Int32Sizer = Int64Sizer = EnumSizer = _SimpleSizer(_SignedVarintSize)
+
+UInt32Sizer = UInt64Sizer = _SimpleSizer(_VarintSize)
+
+SInt32Sizer = SInt64Sizer = _ModifiedSizer(
+    _SignedVarintSize, wire_format.ZigZagEncode)
+
+Fixed32Sizer = SFixed32Sizer = FloatSizer  = _FixedSizer(4)
+Fixed64Sizer = SFixed64Sizer = DoubleSizer = _FixedSizer(8)
+
+BoolSizer = _FixedSizer(1)
+
+
+def StringSizer(field_number, is_repeated, is_packed):
+  """Returns a sizer for a string field."""
+
+  tag_size = _TagSize(field_number)
+  local_VarintSize = _VarintSize
+  local_len = len
+  assert not is_packed
+  if is_repeated:
+    def RepeatedFieldSize(value):
+      result = tag_size * len(value)
+      for element in value:
+        l = local_len(element.encode('utf-8'))
+        result += local_VarintSize(l) + l
+      return result
+    return RepeatedFieldSize
+  else:
+    def FieldSize(value):
+      l = local_len(value.encode('utf-8'))
+      return tag_size + local_VarintSize(l) + l
+    return FieldSize
+
+
+def BytesSizer(field_number, is_repeated, is_packed):
+  """Returns a sizer for a bytes field."""
+
+  tag_size = _TagSize(field_number)
+  local_VarintSize = _VarintSize
+  local_len = len
+  assert not is_packed
+  if is_repeated:
+    def RepeatedFieldSize(value):
+      result = tag_size * len(value)
+      for element in value:
+        l = local_len(element)
+        result += local_VarintSize(l) + l
+      return result
+    return RepeatedFieldSize
+  else:
+    def FieldSize(value):
+      l = local_len(value)
+      return tag_size + local_VarintSize(l) + l
+    return FieldSize
+
+
+def GroupSizer(field_number, is_repeated, is_packed):
+  """Returns a sizer for a group field."""
+
+  tag_size = _TagSize(field_number) * 2
+  assert not is_packed
+  if is_repeated:
+    def RepeatedFieldSize(value):
+      result = tag_size * len(value)
+      for element in value:
+        result += element.ByteSize()
+      return result
+    return RepeatedFieldSize
+  else:
+    def FieldSize(value):
+      return tag_size + value.ByteSize()
+    return FieldSize
+
+
+def MessageSizer(field_number, is_repeated, is_packed):
+  """Returns a sizer for a message field."""
+
+  tag_size = _TagSize(field_number)
+  local_VarintSize = _VarintSize
+  assert not is_packed
+  if is_repeated:
+    def RepeatedFieldSize(value):
+      result = tag_size * len(value)
+      for element in value:
+        l = element.ByteSize()
+        result += local_VarintSize(l) + l
+      return result
+    return RepeatedFieldSize
+  else:
+    def FieldSize(value):
+      l = value.ByteSize()
+      return tag_size + local_VarintSize(l) + l
+    return FieldSize
+
+
+# --------------------------------------------------------------------
+# MessageSet is special.
+
+
+def MessageSetItemSizer(field_number):
+  """Returns a sizer for extensions of MessageSet.
+
+  The message set message looks like this:
+    message MessageSet {
+      repeated group Item = 1 {
+        required int32 type_id = 2;
+        required string message = 3;
+      }
+    }
+  """
+  static_size = (_TagSize(1) * 2 + _TagSize(2) + _VarintSize(field_number) +
+                 _TagSize(3))
+  local_VarintSize = _VarintSize
+
+  def FieldSize(value):
+    l = value.ByteSize()
+    return static_size + local_VarintSize(l) + l
+
+  return FieldSize
+
+
+# ====================================================================
+# Encoders!
+
+
+def _VarintEncoder():
+  """Return an encoder for a basic varint value (does not include tag)."""
+
+  local_chr = chr
+  def EncodeVarint(write, value):
+    bits = value & 0x7f
+    value >>= 7
+    while value:
+      write(local_chr(0x80|bits))
+      bits = value & 0x7f
+      value >>= 7
+    return write(local_chr(bits))
+
+  return EncodeVarint
+
+
+def _SignedVarintEncoder():
+  """Return an encoder for a basic signed varint value (does not include
+  tag)."""
+
+  local_chr = chr
+  def EncodeSignedVarint(write, value):
+    if value < 0:
+      value += (1 << 64)
+    bits = value & 0x7f
+    value >>= 7
+    while value:
+      write(local_chr(0x80|bits))
+      bits = value & 0x7f
+      value >>= 7
+    return write(local_chr(bits))
+
+  return EncodeSignedVarint
+
+
+_EncodeVarint = _VarintEncoder()
+_EncodeSignedVarint = _SignedVarintEncoder()
+
+
+def _VarintBytes(value):
+  """Encode the given integer as a varint and return the bytes.  This is only
+  called at startup time so it doesn't need to be fast."""
+
+  pieces = []
+  _EncodeVarint(pieces.append, value)
+  return "".join(pieces)
+
+
+def TagBytes(field_number, wire_type):
+  """Encode the given tag and return the bytes.  Only called at startup."""
+
+  return _VarintBytes(wire_format.PackTag(field_number, wire_type))
+
+# --------------------------------------------------------------------
+# As with sizers (see above), we have a number of common encoder
+# implementations.
+
+
+def _SimpleEncoder(wire_type, encode_value, compute_value_size):
+  """Return a constructor for an encoder for fields of a particular type.
+
+  Args:
+      wire_type:  The field's wire type, for encoding tags.
+      encode_value:  A function which encodes an individual value, e.g.
+        _EncodeVarint().
+      compute_value_size:  A function which computes the size of an individual
+        value, e.g. _VarintSize().
+  """
+
+  def SpecificEncoder(field_number, is_repeated, is_packed):
+    if is_packed:
+      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+      local_EncodeVarint = _EncodeVarint
+      def EncodePackedField(write, value):
+        write(tag_bytes)
+        size = 0
+        for element in value:
+          size += compute_value_size(element)
+        local_EncodeVarint(write, size)
+        for element in value:
+          encode_value(write, element)
+      return EncodePackedField
+    elif is_repeated:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeRepeatedField(write, value):
+        for element in value:
+          write(tag_bytes)
+          encode_value(write, element)
+      return EncodeRepeatedField
+    else:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeField(write, value):
+        write(tag_bytes)
+        return encode_value(write, value)
+      return EncodeField
+
+  return SpecificEncoder
+
+
+def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value):
+  """Like SimpleEncoder but additionally invokes modify_value on every value
+  before passing it to encode_value.  Usually modify_value is ZigZagEncode."""
+
+  def SpecificEncoder(field_number, is_repeated, is_packed):
+    if is_packed:
+      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+      local_EncodeVarint = _EncodeVarint
+      def EncodePackedField(write, value):
+        write(tag_bytes)
+        size = 0
+        for element in value:
+          size += compute_value_size(modify_value(element))
+        local_EncodeVarint(write, size)
+        for element in value:
+          encode_value(write, modify_value(element))
+      return EncodePackedField
+    elif is_repeated:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeRepeatedField(write, value):
+        for element in value:
+          write(tag_bytes)
+          encode_value(write, modify_value(element))
+      return EncodeRepeatedField
+    else:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeField(write, value):
+        write(tag_bytes)
+        return encode_value(write, modify_value(value))
+      return EncodeField
+
+  return SpecificEncoder
+
+
+def _StructPackEncoder(wire_type, format):
+  """Return a constructor for an encoder for a fixed-width field.
+
+  Args:
+      wire_type:  The field's wire type, for encoding tags.
+      format:  The format string to pass to struct.pack().
+  """
+
+  value_size = struct.calcsize(format)
+
+  def SpecificEncoder(field_number, is_repeated, is_packed):
+    local_struct_pack = struct.pack
+    if is_packed:
+      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+      local_EncodeVarint = _EncodeVarint
+      def EncodePackedField(write, value):
+        write(tag_bytes)
+        local_EncodeVarint(write, len(value) * value_size)
+        for element in value:
+          write(local_struct_pack(format, element))
+      return EncodePackedField
+    elif is_repeated:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeRepeatedField(write, value):
+        for element in value:
+          write(tag_bytes)
+          write(local_struct_pack(format, element))
+      return EncodeRepeatedField
+    else:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeField(write, value):
+        write(tag_bytes)
+        return write(local_struct_pack(format, value))
+      return EncodeField
+
+  return SpecificEncoder
+
+
+def _FloatingPointEncoder(wire_type, format):
+  """Return a constructor for an encoder for float fields.
+
+  This is like StructPackEncoder, but catches errors that may be due to
+  passing non-finite floating-point values to struct.pack, and makes a
+  second attempt to encode those values.
+
+  Args:
+      wire_type:  The field's wire type, for encoding tags.
+      format:  The format string to pass to struct.pack().
+  """
+
+  value_size = struct.calcsize(format)
+  if value_size == 4:
+    def EncodeNonFiniteOrRaise(write, value):
+      # Remember that the serialized form uses little-endian byte order.
+      if value == _POS_INF:
+        write('\x00\x00\x80\x7F')
+      elif value == _NEG_INF:
+        write('\x00\x00\x80\xFF')
+      elif value != value:           # NaN
+        write('\x00\x00\xC0\x7F')
+      else:
+        raise
+  elif value_size == 8:
+    def EncodeNonFiniteOrRaise(write, value):
+      if value == _POS_INF:
+        write('\x00\x00\x00\x00\x00\x00\xF0\x7F')
+      elif value == _NEG_INF:
+        write('\x00\x00\x00\x00\x00\x00\xF0\xFF')
+      elif value != value:                         # NaN
+        write('\x00\x00\x00\x00\x00\x00\xF8\x7F')
+      else:
+        raise
+  else:
+    raise ValueError('Can\'t encode floating-point values that are '
+                     '%d bytes long (only 4 or 8)' % value_size)
+
+  def SpecificEncoder(field_number, is_repeated, is_packed):
+    local_struct_pack = struct.pack
+    if is_packed:
+      tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+      local_EncodeVarint = _EncodeVarint
+      def EncodePackedField(write, value):
+        write(tag_bytes)
+        local_EncodeVarint(write, len(value) * value_size)
+        for element in value:
+          # This try/except block is going to be faster than any code that
+          # we could write to check whether element is finite.
+          try:
+            write(local_struct_pack(format, element))
+          except SystemError:
+            EncodeNonFiniteOrRaise(write, element)
+      return EncodePackedField
+    elif is_repeated:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeRepeatedField(write, value):
+        for element in value:
+          write(tag_bytes)
+          try:
+            write(local_struct_pack(format, element))
+          except SystemError:
+            EncodeNonFiniteOrRaise(write, element)
+      return EncodeRepeatedField
+    else:
+      tag_bytes = TagBytes(field_number, wire_type)
+      def EncodeField(write, value):
+        write(tag_bytes)
+        try:
+          write(local_struct_pack(format, value))
+        except SystemError:
+          EncodeNonFiniteOrRaise(write, value)
+      return EncodeField
+
+  return SpecificEncoder
+
+
+# ====================================================================
+# Here we declare an encoder constructor for each field type.  These work
+# very similarly to sizer constructors, described earlier.
+
+
+Int32Encoder = Int64Encoder = EnumEncoder = _SimpleEncoder(
+    wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize)
+
+UInt32Encoder = UInt64Encoder = _SimpleEncoder(
+    wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize)
+
+SInt32Encoder = SInt64Encoder = _ModifiedEncoder(
+    wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize,
+    wire_format.ZigZagEncode)
+
+# Note that Python conveniently guarantees that when using the '<' prefix on
+# formats, they will also have the same size across all platforms (as opposed
+# to without the prefix, where their sizes depend on the C compiler's basic
+# type sizes).
+Fixed32Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<I')
+Fixed64Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<Q')
+SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<i')
+SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<q')
+FloatEncoder    = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, '<f')
+DoubleEncoder   = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d')
+
+
+def BoolEncoder(field_number, is_repeated, is_packed):
+  """Returns an encoder for a boolean field."""
+
+  false_byte = chr(0)
+  true_byte = chr(1)
+  if is_packed:
+    tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+    local_EncodeVarint = _EncodeVarint
+    def EncodePackedField(write, value):
+      write(tag_bytes)
+      local_EncodeVarint(write, len(value))
+      for element in value:
+        if element:
+          write(true_byte)
+        else:
+          write(false_byte)
+    return EncodePackedField
+  elif is_repeated:
+    tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
+    def EncodeRepeatedField(write, value):
+      for element in value:
+        write(tag_bytes)
+        if element:
+          write(true_byte)
+        else:
+          write(false_byte)
+    return EncodeRepeatedField
+  else:
+    tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT)
+    def EncodeField(write, value):
+      write(tag_bytes)
+      if value:
+        return write(true_byte)
+      return write(false_byte)
+    return EncodeField
+
+
+def StringEncoder(field_number, is_repeated, is_packed):
+  """Returns an encoder for a string field."""
+
+  tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+  local_EncodeVarint = _EncodeVarint
+  local_len = len
+  assert not is_packed
+  if is_repeated:
+    def EncodeRepeatedField(write, value):
+      for element in value:
+        encoded = element.encode('utf-8')
+        write(tag)
+        local_EncodeVarint(write, local_len(encoded))
+        write(encoded)
+    return EncodeRepeatedField
+  else:
+    def EncodeField(write, value):
+      encoded = value.encode('utf-8')
+      write(tag)
+      local_EncodeVarint(write, local_len(encoded))
+      return write(encoded)
+    return EncodeField
+
+
+def BytesEncoder(field_number, is_repeated, is_packed):
+  """Returns an encoder for a bytes field."""
+
+  tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+  local_EncodeVarint = _EncodeVarint
+  local_len = len
+  assert not is_packed
+  if is_repeated:
+    def EncodeRepeatedField(write, value):
+      for element in value:
+        write(tag)
+        local_EncodeVarint(write, local_len(element))
+        write(element)
+    return EncodeRepeatedField
+  else:
+    def EncodeField(write, value):
+      write(tag)
+      local_EncodeVarint(write, local_len(value))
+      return write(value)
+    return EncodeField
+
+
+def GroupEncoder(field_number, is_repeated, is_packed):
+  """Returns an encoder for a group field."""
+
+  start_tag = TagBytes(field_number, wire_format.WIRETYPE_START_GROUP)
+  end_tag = TagBytes(field_number, wire_format.WIRETYPE_END_GROUP)
+  assert not is_packed
+  if is_repeated:
+    def EncodeRepeatedField(write, value):
+      for element in value:
+        write(start_tag)
+        element._InternalSerialize(write)
+        write(end_tag)
+    return EncodeRepeatedField
+  else:
+    def EncodeField(write, value):
+      write(start_tag)
+      value._InternalSerialize(write)
+      return write(end_tag)
+    return EncodeField
+
+
+def MessageEncoder(field_number, is_repeated, is_packed):
+  """Returns an encoder for a message field."""
+
+  tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
+  local_EncodeVarint = _EncodeVarint
+  assert not is_packed
+  if is_repeated:
+    def EncodeRepeatedField(write, value):
+      for element in value:
+        write(tag)
+        local_EncodeVarint(write, element.ByteSize())
+        element._InternalSerialize(write)
+    return EncodeRepeatedField
+  else:
+    def EncodeField(write, value):
+      write(tag)
+      local_EncodeVarint(write, value.ByteSize())
+      return value._InternalSerialize(write)
+    return EncodeField
+
+
+# --------------------------------------------------------------------
+# As before, MessageSet is special.
+
+
+def MessageSetItemEncoder(field_number):
+  """Encoder for extensions of MessageSet.
+
+  The message set message looks like this:
+    message MessageSet {
+      repeated group Item = 1 {
+        required int32 type_id = 2;
+        required string message = 3;
+      }
+    }
+  """
+  start_bytes = "".join([
+      TagBytes(1, wire_format.WIRETYPE_START_GROUP),
+      TagBytes(2, wire_format.WIRETYPE_VARINT),
+      _VarintBytes(field_number),
+      TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)])
+  end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP)
+  local_EncodeVarint = _EncodeVarint
+
+  def EncodeField(write, value):
+    write(start_bytes)
+    local_EncodeVarint(write, value.ByteSize())
+    value._InternalSerialize(write)
+    return write(end_bytes)
+
+  return EncodeField

python/google/protobuf/internal/enum_type_wrapper.py

@@ -0,0 +1,89 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A simple wrapper around enum types to expose utility functions.
+
+Instances are created as properties with the same name as the enum they wrap
+on proto classes.  For usage, see:
+  reflection_test.py
+"""
+
+__author__ = 'rabsatt@google.com (Kevin Rabsatt)'
+
+
+class EnumTypeWrapper(object):
+  """A utility for finding the names of enum values."""
+
+  DESCRIPTOR = None
+
+  def __init__(self, enum_type):
+    """Inits EnumTypeWrapper with an EnumDescriptor."""
+    self._enum_type = enum_type
+    self.DESCRIPTOR = enum_type;
+
+  def Name(self, number):
+    """Returns a string containing the name of an enum value."""
+    if number in self._enum_type.values_by_number:
+      return self._enum_type.values_by_number[number].name
+    raise ValueError('Enum %s has no name defined for value %d' % (
+        self._enum_type.name, number))
+
+  def Value(self, name):
+    """Returns the value coresponding to the given enum name."""
+    if name in self._enum_type.values_by_name:
+      return self._enum_type.values_by_name[name].number
+    raise ValueError('Enum %s has no value defined for name %s' % (
+        self._enum_type.name, name))
+
+  def keys(self):
+    """Return a list of the string names in the enum.
+
+    These are returned in the order they were defined in the .proto file.
+    """
+
+    return [value_descriptor.name
+            for value_descriptor in self._enum_type.values]
+
+  def values(self):
+    """Return a list of the integer values in the enum.
+
+    These are returned in the order they were defined in the .proto file.
+    """
+
+    return [value_descriptor.number
+            for value_descriptor in self._enum_type.values]
+
+  def items(self):
+    """Return a list of the (name, value) pairs of the enum.
+
+    These are returned in the order they were defined in the .proto file.
+    """
+    return [(value_descriptor.name, value_descriptor.number)
+            for value_descriptor in self._enum_type.values]

python/google/protobuf/internal/factory_test1.proto

@@ -0,0 +1,55 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: matthewtoia@google.com (Matt Toia)
+
+
+package google.protobuf.python.internal;
+
+
+enum Factory1Enum {
+  FACTORY_1_VALUE_0 = 0;
+  FACTORY_1_VALUE_1 = 1;
+}
+
+message Factory1Message {
+  optional Factory1Enum factory_1_enum = 1;
+  enum NestedFactory1Enum {
+    NESTED_FACTORY_1_VALUE_0 = 0;
+    NESTED_FACTORY_1_VALUE_1 = 1;
+  }
+  optional NestedFactory1Enum nested_factory_1_enum = 2;
+  message NestedFactory1Message {
+    optional string value = 1;
+  }
+  optional NestedFactory1Message nested_factory_1_message = 3;
+  optional int32 scalar_value = 4;
+  repeated string list_value = 5;
+}

python/google/protobuf/internal/factory_test2.proto

@@ -0,0 +1,77 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: matthewtoia@google.com (Matt Toia)
+
+
+package google.protobuf.python.internal;
+
+import "google/protobuf/internal/factory_test1.proto";
+
+
+enum Factory2Enum {
+  FACTORY_2_VALUE_0 = 0;
+  FACTORY_2_VALUE_1 = 1;
+}
+
+message Factory2Message {
+  required int32 mandatory = 1;
+  optional Factory2Enum factory_2_enum = 2;
+  enum NestedFactory2Enum {
+    NESTED_FACTORY_2_VALUE_0 = 0;
+    NESTED_FACTORY_2_VALUE_1 = 1;
+  }
+  optional NestedFactory2Enum nested_factory_2_enum = 3;
+  message NestedFactory2Message {
+    optional string value = 1;
+  }
+  optional NestedFactory2Message nested_factory_2_message = 4;
+  optional Factory1Message factory_1_message = 5;
+  optional Factory1Enum factory_1_enum = 6;
+  optional Factory1Message.NestedFactory1Enum nested_factory_1_enum = 7;
+  optional Factory1Message.NestedFactory1Message nested_factory_1_message = 8;
+  optional Factory2Message circular_message = 9;
+  optional string scalar_value = 10;
+  repeated string list_value = 11;
+  repeated group Grouped = 12 {
+    optional string part_1 = 13;
+    optional string part_2 = 14;
+  }
+  optional LoopMessage loop = 15;
+  optional int32 int_with_default = 16 [default = 1776];
+  optional double double_with_default = 17 [default = 9.99];
+  optional string string_with_default = 18 [default = "hello world"];
+  optional bool bool_with_default = 19 [default = false];
+  optional Factory2Enum enum_with_default = 20 [default = FACTORY_2_VALUE_1];
+}
+
+message LoopMessage {
+  optional Factory2Message loop = 1;
+}

python/google/protobuf/internal/generator_test.py

@@ -0,0 +1,269 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# TODO(robinson): Flesh this out considerably.  We focused on reflection_test.py
+# first, since it's testing the subtler code, and since it provides decent
+# indirect testing of the protocol compiler output.
+
+"""Unittest that directly tests the output of the pure-Python protocol
+compiler.  See //google/protobuf/reflection_test.py for a test which
+further ensures that we can use Python protocol message objects as we expect.
+"""
+
+__author__ = 'robinson@google.com (Will Robinson)'
+
+import unittest
+from google.protobuf.internal import test_bad_identifiers_pb2
+from google.protobuf import unittest_custom_options_pb2
+from google.protobuf import unittest_import_pb2
+from google.protobuf import unittest_import_public_pb2
+from google.protobuf import unittest_mset_pb2
+from google.protobuf import unittest_pb2
+from google.protobuf import unittest_no_generic_services_pb2
+from google.protobuf import service
+
+MAX_EXTENSION = 536870912
+
+
+class GeneratorTest(unittest.TestCase):
+
+  def testNestedMessageDescriptor(self):
+    field_name = 'optional_nested_message'
+    proto_type = unittest_pb2.TestAllTypes
+    self.assertEqual(
+        proto_type.NestedMessage.DESCRIPTOR,
+        proto_type.DESCRIPTOR.fields_by_name[field_name].message_type)
+
+  def testEnums(self):
+    # We test only module-level enums here.
+    # TODO(robinson): Examine descriptors directly to check
+    # enum descriptor output.
+    self.assertEqual(4, unittest_pb2.FOREIGN_FOO)
+    self.assertEqual(5, unittest_pb2.FOREIGN_BAR)
+    self.assertEqual(6, unittest_pb2.FOREIGN_BAZ)
+
+    proto = unittest_pb2.TestAllTypes()
+    self.assertEqual(1, proto.FOO)
+    self.assertEqual(1, unittest_pb2.TestAllTypes.FOO)
+    self.assertEqual(2, proto.BAR)
+    self.assertEqual(2, unittest_pb2.TestAllTypes.BAR)
+    self.assertEqual(3, proto.BAZ)
+    self.assertEqual(3, unittest_pb2.TestAllTypes.BAZ)
+
+  def testExtremeDefaultValues(self):
+    message = unittest_pb2.TestExtremeDefaultValues()
+
+    # Python pre-2.6 does not have isinf() or isnan() functions, so we have
+    # to provide our own.
+    def isnan(val):
+      # NaN is never equal to itself.
+      return val != val
+    def isinf(val):
+      # Infinity times zero equals NaN.
+      return not isnan(val) and isnan(val * 0)
+
+    self.assertTrue(isinf(message.inf_double))
+    self.assertTrue(message.inf_double > 0)
+    self.assertTrue(isinf(message.neg_inf_double))
+    self.assertTrue(message.neg_inf_double < 0)
+    self.assertTrue(isnan(message.nan_double))
+
+    self.assertTrue(isinf(message.inf_float))
+    self.assertTrue(message.inf_float > 0)
+    self.assertTrue(isinf(message.neg_inf_float))
+    self.assertTrue(message.neg_inf_float < 0)
+    self.assertTrue(isnan(message.nan_float))
+    self.assertEqual("? ? ?? ?? ??? ??/ ??-", message.cpp_trigraph)
+
+  def testHasDefaultValues(self):
+    desc = unittest_pb2.TestAllTypes.DESCRIPTOR
+
+    expected_has_default_by_name = {
+        'optional_int32': False,
+        'repeated_int32': False,
+        'optional_nested_message': False,
+        'default_int32': True,
+    }
+
+    has_default_by_name = dict(
+        [(f.name, f.has_default_value)
+         for f in desc.fields
+         if f.name in expected_has_default_by_name])
+    self.assertEqual(expected_has_default_by_name, has_default_by_name)
+
+  def testContainingTypeBehaviorForExtensions(self):
+    self.assertEqual(unittest_pb2.optional_int32_extension.containing_type,
+                     unittest_pb2.TestAllExtensions.DESCRIPTOR)
+    self.assertEqual(unittest_pb2.TestRequired.single.containing_type,
+                     unittest_pb2.TestAllExtensions.DESCRIPTOR)
+
+  def testExtensionScope(self):
+    self.assertEqual(unittest_pb2.optional_int32_extension.extension_scope,
+                     None)
+    self.assertEqual(unittest_pb2.TestRequired.single.extension_scope,
+                     unittest_pb2.TestRequired.DESCRIPTOR)
+
+  def testIsExtension(self):
+    self.assertTrue(unittest_pb2.optional_int32_extension.is_extension)
+    self.assertTrue(unittest_pb2.TestRequired.single.is_extension)
+
+    message_descriptor = unittest_pb2.TestRequired.DESCRIPTOR
+    non_extension_descriptor = message_descriptor.fields_by_name['a']
+    self.assertTrue(not non_extension_descriptor.is_extension)
+
+  def testOptions(self):
+    proto = unittest_mset_pb2.TestMessageSet()
+    self.assertTrue(proto.DESCRIPTOR.GetOptions().message_set_wire_format)
+
+  def testMessageWithCustomOptions(self):
+    proto = unittest_custom_options_pb2.TestMessageWithCustomOptions()
+    enum_options = proto.DESCRIPTOR.enum_types_by_name['AnEnum'].GetOptions()
+    self.assertTrue(enum_options is not None)
+    # TODO(gps): We really should test for the presense of the enum_opt1
+    # extension and for its value to be set to -789.
+
+  def testNestedTypes(self):
+    self.assertEquals(
+        set(unittest_pb2.TestAllTypes.DESCRIPTOR.nested_types),
+        set([
+            unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR,
+            unittest_pb2.TestAllTypes.OptionalGroup.DESCRIPTOR,
+            unittest_pb2.TestAllTypes.RepeatedGroup.DESCRIPTOR,
+        ]))
+    self.assertEqual(unittest_pb2.TestEmptyMessage.DESCRIPTOR.nested_types, [])
+    self.assertEqual(
+        unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.nested_types, [])
+
+  def testContainingType(self):
+    self.assertTrue(
+        unittest_pb2.TestEmptyMessage.DESCRIPTOR.containing_type is None)
+    self.assertTrue(
+        unittest_pb2.TestAllTypes.DESCRIPTOR.containing_type is None)
+    self.assertEqual(
+        unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.containing_type,
+        unittest_pb2.TestAllTypes.DESCRIPTOR)
+    self.assertEqual(
+        unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR.containing_type,
+        unittest_pb2.TestAllTypes.DESCRIPTOR)
+    self.assertEqual(
+        unittest_pb2.TestAllTypes.RepeatedGroup.DESCRIPTOR.containing_type,
+        unittest_pb2.TestAllTypes.DESCRIPTOR)
+
+  def testContainingTypeInEnumDescriptor(self):
+    self.assertTrue(unittest_pb2._FOREIGNENUM.containing_type is None)
+    self.assertEqual(unittest_pb2._TESTALLTYPES_NESTEDENUM.containing_type,
+                     unittest_pb2.TestAllTypes.DESCRIPTOR)
+
+  def testPackage(self):
+    self.assertEqual(
+        unittest_pb2.TestAllTypes.DESCRIPTOR.file.package,
+        'protobuf_unittest')
+    desc = unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR
+    self.assertEqual(desc.file.package, 'protobuf_unittest')
+    self.assertEqual(
+        unittest_import_pb2.ImportMessage.DESCRIPTOR.file.package,
+        'protobuf_unittest_import')
+
+    self.assertEqual(
+        unittest_pb2._FOREIGNENUM.file.package, 'protobuf_unittest')
+    self.assertEqual(
+        unittest_pb2._TESTALLTYPES_NESTEDENUM.file.package,
+        'protobuf_unittest')
+    self.assertEqual(
+        unittest_import_pb2._IMPORTENUM.file.package,
+        'protobuf_unittest_import')
+
+  def testExtensionRange(self):
+    self.assertEqual(
+        unittest_pb2.TestAllTypes.DESCRIPTOR.extension_ranges, [])
+    self.assertEqual(
+        unittest_pb2.TestAllExtensions.DESCRIPTOR.extension_ranges,
+        [(1, MAX_EXTENSION)])
+    self.assertEqual(
+        unittest_pb2.TestMultipleExtensionRanges.DESCRIPTOR.extension_ranges,
+        [(42, 43), (4143, 4244), (65536, MAX_EXTENSION)])
+
+  def testFileDescriptor(self):
+    self.assertEqual(unittest_pb2.DESCRIPTOR.name,
+                     'google/protobuf/unittest.proto')
+    self.assertEqual(unittest_pb2.DESCRIPTOR.package, 'protobuf_unittest')
+    self.assertFalse(unittest_pb2.DESCRIPTOR.serialized_pb is None)
+
+  def testNoGenericServices(self):
+    self.assertTrue(hasattr(unittest_no_generic_services_pb2, "TestMessage"))
+    self.assertTrue(hasattr(unittest_no_generic_services_pb2, "FOO"))
+    self.assertTrue(hasattr(unittest_no_generic_services_pb2, "test_extension"))
+
+    # Make sure unittest_no_generic_services_pb2 has no services subclassing
+    # Proto2 Service class.
+    if hasattr(unittest_no_generic_services_pb2, "TestService"):
+      self.assertFalse(issubclass(unittest_no_generic_services_pb2.TestService,
+                                  service.Service))
+
+  def testMessageTypesByName(self):
+    file_type = unittest_pb2.DESCRIPTOR
+    self.assertEqual(
+        unittest_pb2._TESTALLTYPES,
+        file_type.message_types_by_name[unittest_pb2._TESTALLTYPES.name])
+
+    # Nested messages shouldn't be included in the message_types_by_name
+    # dictionary (like in the C++ API).
+    self.assertFalse(
+        unittest_pb2._TESTALLTYPES_NESTEDMESSAGE.name in
+        file_type.message_types_by_name)
+
+  def testPublicImports(self):
+    # Test public imports as embedded message.
+    all_type_proto = unittest_pb2.TestAllTypes()
+    self.assertEqual(0, all_type_proto.optional_public_import_message.e)
+
+    # PublicImportMessage is actually defined in unittest_import_public_pb2
+    # module, and is public imported by unittest_import_pb2 module.
+    public_import_proto = unittest_import_pb2.PublicImportMessage()
+    self.assertEqual(0, public_import_proto.e)
+    self.assertTrue(unittest_import_public_pb2.PublicImportMessage is
+                    unittest_import_pb2.PublicImportMessage)
+
+  def testBadIdentifiers(self):
+    # We're just testing that the code was imported without problems.
+    message = test_bad_identifiers_pb2.TestBadIdentifiers()
+    self.assertEqual(message.Extensions[test_bad_identifiers_pb2.message],
+                     "foo")
+    self.assertEqual(message.Extensions[test_bad_identifiers_pb2.descriptor],
+                     "bar")
+    self.assertEqual(message.Extensions[test_bad_identifiers_pb2.reflection],
+                     "baz")
+    self.assertEqual(message.Extensions[test_bad_identifiers_pb2.service],
+                     "qux")
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/message_cpp_test.py

@@ -0,0 +1,45 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests for google.protobuf.internal.message_cpp."""
+
+__author__ = 'shahms@google.com (Shahms King)'
+
+import os
+os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
+
+import unittest
+from google.protobuf.internal.message_test import *
+
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/message_factory_test.py

@@ -0,0 +1,113 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests for google.protobuf.message_factory."""
+
+__author__ = 'matthewtoia@google.com (Matt Toia)'
+
+import unittest
+from google.protobuf import descriptor_pb2
+from google.protobuf.internal import factory_test1_pb2
+from google.protobuf.internal import factory_test2_pb2
+from google.protobuf import descriptor_database
+from google.protobuf import descriptor_pool
+from google.protobuf import message_factory
+
+
+class MessageFactoryTest(unittest.TestCase):
+
+  def setUp(self):
+    self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test1_pb2.DESCRIPTOR.serialized_pb)
+    self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        factory_test2_pb2.DESCRIPTOR.serialized_pb)
+
+  def _ExerciseDynamicClass(self, cls):
+    msg = cls()
+    msg.mandatory = 42
+    msg.nested_factory_2_enum = 0
+    msg.nested_factory_2_message.value = 'nested message value'
+    msg.factory_1_message.factory_1_enum = 1
+    msg.factory_1_message.nested_factory_1_enum = 0
+    msg.factory_1_message.nested_factory_1_message.value = (
+        'nested message value')
+    msg.factory_1_message.scalar_value = 22
+    msg.factory_1_message.list_value.extend(['one', 'two', 'three'])
+    msg.factory_1_message.list_value.append('four')
+    msg.factory_1_enum = 1
+    msg.nested_factory_1_enum = 0
+    msg.nested_factory_1_message.value = 'nested message value'
+    msg.circular_message.mandatory = 1
+    msg.circular_message.circular_message.mandatory = 2
+    msg.circular_message.scalar_value = 'one deep'
+    msg.scalar_value = 'zero deep'
+    msg.list_value.extend(['four', 'three', 'two'])
+    msg.list_value.append('one')
+    msg.grouped.add()
+    msg.grouped[0].part_1 = 'hello'
+    msg.grouped[0].part_2 = 'world'
+    msg.grouped.add(part_1='testing', part_2='123')
+    msg.loop.loop.mandatory = 2
+    msg.loop.loop.loop.loop.mandatory = 4
+    serialized = msg.SerializeToString()
+    converted = factory_test2_pb2.Factory2Message.FromString(serialized)
+    reserialized = converted.SerializeToString()
+    self.assertEquals(serialized, reserialized)
+    result = cls.FromString(reserialized)
+    self.assertEquals(msg, result)
+
+  def testGetPrototype(self):
+    db = descriptor_database.DescriptorDatabase()
+    pool = descriptor_pool.DescriptorPool(db)
+    db.Add(self.factory_test1_fd)
+    db.Add(self.factory_test2_fd)
+    factory = message_factory.MessageFactory()
+    cls = factory.GetPrototype(pool.FindMessageTypeByName(
+        'net.proto2.python.internal.Factory2Message'))
+    self.assertIsNot(cls, factory_test2_pb2.Factory2Message)
+    self._ExerciseDynamicClass(cls)
+    cls2 = factory.GetPrototype(pool.FindMessageTypeByName(
+        'net.proto2.python.internal.Factory2Message'))
+    self.assertIs(cls, cls2)
+
+  def testGetMessages(self):
+    messages = message_factory.GetMessages([self.factory_test2_fd,
+                                            self.factory_test1_fd])
+    self.assertContainsSubset(
+        ['net.proto2.python.internal.Factory2Message',
+         'net.proto2.python.internal.Factory1Message'],
+        messages.keys())
+    self._ExerciseDynamicClass(
+        messages['net.proto2.python.internal.Factory2Message'])
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/message_listener.py

@@ -0,0 +1,78 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Defines a listener interface for observing certain
+state transitions on Message objects.
+
+Also defines a null implementation of this interface.
+"""
+
+__author__ = 'robinson@google.com (Will Robinson)'
+
+
+class MessageListener(object):
+
+  """Listens for modifications made to a message.  Meant to be registered via
+  Message._SetListener().
+
+  Attributes:
+    dirty:  If True, then calling Modified() would be a no-op.  This can be
+            used to avoid these calls entirely in the common case.
+  """
+
+  def Modified(self):
+    """Called every time the message is modified in such a way that the parent
+    message may need to be updated.  This currently means either:
+    (a) The message was modified for the first time, so the parent message
+        should henceforth mark the message as present.
+    (b) The message's cached byte size became dirty -- i.e. the message was
+        modified for the first time after a previous call to ByteSize().
+        Therefore the parent should also mark its byte size as dirty.
+    Note that (a) implies (b), since new objects start out with a client cached
+    size (zero).  However, we document (a) explicitly because it is important.
+
+    Modified() will *only* be called in response to one of these two events --
+    not every time the sub-message is modified.
+
+    Note that if the listener's |dirty| attribute is true, then calling
+    Modified at the moment would be a no-op, so it can be skipped.  Performance-
+    sensitive callers should check this attribute directly before calling since
+    it will be true most of the time.
+    """
+
+    raise NotImplementedError
+
+
+class NullMessageListener(object):
+
+  """No-op MessageListener implementation."""
+
+  def Modified(self):
+    pass

python/google/protobuf/internal/message_test.py

@@ -0,0 +1,494 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests python protocol buffers against the golden message.
+
+Note that the golden messages exercise every known field type, thus this
+test ends up exercising and verifying nearly all of the parsing and
+serialization code in the whole library.
+
+TODO(kenton):  Merge with wire_format_test?  It doesn't make a whole lot of
+sense to call this a test of the "message" module, which only declares an
+abstract interface.
+"""
+
+__author__ = 'gps@google.com (Gregory P. Smith)'
+
+import copy
+import math
+import operator
+import pickle
+
+import unittest
+from google.protobuf import unittest_import_pb2
+from google.protobuf import unittest_pb2
+from google.protobuf.internal import api_implementation
+from google.protobuf.internal import test_util
+from google.protobuf import message
+
+# Python pre-2.6 does not have isinf() or isnan() functions, so we have
+# to provide our own.
+def isnan(val):
+  # NaN is never equal to itself.
+  return val != val
+def isinf(val):
+  # Infinity times zero equals NaN.
+  return not isnan(val) and isnan(val * 0)
+def IsPosInf(val):
+  return isinf(val) and (val > 0)
+def IsNegInf(val):
+  return isinf(val) and (val < 0)
+
+class MessageTest(unittest.TestCase):
+
+  def testGoldenMessage(self):
+    golden_data = test_util.GoldenFile('golden_message').read()
+    golden_message = unittest_pb2.TestAllTypes()
+    golden_message.ParseFromString(golden_data)
+    test_util.ExpectAllFieldsSet(self, golden_message)
+    self.assertEqual(golden_data, golden_message.SerializeToString())
+    golden_copy = copy.deepcopy(golden_message)
+    self.assertEqual(golden_data, golden_copy.SerializeToString())
+
+  def testGoldenExtensions(self):
+    golden_data = test_util.GoldenFile('golden_message').read()
+    golden_message = unittest_pb2.TestAllExtensions()
+    golden_message.ParseFromString(golden_data)
+    all_set = unittest_pb2.TestAllExtensions()
+    test_util.SetAllExtensions(all_set)
+    self.assertEquals(all_set, golden_message)
+    self.assertEqual(golden_data, golden_message.SerializeToString())
+    golden_copy = copy.deepcopy(golden_message)
+    self.assertEqual(golden_data, golden_copy.SerializeToString())
+
+  def testGoldenPackedMessage(self):
+    golden_data = test_util.GoldenFile('golden_packed_fields_message').read()
+    golden_message = unittest_pb2.TestPackedTypes()
+    golden_message.ParseFromString(golden_data)
+    all_set = unittest_pb2.TestPackedTypes()
+    test_util.SetAllPackedFields(all_set)
+    self.assertEquals(all_set, golden_message)
+    self.assertEqual(golden_data, all_set.SerializeToString())
+    golden_copy = copy.deepcopy(golden_message)
+    self.assertEqual(golden_data, golden_copy.SerializeToString())
+
+  def testGoldenPackedExtensions(self):
+    golden_data = test_util.GoldenFile('golden_packed_fields_message').read()
+    golden_message = unittest_pb2.TestPackedExtensions()
+    golden_message.ParseFromString(golden_data)
+    all_set = unittest_pb2.TestPackedExtensions()
+    test_util.SetAllPackedExtensions(all_set)
+    self.assertEquals(all_set, golden_message)
+    self.assertEqual(golden_data, all_set.SerializeToString())
+    golden_copy = copy.deepcopy(golden_message)
+    self.assertEqual(golden_data, golden_copy.SerializeToString())
+
+  def testPickleSupport(self):
+    golden_data = test_util.GoldenFile('golden_message').read()
+    golden_message = unittest_pb2.TestAllTypes()
+    golden_message.ParseFromString(golden_data)
+    pickled_message = pickle.dumps(golden_message)
+
+    unpickled_message = pickle.loads(pickled_message)
+    self.assertEquals(unpickled_message, golden_message)
+
+  def testPickleIncompleteProto(self):
+    golden_message = unittest_pb2.TestRequired(a=1)
+    pickled_message = pickle.dumps(golden_message)
+
+    unpickled_message = pickle.loads(pickled_message)
+    self.assertEquals(unpickled_message, golden_message)
+    self.assertEquals(unpickled_message.a, 1)
+    # This is still an incomplete proto - so serializing should fail
+    self.assertRaises(message.EncodeError, unpickled_message.SerializeToString)
+
+  def testPositiveInfinity(self):
+    golden_data = ('\x5D\x00\x00\x80\x7F'
+                   '\x61\x00\x00\x00\x00\x00\x00\xF0\x7F'
+                   '\xCD\x02\x00\x00\x80\x7F'
+                   '\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\x7F')
+    golden_message = unittest_pb2.TestAllTypes()
+    golden_message.ParseFromString(golden_data)
+    self.assertTrue(IsPosInf(golden_message.optional_float))
+    self.assertTrue(IsPosInf(golden_message.optional_double))
+    self.assertTrue(IsPosInf(golden_message.repeated_float[0]))
+    self.assertTrue(IsPosInf(golden_message.repeated_double[0]))
+    self.assertEqual(golden_data, golden_message.SerializeToString())
+
+  def testNegativeInfinity(self):
+    golden_data = ('\x5D\x00\x00\x80\xFF'
+                   '\x61\x00\x00\x00\x00\x00\x00\xF0\xFF'
+                   '\xCD\x02\x00\x00\x80\xFF'
+                   '\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\xFF')
+    golden_message = unittest_pb2.TestAllTypes()
+    golden_message.ParseFromString(golden_data)
+    self.assertTrue(IsNegInf(golden_message.optional_float))
+    self.assertTrue(IsNegInf(golden_message.optional_double))
+    self.assertTrue(IsNegInf(golden_message.repeated_float[0]))
+    self.assertTrue(IsNegInf(golden_message.repeated_double[0]))
+    self.assertEqual(golden_data, golden_message.SerializeToString())
+
+  def testNotANumber(self):
+    golden_data = ('\x5D\x00\x00\xC0\x7F'
+                   '\x61\x00\x00\x00\x00\x00\x00\xF8\x7F'
+                   '\xCD\x02\x00\x00\xC0\x7F'
+                   '\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F')
+    golden_message = unittest_pb2.TestAllTypes()
+    golden_message.ParseFromString(golden_data)
+    self.assertTrue(isnan(golden_message.optional_float))
+    self.assertTrue(isnan(golden_message.optional_double))
+    self.assertTrue(isnan(golden_message.repeated_float[0]))
+    self.assertTrue(isnan(golden_message.repeated_double[0]))
+
+    # The protocol buffer may serialize to any one of multiple different
+    # representations of a NaN.  Rather than verify a specific representation,
+    # verify the serialized string can be converted into a correctly
+    # behaving protocol buffer.
+    serialized = golden_message.SerializeToString()
+    message = unittest_pb2.TestAllTypes()
+    message.ParseFromString(serialized)
+    self.assertTrue(isnan(message.optional_float))
+    self.assertTrue(isnan(message.optional_double))
+    self.assertTrue(isnan(message.repeated_float[0]))
+    self.assertTrue(isnan(message.repeated_double[0]))
+
+  def testPositiveInfinityPacked(self):
+    golden_data = ('\xA2\x06\x04\x00\x00\x80\x7F'
+                   '\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\x7F')
+    golden_message = unittest_pb2.TestPackedTypes()
+    golden_message.ParseFromString(golden_data)
+    self.assertTrue(IsPosInf(golden_message.packed_float[0]))
+    self.assertTrue(IsPosInf(golden_message.packed_double[0]))
+    self.assertEqual(golden_data, golden_message.SerializeToString())
+
+  def testNegativeInfinityPacked(self):
+    golden_data = ('\xA2\x06\x04\x00\x00\x80\xFF'
+                   '\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\xFF')
+    golden_message = unittest_pb2.TestPackedTypes()
+    golden_message.ParseFromString(golden_data)
+    self.assertTrue(IsNegInf(golden_message.packed_float[0]))
+    self.assertTrue(IsNegInf(golden_message.packed_double[0]))
+    self.assertEqual(golden_data, golden_message.SerializeToString())
+
+  def testNotANumberPacked(self):
+    golden_data = ('\xA2\x06\x04\x00\x00\xC0\x7F'
+                   '\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F')
+    golden_message = unittest_pb2.TestPackedTypes()
+    golden_message.ParseFromString(golden_data)
+    self.assertTrue(isnan(golden_message.packed_float[0]))
+    self.assertTrue(isnan(golden_message.packed_double[0]))
+
+    serialized = golden_message.SerializeToString()
+    message = unittest_pb2.TestPackedTypes()
+    message.ParseFromString(serialized)
+    self.assertTrue(isnan(message.packed_float[0]))
+    self.assertTrue(isnan(message.packed_double[0]))
+
+  def testExtremeFloatValues(self):
+    message = unittest_pb2.TestAllTypes()
+
+    # Most positive exponent, no significand bits set.
+    kMostPosExponentNoSigBits = math.pow(2, 127)
+    message.optional_float = kMostPosExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == kMostPosExponentNoSigBits)
+
+    # Most positive exponent, one significand bit set.
+    kMostPosExponentOneSigBit = 1.5 * math.pow(2, 127)
+    message.optional_float = kMostPosExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == kMostPosExponentOneSigBit)
+
+    # Repeat last two cases with values of same magnitude, but negative.
+    message.optional_float = -kMostPosExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == -kMostPosExponentNoSigBits)
+
+    message.optional_float = -kMostPosExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == -kMostPosExponentOneSigBit)
+
+    # Most negative exponent, no significand bits set.
+    kMostNegExponentNoSigBits = math.pow(2, -127)
+    message.optional_float = kMostNegExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == kMostNegExponentNoSigBits)
+
+    # Most negative exponent, one significand bit set.
+    kMostNegExponentOneSigBit = 1.5 * math.pow(2, -127)
+    message.optional_float = kMostNegExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == kMostNegExponentOneSigBit)
+
+    # Repeat last two cases with values of the same magnitude, but negative.
+    message.optional_float = -kMostNegExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == -kMostNegExponentNoSigBits)
+
+    message.optional_float = -kMostNegExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_float == -kMostNegExponentOneSigBit)
+
+  def testExtremeDoubleValues(self):
+    message = unittest_pb2.TestAllTypes()
+
+    # Most positive exponent, no significand bits set.
+    kMostPosExponentNoSigBits = math.pow(2, 1023)
+    message.optional_double = kMostPosExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == kMostPosExponentNoSigBits)
+
+    # Most positive exponent, one significand bit set.
+    kMostPosExponentOneSigBit = 1.5 * math.pow(2, 1023)
+    message.optional_double = kMostPosExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == kMostPosExponentOneSigBit)
+
+    # Repeat last two cases with values of same magnitude, but negative.
+    message.optional_double = -kMostPosExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == -kMostPosExponentNoSigBits)
+
+    message.optional_double = -kMostPosExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == -kMostPosExponentOneSigBit)
+
+    # Most negative exponent, no significand bits set.
+    kMostNegExponentNoSigBits = math.pow(2, -1023)
+    message.optional_double = kMostNegExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == kMostNegExponentNoSigBits)
+
+    # Most negative exponent, one significand bit set.
+    kMostNegExponentOneSigBit = 1.5 * math.pow(2, -1023)
+    message.optional_double = kMostNegExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == kMostNegExponentOneSigBit)
+
+    # Repeat last two cases with values of the same magnitude, but negative.
+    message.optional_double = -kMostNegExponentNoSigBits
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == -kMostNegExponentNoSigBits)
+
+    message.optional_double = -kMostNegExponentOneSigBit
+    message.ParseFromString(message.SerializeToString())
+    self.assertTrue(message.optional_double == -kMostNegExponentOneSigBit)
+
+  def testSortingRepeatedScalarFieldsDefaultComparator(self):
+    """Check some different types with the default comparator."""
+    message = unittest_pb2.TestAllTypes()
+
+    # TODO(mattp): would testing more scalar types strengthen test?
+    message.repeated_int32.append(1)
+    message.repeated_int32.append(3)
+    message.repeated_int32.append(2)
+    message.repeated_int32.sort()
+    self.assertEqual(message.repeated_int32[0], 1)
+    self.assertEqual(message.repeated_int32[1], 2)
+    self.assertEqual(message.repeated_int32[2], 3)
+
+    message.repeated_float.append(1.1)
+    message.repeated_float.append(1.3)
+    message.repeated_float.append(1.2)
+    message.repeated_float.sort()
+    self.assertAlmostEqual(message.repeated_float[0], 1.1)
+    self.assertAlmostEqual(message.repeated_float[1], 1.2)
+    self.assertAlmostEqual(message.repeated_float[2], 1.3)
+
+    message.repeated_string.append('a')
+    message.repeated_string.append('c')
+    message.repeated_string.append('b')
+    message.repeated_string.sort()
+    self.assertEqual(message.repeated_string[0], 'a')
+    self.assertEqual(message.repeated_string[1], 'b')
+    self.assertEqual(message.repeated_string[2], 'c')
+
+    message.repeated_bytes.append('a')
+    message.repeated_bytes.append('c')
+    message.repeated_bytes.append('b')
+    message.repeated_bytes.sort()
+    self.assertEqual(message.repeated_bytes[0], 'a')
+    self.assertEqual(message.repeated_bytes[1], 'b')
+    self.assertEqual(message.repeated_bytes[2], 'c')
+
+  def testSortingRepeatedScalarFieldsCustomComparator(self):
+    """Check some different types with custom comparator."""
+    message = unittest_pb2.TestAllTypes()
+
+    message.repeated_int32.append(-3)
+    message.repeated_int32.append(-2)
+    message.repeated_int32.append(-1)
+    message.repeated_int32.sort(lambda x,y: cmp(abs(x), abs(y)))
+    self.assertEqual(message.repeated_int32[0], -1)
+    self.assertEqual(message.repeated_int32[1], -2)
+    self.assertEqual(message.repeated_int32[2], -3)
+
+    message.repeated_string.append('aaa')
+    message.repeated_string.append('bb')
+    message.repeated_string.append('c')
+    message.repeated_string.sort(lambda x,y: cmp(len(x), len(y)))
+    self.assertEqual(message.repeated_string[0], 'c')
+    self.assertEqual(message.repeated_string[1], 'bb')
+    self.assertEqual(message.repeated_string[2], 'aaa')
+
+  def testSortingRepeatedCompositeFieldsCustomComparator(self):
+    """Check passing a custom comparator to sort a repeated composite field."""
+    message = unittest_pb2.TestAllTypes()
+
+    message.repeated_nested_message.add().bb = 1
+    message.repeated_nested_message.add().bb = 3
+    message.repeated_nested_message.add().bb = 2
+    message.repeated_nested_message.add().bb = 6
+    message.repeated_nested_message.add().bb = 5
+    message.repeated_nested_message.add().bb = 4
+    message.repeated_nested_message.sort(lambda x,y: cmp(x.bb, y.bb))
+    self.assertEqual(message.repeated_nested_message[0].bb, 1)
+    self.assertEqual(message.repeated_nested_message[1].bb, 2)
+    self.assertEqual(message.repeated_nested_message[2].bb, 3)
+    self.assertEqual(message.repeated_nested_message[3].bb, 4)
+    self.assertEqual(message.repeated_nested_message[4].bb, 5)
+    self.assertEqual(message.repeated_nested_message[5].bb, 6)
+
+  def testRepeatedCompositeFieldSortArguments(self):
+    """Check sorting a repeated composite field using list.sort() arguments."""
+    message = unittest_pb2.TestAllTypes()
+
+    get_bb = operator.attrgetter('bb')
+    cmp_bb = lambda a, b: cmp(a.bb, b.bb)
+    message.repeated_nested_message.add().bb = 1
+    message.repeated_nested_message.add().bb = 3
+    message.repeated_nested_message.add().bb = 2
+    message.repeated_nested_message.add().bb = 6
+    message.repeated_nested_message.add().bb = 5
+    message.repeated_nested_message.add().bb = 4
+    message.repeated_nested_message.sort(key=get_bb)
+    self.assertEqual([k.bb for k in message.repeated_nested_message],
+                     [1, 2, 3, 4, 5, 6])
+    message.repeated_nested_message.sort(key=get_bb, reverse=True)
+    self.assertEqual([k.bb for k in message.repeated_nested_message],
+                     [6, 5, 4, 3, 2, 1])
+    message.repeated_nested_message.sort(sort_function=cmp_bb)
+    self.assertEqual([k.bb for k in message.repeated_nested_message],
+                     [1, 2, 3, 4, 5, 6])
+    message.repeated_nested_message.sort(cmp=cmp_bb, reverse=True)
+    self.assertEqual([k.bb for k in message.repeated_nested_message],
+                     [6, 5, 4, 3, 2, 1])
+
+  def testRepeatedScalarFieldSortArguments(self):
+    """Check sorting a scalar field using list.sort() arguments."""
+    message = unittest_pb2.TestAllTypes()
+
+    abs_cmp = lambda a, b: cmp(abs(a), abs(b))
+    message.repeated_int32.append(-3)
+    message.repeated_int32.append(-2)
+    message.repeated_int32.append(-1)
+    message.repeated_int32.sort(key=abs)
+    self.assertEqual(list(message.repeated_int32), [-1, -2, -3])
+    message.repeated_int32.sort(key=abs, reverse=True)
+    self.assertEqual(list(message.repeated_int32), [-3, -2, -1])
+    message.repeated_int32.sort(sort_function=abs_cmp)
+    self.assertEqual(list(message.repeated_int32), [-1, -2, -3])
+    message.repeated_int32.sort(cmp=abs_cmp, reverse=True)
+    self.assertEqual(list(message.repeated_int32), [-3, -2, -1])
+
+    len_cmp = lambda a, b: cmp(len(a), len(b))
+    message.repeated_string.append('aaa')
+    message.repeated_string.append('bb')
+    message.repeated_string.append('c')
+    message.repeated_string.sort(key=len)
+    self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa'])
+    message.repeated_string.sort(key=len, reverse=True)
+    self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c'])
+    message.repeated_string.sort(sort_function=len_cmp)
+    self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa'])
+    message.repeated_string.sort(cmp=len_cmp, reverse=True)
+    self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c'])
+
+  def testParsingMerge(self):
+    """Check the merge behavior when a required or optional field appears
+    multiple times in the input."""
+    messages = [
+        unittest_pb2.TestAllTypes(),
+        unittest_pb2.TestAllTypes(),
+        unittest_pb2.TestAllTypes() ]
+    messages[0].optional_int32 = 1
+    messages[1].optional_int64 = 2
+    messages[2].optional_int32 = 3
+    messages[2].optional_string = 'hello'
+
+    merged_message = unittest_pb2.TestAllTypes()
+    merged_message.optional_int32 = 3
+    merged_message.optional_int64 = 2
+    merged_message.optional_string = 'hello'
+
+    generator = unittest_pb2.TestParsingMerge.RepeatedFieldsGenerator()
+    generator.field1.extend(messages)
+    generator.field2.extend(messages)
+    generator.field3.extend(messages)
+    generator.ext1.extend(messages)
+    generator.ext2.extend(messages)
+    generator.group1.add().field1.MergeFrom(messages[0])
+    generator.group1.add().field1.MergeFrom(messages[1])
+    generator.group1.add().field1.MergeFrom(messages[2])
+    generator.group2.add().field1.MergeFrom(messages[0])
+    generator.group2.add().field1.MergeFrom(messages[1])
+    generator.group2.add().field1.MergeFrom(messages[2])
+
+    data = generator.SerializeToString()
+    parsing_merge = unittest_pb2.TestParsingMerge()
+    parsing_merge.ParseFromString(data)
+
+    # Required and optional fields should be merged.
+    self.assertEqual(parsing_merge.required_all_types, merged_message)
+    self.assertEqual(parsing_merge.optional_all_types, merged_message)
+    self.assertEqual(parsing_merge.optionalgroup.optional_group_all_types,
+                     merged_message)
+    self.assertEqual(parsing_merge.Extensions[
+                     unittest_pb2.TestParsingMerge.optional_ext],
+                     merged_message)
+
+    # Repeated fields should not be merged.
+    self.assertEqual(len(parsing_merge.repeated_all_types), 3)
+    self.assertEqual(len(parsing_merge.repeatedgroup), 3)
+    self.assertEqual(len(parsing_merge.Extensions[
+        unittest_pb2.TestParsingMerge.repeated_ext]), 3)
+
+
+  def testSortEmptyRepeatedCompositeContainer(self):
+    """Exercise a scenario that has led to segfaults in the past.
+    """
+    m = unittest_pb2.TestAllTypes()
+    m.repeated_nested_message.sort()
+
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/more_extensions.proto

@@ -0,0 +1,58 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: robinson@google.com (Will Robinson)
+
+
+package google.protobuf.internal;
+
+
+message TopLevelMessage {
+  optional ExtendedMessage submessage = 1;
+}
+
+
+message ExtendedMessage {
+  extensions 1 to max;
+}
+
+
+message ForeignMessage {
+  optional int32 foreign_message_int = 1;
+}
+
+
+extend ExtendedMessage {
+  optional int32 optional_int_extension = 1;
+  optional ForeignMessage optional_message_extension = 2;
+
+  repeated int32 repeated_int_extension = 3;
+  repeated ForeignMessage repeated_message_extension = 4;
+}

python/google/protobuf/internal/more_extensions_dynamic.proto

@@ -0,0 +1,49 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: jasonh@google.com (Jason Hsueh)
+//
+// This file is used to test a corner case in the CPP implementation where the
+// generated C++ type is available for the extendee, but the extension is
+// defined in a file whose C++ type is not in the binary.
+
+
+import "google/protobuf/internal/more_extensions.proto";
+
+package google.protobuf.internal;
+
+message DynamicMessageType {
+  optional int32 a = 1;
+}
+
+extend ExtendedMessage {
+  optional int32 dynamic_int32_extension = 100;
+  optional DynamicMessageType dynamic_message_extension = 101;
+}

python/google/protobuf/internal/more_messages.proto

@@ -0,0 +1,51 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: robinson@google.com (Will Robinson)
+
+
+package google.protobuf.internal;
+
+// A message where tag numbers are listed out of order, to allow us to test our
+// canonicalization of serialized output, which should always be in tag order.
+// We also mix in some extensions for extra fun.
+message OutOfOrderFields {
+  optional   sint32 optional_sint32   =  5;
+  extensions 4 to 4;
+  optional   uint32 optional_uint32   =  3;
+  extensions 2 to 2;
+  optional    int32 optional_int32    =  1;
+};
+
+
+extend OutOfOrderFields {
+  optional   uint64 optional_uint64   =  4;
+  optional    int64 optional_int64    =  2;
+}

python/google/protobuf/internal/reflection_cpp_generated_test.py

@@ -0,0 +1,91 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unittest for reflection.py, which tests the generated C++ implementation."""
+
+__author__ = 'jasonh@google.com (Jason Hsueh)'
+
+import os
+os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
+
+import unittest
+from google.protobuf.internal import api_implementation
+from google.protobuf.internal import more_extensions_dynamic_pb2
+from google.protobuf.internal import more_extensions_pb2
+from google.protobuf.internal.reflection_test import *
+
+
+class ReflectionCppTest(unittest.TestCase):
+  def testImplementationSetting(self):
+    self.assertEqual('cpp', api_implementation.Type())
+
+  def testExtensionOfGeneratedTypeInDynamicFile(self):
+    """Tests that a file built dynamically can extend a generated C++ type.
+
+    The C++ implementation uses a DescriptorPool that has the generated
+    DescriptorPool as an underlay. Typically, a type can only find
+    extensions in its own pool. With the python C-extension, the generated C++
+    extendee may be available, but not the extension. This tests that the
+    C-extension implements the correct special handling to make such extensions
+    available.
+    """
+    pb1 = more_extensions_pb2.ExtendedMessage()
+    # Test that basic accessors work.
+    self.assertFalse(
+        pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
+    self.assertFalse(
+        pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
+    pb1.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension] = 17
+    pb1.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a = 24
+    self.assertTrue(
+        pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
+    self.assertTrue(
+        pb1.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
+
+    # Now serialize the data and parse to a new message.
+    pb2 = more_extensions_pb2.ExtendedMessage()
+    pb2.MergeFromString(pb1.SerializeToString())
+
+    self.assertTrue(
+        pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_int32_extension))
+    self.assertTrue(
+        pb2.HasExtension(more_extensions_dynamic_pb2.dynamic_message_extension))
+    self.assertEqual(
+        17, pb2.Extensions[more_extensions_dynamic_pb2.dynamic_int32_extension])
+    self.assertEqual(
+        24,
+        pb2.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a)
+
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/service_reflection_test.py

@@ -0,0 +1,136 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests for google.protobuf.internal.service_reflection."""
+
+__author__ = 'petar@google.com (Petar Petrov)'
+
+import unittest
+from google.protobuf import unittest_pb2
+from google.protobuf import service_reflection
+from google.protobuf import service
+
+
+class FooUnitTest(unittest.TestCase):
+
+  def testService(self):
+    class MockRpcChannel(service.RpcChannel):
+      def CallMethod(self, method, controller, request, response, callback):
+        self.method = method
+        self.controller = controller
+        self.request = request
+        callback(response)
+
+    class MockRpcController(service.RpcController):
+      def SetFailed(self, msg):
+        self.failure_message = msg
+
+    self.callback_response = None
+
+    class MyService(unittest_pb2.TestService):
+      pass
+
+    self.callback_response = None
+
+    def MyCallback(response):
+      self.callback_response = response
+
+    rpc_controller = MockRpcController()
+    channel = MockRpcChannel()
+    srvc = MyService()
+    srvc.Foo(rpc_controller, unittest_pb2.FooRequest(), MyCallback)
+    self.assertEqual('Method Foo not implemented.',
+                     rpc_controller.failure_message)
+    self.assertEqual(None, self.callback_response)
+
+    rpc_controller.failure_message = None
+
+    service_descriptor = unittest_pb2.TestService.GetDescriptor()
+    srvc.CallMethod(service_descriptor.methods[1], rpc_controller,
+                    unittest_pb2.BarRequest(), MyCallback)
+    self.assertEqual('Method Bar not implemented.',
+                     rpc_controller.failure_message)
+    self.assertEqual(None, self.callback_response)
+    
+    class MyServiceImpl(unittest_pb2.TestService):
+      def Foo(self, rpc_controller, request, done):
+        self.foo_called = True
+      def Bar(self, rpc_controller, request, done):
+        self.bar_called = True
+
+    srvc = MyServiceImpl()
+    rpc_controller.failure_message = None
+    srvc.Foo(rpc_controller, unittest_pb2.FooRequest(), MyCallback)
+    self.assertEqual(None, rpc_controller.failure_message)
+    self.assertEqual(True, srvc.foo_called)
+
+    rpc_controller.failure_message = None
+    srvc.CallMethod(service_descriptor.methods[1], rpc_controller,
+                    unittest_pb2.BarRequest(), MyCallback)
+    self.assertEqual(None, rpc_controller.failure_message)
+    self.assertEqual(True, srvc.bar_called)
+
+  def testServiceStub(self):
+    class MockRpcChannel(service.RpcChannel):
+      def CallMethod(self, method, controller, request,
+                     response_class, callback):
+        self.method = method
+        self.controller = controller
+        self.request = request
+        callback(response_class())
+
+    self.callback_response = None
+
+    def MyCallback(response):
+      self.callback_response = response
+
+    channel = MockRpcChannel()
+    stub = unittest_pb2.TestService_Stub(channel)
+    rpc_controller = 'controller'
+    request = 'request'
+
+    # GetDescriptor now static, still works as instance method for compatability
+    self.assertEqual(unittest_pb2.TestService_Stub.GetDescriptor(),
+                     stub.GetDescriptor())
+
+    # Invoke method.
+    stub.Foo(rpc_controller, request, MyCallback)
+
+    self.assertTrue(isinstance(self.callback_response,
+                               unittest_pb2.FooResponse))
+    self.assertEqual(request, channel.request)
+    self.assertEqual(rpc_controller, channel.controller)
+    self.assertEqual(stub.GetDescriptor().methods[0], channel.method)
+
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/test_bad_identifiers.proto

@@ -0,0 +1,52 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// http://code.google.com/p/protobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+
+
+package protobuf_unittest;
+
+option py_generic_services = true;
+
+message TestBadIdentifiers {
+  extensions 100 to max;
+}
+
+// Make sure these reasonable extension names don't conflict with internal
+// variables.
+extend TestBadIdentifiers {
+  optional string message = 100 [default="foo"];
+  optional string descriptor = 101 [default="bar"];
+  optional string reflection = 102 [default="baz"];
+  optional string service = 103 [default="qux"];
+}
+
+message AnotherMessage {}
+service AnotherService {}

python/google/protobuf/internal/test_util.py

@@ -0,0 +1,651 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Utilities for Python proto2 tests.
+
+This is intentionally modeled on C++ code in
+//google/protobuf/test_util.*.
+"""
+
+__author__ = 'robinson@google.com (Will Robinson)'
+
+import os.path
+
+from google.protobuf import unittest_import_pb2
+from google.protobuf import unittest_pb2
+
+
+def SetAllNonLazyFields(message):
+  """Sets every non-lazy field in the message to a unique value.
+
+  Args:
+    message: A unittest_pb2.TestAllTypes instance.
+  """
+
+  #
+  # Optional fields.
+  #
+
+  message.optional_int32    = 101
+  message.optional_int64    = 102
+  message.optional_uint32   = 103
+  message.optional_uint64   = 104
+  message.optional_sint32   = 105
+  message.optional_sint64   = 106
+  message.optional_fixed32  = 107
+  message.optional_fixed64  = 108
+  message.optional_sfixed32 = 109
+  message.optional_sfixed64 = 110
+  message.optional_float    = 111
+  message.optional_double   = 112
+  message.optional_bool     = True
+  # TODO(robinson): Firmly spec out and test how
+  # protos interact with unicode.  One specific example:
+  # what happens if we change the literal below to
+  # u'115'?  What *should* happen?  Still some discussion
+  # to finish with Kenton about bytes vs. strings
+  # and forcing everything to be utf8. :-/
+  message.optional_string   = '115'
+  message.optional_bytes    = '116'
+
+  message.optionalgroup.a = 117
+  message.optional_nested_message.bb = 118
+  message.optional_foreign_message.c = 119
+  message.optional_import_message.d = 120
+  message.optional_public_import_message.e = 126
+
+  message.optional_nested_enum = unittest_pb2.TestAllTypes.BAZ
+  message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ
+  message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ
+
+  message.optional_string_piece = '124'
+  message.optional_cord = '125'
+
+  #
+  # Repeated fields.
+  #
+
+  message.repeated_int32.append(201)
+  message.repeated_int64.append(202)
+  message.repeated_uint32.append(203)
+  message.repeated_uint64.append(204)
+  message.repeated_sint32.append(205)
+  message.repeated_sint64.append(206)
+  message.repeated_fixed32.append(207)
+  message.repeated_fixed64.append(208)
+  message.repeated_sfixed32.append(209)
+  message.repeated_sfixed64.append(210)
+  message.repeated_float.append(211)
+  message.repeated_double.append(212)
+  message.repeated_bool.append(True)
+  message.repeated_string.append('215')
+  message.repeated_bytes.append('216')
+
+  message.repeatedgroup.add().a = 217
+  message.repeated_nested_message.add().bb = 218
+  message.repeated_foreign_message.add().c = 219
+  message.repeated_import_message.add().d = 220
+  message.repeated_lazy_message.add().bb = 227
+
+  message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAR)
+  message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR)
+  message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR)
+
+  message.repeated_string_piece.append('224')
+  message.repeated_cord.append('225')
+
+  # Add a second one of each field.
+  message.repeated_int32.append(301)
+  message.repeated_int64.append(302)
+  message.repeated_uint32.append(303)
+  message.repeated_uint64.append(304)
+  message.repeated_sint32.append(305)
+  message.repeated_sint64.append(306)
+  message.repeated_fixed32.append(307)
+  message.repeated_fixed64.append(308)
+  message.repeated_sfixed32.append(309)
+  message.repeated_sfixed64.append(310)
+  message.repeated_float.append(311)
+  message.repeated_double.append(312)
+  message.repeated_bool.append(False)
+  message.repeated_string.append('315')
+  message.repeated_bytes.append('316')
+
+  message.repeatedgroup.add().a = 317
+  message.repeated_nested_message.add().bb = 318
+  message.repeated_foreign_message.add().c = 319
+  message.repeated_import_message.add().d = 320
+  message.repeated_lazy_message.add().bb = 327
+
+  message.repeated_nested_enum.append(unittest_pb2.TestAllTypes.BAZ)
+  message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ)
+  message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ)
+
+  message.repeated_string_piece.append('324')
+  message.repeated_cord.append('325')
+
+  #
+  # Fields that have defaults.
+  #
+
+  message.default_int32 = 401
+  message.default_int64 = 402
+  message.default_uint32 = 403
+  message.default_uint64 = 404
+  message.default_sint32 = 405
+  message.default_sint64 = 406
+  message.default_fixed32 = 407
+  message.default_fixed64 = 408
+  message.default_sfixed32 = 409
+  message.default_sfixed64 = 410
+  message.default_float = 411
+  message.default_double = 412
+  message.default_bool = False
+  message.default_string = '415'
+  message.default_bytes = '416'
+
+  message.default_nested_enum = unittest_pb2.TestAllTypes.FOO
+  message.default_foreign_enum = unittest_pb2.FOREIGN_FOO
+  message.default_import_enum = unittest_import_pb2.IMPORT_FOO
+
+  message.default_string_piece = '424'
+  message.default_cord = '425'
+
+
+def SetAllFields(message):
+  SetAllNonLazyFields(message)
+  message.optional_lazy_message.bb = 127
+
+
+def SetAllExtensions(message):
+  """Sets every extension in the message to a unique value.
+
+  Args:
+    message: A unittest_pb2.TestAllExtensions instance.
+  """
+
+  extensions = message.Extensions
+  pb2 = unittest_pb2
+  import_pb2 = unittest_import_pb2
+
+  #
+  # Optional fields.
+  #
+
+  extensions[pb2.optional_int32_extension] = 101
+  extensions[pb2.optional_int64_extension] = 102
+  extensions[pb2.optional_uint32_extension] = 103
+  extensions[pb2.optional_uint64_extension] = 104
+  extensions[pb2.optional_sint32_extension] = 105
+  extensions[pb2.optional_sint64_extension] = 106
+  extensions[pb2.optional_fixed32_extension] = 107
+  extensions[pb2.optional_fixed64_extension] = 108
+  extensions[pb2.optional_sfixed32_extension] = 109
+  extensions[pb2.optional_sfixed64_extension] = 110
+  extensions[pb2.optional_float_extension] = 111
+  extensions[pb2.optional_double_extension] = 112
+  extensions[pb2.optional_bool_extension] = True
+  extensions[pb2.optional_string_extension] = '115'
+  extensions[pb2.optional_bytes_extension] = '116'
+
+  extensions[pb2.optionalgroup_extension].a = 117
+  extensions[pb2.optional_nested_message_extension].bb = 118
+  extensions[pb2.optional_foreign_message_extension].c = 119
+  extensions[pb2.optional_import_message_extension].d = 120
+  extensions[pb2.optional_public_import_message_extension].e = 126
+  extensions[pb2.optional_lazy_message_extension].bb = 127
+
+  extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ
+  extensions[pb2.optional_nested_enum_extension] = pb2.TestAllTypes.BAZ
+  extensions[pb2.optional_foreign_enum_extension] = pb2.FOREIGN_BAZ
+  extensions[pb2.optional_import_enum_extension] = import_pb2.IMPORT_BAZ
+
+  extensions[pb2.optional_string_piece_extension] = '124'
+  extensions[pb2.optional_cord_extension] = '125'
+
+  #
+  # Repeated fields.
+  #
+
+  extensions[pb2.repeated_int32_extension].append(201)
+  extensions[pb2.repeated_int64_extension].append(202)
+  extensions[pb2.repeated_uint32_extension].append(203)
+  extensions[pb2.repeated_uint64_extension].append(204)
+  extensions[pb2.repeated_sint32_extension].append(205)
+  extensions[pb2.repeated_sint64_extension].append(206)
+  extensions[pb2.repeated_fixed32_extension].append(207)
+  extensions[pb2.repeated_fixed64_extension].append(208)
+  extensions[pb2.repeated_sfixed32_extension].append(209)
+  extensions[pb2.repeated_sfixed64_extension].append(210)
+  extensions[pb2.repeated_float_extension].append(211)
+  extensions[pb2.repeated_double_extension].append(212)
+  extensions[pb2.repeated_bool_extension].append(True)
+  extensions[pb2.repeated_string_extension].append('215')
+  extensions[pb2.repeated_bytes_extension].append('216')
+
+  extensions[pb2.repeatedgroup_extension].add().a = 217
+  extensions[pb2.repeated_nested_message_extension].add().bb = 218
+  extensions[pb2.repeated_foreign_message_extension].add().c = 219
+  extensions[pb2.repeated_import_message_extension].add().d = 220
+  extensions[pb2.repeated_lazy_message_extension].add().bb = 227
+
+  extensions[pb2.repeated_nested_enum_extension].append(pb2.TestAllTypes.BAR)
+  extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAR)
+  extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAR)
+
+  extensions[pb2.repeated_string_piece_extension].append('224')
+  extensions[pb2.repeated_cord_extension].append('225')
+
+  # Append a second one of each field.
+  extensions[pb2.repeated_int32_extension].append(301)
+  extensions[pb2.repeated_int64_extension].append(302)
+  extensions[pb2.repeated_uint32_extension].append(303)
+  extensions[pb2.repeated_uint64_extension].append(304)
+  extensions[pb2.repeated_sint32_extension].append(305)
+  extensions[pb2.repeated_sint64_extension].append(306)
+  extensions[pb2.repeated_fixed32_extension].append(307)
+  extensions[pb2.repeated_fixed64_extension].append(308)
+  extensions[pb2.repeated_sfixed32_extension].append(309)
+  extensions[pb2.repeated_sfixed64_extension].append(310)
+  extensions[pb2.repeated_float_extension].append(311)
+  extensions[pb2.repeated_double_extension].append(312)
+  extensions[pb2.repeated_bool_extension].append(False)
+  extensions[pb2.repeated_string_extension].append('315')
+  extensions[pb2.repeated_bytes_extension].append('316')
+
+  extensions[pb2.repeatedgroup_extension].add().a = 317
+  extensions[pb2.repeated_nested_message_extension].add().bb = 318
+  extensions[pb2.repeated_foreign_message_extension].add().c = 319
+  extensions[pb2.repeated_import_message_extension].add().d = 320
+  extensions[pb2.repeated_lazy_message_extension].add().bb = 327
+
+  extensions[pb2.repeated_nested_enum_extension].append(pb2.TestAllTypes.BAZ)
+  extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAZ)
+  extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAZ)
+
+  extensions[pb2.repeated_string_piece_extension].append('324')
+  extensions[pb2.repeated_cord_extension].append('325')
+
+  #
+  # Fields with defaults.
+  #
+
+  extensions[pb2.default_int32_extension] = 401
+  extensions[pb2.default_int64_extension] = 402
+  extensions[pb2.default_uint32_extension] = 403
+  extensions[pb2.default_uint64_extension] = 404
+  extensions[pb2.default_sint32_extension] = 405
+  extensions[pb2.default_sint64_extension] = 406
+  extensions[pb2.default_fixed32_extension] = 407
+  extensions[pb2.default_fixed64_extension] = 408
+  extensions[pb2.default_sfixed32_extension] = 409
+  extensions[pb2.default_sfixed64_extension] = 410
+  extensions[pb2.default_float_extension] = 411
+  extensions[pb2.default_double_extension] = 412
+  extensions[pb2.default_bool_extension] = False
+  extensions[pb2.default_string_extension] = '415'
+  extensions[pb2.default_bytes_extension] = '416'
+
+  extensions[pb2.default_nested_enum_extension] = pb2.TestAllTypes.FOO
+  extensions[pb2.default_foreign_enum_extension] = pb2.FOREIGN_FOO
+  extensions[pb2.default_import_enum_extension] = import_pb2.IMPORT_FOO
+
+  extensions[pb2.default_string_piece_extension] = '424'
+  extensions[pb2.default_cord_extension] = '425'
+
+
+def SetAllFieldsAndExtensions(message):
+  """Sets every field and extension in the message to a unique value.
+
+  Args:
+    message: A unittest_pb2.TestAllExtensions message.
+  """
+  message.my_int = 1
+  message.my_string = 'foo'
+  message.my_float = 1.0
+  message.Extensions[unittest_pb2.my_extension_int] = 23
+  message.Extensions[unittest_pb2.my_extension_string] = 'bar'
+
+
+def ExpectAllFieldsAndExtensionsInOrder(serialized):
+  """Ensures that serialized is the serialization we expect for a message
+  filled with SetAllFieldsAndExtensions().  (Specifically, ensures that the
+  serialization is in canonical, tag-number order).
+  """
+  my_extension_int = unittest_pb2.my_extension_int
+  my_extension_string = unittest_pb2.my_extension_string
+  expected_strings = []
+  message = unittest_pb2.TestFieldOrderings()
+  message.my_int = 1  # Field 1.
+  expected_strings.append(message.SerializeToString())
+  message.Clear()
+  message.Extensions[my_extension_int] = 23  # Field 5.
+  expected_strings.append(message.SerializeToString())
+  message.Clear()
+  message.my_string = 'foo'  # Field 11.
+  expected_strings.append(message.SerializeToString())
+  message.Clear()
+  message.Extensions[my_extension_string] = 'bar'  # Field 50.
+  expected_strings.append(message.SerializeToString())
+  message.Clear()
+  message.my_float = 1.0
+  expected_strings.append(message.SerializeToString())
+  message.Clear()
+  expected = ''.join(expected_strings)
+
+  if expected != serialized:
+    raise ValueError('Expected %r, found %r' % (expected, serialized))
+
+
+def ExpectAllFieldsSet(test_case, message):
+  """Check all fields for correct values have after Set*Fields() is called."""
+  test_case.assertTrue(message.HasField('optional_int32'))
+  test_case.assertTrue(message.HasField('optional_int64'))
+  test_case.assertTrue(message.HasField('optional_uint32'))
+  test_case.assertTrue(message.HasField('optional_uint64'))
+  test_case.assertTrue(message.HasField('optional_sint32'))
+  test_case.assertTrue(message.HasField('optional_sint64'))
+  test_case.assertTrue(message.HasField('optional_fixed32'))
+  test_case.assertTrue(message.HasField('optional_fixed64'))
+  test_case.assertTrue(message.HasField('optional_sfixed32'))
+  test_case.assertTrue(message.HasField('optional_sfixed64'))
+  test_case.assertTrue(message.HasField('optional_float'))
+  test_case.assertTrue(message.HasField('optional_double'))
+  test_case.assertTrue(message.HasField('optional_bool'))
+  test_case.assertTrue(message.HasField('optional_string'))
+  test_case.assertTrue(message.HasField('optional_bytes'))
+
+  test_case.assertTrue(message.HasField('optionalgroup'))
+  test_case.assertTrue(message.HasField('optional_nested_message'))
+  test_case.assertTrue(message.HasField('optional_foreign_message'))
+  test_case.assertTrue(message.HasField('optional_import_message'))
+
+  test_case.assertTrue(message.optionalgroup.HasField('a'))
+  test_case.assertTrue(message.optional_nested_message.HasField('bb'))
+  test_case.assertTrue(message.optional_foreign_message.HasField('c'))
+  test_case.assertTrue(message.optional_import_message.HasField('d'))
+
+  test_case.assertTrue(message.HasField('optional_nested_enum'))
+  test_case.assertTrue(message.HasField('optional_foreign_enum'))
+  test_case.assertTrue(message.HasField('optional_import_enum'))
+
+  test_case.assertTrue(message.HasField('optional_string_piece'))
+  test_case.assertTrue(message.HasField('optional_cord'))
+
+  test_case.assertEqual(101, message.optional_int32)
+  test_case.assertEqual(102, message.optional_int64)
+  test_case.assertEqual(103, message.optional_uint32)
+  test_case.assertEqual(104, message.optional_uint64)
+  test_case.assertEqual(105, message.optional_sint32)
+  test_case.assertEqual(106, message.optional_sint64)
+  test_case.assertEqual(107, message.optional_fixed32)
+  test_case.assertEqual(108, message.optional_fixed64)
+  test_case.assertEqual(109, message.optional_sfixed32)
+  test_case.assertEqual(110, message.optional_sfixed64)
+  test_case.assertEqual(111, message.optional_float)
+  test_case.assertEqual(112, message.optional_double)
+  test_case.assertEqual(True, message.optional_bool)
+  test_case.assertEqual('115', message.optional_string)
+  test_case.assertEqual('116', message.optional_bytes)
+
+  test_case.assertEqual(117, message.optionalgroup.a)
+  test_case.assertEqual(118, message.optional_nested_message.bb)
+  test_case.assertEqual(119, message.optional_foreign_message.c)
+  test_case.assertEqual(120, message.optional_import_message.d)
+  test_case.assertEqual(126, message.optional_public_import_message.e)
+  test_case.assertEqual(127, message.optional_lazy_message.bb)
+
+  test_case.assertEqual(unittest_pb2.TestAllTypes.BAZ,
+                        message.optional_nested_enum)
+  test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
+                        message.optional_foreign_enum)
+  test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
+                        message.optional_import_enum)
+
+  # -----------------------------------------------------------------
+
+  test_case.assertEqual(2, len(message.repeated_int32))
+  test_case.assertEqual(2, len(message.repeated_int64))
+  test_case.assertEqual(2, len(message.repeated_uint32))
+  test_case.assertEqual(2, len(message.repeated_uint64))
+  test_case.assertEqual(2, len(message.repeated_sint32))
+  test_case.assertEqual(2, len(message.repeated_sint64))
+  test_case.assertEqual(2, len(message.repeated_fixed32))
+  test_case.assertEqual(2, len(message.repeated_fixed64))
+  test_case.assertEqual(2, len(message.repeated_sfixed32))
+  test_case.assertEqual(2, len(message.repeated_sfixed64))
+  test_case.assertEqual(2, len(message.repeated_float))
+  test_case.assertEqual(2, len(message.repeated_double))
+  test_case.assertEqual(2, len(message.repeated_bool))
+  test_case.assertEqual(2, len(message.repeated_string))
+  test_case.assertEqual(2, len(message.repeated_bytes))
+
+  test_case.assertEqual(2, len(message.repeatedgroup))
+  test_case.assertEqual(2, len(message.repeated_nested_message))
+  test_case.assertEqual(2, len(message.repeated_foreign_message))
+  test_case.assertEqual(2, len(message.repeated_import_message))
+  test_case.assertEqual(2, len(message.repeated_nested_enum))
+  test_case.assertEqual(2, len(message.repeated_foreign_enum))
+  test_case.assertEqual(2, len(message.repeated_import_enum))
+
+  test_case.assertEqual(2, len(message.repeated_string_piece))
+  test_case.assertEqual(2, len(message.repeated_cord))
+
+  test_case.assertEqual(201, message.repeated_int32[0])
+  test_case.assertEqual(202, message.repeated_int64[0])
+  test_case.assertEqual(203, message.repeated_uint32[0])
+  test_case.assertEqual(204, message.repeated_uint64[0])
+  test_case.assertEqual(205, message.repeated_sint32[0])
+  test_case.assertEqual(206, message.repeated_sint64[0])
+  test_case.assertEqual(207, message.repeated_fixed32[0])
+  test_case.assertEqual(208, message.repeated_fixed64[0])
+  test_case.assertEqual(209, message.repeated_sfixed32[0])
+  test_case.assertEqual(210, message.repeated_sfixed64[0])
+  test_case.assertEqual(211, message.repeated_float[0])
+  test_case.assertEqual(212, message.repeated_double[0])
+  test_case.assertEqual(True, message.repeated_bool[0])
+  test_case.assertEqual('215', message.repeated_string[0])
+  test_case.assertEqual('216', message.repeated_bytes[0])
+
+  test_case.assertEqual(217, message.repeatedgroup[0].a)
+  test_case.assertEqual(218, message.repeated_nested_message[0].bb)
+  test_case.assertEqual(219, message.repeated_foreign_message[0].c)
+  test_case.assertEqual(220, message.repeated_import_message[0].d)
+  test_case.assertEqual(227, message.repeated_lazy_message[0].bb)
+
+  test_case.assertEqual(unittest_pb2.TestAllTypes.BAR,
+                        message.repeated_nested_enum[0])
+  test_case.assertEqual(unittest_pb2.FOREIGN_BAR,
+                        message.repeated_foreign_enum[0])
+  test_case.assertEqual(unittest_import_pb2.IMPORT_BAR,
+                        message.repeated_import_enum[0])
+
+  test_case.assertEqual(301, message.repeated_int32[1])
+  test_case.assertEqual(302, message.repeated_int64[1])
+  test_case.assertEqual(303, message.repeated_uint32[1])
+  test_case.assertEqual(304, message.repeated_uint64[1])
+  test_case.assertEqual(305, message.repeated_sint32[1])
+  test_case.assertEqual(306, message.repeated_sint64[1])
+  test_case.assertEqual(307, message.repeated_fixed32[1])
+  test_case.assertEqual(308, message.repeated_fixed64[1])
+  test_case.assertEqual(309, message.repeated_sfixed32[1])
+  test_case.assertEqual(310, message.repeated_sfixed64[1])
+  test_case.assertEqual(311, message.repeated_float[1])
+  test_case.assertEqual(312, message.repeated_double[1])
+  test_case.assertEqual(False, message.repeated_bool[1])
+  test_case.assertEqual('315', message.repeated_string[1])
+  test_case.assertEqual('316', message.repeated_bytes[1])
+
+  test_case.assertEqual(317, message.repeatedgroup[1].a)
+  test_case.assertEqual(318, message.repeated_nested_message[1].bb)
+  test_case.assertEqual(319, message.repeated_foreign_message[1].c)
+  test_case.assertEqual(320, message.repeated_import_message[1].d)
+  test_case.assertEqual(327, message.repeated_lazy_message[1].bb)
+
+  test_case.assertEqual(unittest_pb2.TestAllTypes.BAZ,
+                        message.repeated_nested_enum[1])
+  test_case.assertEqual(unittest_pb2.FOREIGN_BAZ,
+                        message.repeated_foreign_enum[1])
+  test_case.assertEqual(unittest_import_pb2.IMPORT_BAZ,
+                        message.repeated_import_enum[1])
+
+  # -----------------------------------------------------------------
+
+  test_case.assertTrue(message.HasField('default_int32'))
+  test_case.assertTrue(message.HasField('default_int64'))
+  test_case.assertTrue(message.HasField('default_uint32'))
+  test_case.assertTrue(message.HasField('default_uint64'))
+  test_case.assertTrue(message.HasField('default_sint32'))
+  test_case.assertTrue(message.HasField('default_sint64'))
+  test_case.assertTrue(message.HasField('default_fixed32'))
+  test_case.assertTrue(message.HasField('default_fixed64'))
+  test_case.assertTrue(message.HasField('default_sfixed32'))
+  test_case.assertTrue(message.HasField('default_sfixed64'))
+  test_case.assertTrue(message.HasField('default_float'))
+  test_case.assertTrue(message.HasField('default_double'))
+  test_case.assertTrue(message.HasField('default_bool'))
+  test_case.assertTrue(message.HasField('default_string'))
+  test_case.assertTrue(message.HasField('default_bytes'))
+
+  test_case.assertTrue(message.HasField('default_nested_enum'))
+  test_case.assertTrue(message.HasField('default_foreign_enum'))
+  test_case.assertTrue(message.HasField('default_import_enum'))
+
+  test_case.assertEqual(401, message.default_int32)
+  test_case.assertEqual(402, message.default_int64)
+  test_case.assertEqual(403, message.default_uint32)
+  test_case.assertEqual(404, message.default_uint64)
+  test_case.assertEqual(405, message.default_sint32)
+  test_case.assertEqual(406, message.default_sint64)
+  test_case.assertEqual(407, message.default_fixed32)
+  test_case.assertEqual(408, message.default_fixed64)
+  test_case.assertEqual(409, message.default_sfixed32)
+  test_case.assertEqual(410, message.default_sfixed64)
+  test_case.assertEqual(411, message.default_float)
+  test_case.assertEqual(412, message.default_double)
+  test_case.assertEqual(False, message.default_bool)
+  test_case.assertEqual('415', message.default_string)
+  test_case.assertEqual('416', message.default_bytes)
+
+  test_case.assertEqual(unittest_pb2.TestAllTypes.FOO,
+                        message.default_nested_enum)
+  test_case.assertEqual(unittest_pb2.FOREIGN_FOO,
+                        message.default_foreign_enum)
+  test_case.assertEqual(unittest_import_pb2.IMPORT_FOO,
+                        message.default_import_enum)
+
+def GoldenFile(filename):
+  """Finds the given golden file and returns a file object representing it."""
+
+  # Search up the directory tree looking for the C++ protobuf source code.
+  path = '.'
+  while os.path.exists(path):
+    if os.path.exists(os.path.join(path, 'src/google/protobuf')):
+      # Found it.  Load the golden file from the testdata directory.
+      full_path = os.path.join(path, 'src/google/protobuf/testdata', filename)
+      return open(full_path, 'rb')
+    path = os.path.join(path, '..')
+
+  raise RuntimeError(
+    'Could not find golden files.  This test must be run from within the '
+    'protobuf source package so that it can read test data files from the '
+    'C++ source tree.')
+
+
+def SetAllPackedFields(message):
+  """Sets every field in the message to a unique value.
+
+  Args:
+    message: A unittest_pb2.TestPackedTypes instance.
+  """
+  message.packed_int32.extend([601, 701])
+  message.packed_int64.extend([602, 702])
+  message.packed_uint32.extend([603, 703])
+  message.packed_uint64.extend([604, 704])
+  message.packed_sint32.extend([605, 705])
+  message.packed_sint64.extend([606, 706])
+  message.packed_fixed32.extend([607, 707])
+  message.packed_fixed64.extend([608, 708])
+  message.packed_sfixed32.extend([609, 709])
+  message.packed_sfixed64.extend([610, 710])
+  message.packed_float.extend([611.0, 711.0])
+  message.packed_double.extend([612.0, 712.0])
+  message.packed_bool.extend([True, False])
+  message.packed_enum.extend([unittest_pb2.FOREIGN_BAR,
+                              unittest_pb2.FOREIGN_BAZ])
+
+
+def SetAllPackedExtensions(message):
+  """Sets every extension in the message to a unique value.
+
+  Args:
+    message: A unittest_pb2.TestPackedExtensions instance.
+  """
+  extensions = message.Extensions
+  pb2 = unittest_pb2
+
+  extensions[pb2.packed_int32_extension].extend([601, 701])
+  extensions[pb2.packed_int64_extension].extend([602, 702])
+  extensions[pb2.packed_uint32_extension].extend([603, 703])
+  extensions[pb2.packed_uint64_extension].extend([604, 704])
+  extensions[pb2.packed_sint32_extension].extend([605, 705])
+  extensions[pb2.packed_sint64_extension].extend([606, 706])
+  extensions[pb2.packed_fixed32_extension].extend([607, 707])
+  extensions[pb2.packed_fixed64_extension].extend([608, 708])
+  extensions[pb2.packed_sfixed32_extension].extend([609, 709])
+  extensions[pb2.packed_sfixed64_extension].extend([610, 710])
+  extensions[pb2.packed_float_extension].extend([611.0, 711.0])
+  extensions[pb2.packed_double_extension].extend([612.0, 712.0])
+  extensions[pb2.packed_bool_extension].extend([True, False])
+  extensions[pb2.packed_enum_extension].extend([unittest_pb2.FOREIGN_BAR,
+                                                unittest_pb2.FOREIGN_BAZ])
+
+
+def SetAllUnpackedFields(message):
+  """Sets every field in the message to a unique value.
+
+  Args:
+    message: A unittest_pb2.TestUnpackedTypes instance.
+  """
+  message.unpacked_int32.extend([601, 701])
+  message.unpacked_int64.extend([602, 702])
+  message.unpacked_uint32.extend([603, 703])
+  message.unpacked_uint64.extend([604, 704])
+  message.unpacked_sint32.extend([605, 705])
+  message.unpacked_sint64.extend([606, 706])
+  message.unpacked_fixed32.extend([607, 707])
+  message.unpacked_fixed64.extend([608, 708])
+  message.unpacked_sfixed32.extend([609, 709])
+  message.unpacked_sfixed64.extend([610, 710])
+  message.unpacked_float.extend([611.0, 711.0])
+  message.unpacked_double.extend([612.0, 712.0])
+  message.unpacked_bool.extend([True, False])
+  message.unpacked_enum.extend([unittest_pb2.FOREIGN_BAR,
+                                unittest_pb2.FOREIGN_BAZ])

python/google/protobuf/internal/text_format_test.py

@@ -0,0 +1,617 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test for google.protobuf.text_format."""
+
+__author__ = 'kenton@google.com (Kenton Varda)'
+
+import difflib
+import re
+
+import unittest
+from google.protobuf import text_format
+from google.protobuf.internal import test_util
+from google.protobuf import unittest_pb2
+from google.protobuf import unittest_mset_pb2
+
+
+class TextFormatTest(unittest.TestCase):
+  def ReadGolden(self, golden_filename):
+    f = test_util.GoldenFile(golden_filename)
+    golden_lines = f.readlines()
+    f.close()
+    return golden_lines
+
+  def CompareToGoldenFile(self, text, golden_filename):
+    golden_lines = self.ReadGolden(golden_filename)
+    self.CompareToGoldenLines(text, golden_lines)
+
+  def CompareToGoldenText(self, text, golden_text):
+    self.CompareToGoldenLines(text, golden_text.splitlines(1))
+
+  def CompareToGoldenLines(self, text, golden_lines):
+    actual_lines = text.splitlines(1)
+    self.assertEqual(golden_lines, actual_lines,
+      "Text doesn't match golden.  Diff:\n" +
+      ''.join(difflib.ndiff(golden_lines, actual_lines)))
+
+  def testPrintAllFields(self):
+    message = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(message)
+    self.CompareToGoldenFile(
+      self.RemoveRedundantZeros(text_format.MessageToString(message)),
+      'text_format_unittest_data.txt')
+
+  def testPrintAllExtensions(self):
+    message = unittest_pb2.TestAllExtensions()
+    test_util.SetAllExtensions(message)
+    self.CompareToGoldenFile(
+      self.RemoveRedundantZeros(text_format.MessageToString(message)),
+      'text_format_unittest_extensions_data.txt')
+
+  def testPrintMessageSet(self):
+    message = unittest_mset_pb2.TestMessageSetContainer()
+    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
+    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
+    message.message_set.Extensions[ext1].i = 23
+    message.message_set.Extensions[ext2].str = 'foo'
+    self.CompareToGoldenText(text_format.MessageToString(message),
+      'message_set {\n'
+      '  [protobuf_unittest.TestMessageSetExtension1] {\n'
+      '    i: 23\n'
+      '  }\n'
+      '  [protobuf_unittest.TestMessageSetExtension2] {\n'
+      '    str: \"foo\"\n'
+      '  }\n'
+      '}\n')
+
+  def testPrintBadEnumValue(self):
+    message = unittest_pb2.TestAllTypes()
+    message.optional_nested_enum = 100
+    message.optional_foreign_enum = 101
+    message.optional_import_enum = 102
+    self.CompareToGoldenText(
+        text_format.MessageToString(message),
+        'optional_nested_enum: 100\n'
+        'optional_foreign_enum: 101\n'
+        'optional_import_enum: 102\n')
+
+  def testPrintBadEnumValueExtensions(self):
+    message = unittest_pb2.TestAllExtensions()
+    message.Extensions[unittest_pb2.optional_nested_enum_extension] = 100
+    message.Extensions[unittest_pb2.optional_foreign_enum_extension] = 101
+    message.Extensions[unittest_pb2.optional_import_enum_extension] = 102
+    self.CompareToGoldenText(
+        text_format.MessageToString(message),
+        '[protobuf_unittest.optional_nested_enum_extension]: 100\n'
+        '[protobuf_unittest.optional_foreign_enum_extension]: 101\n'
+        '[protobuf_unittest.optional_import_enum_extension]: 102\n')
+
+  def testPrintExotic(self):
+    message = unittest_pb2.TestAllTypes()
+    message.repeated_int64.append(-9223372036854775808)
+    message.repeated_uint64.append(18446744073709551615)
+    message.repeated_double.append(123.456)
+    message.repeated_double.append(1.23e22)
+    message.repeated_double.append(1.23e-18)
+    message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"')
+    message.repeated_string.append(u'\u00fc\ua71f')
+    self.CompareToGoldenText(
+      self.RemoveRedundantZeros(text_format.MessageToString(message)),
+      'repeated_int64: -9223372036854775808\n'
+      'repeated_uint64: 18446744073709551615\n'
+      'repeated_double: 123.456\n'
+      'repeated_double: 1.23e+22\n'
+      'repeated_double: 1.23e-18\n'
+      'repeated_string: '
+        '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n'
+      'repeated_string: "\\303\\274\\352\\234\\237"\n')
+
+  def testPrintNestedMessageAsOneLine(self):
+    message = unittest_pb2.TestAllTypes()
+    msg = message.repeated_nested_message.add()
+    msg.bb = 42;
+    self.CompareToGoldenText(
+        text_format.MessageToString(message, as_one_line=True),
+        'repeated_nested_message { bb: 42 }')
+
+  def testPrintRepeatedFieldsAsOneLine(self):
+    message = unittest_pb2.TestAllTypes()
+    message.repeated_int32.append(1)
+    message.repeated_int32.append(1)
+    message.repeated_int32.append(3)
+    message.repeated_string.append("Google")
+    message.repeated_string.append("Zurich")
+    self.CompareToGoldenText(
+        text_format.MessageToString(message, as_one_line=True),
+        'repeated_int32: 1 repeated_int32: 1 repeated_int32: 3 '
+        'repeated_string: "Google" repeated_string: "Zurich"')
+
+  def testPrintNestedNewLineInStringAsOneLine(self):
+    message = unittest_pb2.TestAllTypes()
+    message.optional_string = "a\nnew\nline"
+    self.CompareToGoldenText(
+        text_format.MessageToString(message, as_one_line=True),
+        'optional_string: "a\\nnew\\nline"')
+
+  def testPrintMessageSetAsOneLine(self):
+    message = unittest_mset_pb2.TestMessageSetContainer()
+    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
+    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
+    message.message_set.Extensions[ext1].i = 23
+    message.message_set.Extensions[ext2].str = 'foo'
+    self.CompareToGoldenText(
+        text_format.MessageToString(message, as_one_line=True),
+        'message_set {'
+        ' [protobuf_unittest.TestMessageSetExtension1] {'
+        ' i: 23'
+        ' }'
+        ' [protobuf_unittest.TestMessageSetExtension2] {'
+        ' str: \"foo\"'
+        ' }'
+        ' }')
+
+  def testPrintExoticAsOneLine(self):
+    message = unittest_pb2.TestAllTypes()
+    message.repeated_int64.append(-9223372036854775808)
+    message.repeated_uint64.append(18446744073709551615)
+    message.repeated_double.append(123.456)
+    message.repeated_double.append(1.23e22)
+    message.repeated_double.append(1.23e-18)
+    message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"')
+    message.repeated_string.append(u'\u00fc\ua71f')
+    self.CompareToGoldenText(
+      self.RemoveRedundantZeros(
+          text_format.MessageToString(message, as_one_line=True)),
+      'repeated_int64: -9223372036854775808'
+      ' repeated_uint64: 18446744073709551615'
+      ' repeated_double: 123.456'
+      ' repeated_double: 1.23e+22'
+      ' repeated_double: 1.23e-18'
+      ' repeated_string: '
+      '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""'
+      ' repeated_string: "\\303\\274\\352\\234\\237"')
+
+  def testRoundTripExoticAsOneLine(self):
+    message = unittest_pb2.TestAllTypes()
+    message.repeated_int64.append(-9223372036854775808)
+    message.repeated_uint64.append(18446744073709551615)
+    message.repeated_double.append(123.456)
+    message.repeated_double.append(1.23e22)
+    message.repeated_double.append(1.23e-18)
+    message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"')
+    message.repeated_string.append(u'\u00fc\ua71f')
+
+    # Test as_utf8 = False.
+    wire_text = text_format.MessageToString(
+        message, as_one_line=True, as_utf8=False)
+    parsed_message = unittest_pb2.TestAllTypes()
+    text_format.Merge(wire_text, parsed_message)
+    self.assertEquals(message, parsed_message)
+
+    # Test as_utf8 = True.
+    wire_text = text_format.MessageToString(
+        message, as_one_line=True, as_utf8=True)
+    parsed_message = unittest_pb2.TestAllTypes()
+    text_format.Merge(wire_text, parsed_message)
+    self.assertEquals(message, parsed_message)
+
+  def testPrintRawUtf8String(self):
+    message = unittest_pb2.TestAllTypes()
+    message.repeated_string.append(u'\u00fc\ua71f')
+    text = text_format.MessageToString(message, as_utf8 = True)
+    self.CompareToGoldenText(text, 'repeated_string: "\303\274\352\234\237"\n')
+    parsed_message = unittest_pb2.TestAllTypes()
+    text_format.Merge(text, parsed_message)
+    self.assertEquals(message, parsed_message)
+
+  def testMessageToString(self):
+    message = unittest_pb2.ForeignMessage()
+    message.c = 123
+    self.assertEqual('c: 123\n', str(message))
+
+  def RemoveRedundantZeros(self, text):
+    # Some platforms print 1e+5 as 1e+005.  This is fine, but we need to remove
+    # these zeros in order to match the golden file.
+    text = text.replace('e+0','e+').replace('e+0','e+') \
+               .replace('e-0','e-').replace('e-0','e-')
+    # Floating point fields are printed with .0 suffix even if they are
+    # actualy integer numbers.
+    text = re.compile('\.0$', re.MULTILINE).sub('', text)
+    return text
+
+  def testMergeGolden(self):
+    golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
+    parsed_message = unittest_pb2.TestAllTypes()
+    text_format.Merge(golden_text, parsed_message)
+
+    message = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(message)
+    self.assertEquals(message, parsed_message)
+
+  def testMergeGoldenExtensions(self):
+    golden_text = '\n'.join(self.ReadGolden(
+        'text_format_unittest_extensions_data.txt'))
+    parsed_message = unittest_pb2.TestAllExtensions()
+    text_format.Merge(golden_text, parsed_message)
+
+    message = unittest_pb2.TestAllExtensions()
+    test_util.SetAllExtensions(message)
+    self.assertEquals(message, parsed_message)
+
+  def testMergeAllFields(self):
+    message = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(message)
+    ascii_text = text_format.MessageToString(message)
+
+    parsed_message = unittest_pb2.TestAllTypes()
+    text_format.Merge(ascii_text, parsed_message)
+    self.assertEqual(message, parsed_message)
+    test_util.ExpectAllFieldsSet(self, message)
+
+  def testMergeAllExtensions(self):
+    message = unittest_pb2.TestAllExtensions()
+    test_util.SetAllExtensions(message)
+    ascii_text = text_format.MessageToString(message)
+
+    parsed_message = unittest_pb2.TestAllExtensions()
+    text_format.Merge(ascii_text, parsed_message)
+    self.assertEqual(message, parsed_message)
+
+  def testMergeMessageSet(self):
+    message = unittest_pb2.TestAllTypes()
+    text = ('repeated_uint64: 1\n'
+            'repeated_uint64: 2\n')
+    text_format.Merge(text, message)
+    self.assertEqual(1, message.repeated_uint64[0])
+    self.assertEqual(2, message.repeated_uint64[1])
+
+    message = unittest_mset_pb2.TestMessageSetContainer()
+    text = ('message_set {\n'
+            '  [protobuf_unittest.TestMessageSetExtension1] {\n'
+            '    i: 23\n'
+            '  }\n'
+            '  [protobuf_unittest.TestMessageSetExtension2] {\n'
+            '    str: \"foo\"\n'
+            '  }\n'
+            '}\n')
+    text_format.Merge(text, message)
+    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
+    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
+    self.assertEquals(23, message.message_set.Extensions[ext1].i)
+    self.assertEquals('foo', message.message_set.Extensions[ext2].str)
+
+  def testMergeExotic(self):
+    message = unittest_pb2.TestAllTypes()
+    text = ('repeated_int64: -9223372036854775808\n'
+            'repeated_uint64: 18446744073709551615\n'
+            'repeated_double: 123.456\n'
+            'repeated_double: 1.23e+22\n'
+            'repeated_double: 1.23e-18\n'
+            'repeated_string: \n'
+            '"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n'
+            'repeated_string: "foo" \'corge\' "grault"\n'
+            'repeated_string: "\\303\\274\\352\\234\\237"\n'
+            'repeated_string: "\\xc3\\xbc"\n'
+            'repeated_string: "\xc3\xbc"\n')
+    text_format.Merge(text, message)
+
+    self.assertEqual(-9223372036854775808, message.repeated_int64[0])
+    self.assertEqual(18446744073709551615, message.repeated_uint64[0])
+    self.assertEqual(123.456, message.repeated_double[0])
+    self.assertEqual(1.23e22, message.repeated_double[1])
+    self.assertEqual(1.23e-18, message.repeated_double[2])
+    self.assertEqual(
+        '\000\001\a\b\f\n\r\t\v\\\'"', message.repeated_string[0])
+    self.assertEqual('foocorgegrault', message.repeated_string[1])
+    self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2])
+    self.assertEqual(u'\u00fc', message.repeated_string[3])
+
+  def testMergeEmptyText(self):
+    message = unittest_pb2.TestAllTypes()
+    text = ''
+    text_format.Merge(text, message)
+    self.assertEquals(unittest_pb2.TestAllTypes(), message)
+
+  def testMergeInvalidUtf8(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'repeated_string: "\\xc3\\xc3"'
+    self.assertRaises(text_format.ParseError, text_format.Merge, text, message)
+
+  def testMergeSingleWord(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'foo'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named '
+         '"foo".'),
+        text_format.Merge, text, message)
+
+  def testMergeUnknownField(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'unknown_field: 8\n'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named '
+         '"unknown_field".'),
+        text_format.Merge, text, message)
+
+  def testMergeBadExtension(self):
+    message = unittest_pb2.TestAllExtensions()
+    text = '[unknown_extension]: 8\n'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        '1:2 : Extension "unknown_extension" not registered.',
+        text_format.Merge, text, message)
+    message = unittest_pb2.TestAllTypes()
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
+         'extensions.'),
+        text_format.Merge, text, message)
+
+  def testMergeGroupNotClosed(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'RepeatedGroup: <'
+    self.assertRaisesWithMessage(
+        text_format.ParseError, '1:16 : Expected ">".',
+        text_format.Merge, text, message)
+
+    text = 'RepeatedGroup: {'
+    self.assertRaisesWithMessage(
+        text_format.ParseError, '1:16 : Expected "}".',
+        text_format.Merge, text, message)
+
+  def testMergeEmptyGroup(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'OptionalGroup: {}'
+    text_format.Merge(text, message)
+    self.assertTrue(message.HasField('optionalgroup'))
+
+    message.Clear()
+
+    message = unittest_pb2.TestAllTypes()
+    text = 'OptionalGroup: <>'
+    text_format.Merge(text, message)
+    self.assertTrue(message.HasField('optionalgroup'))
+
+  def testMergeBadEnumValue(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'optional_nested_enum: BARR'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
+         'has no value named BARR.'),
+        text_format.Merge, text, message)
+
+    message = unittest_pb2.TestAllTypes()
+    text = 'optional_nested_enum: 100'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
+         'has no value with number 100.'),
+        text_format.Merge, text, message)
+
+  def testMergeBadIntValue(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'optional_int32: bork'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:17 : Couldn\'t parse integer: bork'),
+        text_format.Merge, text, message)
+
+  def testMergeStringFieldUnescape(self):
+    message = unittest_pb2.TestAllTypes()
+    text = r'''repeated_string: "\xf\x62"
+               repeated_string: "\\xf\\x62"
+               repeated_string: "\\\xf\\\x62"
+               repeated_string: "\\\\xf\\\\x62"
+               repeated_string: "\\\\\xf\\\\\x62"
+               repeated_string: "\x5cx20"'''
+    text_format.Merge(text, message)
+
+    SLASH = '\\'
+    self.assertEqual('\x0fb', message.repeated_string[0])
+    self.assertEqual(SLASH + 'xf' + SLASH + 'x62', message.repeated_string[1])
+    self.assertEqual(SLASH + '\x0f' + SLASH + 'b', message.repeated_string[2])
+    self.assertEqual(SLASH + SLASH + 'xf' + SLASH + SLASH + 'x62',
+                     message.repeated_string[3])
+    self.assertEqual(SLASH + SLASH + '\x0f' + SLASH + SLASH + 'b',
+                     message.repeated_string[4])
+    self.assertEqual(SLASH + 'x20', message.repeated_string[5])
+
+  def assertRaisesWithMessage(self, e_class, e, func, *args, **kwargs):
+    """Same as assertRaises, but also compares the exception message."""
+    if hasattr(e_class, '__name__'):
+      exc_name = e_class.__name__
+    else:
+      exc_name = str(e_class)
+
+    try:
+      func(*args, **kwargs)
+    except e_class as expr:
+      if str(expr) != e:
+        msg = '%s raised, but with wrong message: "%s" instead of "%s"'
+        raise self.failureException(msg % (exc_name,
+                                           str(expr).encode('string_escape'),
+                                           e.encode('string_escape')))
+      return
+    else:
+      raise self.failureException('%s not raised' % exc_name)
+
+
+class TokenizerTest(unittest.TestCase):
+
+  def testSimpleTokenCases(self):
+    text = ('identifier1:"string1"\n     \n\n'
+            'identifier2 : \n \n123  \n  identifier3 :\'string\'\n'
+            'identifiER_4 : 1.1e+2 ID5:-0.23 ID6:\'aaaa\\\'bbbb\'\n'
+            'ID7 : "aa\\"bb"\n\n\n\n ID8: {A:inf B:-inf C:true D:false}\n'
+            'ID9: 22 ID10: -111111111111111111 ID11: -22\n'
+            'ID12: 2222222222222222222 ID13: 1.23456f ID14: 1.2e+2f '
+            'false_bool:  0 true_BOOL:t \n true_bool1:  1 false_BOOL1:f ' )
+    tokenizer = text_format._Tokenizer(text)
+    methods = [(tokenizer.ConsumeIdentifier, 'identifier1'),
+               ':',
+               (tokenizer.ConsumeString, 'string1'),
+               (tokenizer.ConsumeIdentifier, 'identifier2'),
+               ':',
+               (tokenizer.ConsumeInt32, 123),
+               (tokenizer.ConsumeIdentifier, 'identifier3'),
+               ':',
+               (tokenizer.ConsumeString, 'string'),
+               (tokenizer.ConsumeIdentifier, 'identifiER_4'),
+               ':',
+               (tokenizer.ConsumeFloat, 1.1e+2),
+               (tokenizer.ConsumeIdentifier, 'ID5'),
+               ':',
+               (tokenizer.ConsumeFloat, -0.23),
+               (tokenizer.ConsumeIdentifier, 'ID6'),
+               ':',
+               (tokenizer.ConsumeString, 'aaaa\'bbbb'),
+               (tokenizer.ConsumeIdentifier, 'ID7'),
+               ':',
+               (tokenizer.ConsumeString, 'aa\"bb'),
+               (tokenizer.ConsumeIdentifier, 'ID8'),
+               ':',
+               '{',
+               (tokenizer.ConsumeIdentifier, 'A'),
+               ':',
+               (tokenizer.ConsumeFloat, float('inf')),
+               (tokenizer.ConsumeIdentifier, 'B'),
+               ':',
+               (tokenizer.ConsumeFloat, -float('inf')),
+               (tokenizer.ConsumeIdentifier, 'C'),
+               ':',
+               (tokenizer.ConsumeBool, True),
+               (tokenizer.ConsumeIdentifier, 'D'),
+               ':',
+               (tokenizer.ConsumeBool, False),
+               '}',
+               (tokenizer.ConsumeIdentifier, 'ID9'),
+               ':',
+               (tokenizer.ConsumeUint32, 22),
+               (tokenizer.ConsumeIdentifier, 'ID10'),
+               ':',
+               (tokenizer.ConsumeInt64, -111111111111111111),
+               (tokenizer.ConsumeIdentifier, 'ID11'),
+               ':',
+               (tokenizer.ConsumeInt32, -22),
+               (tokenizer.ConsumeIdentifier, 'ID12'),
+               ':',
+               (tokenizer.ConsumeUint64, 2222222222222222222),
+               (tokenizer.ConsumeIdentifier, 'ID13'),
+               ':',
+               (tokenizer.ConsumeFloat, 1.23456),
+               (tokenizer.ConsumeIdentifier, 'ID14'),
+               ':',
+               (tokenizer.ConsumeFloat, 1.2e+2),
+               (tokenizer.ConsumeIdentifier, 'false_bool'),
+               ':',
+               (tokenizer.ConsumeBool, False),
+               (tokenizer.ConsumeIdentifier, 'true_BOOL'),
+               ':',
+               (tokenizer.ConsumeBool, True),
+               (tokenizer.ConsumeIdentifier, 'true_bool1'),
+               ':',
+               (tokenizer.ConsumeBool, True),
+               (tokenizer.ConsumeIdentifier, 'false_BOOL1'),
+               ':',
+               (tokenizer.ConsumeBool, False)]
+
+    i = 0
+    while not tokenizer.AtEnd():
+      m = methods[i]
+      if type(m) == str:
+        token = tokenizer.token
+        self.assertEqual(token, m)
+        tokenizer.NextToken()
+      else:
+        self.assertEqual(m[1], m[0]())
+      i += 1
+
+  def testConsumeIntegers(self):
+    # This test only tests the failures in the integer parsing methods as well
+    # as the '0' special cases.
+    int64_max = (1 << 63) - 1
+    uint32_max = (1 << 32) - 1
+    text = '-1 %d %d' % (uint32_max + 1, int64_max + 1)
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64)
+    self.assertEqual(-1, tokenizer.ConsumeInt32())
+
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt32)
+    self.assertEqual(uint32_max + 1, tokenizer.ConsumeInt64())
+
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt64)
+    self.assertEqual(int64_max + 1, tokenizer.ConsumeUint64())
+    self.assertTrue(tokenizer.AtEnd())
+
+    text = '-0 -0 0 0'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertEqual(0, tokenizer.ConsumeUint32())
+    self.assertEqual(0, tokenizer.ConsumeUint64())
+    self.assertEqual(0, tokenizer.ConsumeUint32())
+    self.assertEqual(0, tokenizer.ConsumeUint64())
+    self.assertTrue(tokenizer.AtEnd())
+
+  def testConsumeByteString(self):
+    text = '"string1\''
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = 'string1"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = '\n"\\xt"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = '\n"\\"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = '\n"\\x"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+  def testConsumeBool(self):
+    text = 'not-a-bool'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool)
+
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/type_checkers.py

@@ -0,0 +1,286 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Provides type checking routines.
+
+This module defines type checking utilities in the forms of dictionaries:
+
+VALUE_CHECKERS: A dictionary of field types and a value validation object.
+TYPE_TO_BYTE_SIZE_FN: A dictionary with field types and a size computing
+  function.
+TYPE_TO_SERIALIZE_METHOD: A dictionary with field types and serialization
+  function.
+FIELD_TYPE_TO_WIRE_TYPE: A dictionary with field typed and their
+  coresponding wire types.
+TYPE_TO_DESERIALIZE_METHOD: A dictionary with field types and deserialization
+  function.
+"""
+
+__author__ = 'robinson@google.com (Will Robinson)'
+
+from google.protobuf.internal import decoder
+from google.protobuf.internal import encoder
+from google.protobuf.internal import wire_format
+from google.protobuf import descriptor
+
+_FieldDescriptor = descriptor.FieldDescriptor
+
+
+def GetTypeChecker(cpp_type, field_type):
+  """Returns a type checker for a message field of the specified types.
+
+  Args:
+    cpp_type: C++ type of the field (see descriptor.py).
+    field_type: Protocol message field type (see descriptor.py).
+
+  Returns:
+    An instance of TypeChecker which can be used to verify the types
+    of values assigned to a field of the specified type.
+  """
+  if (cpp_type == _FieldDescriptor.CPPTYPE_STRING and
+      field_type == _FieldDescriptor.TYPE_STRING):
+    return UnicodeValueChecker()
+  return _VALUE_CHECKERS[cpp_type]
+
+
+# None of the typecheckers below make any attempt to guard against people
+# subclassing builtin types and doing weird things.  We're not trying to
+# protect against malicious clients here, just people accidentally shooting
+# themselves in the foot in obvious ways.
+
+class TypeChecker(object):
+
+  """Type checker used to catch type errors as early as possible
+  when the client is setting scalar fields in protocol messages.
+  """
+
+  def __init__(self, *acceptable_types):
+    self._acceptable_types = acceptable_types
+
+  def CheckValue(self, proposed_value):
+    if not isinstance(proposed_value, self._acceptable_types):
+      message = ('%.1024r has type %s, but expected one of: %s' %
+                 (proposed_value, type(proposed_value), self._acceptable_types))
+      raise TypeError(message)
+
+
+# IntValueChecker and its subclasses perform integer type-checks
+# and bounds-checks.
+class IntValueChecker(object):
+
+  """Checker used for integer fields.  Performs type-check and range check."""
+
+  def CheckValue(self, proposed_value):
+    if not isinstance(proposed_value, (int, long)):
+      message = ('%.1024r has type %s, but expected one of: %s' %
+                 (proposed_value, type(proposed_value), (int, long)))
+      raise TypeError(message)
+    if not self._MIN <= proposed_value <= self._MAX:
+      raise ValueError('Value out of range: %d' % proposed_value)
+
+
+class UnicodeValueChecker(object):
+
+  """Checker used for string fields."""
+
+  def CheckValue(self, proposed_value):
+    if not isinstance(proposed_value, (str, unicode)):
+      message = ('%.1024r has type %s, but expected one of: %s' %
+                 (proposed_value, type(proposed_value), (str, unicode)))
+      raise TypeError(message)
+
+    # If the value is of type 'str' make sure that it is in 7-bit ASCII
+    # encoding.
+    if isinstance(proposed_value, str):
+      try:
+        unicode(proposed_value, 'ascii')
+      except UnicodeDecodeError:
+        raise ValueError('%.1024r has type str, but isn\'t in 7-bit ASCII '
+                         'encoding. Non-ASCII strings must be converted to '
+                         'unicode objects before being added.' %
+                         (proposed_value))
+
+
+class Int32ValueChecker(IntValueChecker):
+  # We're sure to use ints instead of longs here since comparison may be more
+  # efficient.
+  _MIN = -2147483648
+  _MAX = 2147483647
+
+
+class Uint32ValueChecker(IntValueChecker):
+  _MIN = 0
+  _MAX = (1 << 32) - 1
+
+
+class Int64ValueChecker(IntValueChecker):
+  _MIN = -(1 << 63)
+  _MAX = (1 << 63) - 1
+
+
+class Uint64ValueChecker(IntValueChecker):
+  _MIN = 0
+  _MAX = (1 << 64) - 1
+
+
+# Type-checkers for all scalar CPPTYPEs.
+_VALUE_CHECKERS = {
+    _FieldDescriptor.CPPTYPE_INT32: Int32ValueChecker(),
+    _FieldDescriptor.CPPTYPE_INT64: Int64ValueChecker(),
+    _FieldDescriptor.CPPTYPE_UINT32: Uint32ValueChecker(),
+    _FieldDescriptor.CPPTYPE_UINT64: Uint64ValueChecker(),
+    _FieldDescriptor.CPPTYPE_DOUBLE: TypeChecker(
+        float, int, long),
+    _FieldDescriptor.CPPTYPE_FLOAT: TypeChecker(
+        float, int, long),
+    _FieldDescriptor.CPPTYPE_BOOL: TypeChecker(bool, int),
+    _FieldDescriptor.CPPTYPE_ENUM: Int32ValueChecker(),
+    _FieldDescriptor.CPPTYPE_STRING: TypeChecker(str),
+    }
+
+
+# Map from field type to a function F, such that F(field_num, value)
+# gives the total byte size for a value of the given type.  This
+# byte size includes tag information and any other additional space
+# associated with serializing "value".
+TYPE_TO_BYTE_SIZE_FN = {
+    _FieldDescriptor.TYPE_DOUBLE: wire_format.DoubleByteSize,
+    _FieldDescriptor.TYPE_FLOAT: wire_format.FloatByteSize,
+    _FieldDescriptor.TYPE_INT64: wire_format.Int64ByteSize,
+    _FieldDescriptor.TYPE_UINT64: wire_format.UInt64ByteSize,
+    _FieldDescriptor.TYPE_INT32: wire_format.Int32ByteSize,
+    _FieldDescriptor.TYPE_FIXED64: wire_format.Fixed64ByteSize,
+    _FieldDescriptor.TYPE_FIXED32: wire_format.Fixed32ByteSize,
+    _FieldDescriptor.TYPE_BOOL: wire_format.BoolByteSize,
+    _FieldDescriptor.TYPE_STRING: wire_format.StringByteSize,
+    _FieldDescriptor.TYPE_GROUP: wire_format.GroupByteSize,
+    _FieldDescriptor.TYPE_MESSAGE: wire_format.MessageByteSize,
+    _FieldDescriptor.TYPE_BYTES: wire_format.BytesByteSize,
+    _FieldDescriptor.TYPE_UINT32: wire_format.UInt32ByteSize,
+    _FieldDescriptor.TYPE_ENUM: wire_format.EnumByteSize,
+    _FieldDescriptor.TYPE_SFIXED32: wire_format.SFixed32ByteSize,
+    _FieldDescriptor.TYPE_SFIXED64: wire_format.SFixed64ByteSize,
+    _FieldDescriptor.TYPE_SINT32: wire_format.SInt32ByteSize,
+    _FieldDescriptor.TYPE_SINT64: wire_format.SInt64ByteSize
+    }
+
+
+# Maps from field types to encoder constructors.
+TYPE_TO_ENCODER = {
+    _FieldDescriptor.TYPE_DOUBLE: encoder.DoubleEncoder,
+    _FieldDescriptor.TYPE_FLOAT: encoder.FloatEncoder,
+    _FieldDescriptor.TYPE_INT64: encoder.Int64Encoder,
+    _FieldDescriptor.TYPE_UINT64: encoder.UInt64Encoder,
+    _FieldDescriptor.TYPE_INT32: encoder.Int32Encoder,
+    _FieldDescriptor.TYPE_FIXED64: encoder.Fixed64Encoder,
+    _FieldDescriptor.TYPE_FIXED32: encoder.Fixed32Encoder,
+    _FieldDescriptor.TYPE_BOOL: encoder.BoolEncoder,
+    _FieldDescriptor.TYPE_STRING: encoder.StringEncoder,
+    _FieldDescriptor.TYPE_GROUP: encoder.GroupEncoder,
+    _FieldDescriptor.TYPE_MESSAGE: encoder.MessageEncoder,
+    _FieldDescriptor.TYPE_BYTES: encoder.BytesEncoder,
+    _FieldDescriptor.TYPE_UINT32: encoder.UInt32Encoder,
+    _FieldDescriptor.TYPE_ENUM: encoder.EnumEncoder,
+    _FieldDescriptor.TYPE_SFIXED32: encoder.SFixed32Encoder,
+    _FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Encoder,
+    _FieldDescriptor.TYPE_SINT32: encoder.SInt32Encoder,
+    _FieldDescriptor.TYPE_SINT64: encoder.SInt64Encoder,
+    }
+
+
+# Maps from field types to sizer constructors.
+TYPE_TO_SIZER = {
+    _FieldDescriptor.TYPE_DOUBLE: encoder.DoubleSizer,
+    _FieldDescriptor.TYPE_FLOAT: encoder.FloatSizer,
+    _FieldDescriptor.TYPE_INT64: encoder.Int64Sizer,
+    _FieldDescriptor.TYPE_UINT64: encoder.UInt64Sizer,
+    _FieldDescriptor.TYPE_INT32: encoder.Int32Sizer,
+    _FieldDescriptor.TYPE_FIXED64: encoder.Fixed64Sizer,
+    _FieldDescriptor.TYPE_FIXED32: encoder.Fixed32Sizer,
+    _FieldDescriptor.TYPE_BOOL: encoder.BoolSizer,
+    _FieldDescriptor.TYPE_STRING: encoder.StringSizer,
+    _FieldDescriptor.TYPE_GROUP: encoder.GroupSizer,
+    _FieldDescriptor.TYPE_MESSAGE: encoder.MessageSizer,
+    _FieldDescriptor.TYPE_BYTES: encoder.BytesSizer,
+    _FieldDescriptor.TYPE_UINT32: encoder.UInt32Sizer,
+    _FieldDescriptor.TYPE_ENUM: encoder.EnumSizer,
+    _FieldDescriptor.TYPE_SFIXED32: encoder.SFixed32Sizer,
+    _FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Sizer,
+    _FieldDescriptor.TYPE_SINT32: encoder.SInt32Sizer,
+    _FieldDescriptor.TYPE_SINT64: encoder.SInt64Sizer,
+    }
+
+
+# Maps from field type to a decoder constructor.
+TYPE_TO_DECODER = {
+    _FieldDescriptor.TYPE_DOUBLE: decoder.DoubleDecoder,
+    _FieldDescriptor.TYPE_FLOAT: decoder.FloatDecoder,
+    _FieldDescriptor.TYPE_INT64: decoder.Int64Decoder,
+    _FieldDescriptor.TYPE_UINT64: decoder.UInt64Decoder,
+    _FieldDescriptor.TYPE_INT32: decoder.Int32Decoder,
+    _FieldDescriptor.TYPE_FIXED64: decoder.Fixed64Decoder,
+    _FieldDescriptor.TYPE_FIXED32: decoder.Fixed32Decoder,
+    _FieldDescriptor.TYPE_BOOL: decoder.BoolDecoder,
+    _FieldDescriptor.TYPE_STRING: decoder.StringDecoder,
+    _FieldDescriptor.TYPE_GROUP: decoder.GroupDecoder,
+    _FieldDescriptor.TYPE_MESSAGE: decoder.MessageDecoder,
+    _FieldDescriptor.TYPE_BYTES: decoder.BytesDecoder,
+    _FieldDescriptor.TYPE_UINT32: decoder.UInt32Decoder,
+    _FieldDescriptor.TYPE_ENUM: decoder.EnumDecoder,
+    _FieldDescriptor.TYPE_SFIXED32: decoder.SFixed32Decoder,
+    _FieldDescriptor.TYPE_SFIXED64: decoder.SFixed64Decoder,
+    _FieldDescriptor.TYPE_SINT32: decoder.SInt32Decoder,
+    _FieldDescriptor.TYPE_SINT64: decoder.SInt64Decoder,
+    }
+
+# Maps from field type to expected wiretype.
+FIELD_TYPE_TO_WIRE_TYPE = {
+    _FieldDescriptor.TYPE_DOUBLE: wire_format.WIRETYPE_FIXED64,
+    _FieldDescriptor.TYPE_FLOAT: wire_format.WIRETYPE_FIXED32,
+    _FieldDescriptor.TYPE_INT64: wire_format.WIRETYPE_VARINT,
+    _FieldDescriptor.TYPE_UINT64: wire_format.WIRETYPE_VARINT,
+    _FieldDescriptor.TYPE_INT32: wire_format.WIRETYPE_VARINT,
+    _FieldDescriptor.TYPE_FIXED64: wire_format.WIRETYPE_FIXED64,
+    _FieldDescriptor.TYPE_FIXED32: wire_format.WIRETYPE_FIXED32,
+    _FieldDescriptor.TYPE_BOOL: wire_format.WIRETYPE_VARINT,
+    _FieldDescriptor.TYPE_STRING:
+      wire_format.WIRETYPE_LENGTH_DELIMITED,
+    _FieldDescriptor.TYPE_GROUP: wire_format.WIRETYPE_START_GROUP,
+    _FieldDescriptor.TYPE_MESSAGE:
+      wire_format.WIRETYPE_LENGTH_DELIMITED,
+    _FieldDescriptor.TYPE_BYTES:
+      wire_format.WIRETYPE_LENGTH_DELIMITED,
+    _FieldDescriptor.TYPE_UINT32: wire_format.WIRETYPE_VARINT,
+    _FieldDescriptor.TYPE_ENUM: wire_format.WIRETYPE_VARINT,
+    _FieldDescriptor.TYPE_SFIXED32: wire_format.WIRETYPE_FIXED32,
+    _FieldDescriptor.TYPE_SFIXED64: wire_format.WIRETYPE_FIXED64,
+    _FieldDescriptor.TYPE_SINT32: wire_format.WIRETYPE_VARINT,
+    _FieldDescriptor.TYPE_SINT64: wire_format.WIRETYPE_VARINT,
+    }

python/google/protobuf/internal/unknown_fields_test.py

@@ -0,0 +1,170 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test for preservation of unknown fields in the pure Python implementation."""
+
+__author__ = 'bohdank@google.com (Bohdan Koval)'
+
+import unittest
+from google.protobuf import unittest_mset_pb2
+from google.protobuf import unittest_pb2
+from google.protobuf.internal import encoder
+from google.protobuf.internal import test_util
+from google.protobuf.internal import type_checkers
+
+
+class UnknownFieldsTest(unittest.TestCase):
+
+  def setUp(self):
+    self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
+    self.all_fields = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(self.all_fields)
+    self.all_fields_data = self.all_fields.SerializeToString()
+    self.empty_message = unittest_pb2.TestEmptyMessage()
+    self.empty_message.ParseFromString(self.all_fields_data)
+    self.unknown_fields = self.empty_message._unknown_fields
+
+  def GetField(self, name):
+    field_descriptor = self.descriptor.fields_by_name[name]
+    wire_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type]
+    field_tag = encoder.TagBytes(field_descriptor.number, wire_type)
+    for tag_bytes, value in self.unknown_fields:
+      if tag_bytes == field_tag:
+        decoder = unittest_pb2.TestAllTypes._decoders_by_tag[tag_bytes]
+        result_dict = {}
+        decoder(value, 0, len(value), self.all_fields, result_dict)
+        return result_dict[field_descriptor]
+
+  def testVarint(self):
+    value = self.GetField('optional_int32')
+    self.assertEqual(self.all_fields.optional_int32, value)
+
+  def testFixed32(self):
+    value = self.GetField('optional_fixed32')
+    self.assertEqual(self.all_fields.optional_fixed32, value)
+
+  def testFixed64(self):
+    value = self.GetField('optional_fixed64')
+    self.assertEqual(self.all_fields.optional_fixed64, value)
+
+  def testLengthDelimited(self):
+    value = self.GetField('optional_string')
+    self.assertEqual(self.all_fields.optional_string, value)
+
+  def testGroup(self):
+    value = self.GetField('optionalgroup')
+    self.assertEqual(self.all_fields.optionalgroup, value)
+
+  def testSerialize(self):
+    data = self.empty_message.SerializeToString()
+
+    # Don't use assertEqual because we don't want to dump raw binary data to
+    # stdout.
+    self.assertTrue(data == self.all_fields_data)
+
+  def testCopyFrom(self):
+    message = unittest_pb2.TestEmptyMessage()
+    message.CopyFrom(self.empty_message)
+    self.assertEqual(self.unknown_fields, message._unknown_fields)
+
+  def testMergeFrom(self):
+    message = unittest_pb2.TestAllTypes()
+    message.optional_int32 = 1
+    message.optional_uint32 = 2
+    source = unittest_pb2.TestEmptyMessage()
+    source.ParseFromString(message.SerializeToString())
+
+    message.ClearField('optional_int32')
+    message.optional_int64 = 3
+    message.optional_uint32 = 4
+    destination = unittest_pb2.TestEmptyMessage()
+    destination.ParseFromString(message.SerializeToString())
+    unknown_fields = destination._unknown_fields[:]
+
+    destination.MergeFrom(source)
+    self.assertEqual(unknown_fields + source._unknown_fields,
+                     destination._unknown_fields)
+
+  def testClear(self):
+    self.empty_message.Clear()
+    self.assertEqual(0, len(self.empty_message._unknown_fields))
+
+  def testByteSize(self):
+    self.assertEqual(self.all_fields.ByteSize(), self.empty_message.ByteSize())
+
+  def testUnknownExtensions(self):
+    message = unittest_pb2.TestEmptyMessageWithExtensions()
+    message.ParseFromString(self.all_fields_data)
+    self.assertEqual(self.empty_message._unknown_fields,
+                     message._unknown_fields)
+
+  def testListFields(self):
+    # Make sure ListFields doesn't return unknown fields.
+    self.assertEqual(0, len(self.empty_message.ListFields()))
+
+  def testSerializeMessageSetWireFormatUnknownExtension(self):
+    # Create a message using the message set wire format with an unknown
+    # message.
+    raw = unittest_mset_pb2.RawMessageSet()
+
+    # Add an unknown extension.
+    item = raw.item.add()
+    item.type_id = 1545009
+    message1 = unittest_mset_pb2.TestMessageSetExtension1()
+    message1.i = 12345
+    item.message = message1.SerializeToString()
+
+    serialized = raw.SerializeToString()
+
+    # Parse message using the message set wire format.
+    proto = unittest_mset_pb2.TestMessageSet()
+    proto.MergeFromString(serialized)
+
+    # Verify that the unknown extension is serialized unchanged
+    reserialized = proto.SerializeToString()
+    new_raw = unittest_mset_pb2.RawMessageSet()
+    new_raw.MergeFromString(reserialized)
+    self.assertEqual(raw, new_raw)
+
+  def testEquals(self):
+    message = unittest_pb2.TestEmptyMessage()
+    message.ParseFromString(self.all_fields_data)
+    self.assertEqual(self.empty_message, message)
+
+    self.all_fields.ClearField('optional_string')
+    message.ParseFromString(self.all_fields.SerializeToString())
+    self.assertNotEqual(self.empty_message, message)
+
+
+if __name__ == '__main__':
+  unittest.main()

python/google/protobuf/internal/wire_format.py

@@ -0,0 +1,268 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Constants and static functions to support protocol buffer wire format."""
+
+__author__ = 'robinson@google.com (Will Robinson)'
+
+import struct
+from google.protobuf import descriptor
+from google.protobuf import message
+
+
+TAG_TYPE_BITS = 3  # Number of bits used to hold type info in a proto tag.
+TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1  # 0x7
+
+# These numbers identify the wire type of a protocol buffer value.
+# We use the least-significant TAG_TYPE_BITS bits of the varint-encoded
+# tag-and-type to store one of these WIRETYPE_* constants.
+# These values must match WireType enum in google/protobuf/wire_format.h.
+WIRETYPE_VARINT = 0
+WIRETYPE_FIXED64 = 1
+WIRETYPE_LENGTH_DELIMITED = 2
+WIRETYPE_START_GROUP = 3
+WIRETYPE_END_GROUP = 4
+WIRETYPE_FIXED32 = 5
+_WIRETYPE_MAX = 5
+
+
+# Bounds for various integer types.
+INT32_MAX = int((1 << 31) - 1)
+INT32_MIN = int(-(1 << 31))
+UINT32_MAX = (1 << 32) - 1
+
+INT64_MAX = (1 << 63) - 1
+INT64_MIN = -(1 << 63)
+UINT64_MAX = (1 << 64) - 1
+
+# "struct" format strings that will encode/decode the specified formats.
+FORMAT_UINT32_LITTLE_ENDIAN = '<I'
+FORMAT_UINT64_LITTLE_ENDIAN = '<Q'
+FORMAT_FLOAT_LITTLE_ENDIAN = '<f'
+FORMAT_DOUBLE_LITTLE_ENDIAN = '<d'
+
+
+# We'll have to provide alternate implementations of AppendLittleEndian*() on
+# any architectures where these checks fail.
+if struct.calcsize(FORMAT_UINT32_LITTLE_ENDIAN) != 4:
+  raise AssertionError('Format "I" is not a 32-bit number.')
+if struct.calcsize(FORMAT_UINT64_LITTLE_ENDIAN) != 8:
+  raise AssertionError('Format "Q" is not a 64-bit number.')
+
+
+def PackTag(field_number, wire_type):
+  """Returns an unsigned 32-bit integer that encodes the field number and
+  wire type information in standard protocol message wire format.
+
+  Args:
+    field_number: Expected to be an integer in the range [1, 1 << 29)
+    wire_type: One of the WIRETYPE_* constants.
+  """
+  if not 0 <= wire_type <= _WIRETYPE_MAX:
+    raise message.EncodeError('Unknown wire type: %d' % wire_type)
+  return (field_number << TAG_TYPE_BITS) | wire_type
+
+
+def UnpackTag(tag):
+  """The inverse of PackTag().  Given an unsigned 32-bit number,
+  returns a (field_number, wire_type) tuple.
+  """
+  return (tag >> TAG_TYPE_BITS), (tag & TAG_TYPE_MASK)
+
+
+def ZigZagEncode(value):
+  """ZigZag Transform:  Encodes signed integers so that they can be
+  effectively used with varint encoding.  See wire_format.h for
+  more details.
+  """
+  if value >= 0:
+    return value << 1
+  return (value << 1) ^ (~0)
+
+
+def ZigZagDecode(value):
+  """Inverse of ZigZagEncode()."""
+  if not value & 0x1:
+    return value >> 1
+  return (value >> 1) ^ (~0)
+
+
+
+# The *ByteSize() functions below return the number of bytes required to
+# serialize "field number + type" information and then serialize the value.
+
+
+def Int32ByteSize(field_number, int32):
+  return Int64ByteSize(field_number, int32)
+
+
+def Int32ByteSizeNoTag(int32):
+  return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32)
+
+
+def Int64ByteSize(field_number, int64):
+  # Have to convert to uint before calling UInt64ByteSize().
+  return UInt64ByteSize(field_number, 0xffffffffffffffff & int64)
+
+
+def UInt32ByteSize(field_number, uint32):
+  return UInt64ByteSize(field_number, uint32)
+
+
+def UInt64ByteSize(field_number, uint64):
+  return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64)
+
+
+def SInt32ByteSize(field_number, int32):
+  return UInt32ByteSize(field_number, ZigZagEncode(int32))
+
+
+def SInt64ByteSize(field_number, int64):
+  return UInt64ByteSize(field_number, ZigZagEncode(int64))
+
+
+def Fixed32ByteSize(field_number, fixed32):
+  return TagByteSize(field_number) + 4
+
+
+def Fixed64ByteSize(field_number, fixed64):
+  return TagByteSize(field_number) + 8
+
+
+def SFixed32ByteSize(field_number, sfixed32):
+  return TagByteSize(field_number) + 4
+
+
+def SFixed64ByteSize(field_number, sfixed64):
+  return TagByteSize(field_number) + 8
+
+
+def FloatByteSize(field_number, flt):
+  return TagByteSize(field_number) + 4
+
+
+def DoubleByteSize(field_number, double):
+  return TagByteSize(field_number) + 8
+
+
+def BoolByteSize(field_number, b):
+  return TagByteSize(field_number) + 1
+
+
+def EnumByteSize(field_number, enum):
+  return UInt32ByteSize(field_number, enum)
+
+
+def StringByteSize(field_number, string):
+  return BytesByteSize(field_number, string.encode('utf-8'))
+
+
+def BytesByteSize(field_number, b):
+  return (TagByteSize(field_number)
+          + _VarUInt64ByteSizeNoTag(len(b))
+          + len(b))
+
+
+def GroupByteSize(field_number, message):
+  return (2 * TagByteSize(field_number)  # START and END group.
+          + message.ByteSize())
+
+
+def MessageByteSize(field_number, message):
+  return (TagByteSize(field_number)
+          + _VarUInt64ByteSizeNoTag(message.ByteSize())
+          + message.ByteSize())
+
+
+def MessageSetItemByteSize(field_number, msg):
+  # First compute the sizes of the tags.
+  # There are 2 tags for the beginning and ending of the repeated group, that
+  # is field number 1, one with field number 2 (type_id) and one with field
+  # number 3 (message).
+  total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3))
+
+  # Add the number of bytes for type_id.
+  total_size += _VarUInt64ByteSizeNoTag(field_number)
+
+  message_size = msg.ByteSize()
+
+  # The number of bytes for encoding the length of the message.
+  total_size += _VarUInt64ByteSizeNoTag(message_size)
+
+  # The size of the message.
+  total_size += message_size
+  return total_size
+
+
+def TagByteSize(field_number):
+  """Returns the bytes required to serialize a tag with this field number."""
+  # Just pass in type 0, since the type won't affect the tag+type size.
+  return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0))
+
+
+# Private helper function for the *ByteSize() functions above.
+
+def _VarUInt64ByteSizeNoTag(uint64):
+  """Returns the number of bytes required to serialize a single varint
+  using boundary value comparisons. (unrolled loop optimization -WPierce)
+  uint64 must be unsigned.
+  """
+  if uint64 <= 0x7f: return 1
+  if uint64 <= 0x3fff: return 2
+  if uint64 <= 0x1fffff: return 3
+  if uint64 <= 0xfffffff: return 4
+  if uint64 <= 0x7ffffffff: return 5
+  if uint64 <= 0x3ffffffffff: return 6
+  if uint64 <= 0x1ffffffffffff: return 7
+  if uint64 <= 0xffffffffffffff: return 8
+  if uint64 <= 0x7fffffffffffffff: return 9
+  if uint64 > UINT64_MAX:
+    raise message.EncodeError('Value out of range: %d' % uint64)
+  return 10
+
+
+NON_PACKABLE_TYPES = (
+  descriptor.FieldDescriptor.TYPE_STRING,
+  descriptor.FieldDescriptor.TYPE_GROUP,
+  descriptor.FieldDescriptor.TYPE_MESSAGE,
+  descriptor.FieldDescriptor.TYPE_BYTES
+)
+
+
+def IsTypePackable(field_type):
+  """Return true iff packable = true is valid for fields of this type.
+
+  Args:
+    field_type: a FieldDescriptor::Type value.
+
+  Returns:
+    True iff fields of this type are packable.
+  """
+  return field_type not in NON_PACKABLE_TYPES

python/google/protobuf/internal/wire_format_test.py

@@ -0,0 +1,253 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test for google.protobuf.internal.wire_format."""
+
+__author__ = 'robinson@google.com (Will Robinson)'
+
+import unittest
+from google.protobuf import message
+from google.protobuf.internal import wire_format
+
+
+class WireFormatTest(unittest.TestCase):
+
+  def testPackTag(self):
+    field_number = 0xabc
+    tag_type = 2
+    self.assertEqual((field_number << 3) | tag_type,
+                     wire_format.PackTag(field_number, tag_type))
+    PackTag = wire_format.PackTag
+    # Number too high.
+    self.assertRaises(message.EncodeError, PackTag, field_number, 6)
+    # Number too low.
+    self.assertRaises(message.EncodeError, PackTag, field_number, -1)
+
+  def testUnpackTag(self):
+    # Test field numbers that will require various varint sizes.
+    for expected_field_number in (1, 15, 16, 2047, 2048):
+      for expected_wire_type in range(6):  # Highest-numbered wiretype is 5.
+        field_number, wire_type = wire_format.UnpackTag(
+            wire_format.PackTag(expected_field_number, expected_wire_type))
+        self.assertEqual(expected_field_number, field_number)
+        self.assertEqual(expected_wire_type, wire_type)
+
+    self.assertRaises(TypeError, wire_format.UnpackTag, None)
+    self.assertRaises(TypeError, wire_format.UnpackTag, 'abc')
+    self.assertRaises(TypeError, wire_format.UnpackTag, 0.0)
+    self.assertRaises(TypeError, wire_format.UnpackTag, object())
+
+  def testZigZagEncode(self):
+    Z = wire_format.ZigZagEncode
+    self.assertEqual(0, Z(0))
+    self.assertEqual(1, Z(-1))
+    self.assertEqual(2, Z(1))
+    self.assertEqual(3, Z(-2))
+    self.assertEqual(4, Z(2))
+    self.assertEqual(0xfffffffe, Z(0x7fffffff))
+    self.assertEqual(0xffffffff, Z(-0x80000000))
+    self.assertEqual(0xfffffffffffffffe, Z(0x7fffffffffffffff))
+    self.assertEqual(0xffffffffffffffff, Z(-0x8000000000000000))
+
+    self.assertRaises(TypeError, Z, None)
+    self.assertRaises(TypeError, Z, 'abcd')
+    self.assertRaises(TypeError, Z, 0.0)
+    self.assertRaises(TypeError, Z, object())
+
+  def testZigZagDecode(self):
+    Z = wire_format.ZigZagDecode
+    self.assertEqual(0, Z(0))
+    self.assertEqual(-1, Z(1))
+    self.assertEqual(1, Z(2))
+    self.assertEqual(-2, Z(3))
+    self.assertEqual(2, Z(4))
+    self.assertEqual(0x7fffffff, Z(0xfffffffe))
+    self.assertEqual(-0x80000000, Z(0xffffffff))
+    self.assertEqual(0x7fffffffffffffff, Z(0xfffffffffffffffe))
+    self.assertEqual(-0x8000000000000000, Z(0xffffffffffffffff))
+
+    self.assertRaises(TypeError, Z, None)
+    self.assertRaises(TypeError, Z, 'abcd')
+    self.assertRaises(TypeError, Z, 0.0)
+    self.assertRaises(TypeError, Z, object())
+
+  def NumericByteSizeTestHelper(self, byte_size_fn, value, expected_value_size):
+    # Use field numbers that cause various byte sizes for the tag information.
+    for field_number, tag_bytes in ((15, 1), (16, 2), (2047, 2), (2048, 3)):
+      expected_size = expected_value_size + tag_bytes
+      actual_size = byte_size_fn(field_number, value)
+      self.assertEqual(expected_size, actual_size,
+                       'byte_size_fn: %s, field_number: %d, value: %r\n'
+                       'Expected: %d, Actual: %d'% (
+          byte_size_fn, field_number, value, expected_size, actual_size))
+
+  def testByteSizeFunctions(self):
+    # Test all numeric *ByteSize() functions.
+    NUMERIC_ARGS = [
+        # Int32ByteSize().
+        [wire_format.Int32ByteSize, 0, 1],
+        [wire_format.Int32ByteSize, 127, 1],
+        [wire_format.Int32ByteSize, 128, 2],
+        [wire_format.Int32ByteSize, -1, 10],
+        # Int64ByteSize().
+        [wire_format.Int64ByteSize, 0, 1],
+        [wire_format.Int64ByteSize, 127, 1],
+        [wire_format.Int64ByteSize, 128, 2],
+        [wire_format.Int64ByteSize, -1, 10],
+        # UInt32ByteSize().
+        [wire_format.UInt32ByteSize, 0, 1],
+        [wire_format.UInt32ByteSize, 127, 1],
+        [wire_format.UInt32ByteSize, 128, 2],
+        [wire_format.UInt32ByteSize, wire_format.UINT32_MAX, 5],
+        # UInt64ByteSize().
+        [wire_format.UInt64ByteSize, 0, 1],
+        [wire_format.UInt64ByteSize, 127, 1],
+        [wire_format.UInt64ByteSize, 128, 2],
+        [wire_format.UInt64ByteSize, wire_format.UINT64_MAX, 10],
+        # SInt32ByteSize().
+        [wire_format.SInt32ByteSize, 0, 1],
+        [wire_format.SInt32ByteSize, -1, 1],
+        [wire_format.SInt32ByteSize, 1, 1],
+        [wire_format.SInt32ByteSize, -63, 1],
+        [wire_format.SInt32ByteSize, 63, 1],
+        [wire_format.SInt32ByteSize, -64, 1],
+        [wire_format.SInt32ByteSize, 64, 2],
+        # SInt64ByteSize().
+        [wire_format.SInt64ByteSize, 0, 1],
+        [wire_format.SInt64ByteSize, -1, 1],
+        [wire_format.SInt64ByteSize, 1, 1],
+        [wire_format.SInt64ByteSize, -63, 1],
+        [wire_format.SInt64ByteSize, 63, 1],
+        [wire_format.SInt64ByteSize, -64, 1],
+        [wire_format.SInt64ByteSize, 64, 2],
+        # Fixed32ByteSize().
+        [wire_format.Fixed32ByteSize, 0, 4],
+        [wire_format.Fixed32ByteSize, wire_format.UINT32_MAX, 4],
+        # Fixed64ByteSize().
+        [wire_format.Fixed64ByteSize, 0, 8],
+        [wire_format.Fixed64ByteSize, wire_format.UINT64_MAX, 8],
+        # SFixed32ByteSize().
+        [wire_format.SFixed32ByteSize, 0, 4],
+        [wire_format.SFixed32ByteSize, wire_format.INT32_MIN, 4],
+        [wire_format.SFixed32ByteSize, wire_format.INT32_MAX, 4],
+        # SFixed64ByteSize().
+        [wire_format.SFixed64ByteSize, 0, 8],
+        [wire_format.SFixed64ByteSize, wire_format.INT64_MIN, 8],
+        [wire_format.SFixed64ByteSize, wire_format.INT64_MAX, 8],
+        # FloatByteSize().
+        [wire_format.FloatByteSize, 0.0, 4],
+        [wire_format.FloatByteSize, 1000000000.0, 4],
+        [wire_format.FloatByteSize, -1000000000.0, 4],
+        # DoubleByteSize().
+        [wire_format.DoubleByteSize, 0.0, 8],
+        [wire_format.DoubleByteSize, 1000000000.0, 8],
+        [wire_format.DoubleByteSize, -1000000000.0, 8],
+        # BoolByteSize().
+        [wire_format.BoolByteSize, False, 1],
+        [wire_format.BoolByteSize, True, 1],
+        # EnumByteSize().
+        [wire_format.EnumByteSize, 0, 1],
+        [wire_format.EnumByteSize, 127, 1],
+        [wire_format.EnumByteSize, 128, 2],
+        [wire_format.EnumByteSize, wire_format.UINT32_MAX, 5],
+        ]
+    for args in NUMERIC_ARGS:
+      self.NumericByteSizeTestHelper(*args)
+
+    # Test strings and bytes.
+    for byte_size_fn in (wire_format.StringByteSize, wire_format.BytesByteSize):
+      # 1 byte for tag, 1 byte for length, 3 bytes for contents.
+      self.assertEqual(5, byte_size_fn(10, 'abc'))
+      # 2 bytes for tag, 1 byte for length, 3 bytes for contents.
+      self.assertEqual(6, byte_size_fn(16, 'abc'))
+      # 2 bytes for tag, 2 bytes for length, 128 bytes for contents.
+      self.assertEqual(132, byte_size_fn(16, 'a' * 128))
+
+    # Test UTF-8 string byte size calculation.
+    # 1 byte for tag, 1 byte for length, 8 bytes for content.
+    self.assertEqual(10, wire_format.StringByteSize(
+        5, unicode('\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82', 'utf-8')))
+
+    class MockMessage(object):
+      def __init__(self, byte_size):
+        self.byte_size = byte_size
+      def ByteSize(self):
+        return self.byte_size
+
+    message_byte_size = 10
+    mock_message = MockMessage(byte_size=message_byte_size)
+    # Test groups.
+    # (2 * 1) bytes for begin and end tags, plus message_byte_size.
+    self.assertEqual(2 + message_byte_size,
+                     wire_format.GroupByteSize(1, mock_message))
+    # (2 * 2) bytes for begin and end tags, plus message_byte_size.
+    self.assertEqual(4 + message_byte_size,
+                     wire_format.GroupByteSize(16, mock_message))
+
+    # Test messages.
+    # 1 byte for tag, plus 1 byte for length, plus contents.
+    self.assertEqual(2 + mock_message.byte_size,
+                     wire_format.MessageByteSize(1, mock_message))
+    # 2 bytes for tag, plus 1 byte for length, plus contents.
+    self.assertEqual(3 + mock_message.byte_size,
+                     wire_format.MessageByteSize(16, mock_message))
+    # 2 bytes for tag, plus 2 bytes for length, plus contents.
+    mock_message.byte_size = 128
+    self.assertEqual(4 + mock_message.byte_size,
+                     wire_format.MessageByteSize(16, mock_message))
+
+
+    # Test message set item byte size.
+    # 4 bytes for tags, plus 1 byte for length, plus 1 byte for type_id,
+    # plus contents.
+    mock_message.byte_size = 10
+    self.assertEqual(mock_message.byte_size + 6,
+                     wire_format.MessageSetItemByteSize(1, mock_message))
+
+    # 4 bytes for tags, plus 2 bytes for length, plus 1 byte for type_id,
+    # plus contents.
+    mock_message.byte_size = 128
+    self.assertEqual(mock_message.byte_size + 7,
+                     wire_format.MessageSetItemByteSize(1, mock_message))
+
+    # 4 bytes for tags, plus 2 bytes for length, plus 2 byte for type_id,
+    # plus contents.
+    self.assertEqual(mock_message.byte_size + 8,
+                     wire_format.MessageSetItemByteSize(128, mock_message))
+
+    # Too-long varint.
+    self.assertRaises(message.EncodeError,
+                      wire_format.UInt64ByteSize, 1, 1 << 128)
+
+
+if __name__ == '__main__':
+  unittest.main()