text_format.py 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884
  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. # https://developers.google.com/protocol-buffers/
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. # * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. # * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. #PY25 compatible for GAE.
  31. #
  32. # Copyright 2007 Google Inc. All Rights Reserved.
  33. """Contains routines for printing protocol messages in text format."""
  34. __author__ = 'kenton@google.com (Kenton Varda)'
  35. import cStringIO
  36. import re
  37. from google.protobuf.internal import type_checkers
  38. from google.protobuf import descriptor
  39. from google.protobuf import text_encoding
  40. __all__ = ['MessageToString', 'PrintMessage', 'PrintField',
  41. 'PrintFieldValue', 'Merge']
  42. _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
  43. type_checkers.Int32ValueChecker(),
  44. type_checkers.Uint64ValueChecker(),
  45. type_checkers.Int64ValueChecker())
  46. _FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE)
  47. _FLOAT_NAN = re.compile('nanf?', re.IGNORECASE)
  48. _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
  49. descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
  50. class Error(Exception):
  51. """Top-level module error for text_format."""
  52. class ParseError(Error):
  53. """Thrown in case of ASCII parsing error."""
  54. def MessageToString(message, as_utf8=False, as_one_line=False,
  55. pointy_brackets=False, use_index_order=False,
  56. float_format=None):
  57. """Convert protobuf message to text format.
  58. Floating point values can be formatted compactly with 15 digits of
  59. precision (which is the most that IEEE 754 "double" can guarantee)
  60. using float_format='.15g'.
  61. Args:
  62. message: The protocol buffers message.
  63. as_utf8: Produce text output in UTF8 format.
  64. as_one_line: Don't introduce newlines between fields.
  65. pointy_brackets: If True, use angle brackets instead of curly braces for
  66. nesting.
  67. use_index_order: If True, print fields of a proto message using the order
  68. defined in source code instead of the field number. By default, use the
  69. field number order.
  70. float_format: If set, use this to specify floating point number formatting
  71. (per the "Format Specification Mini-Language"); otherwise, str() is used.
  72. Returns:
  73. A string of the text formatted protocol buffer message.
  74. """
  75. out = cStringIO.StringIO()
  76. PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line,
  77. pointy_brackets=pointy_brackets,
  78. use_index_order=use_index_order,
  79. float_format=float_format)
  80. result = out.getvalue()
  81. out.close()
  82. if as_one_line:
  83. return result.rstrip()
  84. return result
  85. def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False,
  86. pointy_brackets=False, use_index_order=False,
  87. float_format=None):
  88. fields = message.ListFields()
  89. if use_index_order:
  90. fields.sort(key=lambda x: x[0].index)
  91. for field, value in fields:
  92. if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  93. for element in value:
  94. PrintField(field, element, out, indent, as_utf8, as_one_line,
  95. pointy_brackets=pointy_brackets,
  96. use_index_order=use_index_order,
  97. float_format=float_format)
  98. else:
  99. PrintField(field, value, out, indent, as_utf8, as_one_line,
  100. pointy_brackets=pointy_brackets,
  101. use_index_order=use_index_order,
  102. float_format=float_format)
  103. def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False,
  104. pointy_brackets=False, use_index_order=False, float_format=None):
  105. """Print a single field name/value pair. For repeated fields, the value
  106. should be a single element."""
  107. out.write(' ' * indent)
  108. if field.is_extension:
  109. out.write('[')
  110. if (field.containing_type.GetOptions().message_set_wire_format and
  111. field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
  112. field.message_type == field.extension_scope and
  113. field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL):
  114. out.write(field.message_type.full_name)
  115. else:
  116. out.write(field.full_name)
  117. out.write(']')
  118. elif field.type == descriptor.FieldDescriptor.TYPE_GROUP:
  119. # For groups, use the capitalized name.
  120. out.write(field.message_type.name)
  121. else:
  122. out.write(field.name)
  123. if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  124. # The colon is optional in this case, but our cross-language golden files
  125. # don't include it.
  126. out.write(': ')
  127. PrintFieldValue(field, value, out, indent, as_utf8, as_one_line,
  128. pointy_brackets=pointy_brackets,
  129. use_index_order=use_index_order,
  130. float_format=float_format)
  131. if as_one_line:
  132. out.write(' ')
  133. else:
  134. out.write('\n')
  135. def PrintFieldValue(field, value, out, indent=0, as_utf8=False,
  136. as_one_line=False, pointy_brackets=False,
  137. use_index_order=False,
  138. float_format=None):
  139. """Print a single field value (not including name). For repeated fields,
  140. the value should be a single element."""
  141. if pointy_brackets:
  142. openb = '<'
  143. closeb = '>'
  144. else:
  145. openb = '{'
  146. closeb = '}'
  147. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  148. if as_one_line:
  149. out.write(' %s ' % openb)
  150. PrintMessage(value, out, indent, as_utf8, as_one_line,
  151. pointy_brackets=pointy_brackets,
  152. use_index_order=use_index_order,
  153. float_format=float_format)
  154. out.write(closeb)
  155. else:
  156. out.write(' %s\n' % openb)
  157. PrintMessage(value, out, indent + 2, as_utf8, as_one_line,
  158. pointy_brackets=pointy_brackets,
  159. use_index_order=use_index_order,
  160. float_format=float_format)
  161. out.write(' ' * indent + closeb)
  162. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  163. enum_value = field.enum_type.values_by_number.get(value, None)
  164. if enum_value is not None:
  165. out.write(enum_value.name)
  166. else:
  167. out.write(str(value))
  168. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  169. out.write('\"')
  170. if isinstance(value, unicode):
  171. out_value = value.encode('utf-8')
  172. else:
  173. out_value = value
  174. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  175. # We need to escape non-UTF8 chars in TYPE_BYTES field.
  176. out_as_utf8 = False
  177. else:
  178. out_as_utf8 = as_utf8
  179. out.write(text_encoding.CEscape(out_value, out_as_utf8))
  180. out.write('\"')
  181. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  182. if value:
  183. out.write('true')
  184. else:
  185. out.write('false')
  186. elif field.cpp_type in _FLOAT_TYPES and float_format is not None:
  187. out.write('{1:{0}}'.format(float_format, value))
  188. else:
  189. out.write(str(value))
  190. def Parse(text, message):
  191. """Parses an ASCII representation of a protocol message into a message.
  192. Args:
  193. text: Message ASCII representation.
  194. message: A protocol buffer message to merge into.
  195. Returns:
  196. The same message passed as argument.
  197. Raises:
  198. ParseError: On ASCII parsing problems.
  199. """
  200. if not isinstance(text, str): text = text.decode('utf-8')
  201. return ParseLines(text.split('\n'), message)
  202. def Merge(text, message):
  203. """Parses an ASCII representation of a protocol message into a message.
  204. Like Parse(), but allows repeated values for a non-repeated field, and uses
  205. the last one.
  206. Args:
  207. text: Message ASCII representation.
  208. message: A protocol buffer message to merge into.
  209. Returns:
  210. The same message passed as argument.
  211. Raises:
  212. ParseError: On ASCII parsing problems.
  213. """
  214. return MergeLines(text.split('\n'), message)
  215. def ParseLines(lines, message):
  216. """Parses an ASCII representation of a protocol message into a message.
  217. Args:
  218. lines: An iterable of lines of a message's ASCII representation.
  219. message: A protocol buffer message to merge into.
  220. Returns:
  221. The same message passed as argument.
  222. Raises:
  223. ParseError: On ASCII parsing problems.
  224. """
  225. _ParseOrMerge(lines, message, False)
  226. return message
  227. def MergeLines(lines, message):
  228. """Parses an ASCII representation of a protocol message into a message.
  229. Args:
  230. lines: An iterable of lines of a message's ASCII representation.
  231. message: A protocol buffer message to merge into.
  232. Returns:
  233. The same message passed as argument.
  234. Raises:
  235. ParseError: On ASCII parsing problems.
  236. """
  237. _ParseOrMerge(lines, message, True)
  238. return message
  239. def _ParseOrMerge(lines, message, allow_multiple_scalars):
  240. """Converts an ASCII representation of a protocol message into a message.
  241. Args:
  242. lines: Lines of a message's ASCII representation.
  243. message: A protocol buffer message to merge into.
  244. allow_multiple_scalars: Determines if repeated values for a non-repeated
  245. field are permitted, e.g., the string "foo: 1 foo: 2" for a
  246. required/optional field named "foo".
  247. Raises:
  248. ParseError: On ASCII parsing problems.
  249. """
  250. tokenizer = _Tokenizer(lines)
  251. while not tokenizer.AtEnd():
  252. _MergeField(tokenizer, message, allow_multiple_scalars)
  253. def _MergeField(tokenizer, message, allow_multiple_scalars):
  254. """Merges a single protocol message field into a message.
  255. Args:
  256. tokenizer: A tokenizer to parse the field name and values.
  257. message: A protocol message to record the data.
  258. allow_multiple_scalars: Determines if repeated values for a non-repeated
  259. field are permitted, e.g., the string "foo: 1 foo: 2" for a
  260. required/optional field named "foo".
  261. Raises:
  262. ParseError: In case of ASCII parsing problems.
  263. """
  264. message_descriptor = message.DESCRIPTOR
  265. if (hasattr(message_descriptor, 'syntax') and
  266. message_descriptor.syntax == 'proto3'):
  267. # Proto3 doesn't represent presence so we can't test if multiple
  268. # scalars have occurred. We have to allow them.
  269. allow_multiple_scalars = True
  270. if tokenizer.TryConsume('['):
  271. name = [tokenizer.ConsumeIdentifier()]
  272. while tokenizer.TryConsume('.'):
  273. name.append(tokenizer.ConsumeIdentifier())
  274. name = '.'.join(name)
  275. if not message_descriptor.is_extendable:
  276. raise tokenizer.ParseErrorPreviousToken(
  277. 'Message type "%s" does not have extensions.' %
  278. message_descriptor.full_name)
  279. # pylint: disable=protected-access
  280. field = message.Extensions._FindExtensionByName(name)
  281. # pylint: enable=protected-access
  282. if not field:
  283. raise tokenizer.ParseErrorPreviousToken(
  284. 'Extension "%s" not registered.' % name)
  285. elif message_descriptor != field.containing_type:
  286. raise tokenizer.ParseErrorPreviousToken(
  287. 'Extension "%s" does not extend message type "%s".' % (
  288. name, message_descriptor.full_name))
  289. tokenizer.Consume(']')
  290. else:
  291. name = tokenizer.ConsumeIdentifier()
  292. field = message_descriptor.fields_by_name.get(name, None)
  293. # Group names are expected to be capitalized as they appear in the
  294. # .proto file, which actually matches their type names, not their field
  295. # names.
  296. if not field:
  297. field = message_descriptor.fields_by_name.get(name.lower(), None)
  298. if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP:
  299. field = None
  300. if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and
  301. field.message_type.name != name):
  302. field = None
  303. if not field:
  304. raise tokenizer.ParseErrorPreviousToken(
  305. 'Message type "%s" has no field named "%s".' % (
  306. message_descriptor.full_name, name))
  307. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  308. tokenizer.TryConsume(':')
  309. if tokenizer.TryConsume('<'):
  310. end_token = '>'
  311. else:
  312. tokenizer.Consume('{')
  313. end_token = '}'
  314. if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  315. if field.is_extension:
  316. sub_message = message.Extensions[field].add()
  317. else:
  318. sub_message = getattr(message, field.name).add()
  319. else:
  320. if field.is_extension:
  321. sub_message = message.Extensions[field]
  322. else:
  323. sub_message = getattr(message, field.name)
  324. sub_message.SetInParent()
  325. while not tokenizer.TryConsume(end_token):
  326. if tokenizer.AtEnd():
  327. raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token))
  328. _MergeField(tokenizer, sub_message, allow_multiple_scalars)
  329. else:
  330. _MergeScalarField(tokenizer, message, field, allow_multiple_scalars)
  331. # For historical reasons, fields may optionally be separated by commas or
  332. # semicolons.
  333. if not tokenizer.TryConsume(','):
  334. tokenizer.TryConsume(';')
  335. def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars):
  336. """Merges a single protocol message scalar field into a message.
  337. Args:
  338. tokenizer: A tokenizer to parse the field value.
  339. message: A protocol message to record the data.
  340. field: The descriptor of the field to be merged.
  341. allow_multiple_scalars: Determines if repeated values for a non-repeated
  342. field are permitted, e.g., the string "foo: 1 foo: 2" for a
  343. required/optional field named "foo".
  344. Raises:
  345. ParseError: In case of ASCII parsing problems.
  346. RuntimeError: On runtime errors.
  347. """
  348. tokenizer.Consume(':')
  349. value = None
  350. if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
  351. descriptor.FieldDescriptor.TYPE_SINT32,
  352. descriptor.FieldDescriptor.TYPE_SFIXED32):
  353. value = tokenizer.ConsumeInt32()
  354. elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
  355. descriptor.FieldDescriptor.TYPE_SINT64,
  356. descriptor.FieldDescriptor.TYPE_SFIXED64):
  357. value = tokenizer.ConsumeInt64()
  358. elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
  359. descriptor.FieldDescriptor.TYPE_FIXED32):
  360. value = tokenizer.ConsumeUint32()
  361. elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
  362. descriptor.FieldDescriptor.TYPE_FIXED64):
  363. value = tokenizer.ConsumeUint64()
  364. elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
  365. descriptor.FieldDescriptor.TYPE_DOUBLE):
  366. value = tokenizer.ConsumeFloat()
  367. elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
  368. value = tokenizer.ConsumeBool()
  369. elif field.type == descriptor.FieldDescriptor.TYPE_STRING:
  370. value = tokenizer.ConsumeString()
  371. elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  372. value = tokenizer.ConsumeByteString()
  373. elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:
  374. value = tokenizer.ConsumeEnum(field)
  375. else:
  376. raise RuntimeError('Unknown field type %d' % field.type)
  377. if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  378. if field.is_extension:
  379. message.Extensions[field].append(value)
  380. else:
  381. getattr(message, field.name).append(value)
  382. else:
  383. if field.is_extension:
  384. if not allow_multiple_scalars and message.HasExtension(field):
  385. raise tokenizer.ParseErrorPreviousToken(
  386. 'Message type "%s" should not have multiple "%s" extensions.' %
  387. (message.DESCRIPTOR.full_name, field.full_name))
  388. else:
  389. message.Extensions[field] = value
  390. else:
  391. if not allow_multiple_scalars and message.HasField(field.name):
  392. raise tokenizer.ParseErrorPreviousToken(
  393. 'Message type "%s" should not have multiple "%s" fields.' %
  394. (message.DESCRIPTOR.full_name, field.name))
  395. else:
  396. setattr(message, field.name, value)
  397. class _Tokenizer(object):
  398. """Protocol buffer ASCII representation tokenizer.
  399. This class handles the lower level string parsing by splitting it into
  400. meaningful tokens.
  401. It was directly ported from the Java protocol buffer API.
  402. """
  403. _WHITESPACE = re.compile('(\\s|(#.*$))+', re.MULTILINE)
  404. _TOKEN = re.compile(
  405. '[a-zA-Z_][0-9a-zA-Z_+-]*|' # an identifier
  406. '[0-9+-][0-9a-zA-Z_.+-]*|' # a number
  407. '\"([^\"\n\\\\]|\\\\.)*(\"|\\\\?$)|' # a double-quoted string
  408. '\'([^\'\n\\\\]|\\\\.)*(\'|\\\\?$)') # a single-quoted string
  409. _IDENTIFIER = re.compile(r'\w+')
  410. def __init__(self, lines):
  411. self._position = 0
  412. self._line = -1
  413. self._column = 0
  414. self._token_start = None
  415. self.token = ''
  416. self._lines = iter(lines)
  417. self._current_line = ''
  418. self._previous_line = 0
  419. self._previous_column = 0
  420. self._more_lines = True
  421. self._SkipWhitespace()
  422. self.NextToken()
  423. def AtEnd(self):
  424. """Checks the end of the text was reached.
  425. Returns:
  426. True iff the end was reached.
  427. """
  428. return not self.token
  429. def _PopLine(self):
  430. while len(self._current_line) <= self._column:
  431. try:
  432. self._current_line = self._lines.next()
  433. except StopIteration:
  434. self._current_line = ''
  435. self._more_lines = False
  436. return
  437. else:
  438. self._line += 1
  439. self._column = 0
  440. def _SkipWhitespace(self):
  441. while True:
  442. self._PopLine()
  443. match = self._WHITESPACE.match(self._current_line, self._column)
  444. if not match:
  445. break
  446. length = len(match.group(0))
  447. self._column += length
  448. def TryConsume(self, token):
  449. """Tries to consume a given piece of text.
  450. Args:
  451. token: Text to consume.
  452. Returns:
  453. True iff the text was consumed.
  454. """
  455. if self.token == token:
  456. self.NextToken()
  457. return True
  458. return False
  459. def Consume(self, token):
  460. """Consumes a piece of text.
  461. Args:
  462. token: Text to consume.
  463. Raises:
  464. ParseError: If the text couldn't be consumed.
  465. """
  466. if not self.TryConsume(token):
  467. raise self._ParseError('Expected "%s".' % token)
  468. def ConsumeIdentifier(self):
  469. """Consumes protocol message field identifier.
  470. Returns:
  471. Identifier string.
  472. Raises:
  473. ParseError: If an identifier couldn't be consumed.
  474. """
  475. result = self.token
  476. if not self._IDENTIFIER.match(result):
  477. raise self._ParseError('Expected identifier.')
  478. self.NextToken()
  479. return result
  480. def ConsumeInt32(self):
  481. """Consumes a signed 32bit integer number.
  482. Returns:
  483. The integer parsed.
  484. Raises:
  485. ParseError: If a signed 32bit integer couldn't be consumed.
  486. """
  487. try:
  488. result = ParseInteger(self.token, is_signed=True, is_long=False)
  489. except ValueError, e:
  490. raise self._ParseError(str(e))
  491. self.NextToken()
  492. return result
  493. def ConsumeUint32(self):
  494. """Consumes an unsigned 32bit integer number.
  495. Returns:
  496. The integer parsed.
  497. Raises:
  498. ParseError: If an unsigned 32bit integer couldn't be consumed.
  499. """
  500. try:
  501. result = ParseInteger(self.token, is_signed=False, is_long=False)
  502. except ValueError, e:
  503. raise self._ParseError(str(e))
  504. self.NextToken()
  505. return result
  506. def ConsumeInt64(self):
  507. """Consumes a signed 64bit integer number.
  508. Returns:
  509. The integer parsed.
  510. Raises:
  511. ParseError: If a signed 64bit integer couldn't be consumed.
  512. """
  513. try:
  514. result = ParseInteger(self.token, is_signed=True, is_long=True)
  515. except ValueError, e:
  516. raise self._ParseError(str(e))
  517. self.NextToken()
  518. return result
  519. def ConsumeUint64(self):
  520. """Consumes an unsigned 64bit integer number.
  521. Returns:
  522. The integer parsed.
  523. Raises:
  524. ParseError: If an unsigned 64bit integer couldn't be consumed.
  525. """
  526. try:
  527. result = ParseInteger(self.token, is_signed=False, is_long=True)
  528. except ValueError, e:
  529. raise self._ParseError(str(e))
  530. self.NextToken()
  531. return result
  532. def ConsumeFloat(self):
  533. """Consumes an floating point number.
  534. Returns:
  535. The number parsed.
  536. Raises:
  537. ParseError: If a floating point number couldn't be consumed.
  538. """
  539. try:
  540. result = ParseFloat(self.token)
  541. except ValueError, e:
  542. raise self._ParseError(str(e))
  543. self.NextToken()
  544. return result
  545. def ConsumeBool(self):
  546. """Consumes a boolean value.
  547. Returns:
  548. The bool parsed.
  549. Raises:
  550. ParseError: If a boolean value couldn't be consumed.
  551. """
  552. try:
  553. result = ParseBool(self.token)
  554. except ValueError, e:
  555. raise self._ParseError(str(e))
  556. self.NextToken()
  557. return result
  558. def ConsumeString(self):
  559. """Consumes a string value.
  560. Returns:
  561. The string parsed.
  562. Raises:
  563. ParseError: If a string value couldn't be consumed.
  564. """
  565. the_bytes = self.ConsumeByteString()
  566. try:
  567. return unicode(the_bytes, 'utf-8')
  568. except UnicodeDecodeError, e:
  569. raise self._StringParseError(e)
  570. def ConsumeByteString(self):
  571. """Consumes a byte array value.
  572. Returns:
  573. The array parsed (as a string).
  574. Raises:
  575. ParseError: If a byte array value couldn't be consumed.
  576. """
  577. the_list = [self._ConsumeSingleByteString()]
  578. while self.token and self.token[0] in ('\'', '"'):
  579. the_list.append(self._ConsumeSingleByteString())
  580. return ''.encode('latin1').join(the_list) ##PY25
  581. ##!PY25 return b''.join(the_list)
  582. def _ConsumeSingleByteString(self):
  583. """Consume one token of a string literal.
  584. String literals (whether bytes or text) can come in multiple adjacent
  585. tokens which are automatically concatenated, like in C or Python. This
  586. method only consumes one token.
  587. """
  588. text = self.token
  589. if len(text) < 1 or text[0] not in ('\'', '"'):
  590. raise self._ParseError('Expected string but found: "%r"' % text)
  591. if len(text) < 2 or text[-1] != text[0]:
  592. raise self._ParseError('String missing ending quote.')
  593. try:
  594. result = text_encoding.CUnescape(text[1:-1])
  595. except ValueError, e:
  596. raise self._ParseError(str(e))
  597. self.NextToken()
  598. return result
  599. def ConsumeEnum(self, field):
  600. try:
  601. result = ParseEnum(field, self.token)
  602. except ValueError, e:
  603. raise self._ParseError(str(e))
  604. self.NextToken()
  605. return result
  606. def ParseErrorPreviousToken(self, message):
  607. """Creates and *returns* a ParseError for the previously read token.
  608. Args:
  609. message: A message to set for the exception.
  610. Returns:
  611. A ParseError instance.
  612. """
  613. return ParseError('%d:%d : %s' % (
  614. self._previous_line + 1, self._previous_column + 1, message))
  615. def _ParseError(self, message):
  616. """Creates and *returns* a ParseError for the current token."""
  617. return ParseError('%d:%d : %s' % (
  618. self._line + 1, self._column + 1, message))
  619. def _StringParseError(self, e):
  620. return self._ParseError('Couldn\'t parse string: ' + str(e))
  621. def NextToken(self):
  622. """Reads the next meaningful token."""
  623. self._previous_line = self._line
  624. self._previous_column = self._column
  625. self._column += len(self.token)
  626. self._SkipWhitespace()
  627. if not self._more_lines:
  628. self.token = ''
  629. return
  630. match = self._TOKEN.match(self._current_line, self._column)
  631. if match:
  632. token = match.group(0)
  633. self.token = token
  634. else:
  635. self.token = self._current_line[self._column]
  636. def ParseInteger(text, is_signed=False, is_long=False):
  637. """Parses an integer.
  638. Args:
  639. text: The text to parse.
  640. is_signed: True if a signed integer must be parsed.
  641. is_long: True if a long integer must be parsed.
  642. Returns:
  643. The integer value.
  644. Raises:
  645. ValueError: Thrown Iff the text is not a valid integer.
  646. """
  647. # Do the actual parsing. Exception handling is propagated to caller.
  648. try:
  649. # We force 32-bit values to int and 64-bit values to long to make
  650. # alternate implementations where the distinction is more significant
  651. # (e.g. the C++ implementation) simpler.
  652. if is_long:
  653. result = long(text, 0)
  654. else:
  655. result = int(text, 0)
  656. except ValueError:
  657. raise ValueError('Couldn\'t parse integer: %s' % text)
  658. # Check if the integer is sane. Exceptions handled by callers.
  659. checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)]
  660. checker.CheckValue(result)
  661. return result
  662. def ParseFloat(text):
  663. """Parse a floating point number.
  664. Args:
  665. text: Text to parse.
  666. Returns:
  667. The number parsed.
  668. Raises:
  669. ValueError: If a floating point number couldn't be parsed.
  670. """
  671. try:
  672. # Assume Python compatible syntax.
  673. return float(text)
  674. except ValueError:
  675. # Check alternative spellings.
  676. if _FLOAT_INFINITY.match(text):
  677. if text[0] == '-':
  678. return float('-inf')
  679. else:
  680. return float('inf')
  681. elif _FLOAT_NAN.match(text):
  682. return float('nan')
  683. else:
  684. # assume '1.0f' format
  685. try:
  686. return float(text.rstrip('f'))
  687. except ValueError:
  688. raise ValueError('Couldn\'t parse float: %s' % text)
  689. def ParseBool(text):
  690. """Parse a boolean value.
  691. Args:
  692. text: Text to parse.
  693. Returns:
  694. Boolean values parsed
  695. Raises:
  696. ValueError: If text is not a valid boolean.
  697. """
  698. if text in ('true', 't', '1'):
  699. return True
  700. elif text in ('false', 'f', '0'):
  701. return False
  702. else:
  703. raise ValueError('Expected "true" or "false".')
  704. def ParseEnum(field, value):
  705. """Parse an enum value.
  706. The value can be specified by a number (the enum value), or by
  707. a string literal (the enum name).
  708. Args:
  709. field: Enum field descriptor.
  710. value: String value.
  711. Returns:
  712. Enum value number.
  713. Raises:
  714. ValueError: If the enum value could not be parsed.
  715. """
  716. enum_descriptor = field.enum_type
  717. try:
  718. number = int(value, 0)
  719. except ValueError:
  720. # Identifier.
  721. enum_value = enum_descriptor.values_by_name.get(value, None)
  722. if enum_value is None:
  723. raise ValueError(
  724. 'Enum type "%s" has no value named %s.' % (
  725. enum_descriptor.full_name, value))
  726. else:
  727. # Numeric value.
  728. enum_value = enum_descriptor.values_by_number.get(number, None)
  729. if enum_value is None:
  730. raise ValueError(
  731. 'Enum type "%s" has no value with number %d.' % (
  732. enum_descriptor.full_name, number))
  733. return enum_value.number