text_format.py 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579
  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. # https://developers.google.com/protocol-buffers/
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. # * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. # * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. """Contains routines for printing protocol messages in text format.
  31. Simple usage example:
  32. # Create a proto object and serialize it to a text proto string.
  33. message = my_proto_pb2.MyMessage(foo='bar')
  34. text_proto = text_format.MessageToString(message)
  35. # Parse a text proto string.
  36. message = text_format.Parse(text_proto, my_proto_pb2.MyMessage())
  37. """
  38. __author__ = 'kenton@google.com (Kenton Varda)'
  39. import io
  40. import re
  41. import six
  42. if six.PY3:
  43. long = int # pylint: disable=redefined-builtin,invalid-name
  44. # pylint: disable=g-import-not-at-top
  45. from google.protobuf.internal import type_checkers
  46. from google.protobuf import descriptor
  47. from google.protobuf import text_encoding
  48. __all__ = ['MessageToString', 'PrintMessage', 'PrintField', 'PrintFieldValue',
  49. 'Merge']
  50. _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
  51. type_checkers.Int32ValueChecker(),
  52. type_checkers.Uint64ValueChecker(),
  53. type_checkers.Int64ValueChecker())
  54. _FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE)
  55. _FLOAT_NAN = re.compile('nanf?', re.IGNORECASE)
  56. _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
  57. descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
  58. _QUOTES = frozenset(("'", '"'))
  59. _ANY_FULL_TYPE_NAME = 'google.protobuf.Any'
  60. class Error(Exception):
  61. """Top-level module error for text_format."""
  62. class ParseError(Error):
  63. """Thrown in case of text parsing or tokenizing error."""
  64. def __init__(self, message=None, line=None, column=None):
  65. if message is not None and line is not None:
  66. loc = str(line)
  67. if column is not None:
  68. loc += ':{0}'.format(column)
  69. message = '{0} : {1}'.format(loc, message)
  70. if message is not None:
  71. super(ParseError, self).__init__(message)
  72. else:
  73. super(ParseError, self).__init__()
  74. self._line = line
  75. self._column = column
  76. def GetLine(self):
  77. return self._line
  78. def GetColumn(self):
  79. return self._column
  80. class TextWriter(object):
  81. def __init__(self, as_utf8):
  82. if six.PY2:
  83. self._writer = io.BytesIO()
  84. else:
  85. self._writer = io.StringIO()
  86. def write(self, val):
  87. if six.PY2:
  88. if isinstance(val, six.text_type):
  89. val = val.encode('utf-8')
  90. return self._writer.write(val)
  91. def close(self):
  92. return self._writer.close()
  93. def getvalue(self):
  94. return self._writer.getvalue()
  95. def MessageToString(message,
  96. as_utf8=False,
  97. as_one_line=False,
  98. pointy_brackets=False,
  99. use_index_order=False,
  100. float_format=None,
  101. use_field_number=False,
  102. descriptor_pool=None,
  103. indent=0,
  104. message_formatter=None):
  105. """Convert protobuf message to text format.
  106. Floating point values can be formatted compactly with 15 digits of
  107. precision (which is the most that IEEE 754 "double" can guarantee)
  108. using float_format='.15g'. To ensure that converting to text and back to a
  109. proto will result in an identical value, float_format='.17g' should be used.
  110. Args:
  111. message: The protocol buffers message.
  112. as_utf8: Produce text output in UTF8 format.
  113. as_one_line: Don't introduce newlines between fields.
  114. pointy_brackets: If True, use angle brackets instead of curly braces for
  115. nesting.
  116. use_index_order: If True, print fields of a proto message using the order
  117. defined in source code instead of the field number. By default, use the
  118. field number order.
  119. float_format: If set, use this to specify floating point number formatting
  120. (per the "Format Specification Mini-Language"); otherwise, str() is used.
  121. use_field_number: If True, print field numbers instead of names.
  122. descriptor_pool: A DescriptorPool used to resolve Any types.
  123. indent: The indent level, in terms of spaces, for pretty print.
  124. message_formatter: A function(message, indent, as_one_line): unicode|None
  125. to custom format selected sub-messages (usually based on message type).
  126. Use to pretty print parts of the protobuf for easier diffing.
  127. Returns:
  128. A string of the text formatted protocol buffer message.
  129. """
  130. out = TextWriter(as_utf8)
  131. printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
  132. use_index_order, float_format, use_field_number,
  133. descriptor_pool, message_formatter)
  134. printer.PrintMessage(message)
  135. result = out.getvalue()
  136. out.close()
  137. if as_one_line:
  138. return result.rstrip()
  139. return result
  140. def _IsMapEntry(field):
  141. return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
  142. field.message_type.has_options and
  143. field.message_type.GetOptions().map_entry)
  144. def PrintMessage(message,
  145. out,
  146. indent=0,
  147. as_utf8=False,
  148. as_one_line=False,
  149. pointy_brackets=False,
  150. use_index_order=False,
  151. float_format=None,
  152. use_field_number=False,
  153. descriptor_pool=None,
  154. message_formatter=None):
  155. printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
  156. use_index_order, float_format, use_field_number,
  157. descriptor_pool, message_formatter)
  158. printer.PrintMessage(message)
  159. def PrintField(field,
  160. value,
  161. out,
  162. indent=0,
  163. as_utf8=False,
  164. as_one_line=False,
  165. pointy_brackets=False,
  166. use_index_order=False,
  167. float_format=None,
  168. message_formatter=None):
  169. """Print a single field name/value pair."""
  170. printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
  171. use_index_order, float_format, message_formatter)
  172. printer.PrintField(field, value)
  173. def PrintFieldValue(field,
  174. value,
  175. out,
  176. indent=0,
  177. as_utf8=False,
  178. as_one_line=False,
  179. pointy_brackets=False,
  180. use_index_order=False,
  181. float_format=None,
  182. message_formatter=None):
  183. """Print a single field value (not including name)."""
  184. printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
  185. use_index_order, float_format, message_formatter)
  186. printer.PrintFieldValue(field, value)
  187. def _BuildMessageFromTypeName(type_name, descriptor_pool):
  188. """Returns a protobuf message instance.
  189. Args:
  190. type_name: Fully-qualified protobuf message type name string.
  191. descriptor_pool: DescriptorPool instance.
  192. Returns:
  193. A Message instance of type matching type_name, or None if the a Descriptor
  194. wasn't found matching type_name.
  195. """
  196. # pylint: disable=g-import-not-at-top
  197. if descriptor_pool is None:
  198. from google.protobuf import descriptor_pool as pool_mod
  199. descriptor_pool = pool_mod.Default()
  200. from google.protobuf import symbol_database
  201. database = symbol_database.Default()
  202. try:
  203. message_descriptor = descriptor_pool.FindMessageTypeByName(type_name)
  204. except KeyError:
  205. return None
  206. message_type = database.GetPrototype(message_descriptor)
  207. return message_type()
  208. class _Printer(object):
  209. """Text format printer for protocol message."""
  210. def __init__(self,
  211. out,
  212. indent=0,
  213. as_utf8=False,
  214. as_one_line=False,
  215. pointy_brackets=False,
  216. use_index_order=False,
  217. float_format=None,
  218. use_field_number=False,
  219. descriptor_pool=None,
  220. message_formatter=None):
  221. """Initialize the Printer.
  222. Floating point values can be formatted compactly with 15 digits of
  223. precision (which is the most that IEEE 754 "double" can guarantee)
  224. using float_format='.15g'. To ensure that converting to text and back to a
  225. proto will result in an identical value, float_format='.17g' should be used.
  226. Args:
  227. out: To record the text format result.
  228. indent: The indent level for pretty print.
  229. as_utf8: Produce text output in UTF8 format.
  230. as_one_line: Don't introduce newlines between fields.
  231. pointy_brackets: If True, use angle brackets instead of curly braces for
  232. nesting.
  233. use_index_order: If True, print fields of a proto message using the order
  234. defined in source code instead of the field number. By default, use the
  235. field number order.
  236. float_format: If set, use this to specify floating point number formatting
  237. (per the "Format Specification Mini-Language"); otherwise, str() is
  238. used.
  239. use_field_number: If True, print field numbers instead of names.
  240. descriptor_pool: A DescriptorPool used to resolve Any types.
  241. message_formatter: A function(message, indent, as_one_line): unicode|None
  242. to custom format selected sub-messages (usually based on message type).
  243. Use to pretty print parts of the protobuf for easier diffing.
  244. """
  245. self.out = out
  246. self.indent = indent
  247. self.as_utf8 = as_utf8
  248. self.as_one_line = as_one_line
  249. self.pointy_brackets = pointy_brackets
  250. self.use_index_order = use_index_order
  251. self.float_format = float_format
  252. self.use_field_number = use_field_number
  253. self.descriptor_pool = descriptor_pool
  254. self.message_formatter = message_formatter
  255. def _TryPrintAsAnyMessage(self, message):
  256. """Serializes if message is a google.protobuf.Any field."""
  257. packed_message = _BuildMessageFromTypeName(message.TypeName(),
  258. self.descriptor_pool)
  259. if packed_message:
  260. packed_message.MergeFromString(message.value)
  261. self.out.write('%s[%s]' % (self.indent * ' ', message.type_url))
  262. self._PrintMessageFieldValue(packed_message)
  263. self.out.write(' ' if self.as_one_line else '\n')
  264. return True
  265. else:
  266. return False
  267. def _TryCustomFormatMessage(self, message):
  268. formatted = self.message_formatter(message, self.indent, self.as_one_line)
  269. if formatted is None:
  270. return False
  271. out = self.out
  272. out.write(' ' * self.indent)
  273. out.write(formatted)
  274. out.write(' ' if self.as_one_line else '\n')
  275. return True
  276. def PrintMessage(self, message):
  277. """Convert protobuf message to text format.
  278. Args:
  279. message: The protocol buffers message.
  280. """
  281. if self.message_formatter and self._TryCustomFormatMessage(message):
  282. return
  283. if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and
  284. self._TryPrintAsAnyMessage(message)):
  285. return
  286. fields = message.ListFields()
  287. if self.use_index_order:
  288. fields.sort(key=lambda x: x[0].index)
  289. for field, value in fields:
  290. if _IsMapEntry(field):
  291. for key in sorted(value):
  292. # This is slow for maps with submessage entires because it copies the
  293. # entire tree. Unfortunately this would take significant refactoring
  294. # of this file to work around.
  295. #
  296. # TODO(haberman): refactor and optimize if this becomes an issue.
  297. entry_submsg = value.GetEntryClass()(key=key, value=value[key])
  298. self.PrintField(field, entry_submsg)
  299. elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  300. for element in value:
  301. self.PrintField(field, element)
  302. else:
  303. self.PrintField(field, value)
  304. def PrintField(self, field, value):
  305. """Print a single field name/value pair."""
  306. out = self.out
  307. out.write(' ' * self.indent)
  308. if self.use_field_number:
  309. out.write(str(field.number))
  310. else:
  311. if field.is_extension:
  312. out.write('[')
  313. if (field.containing_type.GetOptions().message_set_wire_format and
  314. field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
  315. field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL):
  316. out.write(field.message_type.full_name)
  317. else:
  318. out.write(field.full_name)
  319. out.write(']')
  320. elif field.type == descriptor.FieldDescriptor.TYPE_GROUP:
  321. # For groups, use the capitalized name.
  322. out.write(field.message_type.name)
  323. else:
  324. out.write(field.name)
  325. if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  326. # The colon is optional in this case, but our cross-language golden files
  327. # don't include it.
  328. out.write(': ')
  329. self.PrintFieldValue(field, value)
  330. if self.as_one_line:
  331. out.write(' ')
  332. else:
  333. out.write('\n')
  334. def _PrintMessageFieldValue(self, value):
  335. if self.pointy_brackets:
  336. openb = '<'
  337. closeb = '>'
  338. else:
  339. openb = '{'
  340. closeb = '}'
  341. if self.as_one_line:
  342. self.out.write(' %s ' % openb)
  343. self.PrintMessage(value)
  344. self.out.write(closeb)
  345. else:
  346. self.out.write(' %s\n' % openb)
  347. self.indent += 2
  348. self.PrintMessage(value)
  349. self.indent -= 2
  350. self.out.write(' ' * self.indent + closeb)
  351. def PrintFieldValue(self, field, value):
  352. """Print a single field value (not including name).
  353. For repeated fields, the value should be a single element.
  354. Args:
  355. field: The descriptor of the field to be printed.
  356. value: The value of the field.
  357. """
  358. out = self.out
  359. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  360. self._PrintMessageFieldValue(value)
  361. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  362. enum_value = field.enum_type.values_by_number.get(value, None)
  363. if enum_value is not None:
  364. out.write(enum_value.name)
  365. else:
  366. out.write(str(value))
  367. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  368. out.write('\"')
  369. if isinstance(value, six.text_type):
  370. out_value = value.encode('utf-8')
  371. else:
  372. out_value = value
  373. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  374. # We need to escape non-UTF8 chars in TYPE_BYTES field.
  375. out_as_utf8 = False
  376. else:
  377. out_as_utf8 = self.as_utf8
  378. out.write(text_encoding.CEscape(out_value, out_as_utf8))
  379. out.write('\"')
  380. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  381. if value:
  382. out.write('true')
  383. else:
  384. out.write('false')
  385. elif field.cpp_type in _FLOAT_TYPES and self.float_format is not None:
  386. out.write('{1:{0}}'.format(self.float_format, value))
  387. else:
  388. out.write(str(value))
  389. def Parse(text,
  390. message,
  391. allow_unknown_extension=False,
  392. allow_field_number=False,
  393. descriptor_pool=None):
  394. """Parses a text representation of a protocol message into a message.
  395. NOTE: for historical reasons this function does not clear the input
  396. message. This is different from what the binary msg.ParseFrom(...) does.
  397. Example
  398. a = MyProto()
  399. a.repeated_field.append('test')
  400. b = MyProto()
  401. text_format.Parse(repr(a), b)
  402. text_format.Parse(repr(a), b) # repeated_field contains ["test", "test"]
  403. # Binary version:
  404. b.ParseFromString(a.SerializeToString()) # repeated_field is now "test"
  405. Caller is responsible for clearing the message as needed.
  406. Args:
  407. text: Message text representation.
  408. message: A protocol buffer message to merge into.
  409. allow_unknown_extension: if True, skip over missing extensions and keep
  410. parsing
  411. allow_field_number: if True, both field number and field name are allowed.
  412. descriptor_pool: A DescriptorPool used to resolve Any types.
  413. Returns:
  414. The same message passed as argument.
  415. Raises:
  416. ParseError: On text parsing problems.
  417. """
  418. if not isinstance(text, str):
  419. if six.PY3:
  420. text = text.decode('utf-8')
  421. else:
  422. text = text.encode('utf-8')
  423. return ParseLines(text.split('\n'),
  424. message,
  425. allow_unknown_extension,
  426. allow_field_number,
  427. descriptor_pool=descriptor_pool)
  428. def Merge(text,
  429. message,
  430. allow_unknown_extension=False,
  431. allow_field_number=False,
  432. descriptor_pool=None):
  433. """Parses a text representation of a protocol message into a message.
  434. Like Parse(), but allows repeated values for a non-repeated field, and uses
  435. the last one.
  436. Args:
  437. text: Message text representation.
  438. message: A protocol buffer message to merge into.
  439. allow_unknown_extension: if True, skip over missing extensions and keep
  440. parsing
  441. allow_field_number: if True, both field number and field name are allowed.
  442. descriptor_pool: A DescriptorPool used to resolve Any types.
  443. Returns:
  444. The same message passed as argument.
  445. Raises:
  446. ParseError: On text parsing problems.
  447. """
  448. if not isinstance(text, str):
  449. if six.PY3:
  450. text = text.decode('utf-8')
  451. else:
  452. text = text.encode('utf-8')
  453. return MergeLines(
  454. text.split('\n'),
  455. message,
  456. allow_unknown_extension,
  457. allow_field_number,
  458. descriptor_pool=descriptor_pool)
  459. def ParseLines(lines,
  460. message,
  461. allow_unknown_extension=False,
  462. allow_field_number=False,
  463. descriptor_pool=None):
  464. """Parses a text representation of a protocol message into a message.
  465. Args:
  466. lines: An iterable of lines of a message's text representation.
  467. message: A protocol buffer message to merge into.
  468. allow_unknown_extension: if True, skip over missing extensions and keep
  469. parsing
  470. allow_field_number: if True, both field number and field name are allowed.
  471. descriptor_pool: A DescriptorPool used to resolve Any types.
  472. Returns:
  473. The same message passed as argument.
  474. Raises:
  475. ParseError: On text parsing problems.
  476. """
  477. parser = _Parser(allow_unknown_extension,
  478. allow_field_number,
  479. descriptor_pool=descriptor_pool)
  480. return parser.ParseLines(lines, message)
  481. def MergeLines(lines,
  482. message,
  483. allow_unknown_extension=False,
  484. allow_field_number=False,
  485. descriptor_pool=None):
  486. """Parses a text representation of a protocol message into a message.
  487. Args:
  488. lines: An iterable of lines of a message's text representation.
  489. message: A protocol buffer message to merge into.
  490. allow_unknown_extension: if True, skip over missing extensions and keep
  491. parsing
  492. allow_field_number: if True, both field number and field name are allowed.
  493. descriptor_pool: A DescriptorPool used to resolve Any types.
  494. Returns:
  495. The same message passed as argument.
  496. Raises:
  497. ParseError: On text parsing problems.
  498. """
  499. parser = _Parser(allow_unknown_extension,
  500. allow_field_number,
  501. descriptor_pool=descriptor_pool)
  502. return parser.MergeLines(lines, message)
  503. class _Parser(object):
  504. """Text format parser for protocol message."""
  505. def __init__(self,
  506. allow_unknown_extension=False,
  507. allow_field_number=False,
  508. descriptor_pool=None):
  509. self.allow_unknown_extension = allow_unknown_extension
  510. self.allow_field_number = allow_field_number
  511. self.descriptor_pool = descriptor_pool
  512. def ParseFromString(self, text, message):
  513. """Parses a text representation of a protocol message into a message."""
  514. if not isinstance(text, str):
  515. text = text.decode('utf-8')
  516. return self.ParseLines(text.split('\n'), message)
  517. def ParseLines(self, lines, message):
  518. """Parses a text representation of a protocol message into a message."""
  519. self._allow_multiple_scalars = False
  520. self._ParseOrMerge(lines, message)
  521. return message
  522. def MergeFromString(self, text, message):
  523. """Merges a text representation of a protocol message into a message."""
  524. return self._MergeLines(text.split('\n'), message)
  525. def MergeLines(self, lines, message):
  526. """Merges a text representation of a protocol message into a message."""
  527. self._allow_multiple_scalars = True
  528. self._ParseOrMerge(lines, message)
  529. return message
  530. def _ParseOrMerge(self, lines, message):
  531. """Converts a text representation of a protocol message into a message.
  532. Args:
  533. lines: Lines of a message's text representation.
  534. message: A protocol buffer message to merge into.
  535. Raises:
  536. ParseError: On text parsing problems.
  537. """
  538. tokenizer = Tokenizer(lines)
  539. while not tokenizer.AtEnd():
  540. self._MergeField(tokenizer, message)
  541. def _MergeField(self, tokenizer, message):
  542. """Merges a single protocol message field into a message.
  543. Args:
  544. tokenizer: A tokenizer to parse the field name and values.
  545. message: A protocol message to record the data.
  546. Raises:
  547. ParseError: In case of text parsing problems.
  548. """
  549. message_descriptor = message.DESCRIPTOR
  550. if tokenizer.TryConsume('['):
  551. name = [tokenizer.ConsumeIdentifier()]
  552. while tokenizer.TryConsume('.'):
  553. name.append(tokenizer.ConsumeIdentifier())
  554. name = '.'.join(name)
  555. if not message_descriptor.is_extendable:
  556. raise tokenizer.ParseErrorPreviousToken(
  557. 'Message type "%s" does not have extensions.' %
  558. message_descriptor.full_name)
  559. # pylint: disable=protected-access
  560. field = message.Extensions._FindExtensionByName(name)
  561. # pylint: enable=protected-access
  562. if not field:
  563. if self.allow_unknown_extension:
  564. field = None
  565. else:
  566. raise tokenizer.ParseErrorPreviousToken(
  567. 'Extension "%s" not registered. '
  568. 'Did you import the _pb2 module which defines it? '
  569. 'If you are trying to place the extension in the MessageSet '
  570. 'field of another message that is in an Any or MessageSet field, '
  571. 'that message\'s _pb2 module must be imported as well' % name)
  572. elif message_descriptor != field.containing_type:
  573. raise tokenizer.ParseErrorPreviousToken(
  574. 'Extension "%s" does not extend message type "%s".' %
  575. (name, message_descriptor.full_name))
  576. tokenizer.Consume(']')
  577. else:
  578. name = tokenizer.ConsumeIdentifierOrNumber()
  579. if self.allow_field_number and name.isdigit():
  580. number = ParseInteger(name, True, True)
  581. field = message_descriptor.fields_by_number.get(number, None)
  582. if not field and message_descriptor.is_extendable:
  583. field = message.Extensions._FindExtensionByNumber(number)
  584. else:
  585. field = message_descriptor.fields_by_name.get(name, None)
  586. # Group names are expected to be capitalized as they appear in the
  587. # .proto file, which actually matches their type names, not their field
  588. # names.
  589. if not field:
  590. field = message_descriptor.fields_by_name.get(name.lower(), None)
  591. if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP:
  592. field = None
  593. if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and
  594. field.message_type.name != name):
  595. field = None
  596. if not field:
  597. raise tokenizer.ParseErrorPreviousToken(
  598. 'Message type "%s" has no field named "%s".' %
  599. (message_descriptor.full_name, name))
  600. if field:
  601. if not self._allow_multiple_scalars and field.containing_oneof:
  602. # Check if there's a different field set in this oneof.
  603. # Note that we ignore the case if the same field was set before, and we
  604. # apply _allow_multiple_scalars to non-scalar fields as well.
  605. which_oneof = message.WhichOneof(field.containing_oneof.name)
  606. if which_oneof is not None and which_oneof != field.name:
  607. raise tokenizer.ParseErrorPreviousToken(
  608. 'Field "%s" is specified along with field "%s", another member '
  609. 'of oneof "%s" for message type "%s".' %
  610. (field.name, which_oneof, field.containing_oneof.name,
  611. message_descriptor.full_name))
  612. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  613. tokenizer.TryConsume(':')
  614. merger = self._MergeMessageField
  615. else:
  616. tokenizer.Consume(':')
  617. merger = self._MergeScalarField
  618. if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and
  619. tokenizer.TryConsume('[')):
  620. # Short repeated format, e.g. "foo: [1, 2, 3]"
  621. while True:
  622. merger(tokenizer, message, field)
  623. if tokenizer.TryConsume(']'):
  624. break
  625. tokenizer.Consume(',')
  626. else:
  627. merger(tokenizer, message, field)
  628. else: # Proto field is unknown.
  629. assert self.allow_unknown_extension
  630. _SkipFieldContents(tokenizer)
  631. # For historical reasons, fields may optionally be separated by commas or
  632. # semicolons.
  633. if not tokenizer.TryConsume(','):
  634. tokenizer.TryConsume(';')
  635. def _ConsumeAnyTypeUrl(self, tokenizer):
  636. """Consumes a google.protobuf.Any type URL and returns the type name."""
  637. # Consume "type.googleapis.com/".
  638. prefix = [tokenizer.ConsumeIdentifier()]
  639. tokenizer.Consume('.')
  640. prefix.append(tokenizer.ConsumeIdentifier())
  641. tokenizer.Consume('.')
  642. prefix.append(tokenizer.ConsumeIdentifier())
  643. tokenizer.Consume('/')
  644. # Consume the fully-qualified type name.
  645. name = [tokenizer.ConsumeIdentifier()]
  646. while tokenizer.TryConsume('.'):
  647. name.append(tokenizer.ConsumeIdentifier())
  648. return '.'.join(prefix), '.'.join(name)
  649. def _MergeMessageField(self, tokenizer, message, field):
  650. """Merges a single scalar field into a message.
  651. Args:
  652. tokenizer: A tokenizer to parse the field value.
  653. message: The message of which field is a member.
  654. field: The descriptor of the field to be merged.
  655. Raises:
  656. ParseError: In case of text parsing problems.
  657. """
  658. is_map_entry = _IsMapEntry(field)
  659. if tokenizer.TryConsume('<'):
  660. end_token = '>'
  661. else:
  662. tokenizer.Consume('{')
  663. end_token = '}'
  664. if (field.message_type.full_name == _ANY_FULL_TYPE_NAME and
  665. tokenizer.TryConsume('[')):
  666. type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
  667. tokenizer.Consume(']')
  668. tokenizer.TryConsume(':')
  669. if tokenizer.TryConsume('<'):
  670. expanded_any_end_token = '>'
  671. else:
  672. tokenizer.Consume('{')
  673. expanded_any_end_token = '}'
  674. expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name,
  675. self.descriptor_pool)
  676. if not expanded_any_sub_message:
  677. raise ParseError('Type %s not found in descriptor pool' %
  678. packed_type_name)
  679. while not tokenizer.TryConsume(expanded_any_end_token):
  680. if tokenizer.AtEnd():
  681. raise tokenizer.ParseErrorPreviousToken('Expected "%s".' %
  682. (expanded_any_end_token,))
  683. self._MergeField(tokenizer, expanded_any_sub_message)
  684. if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  685. any_message = getattr(message, field.name).add()
  686. else:
  687. any_message = getattr(message, field.name)
  688. any_message.Pack(expanded_any_sub_message,
  689. type_url_prefix=type_url_prefix)
  690. elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  691. if field.is_extension:
  692. sub_message = message.Extensions[field].add()
  693. elif is_map_entry:
  694. sub_message = getattr(message, field.name).GetEntryClass()()
  695. else:
  696. sub_message = getattr(message, field.name).add()
  697. else:
  698. if field.is_extension:
  699. sub_message = message.Extensions[field]
  700. else:
  701. sub_message = getattr(message, field.name)
  702. sub_message.SetInParent()
  703. while not tokenizer.TryConsume(end_token):
  704. if tokenizer.AtEnd():
  705. raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token,))
  706. self._MergeField(tokenizer, sub_message)
  707. if is_map_entry:
  708. value_cpptype = field.message_type.fields_by_name['value'].cpp_type
  709. if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  710. value = getattr(message, field.name)[sub_message.key]
  711. value.MergeFrom(sub_message.value)
  712. else:
  713. getattr(message, field.name)[sub_message.key] = sub_message.value
  714. @staticmethod
  715. def _IsProto3Syntax(message):
  716. message_descriptor = message.DESCRIPTOR
  717. return (hasattr(message_descriptor, 'syntax') and
  718. message_descriptor.syntax == 'proto3')
  719. def _MergeScalarField(self, tokenizer, message, field):
  720. """Merges a single scalar field into a message.
  721. Args:
  722. tokenizer: A tokenizer to parse the field value.
  723. message: A protocol message to record the data.
  724. field: The descriptor of the field to be merged.
  725. Raises:
  726. ParseError: In case of text parsing problems.
  727. RuntimeError: On runtime errors.
  728. """
  729. _ = self.allow_unknown_extension
  730. value = None
  731. if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
  732. descriptor.FieldDescriptor.TYPE_SINT32,
  733. descriptor.FieldDescriptor.TYPE_SFIXED32):
  734. value = _ConsumeInt32(tokenizer)
  735. elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
  736. descriptor.FieldDescriptor.TYPE_SINT64,
  737. descriptor.FieldDescriptor.TYPE_SFIXED64):
  738. value = _ConsumeInt64(tokenizer)
  739. elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
  740. descriptor.FieldDescriptor.TYPE_FIXED32):
  741. value = _ConsumeUint32(tokenizer)
  742. elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
  743. descriptor.FieldDescriptor.TYPE_FIXED64):
  744. value = _ConsumeUint64(tokenizer)
  745. elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
  746. descriptor.FieldDescriptor.TYPE_DOUBLE):
  747. value = tokenizer.ConsumeFloat()
  748. elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
  749. value = tokenizer.ConsumeBool()
  750. elif field.type == descriptor.FieldDescriptor.TYPE_STRING:
  751. value = tokenizer.ConsumeString()
  752. elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  753. value = tokenizer.ConsumeByteString()
  754. elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:
  755. value = tokenizer.ConsumeEnum(field)
  756. else:
  757. raise RuntimeError('Unknown field type %d' % field.type)
  758. if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
  759. if field.is_extension:
  760. message.Extensions[field].append(value)
  761. else:
  762. getattr(message, field.name).append(value)
  763. else:
  764. # Proto3 doesn't represent presence so we can't test if multiple scalars
  765. # have occurred. We have to allow them.
  766. can_check_presence = not self._IsProto3Syntax(message)
  767. if field.is_extension:
  768. if (not self._allow_multiple_scalars and can_check_presence and
  769. message.HasExtension(field)):
  770. raise tokenizer.ParseErrorPreviousToken(
  771. 'Message type "%s" should not have multiple "%s" extensions.' %
  772. (message.DESCRIPTOR.full_name, field.full_name))
  773. else:
  774. message.Extensions[field] = value
  775. else:
  776. if (not self._allow_multiple_scalars and can_check_presence and
  777. message.HasField(field.name)):
  778. raise tokenizer.ParseErrorPreviousToken(
  779. 'Message type "%s" should not have multiple "%s" fields.' %
  780. (message.DESCRIPTOR.full_name, field.name))
  781. else:
  782. setattr(message, field.name, value)
  783. def _SkipFieldContents(tokenizer):
  784. """Skips over contents (value or message) of a field.
  785. Args:
  786. tokenizer: A tokenizer to parse the field name and values.
  787. """
  788. # Try to guess the type of this field.
  789. # If this field is not a message, there should be a ":" between the
  790. # field name and the field value and also the field value should not
  791. # start with "{" or "<" which indicates the beginning of a message body.
  792. # If there is no ":" or there is a "{" or "<" after ":", this field has
  793. # to be a message or the input is ill-formed.
  794. if tokenizer.TryConsume(':') and not tokenizer.LookingAt(
  795. '{') and not tokenizer.LookingAt('<'):
  796. _SkipFieldValue(tokenizer)
  797. else:
  798. _SkipFieldMessage(tokenizer)
  799. def _SkipField(tokenizer):
  800. """Skips over a complete field (name and value/message).
  801. Args:
  802. tokenizer: A tokenizer to parse the field name and values.
  803. """
  804. if tokenizer.TryConsume('['):
  805. # Consume extension name.
  806. tokenizer.ConsumeIdentifier()
  807. while tokenizer.TryConsume('.'):
  808. tokenizer.ConsumeIdentifier()
  809. tokenizer.Consume(']')
  810. else:
  811. tokenizer.ConsumeIdentifierOrNumber()
  812. _SkipFieldContents(tokenizer)
  813. # For historical reasons, fields may optionally be separated by commas or
  814. # semicolons.
  815. if not tokenizer.TryConsume(','):
  816. tokenizer.TryConsume(';')
  817. def _SkipFieldMessage(tokenizer):
  818. """Skips over a field message.
  819. Args:
  820. tokenizer: A tokenizer to parse the field name and values.
  821. """
  822. if tokenizer.TryConsume('<'):
  823. delimiter = '>'
  824. else:
  825. tokenizer.Consume('{')
  826. delimiter = '}'
  827. while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'):
  828. _SkipField(tokenizer)
  829. tokenizer.Consume(delimiter)
  830. def _SkipFieldValue(tokenizer):
  831. """Skips over a field value.
  832. Args:
  833. tokenizer: A tokenizer to parse the field name and values.
  834. Raises:
  835. ParseError: In case an invalid field value is found.
  836. """
  837. # String/bytes tokens can come in multiple adjacent string literals.
  838. # If we can consume one, consume as many as we can.
  839. if tokenizer.TryConsumeByteString():
  840. while tokenizer.TryConsumeByteString():
  841. pass
  842. return
  843. if (not tokenizer.TryConsumeIdentifier() and
  844. not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and
  845. not tokenizer.TryConsumeFloat()):
  846. raise ParseError('Invalid field value: ' + tokenizer.token)
  847. class Tokenizer(object):
  848. """Protocol buffer text representation tokenizer.
  849. This class handles the lower level string parsing by splitting it into
  850. meaningful tokens.
  851. It was directly ported from the Java protocol buffer API.
  852. """
  853. _WHITESPACE = re.compile(r'\s+')
  854. _COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE)
  855. _WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE)
  856. _TOKEN = re.compile('|'.join([
  857. r'[a-zA-Z_][0-9a-zA-Z_+-]*', # an identifier
  858. r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*', # a number
  859. ] + [ # quoted str for each quote mark
  860. r'{qt}([^{qt}\n\\]|\\.)*({qt}|\\?$)'.format(qt=mark) for mark in _QUOTES
  861. ]))
  862. _IDENTIFIER = re.compile(r'[^\d\W]\w*')
  863. _IDENTIFIER_OR_NUMBER = re.compile(r'\w+')
  864. def __init__(self, lines, skip_comments=True):
  865. self._position = 0
  866. self._line = -1
  867. self._column = 0
  868. self._token_start = None
  869. self.token = ''
  870. self._lines = iter(lines)
  871. self._current_line = ''
  872. self._previous_line = 0
  873. self._previous_column = 0
  874. self._more_lines = True
  875. self._skip_comments = skip_comments
  876. self._whitespace_pattern = (skip_comments and self._WHITESPACE_OR_COMMENT
  877. or self._WHITESPACE)
  878. self._SkipWhitespace()
  879. self.NextToken()
  880. def LookingAt(self, token):
  881. return self.token == token
  882. def AtEnd(self):
  883. """Checks the end of the text was reached.
  884. Returns:
  885. True iff the end was reached.
  886. """
  887. return not self.token
  888. def _PopLine(self):
  889. while len(self._current_line) <= self._column:
  890. try:
  891. self._current_line = next(self._lines)
  892. except StopIteration:
  893. self._current_line = ''
  894. self._more_lines = False
  895. return
  896. else:
  897. self._line += 1
  898. self._column = 0
  899. def _SkipWhitespace(self):
  900. while True:
  901. self._PopLine()
  902. match = self._whitespace_pattern.match(self._current_line, self._column)
  903. if not match:
  904. break
  905. length = len(match.group(0))
  906. self._column += length
  907. def TryConsume(self, token):
  908. """Tries to consume a given piece of text.
  909. Args:
  910. token: Text to consume.
  911. Returns:
  912. True iff the text was consumed.
  913. """
  914. if self.token == token:
  915. self.NextToken()
  916. return True
  917. return False
  918. def Consume(self, token):
  919. """Consumes a piece of text.
  920. Args:
  921. token: Text to consume.
  922. Raises:
  923. ParseError: If the text couldn't be consumed.
  924. """
  925. if not self.TryConsume(token):
  926. raise self.ParseError('Expected "%s".' % token)
  927. def ConsumeComment(self):
  928. result = self.token
  929. if not self._COMMENT.match(result):
  930. raise self.ParseError('Expected comment.')
  931. self.NextToken()
  932. return result
  933. def ConsumeCommentOrTrailingComment(self):
  934. """Consumes a comment, returns a 2-tuple (trailing bool, comment str)."""
  935. # Tokenizer initializes _previous_line and _previous_column to 0. As the
  936. # tokenizer starts, it looks like there is a previous token on the line.
  937. just_started = self._line == 0 and self._column == 0
  938. before_parsing = self._previous_line
  939. comment = self.ConsumeComment()
  940. # A trailing comment is a comment on the same line than the previous token.
  941. trailing = (self._previous_line == before_parsing
  942. and not just_started)
  943. return trailing, comment
  944. def TryConsumeIdentifier(self):
  945. try:
  946. self.ConsumeIdentifier()
  947. return True
  948. except ParseError:
  949. return False
  950. def ConsumeIdentifier(self):
  951. """Consumes protocol message field identifier.
  952. Returns:
  953. Identifier string.
  954. Raises:
  955. ParseError: If an identifier couldn't be consumed.
  956. """
  957. result = self.token
  958. if not self._IDENTIFIER.match(result):
  959. raise self.ParseError('Expected identifier.')
  960. self.NextToken()
  961. return result
  962. def TryConsumeIdentifierOrNumber(self):
  963. try:
  964. self.ConsumeIdentifierOrNumber()
  965. return True
  966. except ParseError:
  967. return False
  968. def ConsumeIdentifierOrNumber(self):
  969. """Consumes protocol message field identifier.
  970. Returns:
  971. Identifier string.
  972. Raises:
  973. ParseError: If an identifier couldn't be consumed.
  974. """
  975. result = self.token
  976. if not self._IDENTIFIER_OR_NUMBER.match(result):
  977. raise self.ParseError('Expected identifier or number, got %s.' % result)
  978. self.NextToken()
  979. return result
  980. def TryConsumeInteger(self):
  981. try:
  982. # Note: is_long only affects value type, not whether an error is raised.
  983. self.ConsumeInteger()
  984. return True
  985. except ParseError:
  986. return False
  987. def ConsumeInteger(self, is_long=False):
  988. """Consumes an integer number.
  989. Args:
  990. is_long: True if the value should be returned as a long integer.
  991. Returns:
  992. The integer parsed.
  993. Raises:
  994. ParseError: If an integer couldn't be consumed.
  995. """
  996. try:
  997. result = _ParseAbstractInteger(self.token, is_long=is_long)
  998. except ValueError as e:
  999. raise self.ParseError(str(e))
  1000. self.NextToken()
  1001. return result
  1002. def TryConsumeFloat(self):
  1003. try:
  1004. self.ConsumeFloat()
  1005. return True
  1006. except ParseError:
  1007. return False
  1008. def ConsumeFloat(self):
  1009. """Consumes an floating point number.
  1010. Returns:
  1011. The number parsed.
  1012. Raises:
  1013. ParseError: If a floating point number couldn't be consumed.
  1014. """
  1015. try:
  1016. result = ParseFloat(self.token)
  1017. except ValueError as e:
  1018. raise self.ParseError(str(e))
  1019. self.NextToken()
  1020. return result
  1021. def ConsumeBool(self):
  1022. """Consumes a boolean value.
  1023. Returns:
  1024. The bool parsed.
  1025. Raises:
  1026. ParseError: If a boolean value couldn't be consumed.
  1027. """
  1028. try:
  1029. result = ParseBool(self.token)
  1030. except ValueError as e:
  1031. raise self.ParseError(str(e))
  1032. self.NextToken()
  1033. return result
  1034. def TryConsumeByteString(self):
  1035. try:
  1036. self.ConsumeByteString()
  1037. return True
  1038. except ParseError:
  1039. return False
  1040. def ConsumeString(self):
  1041. """Consumes a string value.
  1042. Returns:
  1043. The string parsed.
  1044. Raises:
  1045. ParseError: If a string value couldn't be consumed.
  1046. """
  1047. the_bytes = self.ConsumeByteString()
  1048. try:
  1049. return six.text_type(the_bytes, 'utf-8')
  1050. except UnicodeDecodeError as e:
  1051. raise self._StringParseError(e)
  1052. def ConsumeByteString(self):
  1053. """Consumes a byte array value.
  1054. Returns:
  1055. The array parsed (as a string).
  1056. Raises:
  1057. ParseError: If a byte array value couldn't be consumed.
  1058. """
  1059. the_list = [self._ConsumeSingleByteString()]
  1060. while self.token and self.token[0] in _QUOTES:
  1061. the_list.append(self._ConsumeSingleByteString())
  1062. return b''.join(the_list)
  1063. def _ConsumeSingleByteString(self):
  1064. """Consume one token of a string literal.
  1065. String literals (whether bytes or text) can come in multiple adjacent
  1066. tokens which are automatically concatenated, like in C or Python. This
  1067. method only consumes one token.
  1068. Returns:
  1069. The token parsed.
  1070. Raises:
  1071. ParseError: When the wrong format data is found.
  1072. """
  1073. text = self.token
  1074. if len(text) < 1 or text[0] not in _QUOTES:
  1075. raise self.ParseError('Expected string but found: %r' % (text,))
  1076. if len(text) < 2 or text[-1] != text[0]:
  1077. raise self.ParseError('String missing ending quote: %r' % (text,))
  1078. try:
  1079. result = text_encoding.CUnescape(text[1:-1])
  1080. except ValueError as e:
  1081. raise self.ParseError(str(e))
  1082. self.NextToken()
  1083. return result
  1084. def ConsumeEnum(self, field):
  1085. try:
  1086. result = ParseEnum(field, self.token)
  1087. except ValueError as e:
  1088. raise self.ParseError(str(e))
  1089. self.NextToken()
  1090. return result
  1091. def ParseErrorPreviousToken(self, message):
  1092. """Creates and *returns* a ParseError for the previously read token.
  1093. Args:
  1094. message: A message to set for the exception.
  1095. Returns:
  1096. A ParseError instance.
  1097. """
  1098. return ParseError(message, self._previous_line + 1,
  1099. self._previous_column + 1)
  1100. def ParseError(self, message):
  1101. """Creates and *returns* a ParseError for the current token."""
  1102. return ParseError(message, self._line + 1, self._column + 1)
  1103. def _StringParseError(self, e):
  1104. return self.ParseError('Couldn\'t parse string: ' + str(e))
  1105. def NextToken(self):
  1106. """Reads the next meaningful token."""
  1107. self._previous_line = self._line
  1108. self._previous_column = self._column
  1109. self._column += len(self.token)
  1110. self._SkipWhitespace()
  1111. if not self._more_lines:
  1112. self.token = ''
  1113. return
  1114. match = self._TOKEN.match(self._current_line, self._column)
  1115. if not match and not self._skip_comments:
  1116. match = self._COMMENT.match(self._current_line, self._column)
  1117. if match:
  1118. token = match.group(0)
  1119. self.token = token
  1120. else:
  1121. self.token = self._current_line[self._column]
  1122. # Aliased so it can still be accessed by current visibility violators.
  1123. # TODO(dbarnett): Migrate violators to textformat_tokenizer.
  1124. _Tokenizer = Tokenizer # pylint: disable=invalid-name
  1125. def _ConsumeInt32(tokenizer):
  1126. """Consumes a signed 32bit integer number from tokenizer.
  1127. Args:
  1128. tokenizer: A tokenizer used to parse the number.
  1129. Returns:
  1130. The integer parsed.
  1131. Raises:
  1132. ParseError: If a signed 32bit integer couldn't be consumed.
  1133. """
  1134. return _ConsumeInteger(tokenizer, is_signed=True, is_long=False)
  1135. def _ConsumeUint32(tokenizer):
  1136. """Consumes an unsigned 32bit integer number from tokenizer.
  1137. Args:
  1138. tokenizer: A tokenizer used to parse the number.
  1139. Returns:
  1140. The integer parsed.
  1141. Raises:
  1142. ParseError: If an unsigned 32bit integer couldn't be consumed.
  1143. """
  1144. return _ConsumeInteger(tokenizer, is_signed=False, is_long=False)
  1145. def _TryConsumeInt64(tokenizer):
  1146. try:
  1147. _ConsumeInt64(tokenizer)
  1148. return True
  1149. except ParseError:
  1150. return False
  1151. def _ConsumeInt64(tokenizer):
  1152. """Consumes a signed 32bit integer number from tokenizer.
  1153. Args:
  1154. tokenizer: A tokenizer used to parse the number.
  1155. Returns:
  1156. The integer parsed.
  1157. Raises:
  1158. ParseError: If a signed 32bit integer couldn't be consumed.
  1159. """
  1160. return _ConsumeInteger(tokenizer, is_signed=True, is_long=True)
  1161. def _TryConsumeUint64(tokenizer):
  1162. try:
  1163. _ConsumeUint64(tokenizer)
  1164. return True
  1165. except ParseError:
  1166. return False
  1167. def _ConsumeUint64(tokenizer):
  1168. """Consumes an unsigned 64bit integer number from tokenizer.
  1169. Args:
  1170. tokenizer: A tokenizer used to parse the number.
  1171. Returns:
  1172. The integer parsed.
  1173. Raises:
  1174. ParseError: If an unsigned 64bit integer couldn't be consumed.
  1175. """
  1176. return _ConsumeInteger(tokenizer, is_signed=False, is_long=True)
  1177. def _TryConsumeInteger(tokenizer, is_signed=False, is_long=False):
  1178. try:
  1179. _ConsumeInteger(tokenizer, is_signed=is_signed, is_long=is_long)
  1180. return True
  1181. except ParseError:
  1182. return False
  1183. def _ConsumeInteger(tokenizer, is_signed=False, is_long=False):
  1184. """Consumes an integer number from tokenizer.
  1185. Args:
  1186. tokenizer: A tokenizer used to parse the number.
  1187. is_signed: True if a signed integer must be parsed.
  1188. is_long: True if a long integer must be parsed.
  1189. Returns:
  1190. The integer parsed.
  1191. Raises:
  1192. ParseError: If an integer with given characteristics couldn't be consumed.
  1193. """
  1194. try:
  1195. result = ParseInteger(tokenizer.token, is_signed=is_signed, is_long=is_long)
  1196. except ValueError as e:
  1197. raise tokenizer.ParseError(str(e))
  1198. tokenizer.NextToken()
  1199. return result
  1200. def ParseInteger(text, is_signed=False, is_long=False):
  1201. """Parses an integer.
  1202. Args:
  1203. text: The text to parse.
  1204. is_signed: True if a signed integer must be parsed.
  1205. is_long: True if a long integer must be parsed.
  1206. Returns:
  1207. The integer value.
  1208. Raises:
  1209. ValueError: Thrown Iff the text is not a valid integer.
  1210. """
  1211. # Do the actual parsing. Exception handling is propagated to caller.
  1212. result = _ParseAbstractInteger(text, is_long=is_long)
  1213. # Check if the integer is sane. Exceptions handled by callers.
  1214. checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)]
  1215. checker.CheckValue(result)
  1216. return result
  1217. def _ParseAbstractInteger(text, is_long=False):
  1218. """Parses an integer without checking size/signedness.
  1219. Args:
  1220. text: The text to parse.
  1221. is_long: True if the value should be returned as a long integer.
  1222. Returns:
  1223. The integer value.
  1224. Raises:
  1225. ValueError: Thrown Iff the text is not a valid integer.
  1226. """
  1227. # Do the actual parsing. Exception handling is propagated to caller.
  1228. try:
  1229. # We force 32-bit values to int and 64-bit values to long to make
  1230. # alternate implementations where the distinction is more significant
  1231. # (e.g. the C++ implementation) simpler.
  1232. if is_long:
  1233. return long(text, 0)
  1234. else:
  1235. return int(text, 0)
  1236. except ValueError:
  1237. raise ValueError('Couldn\'t parse integer: %s' % text)
  1238. def ParseFloat(text):
  1239. """Parse a floating point number.
  1240. Args:
  1241. text: Text to parse.
  1242. Returns:
  1243. The number parsed.
  1244. Raises:
  1245. ValueError: If a floating point number couldn't be parsed.
  1246. """
  1247. try:
  1248. # Assume Python compatible syntax.
  1249. return float(text)
  1250. except ValueError:
  1251. # Check alternative spellings.
  1252. if _FLOAT_INFINITY.match(text):
  1253. if text[0] == '-':
  1254. return float('-inf')
  1255. else:
  1256. return float('inf')
  1257. elif _FLOAT_NAN.match(text):
  1258. return float('nan')
  1259. else:
  1260. # assume '1.0f' format
  1261. try:
  1262. return float(text.rstrip('f'))
  1263. except ValueError:
  1264. raise ValueError('Couldn\'t parse float: %s' % text)
  1265. def ParseBool(text):
  1266. """Parse a boolean value.
  1267. Args:
  1268. text: Text to parse.
  1269. Returns:
  1270. Boolean values parsed
  1271. Raises:
  1272. ValueError: If text is not a valid boolean.
  1273. """
  1274. if text in ('true', 't', '1', 'True'):
  1275. return True
  1276. elif text in ('false', 'f', '0', 'False'):
  1277. return False
  1278. else:
  1279. raise ValueError('Expected "true" or "false".')
  1280. def ParseEnum(field, value):
  1281. """Parse an enum value.
  1282. The value can be specified by a number (the enum value), or by
  1283. a string literal (the enum name).
  1284. Args:
  1285. field: Enum field descriptor.
  1286. value: String value.
  1287. Returns:
  1288. Enum value number.
  1289. Raises:
  1290. ValueError: If the enum value could not be parsed.
  1291. """
  1292. enum_descriptor = field.enum_type
  1293. try:
  1294. number = int(value, 0)
  1295. except ValueError:
  1296. # Identifier.
  1297. enum_value = enum_descriptor.values_by_name.get(value, None)
  1298. if enum_value is None:
  1299. raise ValueError('Enum type "%s" has no value named %s.' %
  1300. (enum_descriptor.full_name, value))
  1301. else:
  1302. # Numeric value.
  1303. if hasattr(field.file, 'syntax'):
  1304. # Attribute is checked for compatibility.
  1305. if field.file.syntax == 'proto3':
  1306. # Proto3 accept numeric unknown enums.
  1307. return number
  1308. enum_value = enum_descriptor.values_by_number.get(number, None)
  1309. if enum_value is None:
  1310. raise ValueError('Enum type "%s" has no value with number %d.' %
  1311. (enum_descriptor.full_name, number))
  1312. return enum_value.number