OmniSciDB  a5dc49c757
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Public Member Functions | Public Attributes | Private Attributes | List of all members
TableFunctionsFactory_parser.Parser Class Reference

Public Member Functions

def __init__
 
def tokens
 
def is_at_end
 
def current_token
 
def advance
 
def expect
 
def consume
 
def current_pos
 
def raise_parser_error
 
def match
 
def lookahead
 
def parse_udtf
 
def parse_args
 
def parse_arg
 
def parse_type
 
def parse_composed
 
def parse_primitive
 
def parse_templates
 
def parse_template
 
def parse_annotation
 
def parse_identifier
 
def parse_string
 
def parse_number
 
def parse_boolean
 
def parse
 

Public Attributes

 line
 

Private Attributes

 _tokens
 
 _curr
 

Detailed Description

Definition at line 255 of file TableFunctionsFactory_parser.py.

Constructor & Destructor Documentation

def TableFunctionsFactory_parser.Parser.__init__ (   self,
  line 
)

Definition at line 256 of file TableFunctionsFactory_parser.py.

256 
257  def __init__(self, line):
258  self._tokens = Tokenize(line).tokens
259  self._curr = 0
260  self.line = line
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Tokenize
Definition: TableFunctionsFactory_parser.py:79
TableFunctionsFactory_parser.Parser.__init__
def __init__
Definition: TableFunctionsFactory_parser.py:256
TableFunctionsFactory_parser.Parser.line
line
Definition: TableFunctionsFactory_parser.py:259
TableFunctionsFactory_parser.Parser._tokens
_tokens
Definition: TableFunctionsFactory_parser.py:257

Member Function Documentation

def TableFunctionsFactory_parser.Parser.advance (   self)

Definition at line 271 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr.

Referenced by TableFunctionsFactory_parser.Parser.consume(), and TableFunctionsFactory_parser.Parser.expect().

271 
272  def advance(self):
273  self._curr += 1
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.advance
def advance
Definition: TableFunctionsFactory_parser.py:271

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.consume (   self,
  expected_type 
) 
consumes the current token iff its type matches the
expected_type. Otherwise, an error is raised

Definition at line 285 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Tokenize._tokens, TableFunctionsFactory_parser.Parser._tokens, TableFunctionsFactory_parser.Tokenize.advance(), TableFunctionsFactory_parser.Parser.advance(), anonymous_namespace{RelAlgDag.cpp}::RANodeIterator.advance(), TableFunctionsFactory_parser.Tokenize.current_token(), TableFunctionsFactory_parser.Parser.current_token(), and TableFunctionsFactory_parser.Parser.raise_parser_error().

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), TableFunctionsFactory_parser.Parser.parse_arg(), TableFunctionsFactory_parser.Parser.parse_args(), TableFunctionsFactory_parser.Parser.parse_boolean(), TableFunctionsFactory_parser.Parser.parse_composed(), TableFunctionsFactory_parser.Parser.parse_identifier(), TableFunctionsFactory_parser.Parser.parse_number(), TableFunctionsFactory_parser.Parser.parse_string(), TableFunctionsFactory_parser.Parser.parse_template(), TableFunctionsFactory_parser.Parser.parse_templates(), and TableFunctionsFactory_parser.Parser.parse_udtf().

285 
286  def consume(self, expected_type):
287  """consumes the current token iff its type matches the
288  expected_type. Otherwise, an error is raised
289  """
290  curr_token = self.current_token()
291  if curr_token.type == expected_type:
292  self.advance()
293  return curr_token
294  else:
295  expected_token = Token.tok_name(expected_type)
296  self.raise_parser_error(
297  'Token mismatch at function consume. '
298  'Expected type "%s" but got token "%s"\n\n'
299  'Tokens: %s\n' % (expected_token, curr_token, self._tokens)
300  )
TableFunctionsFactory_parser.Parser.advance
def advance
Definition: TableFunctionsFactory_parser.py:271
TableFunctionsFactory_parser.Parser.raise_parser_error
def raise_parser_error
Definition: TableFunctionsFactory_parser.py:304
TableFunctionsFactory_parser.Parser.current_token
def current_token
Definition: TableFunctionsFactory_parser.py:268
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285
TableFunctionsFactory_parser.Parser._tokens
_tokens
Definition: TableFunctionsFactory_parser.py:257

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.current_pos (   self)

Definition at line 301 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr.

Referenced by TableFunctionsFactory_parser.Parser.raise_parser_error().

301 
302  def current_pos(self):
303  return self._curr
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.current_pos
def current_pos
Definition: TableFunctionsFactory_parser.py:301

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.current_token (   self)

Definition at line 268 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr, TableFunctionsFactory_parser.Tokenize._tokens, and TableFunctionsFactory_parser.Parser._tokens.

Referenced by TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Parser.expect(), TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_annotation(), and TableFunctionsFactory_parser.Parser.raise_parser_error().

268 
269  def current_token(self):
270  return self._tokens[self._curr]
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.current_token
def current_token
Definition: TableFunctionsFactory_parser.py:268
TableFunctionsFactory_parser.Parser._tokens
_tokens
Definition: TableFunctionsFactory_parser.py:257

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.expect (   self,
  expected_type 
)

Definition at line 274 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr, TableFunctionsFactory_parser.Tokenize._tokens, TableFunctionsFactory_parser.Parser._tokens, TableFunctionsFactory_parser.Tokenize.advance(), TableFunctionsFactory_parser.Parser.advance(), anonymous_namespace{RelAlgDag.cpp}::RANodeIterator.advance(), TableFunctionsFactory_parser.Tokenize.current_token(), and TableFunctionsFactory_parser.Parser.current_token().

Referenced by TableFunctionsFactory_parser.Parser.parse_udtf().

274 
275  def expect(self, expected_type):
276  curr_token = self.current_token()
277  msg = "Expected token %s but got %s at pos %d.\n Tokens: %s" % (
278  curr_token,
279  Token.tok_name(expected_type),
280  self._curr,
281  self._tokens,
282  )
283  assert curr_token.type == expected_type, msg
284  self.advance()
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.advance
def advance
Definition: TableFunctionsFactory_parser.py:271
TableFunctionsFactory_parser.Parser.expect
def expect
Definition: TableFunctionsFactory_parser.py:274
TableFunctionsFactory_parser.Parser.current_token
def current_token
Definition: TableFunctionsFactory_parser.py:268
TableFunctionsFactory_parser.Parser._tokens
_tokens
Definition: TableFunctionsFactory_parser.py:257

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.is_at_end (   self)

Definition at line 265 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr, TableFunctionsFactory_parser.Tokenize._tokens, and TableFunctionsFactory_parser.Parser._tokens.

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), TableFunctionsFactory_parser.Parser.parse_arg(), TableFunctionsFactory_parser.Parser.parse_args(), TableFunctionsFactory_parser.Parser.parse_templates(), TableFunctionsFactory_parser.Parser.parse_type(), and TableFunctionsFactory_parser.Parser.parse_udtf().

265 
266  def is_at_end(self):
267  return self._curr >= len(self._tokens)
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.is_at_end
def is_at_end
Definition: TableFunctionsFactory_parser.py:265
TableFunctionsFactory_parser.Parser._tokens
_tokens
Definition: TableFunctionsFactory_parser.py:257

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.lookahead (   self)

Definition at line 320 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr, TableFunctionsFactory_parser.Tokenize._tokens, and TableFunctionsFactory_parser.Parser._tokens.

Referenced by TableFunctionsFactory_parser.Parser.parse_arg().

320 
321  def lookahead(self):
322  return self._tokens[self._curr + 1]
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.lookahead
def lookahead
Definition: TableFunctionsFactory_parser.py:320
TableFunctionsFactory_parser.Parser._tokens
_tokens
Definition: TableFunctionsFactory_parser.py:257

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.match (   self,
  expected_type 
)

Definition at line 316 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Tokenize.current_token(), and TableFunctionsFactory_parser.Parser.current_token().

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), TableFunctionsFactory_parser.Parser.parse_arg(), TableFunctionsFactory_parser.Parser.parse_args(), TableFunctionsFactory_parser.Parser.parse_composed(), TableFunctionsFactory_parser.Parser.parse_primitive(), TableFunctionsFactory_parser.Parser.parse_template(), TableFunctionsFactory_parser.Parser.parse_templates(), TableFunctionsFactory_parser.Parser.parse_type(), and TableFunctionsFactory_parser.Parser.parse_udtf().

316 
317  def match(self, expected_type):
318  curr_token = self.current_token()
319  return curr_token.type == expected_type
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.current_token
def current_token
Definition: TableFunctionsFactory_parser.py:268

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse (   self) 
fmt: off

udtf: IDENTIFIER "(" (args)? ")" ("|" annotation)* "->" args ("," templates)? ("|" "output_row_size" "=" primitive)?

args: arg ("," arg)*

arg: type IDENTIFIER? ("|" annotation)*

type: composed
    | primitive

composed: "Cursor" "<" arg ("," arg)* ">"
| IDENTIFIER "<" type ("," type)* ">"

primitive: IDENTIFIER
 | NUMBER
 | STRING
 | BOOLEAN

annotation: IDENTIFIER "=" IDENTIFIER ("<" NUMBER ("," NUMBER) ">")?
  | IDENTIFIER "=" "[" PRIMITIVE? ("," PRIMITIVE)* "]"
  | "require" "=" STRING
  | "default" "=" STRING | NUMBER | BOOLEAN

templates: template ("," template)
template: IDENTIFIER "=" "[" IDENTIFIER ("," IDENTIFIER)* "]"

IDENTIFIER: [A-Za-z_][A-Za-z0-9_]*
NUMBER: [0-9]+
STRING: \".*?\"
BOOLEAN: \bTrue\b|\bFalse\b

fmt: on

Definition at line 613 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr, and TableFunctionsFactory_parser.Parser.parse_udtf().

613 
614  def parse(self):
615  """fmt: off
616 
617  udtf: IDENTIFIER "(" (args)? ")" ("|" annotation)* "->" args ("," templates)? ("|" "output_row_size" "=" primitive)?
618 
619  args: arg ("," arg)*
620 
621  arg: type IDENTIFIER? ("|" annotation)*
622 
623  type: composed
624  | primitive
625 
626  composed: "Cursor" "<" arg ("," arg)* ">"
627  | IDENTIFIER "<" type ("," type)* ">"
628 
629  primitive: IDENTIFIER
630  | NUMBER
631  | STRING
632  | BOOLEAN
633 
634  annotation: IDENTIFIER "=" IDENTIFIER ("<" NUMBER ("," NUMBER) ">")?
635  | IDENTIFIER "=" "[" PRIMITIVE? ("," PRIMITIVE)* "]"
636  | "require" "=" STRING
637  | "default" "=" STRING | NUMBER | BOOLEAN
638 
639  templates: template ("," template)
640  template: IDENTIFIER "=" "[" IDENTIFIER ("," IDENTIFIER)* "]"
641 
642  IDENTIFIER: [A-Za-z_][A-Za-z0-9_]*
643  NUMBER: [0-9]+
644  STRING: \".*?\"
645  BOOLEAN: \bTrue\b|\bFalse\b
646 
647  fmt: on
648  """
649  self._curr = 0
650  udtf = self.parse_udtf()
651 
652  # set parent
653  udtf.parent = None
654  d = deque()
655  d.append(udtf)
656  while d:
657  node = d.pop()
658  if isinstance(node, Iterable):
659  for child in node:
660  child.parent = node
661  d.append(child)
662  return udtf
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.parse_udtf
def parse_udtf
Definition: TableFunctionsFactory_parser.py:323
TableFunctionsFactory_parser.Parser.parse
def parse
Definition: TableFunctionsFactory_parser.py:613

+ Here is the call graph for this function:

def TableFunctionsFactory_parser.Parser.parse_annotation (   self) 
fmt: off

annotation: IDENTIFIER "=" IDENTIFIER ("<" NUMBER ("," NUMBER) ">")?
  | IDENTIFIER "=" "[" PRIMITIVE? ("," PRIMITIVE)* "]"
  | "require" "=" STRING
  | "default" "=" STRING | NUMBER | BOOLEAN

fmt: on

Definition at line 514 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Tokenize.current_token(), TableFunctionsFactory_parser.Parser.current_token(), TableFunctionsFactory_parser.Tokenize.is_at_end(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_boolean(), TableFunctionsFactory_parser.Parser.parse_identifier(), TableFunctionsFactory_parser.Parser.parse_number(), TableFunctionsFactory_parser.Parser.parse_primitive(), TableFunctionsFactory_parser.Parser.parse_string(), and TableFunctionsFactory_parser.Parser.raise_parser_error().

Referenced by TableFunctionsFactory_parser.Parser.parse_arg(), and TableFunctionsFactory_parser.Parser.parse_udtf().

514 
515  def parse_annotation(self):
516  """fmt: off
517 
518  annotation: IDENTIFIER "=" IDENTIFIER ("<" NUMBER ("," NUMBER) ">")?
519  | IDENTIFIER "=" "[" PRIMITIVE? ("," PRIMITIVE)* "]"
520  | "require" "=" STRING
521  | "default" "=" STRING | NUMBER | BOOLEAN
522 
523  fmt: on
524  """
525  key = self.parse_identifier()
526  self.consume(Token.EQUAL)
527 
528  if key == "require":
529  value = self.parse_string()
530  elif key == "default":
531  if self.match(Token.NUMBER):
532  value = self.parse_number()
533  elif self.match(Token.STRING):
534  value = self.parse_string()
535  elif self.match(Token.BOOLEAN):
536  value = self.parse_boolean()
537  else:
538  self.raise_parser_error(
539  'Unable to parse value in \"default\" annotation.\n'
540  'Expected type NUMBER, STRING or BOOLEAN.\n'
541  'Found token: "%s" of type "%s" \n'
542  % (self.current_token().lexeme, Token.tok_name(self.current_token().type))
543  )
544  elif not self.is_at_end() and self.match(Token.LSQB):
545  value = []
546  self.consume(Token.LSQB)
547  if not self.match(Token.RSQB):
548  value.append(self.parse_primitive())
549  while self.match(Token.COMMA):
550  self.consume(Token.COMMA)
551  value.append(self.parse_primitive())
552  self.consume(Token.RSQB)
553  else:
554  value = self.parse_identifier()
555  if not self.is_at_end() and self.match(Token.LESS):
556  self.consume(Token.LESS)
557  if self.match(Token.GREATER):
558  value += "<%s>" % (-1) # Signifies no input
559  else:
560  num1 = self.parse_number()
561  if self.match(Token.COMMA):
562  self.consume(Token.COMMA)
563  num2 = self.parse_number()
564  value += "<%s,%s>" % (num1, num2)
565  else:
566  value += "<%s>" % (num1)
567  self.consume(Token.GREATER)
568  return tf_node.AnnotationNode(key, value)
TableFunctionsFactory_parser.Parser.is_at_end
def is_at_end
Definition: TableFunctionsFactory_parser.py:265
TableFunctionsFactory_parser.Parser.parse_identifier
def parse_identifier
Definition: TableFunctionsFactory_parser.py:569
TableFunctionsFactory_parser.Parser.parse_annotation
def parse_annotation
Definition: TableFunctionsFactory_parser.py:514
TableFunctionsFactory_parser.Parser.raise_parser_error
def raise_parser_error
Definition: TableFunctionsFactory_parser.py:304
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.current_token
def current_token
Definition: TableFunctionsFactory_parser.py:268
TableFunctionsFactory_parser.Parser.parse_boolean
def parse_boolean
Definition: TableFunctionsFactory_parser.py:599
TableFunctionsFactory_parser.Parser.parse_primitive
def parse_primitive
Definition: TableFunctionsFactory_parser.py:460
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285
TableFunctionsFactory_parser.Parser.parse_string
def parse_string
Definition: TableFunctionsFactory_parser.py:579
TableFunctionsFactory_parser.Parser.parse_number
def parse_number
Definition: TableFunctionsFactory_parser.py:589

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_arg (   self) 
fmt: off

arg: type IDENTIFIER? ("|" annotation)*

fmt: on

Definition at line 393 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Tokenize.is_at_end(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_parser.Tokenize.lookahead(), TableFunctionsFactory_parser.Parser.lookahead(), TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_annotation(), TableFunctionsFactory_parser.Parser.parse_identifier(), and TableFunctionsFactory_parser.Parser.parse_type().

Referenced by TableFunctionsFactory_parser.Parser.parse_args(), and TableFunctionsFactory_parser.Parser.parse_composed().

393 
394  def parse_arg(self):
395  """fmt: off
396 
397  arg: type IDENTIFIER? ("|" annotation)*
398 
399  fmt: on
400  """
401  typ = self.parse_type()
402 
403  annotations = []
404 
405  if not self.is_at_end() and self.match(Token.IDENTIFIER):
406  name = self.parse_identifier()
407  annotations.append(tf_node.AnnotationNode('name', name))
408 
409  while not self.is_at_end() and self.match(Token.VBAR):
410  ahead = self.lookahead()
411  if ahead.type == Token.IDENTIFIER and ahead.lexeme == 'output_row_size':
412  break
413  self.consume(Token.VBAR)
414  annotations.append(self.parse_annotation())
415 
416  return tf_node.ArgNode(typ, annotations)
TableFunctionsFactory_parser.Parser.is_at_end
def is_at_end
Definition: TableFunctionsFactory_parser.py:265
TableFunctionsFactory_parser.Parser.parse_identifier
def parse_identifier
Definition: TableFunctionsFactory_parser.py:569
TableFunctionsFactory_parser.Parser.parse_type
def parse_type
Definition: TableFunctionsFactory_parser.py:417
TableFunctionsFactory_parser.Parser.parse_annotation
def parse_annotation
Definition: TableFunctionsFactory_parser.py:514
TableFunctionsFactory_parser.Parser.parse_arg
def parse_arg
Definition: TableFunctionsFactory_parser.py:393
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285
TableFunctionsFactory_parser.Parser.lookahead
def lookahead
Definition: TableFunctionsFactory_parser.py:320

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_args (   self) 
fmt: off

args: arg IDENTIFIER ("," arg)*

fmt: on

Definition at line 371 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr, TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Tokenize.is_at_end(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_arg(), and TableFunctionsFactory_parser.Parser.parse_type().

Referenced by TableFunctionsFactory_parser.Parser.parse_udtf().

371 
372  def parse_args(self):
373  """fmt: off
374 
375  args: arg IDENTIFIER ("," arg)*
376 
377  fmt: on
378  """
379  args = []
380  args.append(self.parse_arg())
381  while not self.is_at_end() and self.match(Token.COMMA):
382  curr = self._curr
383  self.consume(Token.COMMA)
384  self.parse_type() # assuming that we are not ending with COMMA
385  if not self.is_at_end() and self.match(Token.EQUAL):
386  # arg type cannot be assigned, so this must be a template specification
387  self._curr = curr # step back and let the code below parse the templates
388  break
389  else:
390  self._curr = curr + 1 # step back from self.parse_type(), parse_arg will parse it again
391  args.append(self.parse_arg())
392  return args
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.is_at_end
def is_at_end
Definition: TableFunctionsFactory_parser.py:265
TableFunctionsFactory_parser.Parser.parse_type
def parse_type
Definition: TableFunctionsFactory_parser.py:417
TableFunctionsFactory_parser.Parser.parse_arg
def parse_arg
Definition: TableFunctionsFactory_parser.py:393
TableFunctionsFactory_parser.Parser.parse_args
def parse_args
Definition: TableFunctionsFactory_parser.py:371
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_boolean (   self) 
fmt: off

BOOLEAN: \bTrue\b|\bFalse\b

fmt: on

Definition at line 599 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume().

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), and TableFunctionsFactory_parser.Parser.parse_primitive().

599 
600  def parse_boolean(self):
601  """ fmt: off
602 
603  BOOLEAN: \bTrue\b|\bFalse\b
604 
605  fmt: on
606  """
607  token = self.consume(Token.BOOLEAN)
608  # Make sure booleans are normalized to "False" or "True" regardless
609  # of original capitalization, so they can be properly parsed during
610  # typechecking
611  new_token = token.lexeme.lower().capitalize()
612  return new_token
TableFunctionsFactory_parser.Parser.parse_boolean
def parse_boolean
Definition: TableFunctionsFactory_parser.py:599
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_composed (   self) 
fmt: off

composed: "Cursor" "<" arg ("," arg)* ">"
| IDENTIFIER "<" type ("," type)* ">"

fmt: on

Definition at line 437 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.is_identifier_cursor(), TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_arg(), TableFunctionsFactory_parser.Parser.parse_identifier(), and TableFunctionsFactory_parser.Parser.parse_type().

Referenced by TableFunctionsFactory_parser.Parser.parse_type().

437 
438  def parse_composed(self):
439  """fmt: off
440 
441  composed: "Cursor" "<" arg ("," arg)* ">"
442  | IDENTIFIER "<" type ("," type)* ">"
443 
444  fmt: on
445  """
446  idtn = self.parse_identifier()
447  self.consume(Token.LESS)
448  if is_identifier_cursor(idtn):
449  inner = [self.parse_arg()]
450  while self.match(Token.COMMA):
451  self.consume(Token.COMMA)
452  inner.append(self.parse_arg())
453  else:
454  inner = [self.parse_type()]
455  while self.match(Token.COMMA):
456  self.consume(Token.COMMA)
457  inner.append(self.parse_type())
458  self.consume(Token.GREATER)
459  return tf_node.ComposedNode(idtn, inner)
TableFunctionsFactory_parser.Parser.parse_identifier
def parse_identifier
Definition: TableFunctionsFactory_parser.py:569
TableFunctionsFactory_parser.Parser.parse_composed
def parse_composed
Definition: TableFunctionsFactory_parser.py:437
TableFunctionsFactory_parser.Parser.parse_type
def parse_type
Definition: TableFunctionsFactory_parser.py:417
TableFunctionsFactory_parser.Parser.parse_arg
def parse_arg
Definition: TableFunctionsFactory_parser.py:393
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285
TableFunctionsFactory_parser.is_identifier_cursor
def is_identifier_cursor
Definition: TableFunctionsFactory_parser.py:251

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_identifier (   self) 
fmt: off

IDENTIFIER: [A-Za-z_][A-Za-z0-9_]*

fmt: on

Definition at line 569 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume().

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), TableFunctionsFactory_parser.Parser.parse_arg(), TableFunctionsFactory_parser.Parser.parse_composed(), TableFunctionsFactory_parser.Parser.parse_primitive(), TableFunctionsFactory_parser.Parser.parse_template(), and TableFunctionsFactory_parser.Parser.parse_udtf().

569 
570  def parse_identifier(self):
571  """ fmt: off
572 
573  IDENTIFIER: [A-Za-z_][A-Za-z0-9_]*
574 
575  fmt: on
576  """
577  token = self.consume(Token.IDENTIFIER)
578  return token.lexeme
TableFunctionsFactory_parser.Parser.parse_identifier
def parse_identifier
Definition: TableFunctionsFactory_parser.py:569
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_number (   self) 
fmt: off

NUMBER: [-]([0-9]*[.])?[0-9]+

fmt: on

Definition at line 589 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume().

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), and TableFunctionsFactory_parser.Parser.parse_primitive().

589 
590  def parse_number(self):
591  """ fmt: off
592 
593  NUMBER: [-]([0-9]*[.])?[0-9]+
594 
595  fmt: on
596  """
597  token = self.consume(Token.NUMBER)
598  return token.lexeme
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285
TableFunctionsFactory_parser.Parser.parse_number
def parse_number
Definition: TableFunctionsFactory_parser.py:589

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_primitive (   self) 
fmt: off

primitive: IDENTIFIER
 | NUMBER
 | STRING
 | BOOLEAN

fmt: on

Definition at line 460 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_boolean(), TableFunctionsFactory_parser.Parser.parse_identifier(), TableFunctionsFactory_parser.Parser.parse_number(), TableFunctionsFactory_parser.Parser.parse_string(), and TableFunctionsFactory_parser.Parser.raise_parser_error().

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), TableFunctionsFactory_parser.Parser.parse_type(), and TableFunctionsFactory_parser.Parser.parse_udtf().

460 
461  def parse_primitive(self):
462  """fmt: off
463 
464  primitive: IDENTIFIER
465  | NUMBER
466  | STRING
467  | BOOLEAN
468 
469  fmt: on
470  """
471  if self.match(Token.IDENTIFIER):
472  lexeme = self.parse_identifier()
473  elif self.match(Token.NUMBER):
474  lexeme = self.parse_number()
475  elif self.match(Token.STRING):
476  lexeme = self.parse_string()
477  elif self.match(Token.BOOLEAN):
478  lexeme = self.parse_boolean()
479  else:
480  raise self.raise_parser_error()
481  return tf_node.PrimitiveNode(lexeme)
TableFunctionsFactory_parser.Parser.parse_identifier
def parse_identifier
Definition: TableFunctionsFactory_parser.py:569
TableFunctionsFactory_parser.Parser.raise_parser_error
def raise_parser_error
Definition: TableFunctionsFactory_parser.py:304
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.parse_boolean
def parse_boolean
Definition: TableFunctionsFactory_parser.py:599
TableFunctionsFactory_parser.Parser.parse_primitive
def parse_primitive
Definition: TableFunctionsFactory_parser.py:460
TableFunctionsFactory_parser.Parser.parse_string
def parse_string
Definition: TableFunctionsFactory_parser.py:579
TableFunctionsFactory_parser.Parser.parse_number
def parse_number
Definition: TableFunctionsFactory_parser.py:589

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_string (   self) 
fmt: off

STRING: \".*?\"

fmt: on

Definition at line 579 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume().

Referenced by TableFunctionsFactory_parser.Parser.parse_annotation(), and TableFunctionsFactory_parser.Parser.parse_primitive().

579 
580  def parse_string(self):
581  """ fmt: off
582 
583  STRING: \".*?\"
584 
585  fmt: on
586  """
587  token = self.consume(Token.STRING)
588  return token.lexeme
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285
TableFunctionsFactory_parser.Parser.parse_string
def parse_string
Definition: TableFunctionsFactory_parser.py:579

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_template (   self) 
fmt: off

template: IDENTIFIER "=" "[" IDENTIFIER ("," IDENTIFIER)* "]"

fmt: on

Definition at line 496 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Parser.match(), and TableFunctionsFactory_parser.Parser.parse_identifier().

Referenced by TableFunctionsFactory_parser.Parser.parse_templates().

496 
497  def parse_template(self):
498  """fmt: off
499 
500  template: IDENTIFIER "=" "[" IDENTIFIER ("," IDENTIFIER)* "]"
501 
502  fmt: on
503  """
504  key = self.parse_identifier()
505  types = []
506  self.consume(Token.EQUAL)
507  self.consume(Token.LSQB)
508  types.append(self.parse_identifier())
509  while self.match(Token.COMMA):
510  self.consume(Token.COMMA)
511  types.append(self.parse_identifier())
512  self.consume(Token.RSQB)
513  return tf_node.TemplateNode(key, tuple(types))
TableFunctionsFactory_parser.Parser.parse_identifier
def parse_identifier
Definition: TableFunctionsFactory_parser.py:569
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.parse_template
def parse_template
Definition: TableFunctionsFactory_parser.py:496
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_templates (   self) 
fmt: off

templates: template ("," template)*

fmt: on

Definition at line 482 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Tokenize.is_at_end(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_parser.Parser.match(), and TableFunctionsFactory_parser.Parser.parse_template().

Referenced by TableFunctionsFactory_parser.Parser.parse_udtf().

482 
483  def parse_templates(self):
484  """fmt: off
485 
486  templates: template ("," template)*
487 
488  fmt: on
489  """
490  T = []
491  T.append(self.parse_template())
492  while not self.is_at_end() and self.match(Token.COMMA):
493  self.consume(Token.COMMA)
494  T.append(self.parse_template())
495  return T
TableFunctionsFactory_parser.Parser.is_at_end
def is_at_end
Definition: TableFunctionsFactory_parser.py:265
TableFunctionsFactory_parser.Parser.parse_templates
def parse_templates
Definition: TableFunctionsFactory_parser.py:482
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.parse_template
def parse_template
Definition: TableFunctionsFactory_parser.py:496
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_type (   self) 
fmt: off

type: composed
    | primitive

fmt: on

Definition at line 417 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser._curr, TableFunctionsFactory_parser.Tokenize.is_at_end(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_composed(), and TableFunctionsFactory_parser.Parser.parse_primitive().

Referenced by TableFunctionsFactory_parser.Parser.parse_arg(), TableFunctionsFactory_parser.Parser.parse_args(), and TableFunctionsFactory_parser.Parser.parse_composed().

417 
418  def parse_type(self):
419  """fmt: off
420 
421  type: composed
422  | primitive
423 
424  fmt: on
425  """
426  curr = self._curr # save state
427  primitive = self.parse_primitive()
428  if self.is_at_end():
429  return primitive
430 
431  if not self.match(Token.LESS):
432  return primitive
433 
434  self._curr = curr # return state
435 
436  return self.parse_composed()
TableFunctionsFactory_parser.Parser._curr
_curr
Definition: TableFunctionsFactory_parser.py:258
TableFunctionsFactory_parser.Parser.is_at_end
def is_at_end
Definition: TableFunctionsFactory_parser.py:265
TableFunctionsFactory_parser.Parser.parse_composed
def parse_composed
Definition: TableFunctionsFactory_parser.py:437
TableFunctionsFactory_parser.Parser.parse_type
def parse_type
Definition: TableFunctionsFactory_parser.py:417
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.parse_primitive
def parse_primitive
Definition: TableFunctionsFactory_parser.py:460

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.parse_udtf (   self) 
fmt: off

udtf: IDENTIFIER "(" (args)? ")" ("|" annotation)* "->" args ("," templates)? ("|" "output_row_size" "=" primitive)?

fmt: on

Definition at line 323 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Parser.expect(), TableFunctionsFactory_parser.Tokenize.is_at_end(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_node.UdtfNode.line, TableFunctionsFactory_parser.Parser.line, TableFunctionsFactory_parser.Parser.match(), TableFunctionsFactory_parser.Parser.parse_annotation(), TableFunctionsFactory_parser.Parser.parse_args(), TableFunctionsFactory_parser.Parser.parse_identifier(), TableFunctionsFactory_parser.Parser.parse_primitive(), and TableFunctionsFactory_parser.Parser.parse_templates().

Referenced by TableFunctionsFactory_parser.Parser.parse().

323 
324  def parse_udtf(self):
325  """fmt: off
326 
327  udtf: IDENTIFIER "(" (args)? ")" ("|" annotation)* "->" args ("," templates)? ("|" "output_row_size" "=" primitive)?
328 
329  fmt: on
330  """
331  name = self.parse_identifier()
332  self.expect(Token.LPAR) # (
333  input_args = []
334  if not self.match(Token.RPAR):
335  input_args = self.parse_args()
336  self.expect(Token.RPAR) # )
337  annotations = []
338  while not self.is_at_end() and self.match(Token.VBAR): # |
339  self.consume(Token.VBAR)
340  annotations.append(self.parse_annotation())
341  self.expect(Token.RARROW) # ->
342  output_args = self.parse_args()
343 
344  templates = None
345  if not self.is_at_end() and self.match(Token.COMMA):
346  self.consume(Token.COMMA)
347  templates = self.parse_templates()
348 
349  sizer = None
350  if not self.is_at_end() and self.match(Token.VBAR):
351  self.consume(Token.VBAR)
352  idtn = self.parse_identifier()
353  assert idtn == "output_row_size", idtn
354  self.consume(Token.EQUAL)
355  node = self.parse_primitive()
356  key = "kPreFlightParameter"
357  sizer = tf_node.AnnotationNode(key, value=node.type)
358 
359  # set arg_pos
360  i = 0
361  for arg in input_args:
362  arg.arg_pos = i
363  arg.kind = "input"
364  i += arg.type.cursor_length() if arg.type.is_cursor() else 1
365 
366  for i, arg in enumerate(output_args):
367  arg.arg_pos = i
368  arg.kind = "output"
369 
370  return tf_node.UdtfNode(name, input_args, output_args, annotations, templates, sizer, self.line)
TableFunctionsFactory_parser.Parser.parse_udtf
def parse_udtf
Definition: TableFunctionsFactory_parser.py:323
TableFunctionsFactory_parser.Parser.is_at_end
def is_at_end
Definition: TableFunctionsFactory_parser.py:265
TableFunctionsFactory_parser.Parser.parse_identifier
def parse_identifier
Definition: TableFunctionsFactory_parser.py:569
TableFunctionsFactory_parser.Parser.parse_annotation
def parse_annotation
Definition: TableFunctionsFactory_parser.py:514
TableFunctionsFactory_parser.Parser.parse_templates
def parse_templates
Definition: TableFunctionsFactory_parser.py:482
TableFunctionsFactory_parser.Parser.parse_args
def parse_args
Definition: TableFunctionsFactory_parser.py:371
TableFunctionsFactory_parser.Parser.match
def match
Definition: TableFunctionsFactory_parser.py:316
TableFunctionsFactory_parser.Parser.expect
def expect
Definition: TableFunctionsFactory_parser.py:274
TableFunctionsFactory_parser.Parser.line
line
Definition: TableFunctionsFactory_parser.py:259
TableFunctionsFactory_parser.Parser.parse_primitive
def parse_primitive
Definition: TableFunctionsFactory_parser.py:460
TableFunctionsFactory_parser.Parser.consume
def consume
Definition: TableFunctionsFactory_parser.py:285

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.raise_parser_error (   self,
  msg = None 
)

Definition at line 304 of file TableFunctionsFactory_parser.py.

References ChunkIter.current_pos, TableFunctionsFactory_parser.Parser.current_pos(), TableFunctionsFactory_parser.Tokenize.current_token(), TableFunctionsFactory_parser.Parser.current_token(), TableFunctionsFactory_parser.Tokenize.tokens(), and TableFunctionsFactory_parser.Parser.tokens().

Referenced by TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Parser.parse_annotation(), and TableFunctionsFactory_parser.Parser.parse_primitive().

304 
305  def raise_parser_error(self, msg=None):
306  if not msg:
307  token = self.current_token()
308  pos = self.current_pos()
309  tokens = self.tokens
310  msg = "\n\nError while trying to parse token %s at pos %d.\n" "Tokens: %s" % (
311  token,
312  pos,
313  tokens,
314  )
315  raise ParserException(msg)
TableFunctionsFactory_parser.Parser.tokens
def tokens
Definition: TableFunctionsFactory_parser.py:262
TableFunctionsFactory_parser.ParserException
Definition: TableFunctionsFactory_parser.py:17
TableFunctionsFactory_parser.Parser.raise_parser_error
def raise_parser_error
Definition: TableFunctionsFactory_parser.py:304
TableFunctionsFactory_parser.Parser.current_pos
def current_pos
Definition: TableFunctionsFactory_parser.py:301
TableFunctionsFactory_parser.Parser.current_token
def current_token
Definition: TableFunctionsFactory_parser.py:268

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

def TableFunctionsFactory_parser.Parser.tokens (   self)

Definition at line 262 of file TableFunctionsFactory_parser.py.

References TableFunctionsFactory_parser.Tokenize._tokens, and TableFunctionsFactory_parser.Parser._tokens.

Referenced by TableFunctionsFactory_parser.Parser.raise_parser_error().

262 
263  def tokens(self):
264  return self._tokens
TableFunctionsFactory_parser.Parser.tokens
def tokens
Definition: TableFunctionsFactory_parser.py:262
TableFunctionsFactory_parser.Parser._tokens
_tokens
Definition: TableFunctionsFactory_parser.py:257

+ Here is the caller graph for this function:

Member Data Documentation

TableFunctionsFactory_parser.Parser._curr
private

Definition at line 258 of file TableFunctionsFactory_parser.py.

Referenced by TableFunctionsFactory_parser.Parser.advance(), TableFunctionsFactory_parser.Parser.current_pos(), TableFunctionsFactory_parser.Parser.current_token(), TableFunctionsFactory_parser.Parser.expect(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_parser.Parser.lookahead(), TableFunctionsFactory_parser.Parser.parse(), TableFunctionsFactory_parser.Parser.parse_args(), and TableFunctionsFactory_parser.Parser.parse_type().

TableFunctionsFactory_parser.Parser._tokens
private

Definition at line 257 of file TableFunctionsFactory_parser.py.

Referenced by TableFunctionsFactory_parser.Parser.consume(), TableFunctionsFactory_parser.Parser.current_token(), TableFunctionsFactory_parser.Parser.expect(), TableFunctionsFactory_parser.Parser.is_at_end(), TableFunctionsFactory_parser.Parser.lookahead(), and TableFunctionsFactory_parser.Parser.tokens().

TableFunctionsFactory_parser.Parser.line

Definition at line 259 of file TableFunctionsFactory_parser.py.

Referenced by TableFunctionsFactory_parser.Tokenize.add_token(), TableFunctionsFactory_parser.Tokenize.current_token(), TableFunctionsFactory_parser.Tokenize.is_at_end(), TableFunctionsFactory_parser.Tokenize.lookahead(), TableFunctionsFactory_parser.Parser.parse_udtf(), TableFunctionsFactory_parser.Tokenize.peek(), and TableFunctionsFactory_parser.Tokenize.raise_tokenize_error().

The documentation for this class was generated from the following file: