vendor/twig/twig/src/Lexer.php line 51

Open in your IDE?
  1. <?php
  3. /*
  4.  * This file is part of Twig.
  5.  *
  6.  * (c) Fabien Potencier
  7.  * (c) Armin Ronacher
  8.  *
  9.  * For the full copyright and license information, please view the LICENSE
  10.  * file that was distributed with this source code.
  11.  */
  13. namespace Twig;
  15. use Twig\Error\SyntaxError;
  17. /**
  18.  * @author Fabien Potencier <fabien@symfony.com>
  19.  */
  20. class Lexer
  21. {
  22.     private $tokens;
  23.     private $code;
  24.     private $cursor;
  25.     private $lineno;
  26.     private $end;
  27.     private $state;
  28.     private $states;
  29.     private $brackets;
  30.     private $env;
  31.     private $source;
  32.     private $options;
  33.     private $regexes;
  34.     private $position;
  35.     private $positions;
  36.     private $currentVarBlockLine;
  38.     public const STATE_DATA = 0;
  39.     public const STATE_BLOCK = 1;
  40.     public const STATE_VAR = 2;
  41.     public const STATE_STRING = 3;
  42.     public const STATE_INTERPOLATION = 4;
  44.     public const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
  45.     public const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?([Ee][\+\-][0-9]+)?/A';
  46.     public const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
  47.     public const REGEX_DQ_STRING_DELIM = '/"/A';
  48.     public const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
  49.     public const PUNCTUATION = '()[]{}?:.,|';
  51.     public function __construct(Environment $env, array $options = [])
  52.     {
  53.         $this->env = $env;
  55.         $this->options = array_merge([
  56.             'tag_comment' => ['{#', '#}'],
  57.             'tag_block' => ['{%', '%}'],
  58.             'tag_variable' => ['{{', '}}'],
  59.             'whitespace_trim' => '-',
  60.             'whitespace_line_trim' => '~',
  61.             'whitespace_line_chars' => ' \t\0\x0B',
  62.             'interpolation' => ['#{', '}'],
  63.         ], $options);
  65.         // when PHP 7.3 is the min version, we will be able to remove the '#' part in preg_quote as it's part of the default
  66.         $this->regexes = [
  67.             // }}
  68.             'lex_var' => '{
  69.                 \s*
  70.                 (?:'.
  71.                     preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '#').'\s*'. // -}}\s*
  72.                     '|'.
  73.                     preg_quote($this->options['whitespace_line_trim'].$this->options['tag_variable'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~}}[ \t\0\x0B]*
  74.                     '|'.
  75.                     preg_quote($this->options['tag_variable'][1], '#'). // }}
  76.                 ')
  77.             }Ax',
  79.             // %}
  80.             'lex_block' => '{
  81.                 \s*
  82.                 (?:'.
  83.                     preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*\n?'. // -%}\s*\n?
  84.                     '|'.
  85.                     preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
  86.                     '|'.
  87.                     preg_quote($this->options['tag_block'][1], '#').'\n?'. // %}\n?
  88.                 ')
  89.             }Ax',
  91.             // {% endverbatim %}
  92.             'lex_raw_data' => '{'.
  93.                 preg_quote($this->options['tag_block'][0], '#'). // {%
  94.                 '('.
  95.                     $this->options['whitespace_trim']. // -
  96.                     '|'.
  97.                     $this->options['whitespace_line_trim']. // ~
  98.                 ')?\s*endverbatim\s*'.
  99.                 '(?:'.
  100.                     preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}
  101.                     '|'.
  102.                     preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
  103.                     '|'.
  104.                     preg_quote($this->options['tag_block'][1], '#'). // %}
  105.                 ')
  106.             }sx',
  108.             'operator' => $this->getOperatorRegex(),
  110.             // #}
  111.             'lex_comment' => '{
  112.                 (?:'.
  113.                     preg_quote($this->options['whitespace_trim'].$this->options['tag_comment'][1], '#').'\s*\n?'. // -#}\s*\n?
  114.                     '|'.
  115.                     preg_quote($this->options['whitespace_line_trim'].$this->options['tag_comment'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~#}[ \t\0\x0B]*
  116.                     '|'.
  117.                     preg_quote($this->options['tag_comment'][1], '#').'\n?'. // #}\n?
  118.                 ')
  119.             }sx',
  121.             // verbatim %}
  122.             'lex_block_raw' => '{
  123.                 \s*verbatim\s*
  124.                 (?:'.
  125.                     preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '#').'\s*'. // -%}\s*
  126.                     '|'.
  127.                     preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
  128.                     '|'.
  129.                     preg_quote($this->options['tag_block'][1], '#'). // %}
  130.                 ')
  131.             }Asx',
  133.             'lex_block_line' => '{\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '#').'}As',
  135.             // {{ or {% or {#
  136.             'lex_tokens_start' => '{
  137.                 ('.
  138.                     preg_quote($this->options['tag_variable'][0], '#'). // {{
  139.                     '|'.
  140.                     preg_quote($this->options['tag_block'][0], '#'). // {%
  141.                     '|'.
  142.                     preg_quote($this->options['tag_comment'][0], '#'). // {#
  143.                 ')('.
  144.                     preg_quote($this->options['whitespace_trim'], '#'). // -
  145.                     '|'.
  146.                     preg_quote($this->options['whitespace_line_trim'], '#'). // ~
  147.                 ')?
  148.             }sx',
  149.             'interpolation_start' => '{'.preg_quote($this->options['interpolation'][0], '#').'\s*}A',
  150.             'interpolation_end' => '{\s*'.preg_quote($this->options['interpolation'][1], '#').'}A',
  151.         ];
  152.     }
  154.     public function tokenize(Source $source): TokenStream
  155.     {
  156.         $this->source = $source;
  157.         $this->code = str_replace(["\r\n", "\r"], "\n", $source->getCode());
  158.         $this->cursor = 0;
  159.         $this->lineno = 1;
  160.         $this->end = \strlen($this->code);
  161.         $this->tokens = [];
  162.         $this->state = self::STATE_DATA;
  163.         $this->states = [];
  164.         $this->brackets = [];
  165.         $this->position = -1;
  167.         // find all token starts in one go
  168.         preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, \PREG_OFFSET_CAPTURE);
  169.         $this->positions = $matches;
  171.         while ($this->cursor < $this->end) {
  172.             // dispatch to the lexing functions depending
  173.             // on the current state
  174.             switch ($this->state) {
  175.                 case self::STATE_DATA:
  176.                     $this->lexData();
  177.                     break;
  179.                 case self::STATE_BLOCK:
  180.                     $this->lexBlock();
  181.                     break;
  183.                 case self::STATE_VAR:
  184.                     $this->lexVar();
  185.                     break;
  187.                 case self::STATE_STRING:
  188.                     $this->lexString();
  189.                     break;
  191.                 case self::STATE_INTERPOLATION:
  192.                     $this->lexInterpolation();
  193.                     break;
  194.             }
  195.         }
  197.         $this->pushToken(/* Token::EOF_TYPE */ -1);
  199.         if (!empty($this->brackets)) {
  200.             list($expect, $lineno) = array_pop($this->brackets);
  201.             throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
  202.         }
  204.         return new TokenStream($this->tokens, $this->source);
  205.     }
  207.     private function lexData(): void
  208.     {
  209.         // if no matches are left we return the rest of the template as simple text token
  210.         if ($this->position == \count($this->positions[0]) - 1) {
  211.             $this->pushToken(/* Token::TEXT_TYPE */ 0, substr($this->code, $this->cursor));
  212.             $this->cursor = $this->end;
  214.             return;
  215.         }
  217.         // Find the first token after the current cursor
  218.         $position = $this->positions[0][++$this->position];
  219.         while ($position[1] < $this->cursor) {
  220.             if ($this->position == \count($this->positions[0]) - 1) {
  221.                 return;
  222.             }
  223.             $position = $this->positions[0][++$this->position];
  224.         }
  226.         // push the template text first
  227.         $text = $textContent = substr($this->code, $this->cursor, $position[1] - $this->cursor);
  229.         // trim?
  230.         if (isset($this->positions[2][$this->position][0])) {
  231.             if ($this->options['whitespace_trim'] === $this->positions[2][$this->position][0]) {
  232.                 // whitespace_trim detected ({%-, {{- or {#-)
  233.                 $text = rtrim($text);
  234.             } elseif ($this->options['whitespace_line_trim'] === $this->positions[2][$this->position][0]) {
  235.                 // whitespace_line_trim detected ({%~, {{~ or {#~)
  236.                 // don't trim \r and \n
  237.                 $text = rtrim($text, " \t\0\x0B");
  238.             }
  239.         }
  240.         $this->pushToken(/* Token::TEXT_TYPE */ 0, $text);
  241.         $this->moveCursor($textContent.$position[0]);
  243.         switch ($this->positions[1][$this->position][0]) {
  244.             case $this->options['tag_comment'][0]:
  245.                 $this->lexComment();
  246.                 break;
  248.             case $this->options['tag_block'][0]:
  249.                 // raw data?
  250.                 if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, 0, $this->cursor)) {
  251.                     $this->moveCursor($match[0]);
  252.                     $this->lexRawData();
  253.                 // {% line \d+ %}
  254.                 } elseif (preg_match($this->regexes['lex_block_line'], $this->code, $match, 0, $this->cursor)) {
  255.                     $this->moveCursor($match[0]);
  256.                     $this->lineno = (int) $match[1];
  257.                 } else {
  258.                     $this->pushToken(/* Token::BLOCK_START_TYPE */ 1);
  259.                     $this->pushState(self::STATE_BLOCK);
  260.                     $this->currentVarBlockLine = $this->lineno;
  261.                 }
  262.                 break;
  264.             case $this->options['tag_variable'][0]:
  265.                 $this->pushToken(/* Token::VAR_START_TYPE */ 2);
  266.                 $this->pushState(self::STATE_VAR);
  267.                 $this->currentVarBlockLine = $this->lineno;
  268.                 break;
  269.         }
  270.     }
  272.     private function lexBlock(): void
  273.     {
  274.         if (empty($this->brackets) && preg_match($this->regexes['lex_block'], $this->code, $match, 0, $this->cursor)) {
  275.             $this->pushToken(/* Token::BLOCK_END_TYPE */ 3);
  276.             $this->moveCursor($match[0]);
  277.             $this->popState();
  278.         } else {
  279.             $this->lexExpression();
  280.         }
  281.     }
  283.     private function lexVar(): void
  284.     {
  285.         if (empty($this->brackets) && preg_match($this->regexes['lex_var'], $this->code, $match, 0, $this->cursor)) {
  286.             $this->pushToken(/* Token::VAR_END_TYPE */ 4);
  287.             $this->moveCursor($match[0]);
  288.             $this->popState();
  289.         } else {
  290.             $this->lexExpression();
  291.         }
  292.     }
  294.     private function lexExpression(): void
  295.     {
  296.         // whitespace
  297.         if (preg_match('/\s+/A', $this->code, $match, 0, $this->cursor)) {
  298.             $this->moveCursor($match[0]);
  300.             if ($this->cursor >= $this->end) {
  301.                 throw new SyntaxError(sprintf('Unclosed "%s".', self::STATE_BLOCK === $this->state ? 'block' : 'variable'), $this->currentVarBlockLine, $this->source);
  302.             }
  303.         }
  305.         // arrow function
  306.         if ('=' === $this->code[$this->cursor] && '>' === $this->code[$this->cursor + 1]) {
  307.             $this->pushToken(Token::ARROW_TYPE, '=>');
  308.             $this->moveCursor('=>');
  309.         }
  310.         // operators
  311.         elseif (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
  312.             $this->pushToken(/* Token::OPERATOR_TYPE */ 8, preg_replace('/\s+/', ' ', $match[0]));
  313.             $this->moveCursor($match[0]);
  314.         }
  315.         // names
  316.         elseif (preg_match(self::REGEX_NAME, $this->code, $match, 0, $this->cursor)) {
  317.             $this->pushToken(/* Token::NAME_TYPE */ 5, $match[0]);
  318.             $this->moveCursor($match[0]);
  319.         }
  320.         // numbers
  321.         elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, 0, $this->cursor)) {
  322.             $number = (float) $match[0];  // floats
  323.             if (ctype_digit($match[0]) && $number <= \PHP_INT_MAX) {
  324.                 $number = (int) $match[0]; // integers lower than the maximum
  325.             }
  326.             $this->pushToken(/* Token::NUMBER_TYPE */ 6, $number);
  327.             $this->moveCursor($match[0]);
  328.         }
  329.         // punctuation
  330.         elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
  331.             // opening bracket
  332.             if (false !== strpos('([{', $this->code[$this->cursor])) {
  333.                 $this->brackets[] = [$this->code[$this->cursor], $this->lineno];
  334.             }
  335.             // closing bracket
  336.             elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
  337.                 if (empty($this->brackets)) {
  338.                     throw new SyntaxError(sprintf('Unexpected "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
  339.                 }
  341.                 list($expect, $lineno) = array_pop($this->brackets);
  342.                 if ($this->code[$this->cursor] != strtr($expect, '([{', ')]}')) {
  343.                     throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
  344.                 }
  345.             }
  347.             $this->pushToken(/* Token::PUNCTUATION_TYPE */ 9, $this->code[$this->cursor]);
  348.             ++$this->cursor;
  349.         }
  350.         // strings
  351.         elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) {
  352.             $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes(substr($match[0], 1, -1)));
  353.             $this->moveCursor($match[0]);
  354.         }
  355.         // opening double quoted string
  356.         elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
  357.             $this->brackets[] = ['"', $this->lineno];
  358.             $this->pushState(self::STATE_STRING);
  359.             $this->moveCursor($match[0]);
  360.         }
  361.         // unlexable
  362.         else {
  363.             throw new SyntaxError(sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
  364.         }
  365.     }
  367.     private function lexRawData(): void
  368.     {
  369.         if (!preg_match($this->regexes['lex_raw_data'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
  370.             throw new SyntaxError('Unexpected end of file: Unclosed "verbatim" block.', $this->lineno, $this->source);
  371.         }
  373.         $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
  374.         $this->moveCursor($text.$match[0][0]);
  376.         // trim?
  377.         if (isset($match[1][0])) {
  378.             if ($this->options['whitespace_trim'] === $match[1][0]) {
  379.                 // whitespace_trim detected ({%-, {{- or {#-)
  380.                 $text = rtrim($text);
  381.             } else {
  382.                 // whitespace_line_trim detected ({%~, {{~ or {#~)
  383.                 // don't trim \r and \n
  384.                 $text = rtrim($text, " \t\0\x0B");
  385.             }
  386.         }
  388.         $this->pushToken(/* Token::TEXT_TYPE */ 0, $text);
  389.     }
  391.     private function lexComment(): void
  392.     {
  393.         if (!preg_match($this->regexes['lex_comment'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
  394.             throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source);
  395.         }
  397.         $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
  398.     }
  400.     private function lexString(): void
  401.     {
  402.         if (preg_match($this->regexes['interpolation_start'], $this->code, $match, 0, $this->cursor)) {
  403.             $this->brackets[] = [$this->options['interpolation'][0], $this->lineno];
  404.             $this->pushToken(/* Token::INTERPOLATION_START_TYPE */ 10);
  405.             $this->moveCursor($match[0]);
  406.             $this->pushState(self::STATE_INTERPOLATION);
  407.         } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && \strlen($match[0]) > 0) {
  408.             $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes($match[0]));
  409.             $this->moveCursor($match[0]);
  410.         } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
  411.             list($expect, $lineno) = array_pop($this->brackets);
  412.             if ('"' != $this->code[$this->cursor]) {
  413.                 throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
  414.             }
  416.             $this->popState();
  417.             ++$this->cursor;
  418.         } else {
  419.             // unlexable
  420.             throw new SyntaxError(sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
  421.         }
  422.     }
  424.     private function lexInterpolation(): void
  425.     {
  426.         $bracket = end($this->brackets);
  427.         if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor)) {
  428.             array_pop($this->brackets);
  429.             $this->pushToken(/* Token::INTERPOLATION_END_TYPE */ 11);
  430.             $this->moveCursor($match[0]);
  431.             $this->popState();
  432.         } else {
  433.             $this->lexExpression();
  434.         }
  435.     }
  437.     private function pushToken($type, $value = ''): void
  438.     {
  439.         // do not push empty text tokens
  440.         if (/* Token::TEXT_TYPE */ 0 === $type && '' === $value) {
  441.             return;
  442.         }
  444.         $this->tokens[] = new Token($type, $value, $this->lineno);
  445.     }
  447.     private function moveCursor($text): void
  448.     {
  449.         $this->cursor += \strlen($text);
  450.         $this->lineno += substr_count($text, "\n");
  451.     }
  453.     private function getOperatorRegex(): string
  454.     {
  455.         $operators = array_merge(
  456.             ['='],
  457.             array_keys($this->env->getUnaryOperators()),
  458.             array_keys($this->env->getBinaryOperators())
  459.         );
  461.         $operators = array_combine($operators, array_map('strlen', $operators));
  462.         arsort($operators);
  464.         $regex = [];
  465.         foreach ($operators as $operator => $length) {
  466.             // an operator that ends with a character must be followed by
  467.             // a whitespace, a parenthesis, an opening map [ or sequence {
  468.             $r = preg_quote($operator, '/');
  469.             if (ctype_alpha($operator[$length - 1])) {
  470.                 $r .= '(?=[\s()\[{])';
  471.             }
  473.             // an operator that begins with a character must not have a dot or pipe before
  474.             if (ctype_alpha($operator[0])) {
  475.                 $r = '(?<![\.\|])'.$r;
  476.             }
  478.             // an operator with a space can be any amount of whitespaces
  479.             $r = preg_replace('/\s+/', '\s+', $r);
  481.             $regex[] = $r;
  482.         }
  484.         return '/'.implode('|', $regex).'/A';
  485.     }
  487.     private function pushState($state): void
  488.     {
  489.         $this->states[] = $this->state;
  490.         $this->state = $state;
  491.     }
  493.     private function popState(): void
  494.     {
  495.         if (0 === \count($this->states)) {
  496.             throw new \LogicException('Cannot pop state without a previous state.');
  497.         }
  499.         $this->state = array_pop($this->states);
  500.     }
  501. }