Lexer.php 16KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405
  1. <?php
  2. /*
  3. * This file is part of Twig.
  4. *
  5. * (c) 2009 Fabien Potencier
  6. * (c) 2009 Armin Ronacher
  7. *
  8. * For the full copyright and license information, please view the LICENSE
  9. * file that was distributed with this source code.
  10. */
  11. /**
  12. * Lexes a template string.
  13. *
  14. * @package twig
  15. * @author Fabien Potencier <fabien@symfony.com>
  16. */
  17. class Twig_Lexer implements Twig_LexerInterface
  18. {
  19. protected $tokens;
  20. protected $code;
  21. protected $cursor;
  22. protected $lineno;
  23. protected $end;
  24. protected $state;
  25. protected $states;
  26. protected $brackets;
  27. protected $env;
  28. protected $filename;
  29. protected $options;
  30. protected $regexes;
  31. const STATE_DATA = 0;
  32. const STATE_BLOCK = 1;
  33. const STATE_VAR = 2;
  34. const STATE_STRING = 3;
  35. const STATE_INTERPOLATION = 4;
  36. const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
  37. const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?/A';
  38. const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
  39. const REGEX_DQ_STRING_DELIM = '/"/A';
  40. const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
  41. const PUNCTUATION = '()[]{}?:.,|';
  42. public function __construct(Twig_Environment $env, array $options = array())
  43. {
  44. $this->env = $env;
  45. $this->options = array_merge(array(
  46. 'tag_comment' => array('{#', '#}'),
  47. 'tag_block' => array('{%', '%}'),
  48. 'tag_variable' => array('{{', '}}'),
  49. 'whitespace_trim' => '-',
  50. 'interpolation' => array('#{', '}'),
  51. ), $options);
  52. $this->regexes = array(
  53. 'lex_var' => '/\s*'.preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_variable'][1], '/').'/A',
  54. 'lex_block' => '/\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')\n?/A',
  55. 'lex_raw_data' => '/('.preg_quote($this->options['tag_block'][0].$this->options['whitespace_trim'], '/').'|'.preg_quote($this->options['tag_block'][0], '/').')\s*endraw\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')/s',
  56. 'operator' => $this->getOperatorRegex(),
  57. 'lex_comment' => '/(?:'.preg_quote($this->options['whitespace_trim'], '/').preg_quote($this->options['tag_comment'][1], '/').'\s*|'.preg_quote($this->options['tag_comment'][1], '/').')\n?/s',
  58. 'lex_block_raw' => '/\s*raw\s*(?:'.preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/').'\s*|\s*'.preg_quote($this->options['tag_block'][1], '/').')/As',
  59. 'lex_block_line' => '/\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '/').'/As',
  60. 'lex_tokens_start' => '/('.preg_quote($this->options['tag_variable'][0], '/').'|'.preg_quote($this->options['tag_block'][0], '/').'|'.preg_quote($this->options['tag_comment'][0], '/').')('.preg_quote($this->options['whitespace_trim'], '/').')?/s',
  61. 'interpolation_start' => '/'.preg_quote($this->options['interpolation'][0], '/').'\s*/A',
  62. 'interpolation_end' => '/\s*'.preg_quote($this->options['interpolation'][1], '/').'/A',
  63. );
  64. }
  65. /**
  66. * Tokenizes a source code.
  67. *
  68. * @param string $code The source code
  69. * @param string $filename A unique identifier for the source code
  70. *
  71. * @return Twig_TokenStream A token stream instance
  72. */
  73. public function tokenize($code, $filename = null)
  74. {
  75. if (function_exists('mb_internal_encoding') && ((int) ini_get('mbstring.func_overload')) & 2) {
  76. $mbEncoding = mb_internal_encoding();
  77. mb_internal_encoding('ASCII');
  78. }
  79. $this->code = str_replace(array("\r\n", "\r"), "\n", $code);
  80. $this->filename = $filename;
  81. $this->cursor = 0;
  82. $this->lineno = 1;
  83. $this->end = strlen($this->code);
  84. $this->tokens = array();
  85. $this->state = self::STATE_DATA;
  86. $this->states = array();
  87. $this->brackets = array();
  88. $this->position = -1;
  89. // find all token starts in one go
  90. preg_match_all($this->regexes['lex_tokens_start'], $this->code, $matches, PREG_OFFSET_CAPTURE);
  91. $this->positions = $matches;
  92. while ($this->cursor < $this->end) {
  93. // dispatch to the lexing functions depending
  94. // on the current state
  95. switch ($this->state) {
  96. case self::STATE_DATA:
  97. $this->lexData();
  98. break;
  99. case self::STATE_BLOCK:
  100. $this->lexBlock();
  101. break;
  102. case self::STATE_VAR:
  103. $this->lexVar();
  104. break;
  105. case self::STATE_STRING:
  106. $this->lexString();
  107. break;
  108. case self::STATE_INTERPOLATION:
  109. $this->lexInterpolation();
  110. break;
  111. }
  112. }
  113. $this->pushToken(Twig_Token::EOF_TYPE);
  114. if (!empty($this->brackets)) {
  115. list($expect, $lineno) = array_pop($this->brackets);
  116. throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  117. }
  118. if (isset($mbEncoding)) {
  119. mb_internal_encoding($mbEncoding);
  120. }
  121. return new Twig_TokenStream($this->tokens, $this->filename);
  122. }
  123. protected function lexData()
  124. {
  125. // if no matches are left we return the rest of the template as simple text token
  126. if ($this->position == count($this->positions[0]) - 1) {
  127. $this->pushToken(Twig_Token::TEXT_TYPE, substr($this->code, $this->cursor));
  128. $this->cursor = $this->end;
  129. return;
  130. }
  131. // Find the first token after the current cursor
  132. $position = $this->positions[0][++$this->position];
  133. while ($position[1] < $this->cursor) {
  134. if ($this->position == count($this->positions[0]) - 1) {
  135. return;
  136. }
  137. $position = $this->positions[0][++$this->position];
  138. }
  139. // push the template text first
  140. $text = $textContent = substr($this->code, $this->cursor, $position[1] - $this->cursor);
  141. if (isset($this->positions[2][$this->position][0])) {
  142. $text = rtrim($text);
  143. }
  144. $this->pushToken(Twig_Token::TEXT_TYPE, $text);
  145. $this->moveCursor($textContent.$position[0]);
  146. switch ($this->positions[1][$this->position][0]) {
  147. case $this->options['tag_comment'][0]:
  148. $this->lexComment();
  149. break;
  150. case $this->options['tag_block'][0]:
  151. // raw data?
  152. if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, null, $this->cursor)) {
  153. $this->moveCursor($match[0]);
  154. $this->lexRawData();
  155. // {% line \d+ %}
  156. } else if (preg_match($this->regexes['lex_block_line'], $this->code, $match, null, $this->cursor)) {
  157. $this->moveCursor($match[0]);
  158. $this->lineno = (int) $match[1];
  159. } else {
  160. $this->pushToken(Twig_Token::BLOCK_START_TYPE);
  161. $this->pushState(self::STATE_BLOCK);
  162. }
  163. break;
  164. case $this->options['tag_variable'][0]:
  165. $this->pushToken(Twig_Token::VAR_START_TYPE);
  166. $this->pushState(self::STATE_VAR);
  167. break;
  168. }
  169. }
  170. protected function lexBlock()
  171. {
  172. if (empty($this->brackets) && preg_match($this->regexes['lex_block'], $this->code, $match, null, $this->cursor)) {
  173. $this->pushToken(Twig_Token::BLOCK_END_TYPE);
  174. $this->moveCursor($match[0]);
  175. $this->popState();
  176. } else {
  177. $this->lexExpression();
  178. }
  179. }
  180. protected function lexVar()
  181. {
  182. if (empty($this->brackets) && preg_match($this->regexes['lex_var'], $this->code, $match, null, $this->cursor)) {
  183. $this->pushToken(Twig_Token::VAR_END_TYPE);
  184. $this->moveCursor($match[0]);
  185. $this->popState();
  186. } else {
  187. $this->lexExpression();
  188. }
  189. }
  190. protected function lexExpression()
  191. {
  192. // whitespace
  193. if (preg_match('/\s+/A', $this->code, $match, null, $this->cursor)) {
  194. $this->moveCursor($match[0]);
  195. if ($this->cursor >= $this->end) {
  196. throw new Twig_Error_Syntax(sprintf('Unexpected end of file: Unclosed "%s"', $this->state === self::STATE_BLOCK ? 'block' : 'variable'));
  197. }
  198. }
  199. // operators
  200. if (preg_match($this->regexes['operator'], $this->code, $match, null, $this->cursor)) {
  201. $this->pushToken(Twig_Token::OPERATOR_TYPE, $match[0]);
  202. $this->moveCursor($match[0]);
  203. }
  204. // names
  205. elseif (preg_match(self::REGEX_NAME, $this->code, $match, null, $this->cursor)) {
  206. $this->pushToken(Twig_Token::NAME_TYPE, $match[0]);
  207. $this->moveCursor($match[0]);
  208. }
  209. // numbers
  210. elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, null, $this->cursor)) {
  211. $number = (float) $match[0]; // floats
  212. if (ctype_digit($match[0]) && $number <= PHP_INT_MAX) {
  213. $number = (int) $match[0]; // integers lower than the maximum
  214. }
  215. $this->pushToken(Twig_Token::NUMBER_TYPE, $number);
  216. $this->moveCursor($match[0]);
  217. }
  218. // punctuation
  219. elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
  220. // opening bracket
  221. if (false !== strpos('([{', $this->code[$this->cursor])) {
  222. $this->brackets[] = array($this->code[$this->cursor], $this->lineno);
  223. }
  224. // closing bracket
  225. elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
  226. if (empty($this->brackets)) {
  227. throw new Twig_Error_Syntax(sprintf('Unexpected "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
  228. }
  229. list($expect, $lineno) = array_pop($this->brackets);
  230. if ($this->code[$this->cursor] != strtr($expect, '([{', ')]}')) {
  231. throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  232. }
  233. }
  234. $this->pushToken(Twig_Token::PUNCTUATION_TYPE, $this->code[$this->cursor]);
  235. ++$this->cursor;
  236. }
  237. // strings
  238. elseif (preg_match(self::REGEX_STRING, $this->code, $match, null, $this->cursor)) {
  239. $this->pushToken(Twig_Token::STRING_TYPE, stripcslashes(substr($match[0], 1, -1)));
  240. $this->moveCursor($match[0]);
  241. }
  242. // opening double quoted string
  243. elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, null, $this->cursor)) {
  244. $this->brackets[] = array('"', $this->lineno);
  245. $this->pushState(self::STATE_STRING);
  246. $this->moveCursor($match[0]);
  247. }
  248. // unlexable
  249. else {
  250. throw new Twig_Error_Syntax(sprintf('Unexpected character "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
  251. }
  252. }
  253. protected function lexRawData()
  254. {
  255. if (!preg_match($this->regexes['lex_raw_data'], $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
  256. throw new Twig_Error_Syntax(sprintf('Unexpected end of file: Unclosed "block"'));
  257. }
  258. $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
  259. $this->moveCursor($text.$match[0][0]);
  260. if (false !== strpos($match[1][0], $this->options['whitespace_trim'])) {
  261. $text = rtrim($text);
  262. }
  263. $this->pushToken(Twig_Token::TEXT_TYPE, $text);
  264. }
  265. protected function lexComment()
  266. {
  267. if (!preg_match($this->regexes['lex_comment'], $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
  268. throw new Twig_Error_Syntax('Unclosed comment', $this->lineno, $this->filename);
  269. }
  270. $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
  271. }
  272. protected function lexString()
  273. {
  274. if (preg_match($this->regexes['interpolation_start'], $this->code, $match, null, $this->cursor)) {
  275. $this->brackets[] = array($this->options['interpolation'][0], $this->lineno);
  276. $this->pushToken(Twig_Token::INTERPOLATION_START_TYPE);
  277. $this->moveCursor($match[0]);
  278. $this->pushState(self::STATE_INTERPOLATION);
  279. } else if (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, null, $this->cursor) && strlen($match[0]) > 0) {
  280. $this->pushToken(Twig_Token::STRING_TYPE, stripcslashes($match[0]));
  281. $this->moveCursor($match[0]);
  282. } else if (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, null, $this->cursor)) {
  283. list($expect, $lineno) = array_pop($this->brackets);
  284. if ($this->code[$this->cursor] != '"') {
  285. throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  286. }
  287. $this->popState();
  288. ++$this->cursor;
  289. return;
  290. }
  291. }
  292. protected function lexInterpolation()
  293. {
  294. $bracket = end($this->brackets);
  295. if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, null, $this->cursor)) {
  296. array_pop($this->brackets);
  297. $this->pushToken(Twig_Token::INTERPOLATION_END_TYPE);
  298. $this->moveCursor($match[0]);
  299. $this->popState();
  300. } else {
  301. $this->lexExpression();
  302. }
  303. }
  304. protected function pushToken($type, $value = '')
  305. {
  306. // do not push empty text tokens
  307. if (Twig_Token::TEXT_TYPE === $type && '' === $value) {
  308. return;
  309. }
  310. $this->tokens[] = new Twig_Token($type, $value, $this->lineno);
  311. }
  312. protected function moveCursor($text)
  313. {
  314. $this->cursor += strlen($text);
  315. $this->lineno += substr_count($text, "\n");
  316. }
  317. protected function getOperatorRegex()
  318. {
  319. $operators = array_merge(
  320. array('='),
  321. array_keys($this->env->getUnaryOperators()),
  322. array_keys($this->env->getBinaryOperators())
  323. );
  324. $operators = array_combine($operators, array_map('strlen', $operators));
  325. arsort($operators);
  326. $regex = array();
  327. foreach ($operators as $operator => $length) {
  328. // an operator that ends with a character must be followed by
  329. // a whitespace or a parenthesis
  330. if (ctype_alpha($operator[$length - 1])) {
  331. $regex[] = preg_quote($operator, '/').'(?=[\s()])';
  332. } else {
  333. $regex[] = preg_quote($operator, '/');
  334. }
  335. }
  336. return '/'.implode('|', $regex).'/A';
  337. }
  338. protected function pushState($state)
  339. {
  340. $this->states[] = $this->state;
  341. $this->state = $state;
  342. }
  343. protected function popState()
  344. {
  345. if (0 === count($this->states)) {
  346. throw new Exception('Cannot pop state without a previous state');
  347. }
  348. $this->state = array_pop($this->states);
  349. }
  350. }