Lexer.php 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329
  1. <?php
  2. /*
  3. * This file is part of Twig.
  4. *
  5. * (c) 2009 Fabien Potencier
  6. * (c) 2009 Armin Ronacher
  7. *
  8. * For the full copyright and license information, please view the LICENSE
  9. * file that was distributed with this source code.
  10. */
  11. /**
  12. * Lexes a template string.
  13. *
  14. * @package twig
  15. * @author Fabien Potencier <fabien@symfony.com>
  16. */
  17. class Twig_Lexer implements Twig_LexerInterface
  18. {
  19. protected $tokens;
  20. protected $code;
  21. protected $cursor;
  22. protected $lineno;
  23. protected $end;
  24. protected $state;
  25. protected $brackets;
  26. protected $env;
  27. protected $filename;
  28. protected $options;
  29. protected $operatorRegex;
  30. const STATE_DATA = 0;
  31. const STATE_BLOCK = 1;
  32. const STATE_VAR = 2;
  33. const REGEX_NAME = '/[A-Za-z_][A-Za-z0-9_]*/A';
  34. const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?/A';
  35. const REGEX_STRING = '/"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
  36. const PUNCTUATION = '()[]{}?:.,|';
  37. public function __construct(Twig_Environment $env, array $options = array())
  38. {
  39. $this->env = $env;
  40. $this->options = array_merge(array(
  41. 'tag_comment' => array('{#', '#}'),
  42. 'tag_block' => array('{%', '%}'),
  43. 'tag_variable' => array('{{', '}}'),
  44. 'whitespace_trim' => '-',
  45. ), $options);
  46. }
  47. /**
  48. * Tokenizes a source code.
  49. *
  50. * @param string $code The source code
  51. * @param string $filename A unique identifier for the source code
  52. *
  53. * @return Twig_TokenStream A token stream instance
  54. */
  55. public function tokenize($code, $filename = null)
  56. {
  57. if (function_exists('mb_internal_encoding') && ((int) ini_get('mbstring.func_overload')) & 2) {
  58. $mbEncoding = mb_internal_encoding();
  59. mb_internal_encoding('ASCII');
  60. }
  61. $this->code = str_replace(array("\r\n", "\r"), "\n", $code);
  62. $this->filename = $filename;
  63. $this->cursor = 0;
  64. $this->lineno = 1;
  65. $this->end = strlen($this->code);
  66. $this->tokens = array();
  67. $this->state = self::STATE_DATA;
  68. $this->brackets = array();
  69. while ($this->cursor < $this->end) {
  70. // dispatch to the lexing functions depending
  71. // on the current state
  72. switch ($this->state) {
  73. case self::STATE_DATA:
  74. $this->lexData();
  75. break;
  76. case self::STATE_BLOCK:
  77. $this->lexBlock();
  78. break;
  79. case self::STATE_VAR:
  80. $this->lexVar();
  81. break;
  82. }
  83. }
  84. $this->pushToken(Twig_Token::EOF_TYPE);
  85. if (!empty($this->brackets)) {
  86. list($expect, $lineno) = array_pop($this->brackets);
  87. throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  88. }
  89. if (isset($mbEncoding)) {
  90. mb_internal_encoding($mbEncoding);
  91. }
  92. return new Twig_TokenStream($this->tokens, $this->filename);
  93. }
  94. protected function lexData()
  95. {
  96. $pos = $this->end;
  97. $append = '';
  98. // Find the first token after the cursor
  99. foreach (array('tag_comment', 'tag_variable', 'tag_block') as $type) {
  100. $tmpPos = strpos($this->code, $this->options[$type][0], $this->cursor);
  101. if (false !== $tmpPos && $tmpPos < $pos) {
  102. $trimBlock = false;
  103. $append = '';
  104. $pos = $tmpPos;
  105. $token = $this->options[$type][0];
  106. if (strpos($this->code, $this->options['whitespace_trim'], $pos) === ($pos + strlen($token))) {
  107. $trimBlock = true;
  108. $append = $this->options['whitespace_trim'];
  109. }
  110. }
  111. }
  112. // if no matches are left we return the rest of the template as simple text token
  113. if ($pos === $this->end) {
  114. $this->pushToken(Twig_Token::TEXT_TYPE, substr($this->code, $this->cursor));
  115. $this->cursor = $this->end;
  116. return;
  117. }
  118. // push the template text first
  119. $text = $textContent = substr($this->code, $this->cursor, $pos - $this->cursor);
  120. if (true === $trimBlock) {
  121. $text = rtrim($text);
  122. }
  123. $this->pushToken(Twig_Token::TEXT_TYPE, $text);
  124. $this->moveCursor($textContent.$token.$append);
  125. switch ($token) {
  126. case $this->options['tag_comment'][0]:
  127. $this->lexComment();
  128. break;
  129. case $this->options['tag_block'][0]:
  130. // raw data?
  131. if (preg_match('/\s*raw\s*'.preg_quote($this->options['tag_block'][1], '/').'/As', $this->code, $match, null, $this->cursor)) {
  132. $this->moveCursor($match[0]);
  133. $this->lexRawData();
  134. $this->state = self::STATE_DATA;
  135. // {% line \d+ %}
  136. } else if (preg_match('/\s*line\s+(\d+)\s*'.preg_quote($this->options['tag_block'][1], '/').'/As', $this->code, $match, null, $this->cursor)) {
  137. $this->moveCursor($match[0]);
  138. $this->lineno = (int) $match[1];
  139. $this->state = self::STATE_DATA;
  140. } else {
  141. $this->pushToken(Twig_Token::BLOCK_START_TYPE);
  142. $this->state = self::STATE_BLOCK;
  143. }
  144. break;
  145. case $this->options['tag_variable'][0]:
  146. $this->pushToken(Twig_Token::VAR_START_TYPE);
  147. $this->state = self::STATE_VAR;
  148. break;
  149. }
  150. }
  151. protected function lexBlock()
  152. {
  153. $trimTag = preg_quote($this->options['whitespace_trim'].$this->options['tag_block'][1], '/');
  154. $endTag = preg_quote($this->options['tag_block'][1], '/');
  155. if (empty($this->brackets) && preg_match('/\s*(?:'.$trimTag.'\s*|\s*'.$endTag.')\n?/A', $this->code, $match, null, $this->cursor)) {
  156. $this->pushToken(Twig_Token::BLOCK_END_TYPE);
  157. $this->moveCursor($match[0]);
  158. $this->state = self::STATE_DATA;
  159. } else {
  160. $this->lexExpression();
  161. }
  162. }
  163. protected function lexVar()
  164. {
  165. $trimTag = preg_quote($this->options['whitespace_trim'].$this->options['tag_variable'][1], '/');
  166. $endTag = preg_quote($this->options['tag_variable'][1], '/');
  167. if (empty($this->brackets) && preg_match('/\s*'.$trimTag.'\s*|\s*'.$endTag.'/A', $this->code, $match, null, $this->cursor)) {
  168. $this->pushToken(Twig_Token::VAR_END_TYPE);
  169. $this->moveCursor($match[0]);
  170. $this->state = self::STATE_DATA;
  171. } else {
  172. $this->lexExpression();
  173. }
  174. }
  175. protected function lexExpression()
  176. {
  177. // whitespace
  178. if (preg_match('/\s+/A', $this->code, $match, null, $this->cursor)) {
  179. $this->moveCursor($match[0]);
  180. if ($this->cursor >= $this->end) {
  181. throw new Twig_Error_Syntax(sprintf('Unexpected end of file: Unclosed "%s"', $this->state === self::STATE_BLOCK ? 'block' : 'variable'));
  182. }
  183. }
  184. // operators
  185. if (preg_match($this->getOperatorRegex(), $this->code, $match, null, $this->cursor)) {
  186. $this->pushToken(Twig_Token::OPERATOR_TYPE, $match[0]);
  187. $this->moveCursor($match[0]);
  188. }
  189. // names
  190. elseif (preg_match(self::REGEX_NAME, $this->code, $match, null, $this->cursor)) {
  191. $this->pushToken(Twig_Token::NAME_TYPE, $match[0]);
  192. $this->moveCursor($match[0]);
  193. }
  194. // numbers
  195. elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, null, $this->cursor)) {
  196. $this->pushToken(Twig_Token::NUMBER_TYPE, ctype_digit($match[0]) ? (int) $match[0] : (float) $match[0]);
  197. $this->moveCursor($match[0]);
  198. }
  199. // punctuation
  200. elseif (false !== strpos(self::PUNCTUATION, $this->code[$this->cursor])) {
  201. // opening bracket
  202. if (false !== strpos('([{', $this->code[$this->cursor])) {
  203. $this->brackets[] = array($this->code[$this->cursor], $this->lineno);
  204. }
  205. // closing bracket
  206. elseif (false !== strpos(')]}', $this->code[$this->cursor])) {
  207. if (empty($this->brackets)) {
  208. throw new Twig_Error_Syntax(sprintf('Unexpected "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
  209. }
  210. list($expect, $lineno) = array_pop($this->brackets);
  211. if ($this->code[$this->cursor] != strtr($expect, '([{', ')]}')) {
  212. throw new Twig_Error_Syntax(sprintf('Unclosed "%s"', $expect), $lineno, $this->filename);
  213. }
  214. }
  215. $this->pushToken(Twig_Token::PUNCTUATION_TYPE, $this->code[$this->cursor]);
  216. ++$this->cursor;
  217. }
  218. // strings
  219. elseif (preg_match(self::REGEX_STRING, $this->code, $match, null, $this->cursor)) {
  220. $this->pushToken(Twig_Token::STRING_TYPE, stripcslashes(substr($match[0], 1, -1)));
  221. $this->moveCursor($match[0]);
  222. }
  223. // unlexable
  224. else {
  225. throw new Twig_Error_Syntax(sprintf('Unexpected character "%s"', $this->code[$this->cursor]), $this->lineno, $this->filename);
  226. }
  227. }
  228. protected function lexRawData()
  229. {
  230. if (!preg_match('/'.preg_quote($this->options['tag_block'][0], '/').'\s*endraw\s*'.preg_quote($this->options['tag_block'][1], '/').'/s', $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
  231. throw new Twig_Error_Syntax(sprintf('Unexpected end of file: Unclosed "block"'));
  232. }
  233. $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
  234. $this->pushToken(Twig_Token::TEXT_TYPE, $text);
  235. $this->moveCursor($text.$match[0][0]);
  236. }
  237. protected function lexComment()
  238. {
  239. $commentEndRegex = '/(?:'.preg_quote($this->options['whitespace_trim'], '/')
  240. .preg_quote($this->options['tag_comment'][1], '/').'\s*|'
  241. .preg_quote($this->options['tag_comment'][1], '/').')\n?/s';
  242. if (!preg_match($commentEndRegex, $this->code, $match, PREG_OFFSET_CAPTURE, $this->cursor)) {
  243. throw new Twig_Error_Syntax('Unclosed comment', $this->lineno, $this->filename);
  244. }
  245. $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
  246. }
  247. protected function pushToken($type, $value = '')
  248. {
  249. // do not push empty text tokens
  250. if (Twig_Token::TEXT_TYPE === $type && '' === $value) {
  251. return;
  252. }
  253. $this->tokens[] = new Twig_Token($type, $value, $this->lineno);
  254. }
  255. protected function moveCursor($text)
  256. {
  257. $this->cursor += strlen($text);
  258. $this->lineno += substr_count($text, "\n");
  259. }
  260. protected function getOperatorRegex()
  261. {
  262. if (null !== $this->operatorRegex) {
  263. return $this->operatorRegex;
  264. }
  265. $operators = array_merge(
  266. array('='),
  267. array_keys($this->env->getUnaryOperators()),
  268. array_keys($this->env->getBinaryOperators())
  269. );
  270. $operators = array_combine($operators, array_map('strlen', $operators));
  271. arsort($operators);
  272. $regex = array();
  273. foreach ($operators as $operator => $length) {
  274. // an operator that ends with a character must be followed by
  275. // a whitespace or a parenthesis
  276. if (ctype_alpha($operator[$length - 1])) {
  277. $regex[] = preg_quote($operator, '/').'(?=[ ()])';
  278. } else {
  279. $regex[] = preg_quote($operator, '/');
  280. }
  281. }
  282. return $this->operatorRegex = '/'.implode('|', $regex).'/A';
  283. }
  284. }