Lexer.php 7.1KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. <?php
  2. /*
  3. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  4. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  5. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  6. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  7. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  8. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  9. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  10. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  11. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  12. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  13. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  14. *
  15. * This software consists of voluntary contributions made by many individuals
  16. * and is licensed under the LGPL. For more information, see
  17. * <http://www.doctrine-project.org>.
  18. */
  19. namespace Doctrine\Common;
  20. /**
  21. * Base class for writing simple lexers, i.e. for creating small DSLs.
  22. *
  23. * @since 2.0
  24. * @author Guilherme Blanco <guilhermeblanco@hotmail.com>
  25. * @author Jonathan Wage <jonwage@gmail.com>
  26. * @author Roman Borschel <roman@code-factory.org>
  27. * @todo Rename: AbstractLexer
  28. */
  29. abstract class Lexer
  30. {
  31. /**
  32. * @var array Array of scanned tokens
  33. */
  34. private $tokens = array();
  35. /**
  36. * @var integer Current lexer position in input string
  37. */
  38. private $position = 0;
  39. /**
  40. * @var integer Current peek of current lexer position
  41. */
  42. private $peek = 0;
  43. /**
  44. * @var array The next token in the input.
  45. */
  46. public $lookahead;
  47. /**
  48. * @var array The last matched/seen token.
  49. */
  50. public $token;
  51. /**
  52. * Sets the input data to be tokenized.
  53. *
  54. * The Lexer is immediately reset and the new input tokenized.
  55. * Any unprocessed tokens from any previous input are lost.
  56. *
  57. * @param string $input The input to be tokenized.
  58. */
  59. public function setInput($input)
  60. {
  61. $this->tokens = array();
  62. $this->reset();
  63. $this->scan($input);
  64. }
  65. /**
  66. * Resets the lexer.
  67. */
  68. public function reset()
  69. {
  70. $this->lookahead = null;
  71. $this->token = null;
  72. $this->peek = 0;
  73. $this->position = 0;
  74. }
  75. /**
  76. * Resets the peek pointer to 0.
  77. */
  78. public function resetPeek()
  79. {
  80. $this->peek = 0;
  81. }
  82. /**
  83. * Resets the lexer position on the input to the given position.
  84. *
  85. * @param integer $position Position to place the lexical scanner
  86. */
  87. public function resetPosition($position = 0)
  88. {
  89. $this->position = $position;
  90. }
  91. /**
  92. * Checks whether a given token matches the current lookahead.
  93. *
  94. * @param integer|string $token
  95. * @return boolean
  96. */
  97. public function isNextToken($token)
  98. {
  99. return null !== $this->lookahead && $this->lookahead['type'] === $token;
  100. }
  101. /**
  102. * Checks whether any of the given tokens matches the current lookahead
  103. *
  104. * @param array $tokens
  105. * @return boolean
  106. */
  107. public function isNextTokenAny(array $tokens)
  108. {
  109. return null !== $this->lookahead && in_array($this->lookahead['type'], $tokens, true);
  110. }
  111. /**
  112. * Moves to the next token in the input string.
  113. *
  114. * A token is an associative array containing three items:
  115. * - 'value' : the string value of the token in the input string
  116. * - 'type' : the type of the token (identifier, numeric, string, input
  117. * parameter, none)
  118. * - 'position' : the position of the token in the input string
  119. *
  120. * @return array|null the next token; null if there is no more tokens left
  121. */
  122. public function moveNext()
  123. {
  124. $this->peek = 0;
  125. $this->token = $this->lookahead;
  126. $this->lookahead = (isset($this->tokens[$this->position]))
  127. ? $this->tokens[$this->position++] : null;
  128. return $this->lookahead !== null;
  129. }
  130. /**
  131. * Tells the lexer to skip input tokens until it sees a token with the given value.
  132. *
  133. * @param $type The token type to skip until.
  134. */
  135. public function skipUntil($type)
  136. {
  137. while ($this->lookahead !== null && $this->lookahead['type'] !== $type) {
  138. $this->moveNext();
  139. }
  140. }
  141. /**
  142. * Checks if given value is identical to the given token
  143. *
  144. * @param mixed $value
  145. * @param integer $token
  146. * @return boolean
  147. */
  148. public function isA($value, $token)
  149. {
  150. return $this->getType($value) === $token;
  151. }
  152. /**
  153. * Moves the lookahead token forward.
  154. *
  155. * @return array | null The next token or NULL if there are no more tokens ahead.
  156. */
  157. public function peek()
  158. {
  159. if (isset($this->tokens[$this->position + $this->peek])) {
  160. return $this->tokens[$this->position + $this->peek++];
  161. } else {
  162. return null;
  163. }
  164. }
  165. /**
  166. * Peeks at the next token, returns it and immediately resets the peek.
  167. *
  168. * @return array|null The next token or NULL if there are no more tokens ahead.
  169. */
  170. public function glimpse()
  171. {
  172. $peek = $this->peek();
  173. $this->peek = 0;
  174. return $peek;
  175. }
  176. /**
  177. * Scans the input string for tokens.
  178. *
  179. * @param string $input a query string
  180. */
  181. protected function scan($input)
  182. {
  183. static $regex;
  184. if ( ! isset($regex)) {
  185. $regex = '/(' . implode(')|(', $this->getCatchablePatterns()) . ')|'
  186. . implode('|', $this->getNonCatchablePatterns()) . '/i';
  187. }
  188. $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
  189. $matches = preg_split($regex, $input, -1, $flags);
  190. foreach ($matches as $match) {
  191. // Must remain before 'value' assignment since it can change content
  192. $type = $this->getType($match[0]);
  193. $this->tokens[] = array(
  194. 'value' => $match[0],
  195. 'type' => $type,
  196. 'position' => $match[1],
  197. );
  198. }
  199. }
  200. /**
  201. * Gets the literal for a given token.
  202. *
  203. * @param integer $token
  204. * @return string
  205. */
  206. public function getLiteral($token)
  207. {
  208. $className = get_class($this);
  209. $reflClass = new \ReflectionClass($className);
  210. $constants = $reflClass->getConstants();
  211. foreach ($constants as $name => $value) {
  212. if ($value === $token) {
  213. return $className . '::' . $name;
  214. }
  215. }
  216. return $token;
  217. }
  218. /**
  219. * Lexical catchable patterns.
  220. *
  221. * @return array
  222. */
  223. abstract protected function getCatchablePatterns();
  224. /**
  225. * Lexical non-catchable patterns.
  226. *
  227. * @return array
  228. */
  229. abstract protected function getNonCatchablePatterns();
  230. /**
  231. * Retrieve token type. Also processes the token value if necessary.
  232. *
  233. * @param string $value
  234. * @return integer
  235. */
  236. abstract protected function getType(&$value);
  237. }