Pelzini

This is the code documentation for the Pelzini project

More info
Project:
Home   coverage processor viewer

Search documentation

Search term:

What to search:





Path or filename:

Info Page:

General

Used by

Extends

Source search

class Token

This class is used to represent a token that has been tokenised using a Lexer. (e.g. the JavascriptLexer)
These tokens are used to create various ParserItems, by passing them to an Analyser (e.g. the JavascriptAnalyser)

Source code (8 results)

/processor/analyser.php   Highlighted file source

Line 31: * Generic language analyser. Analysers are used to tranform the language-specific tokens into a set of {@link CodeParserItem ParserItems}
Line 35: private $tokens;
Line 37: private $token_pos;
Line 44: $this->tokens = array();
Line 46: $this->token_pos = 0;
Line 51: * Processes a set of token and populates a {@link ParserFile}
Line 53: abstract public function process($tokens, $parser_file);
Line 57: * Tells the analyser what tokens it should use
Line 59: final protected function setTokens($tokens)
Line 61: $this->tokens = $tokens;
Line 75: * Moves the internal token pointer forwards
Line 86: * Moves the internal token pointer backwards
Line 97: * Returns a token at a specific position
Line 100: final protected function getToken($pos = null)
Line 103: return $this->tokens[$pos];
Line 117: * Finds a token looking forward from the current position.
Line 118: * Searching starts after the current token.
Line 119: * The token must be of the type specified
Line 121: * @param mixed $token_types A token type constant, or an array of token type constants
Line 122: * @param mixed $stop_list Token(s) that should stop the search process
Line 123: * @return Token The found token, or null if nothing was found
Line 125: final protected function findTokenForwards($token_types, $stop_list = null)
Line 127: if (! is_array($token_types)) $token_types = array($token_types);
Line 134: if (empty($this->tokens[$pos])) break;
Line 136: $tok = $this->tokens[$pos];
Line 141: if (in_array($tok->getType(), $token_types)) {
Line 142: $this->token_pos = $pos;
Line 154: * Finds a token looking backwards from the current position.
Line 155: * Searching starts before the current token.
Line 156: * The token must be of the type specified
Line 158: * @param mixed $token_types A token type constant, or an array of token type constants
Line 159: * @param mixed $stop_list Token(s) that should stop the search process
Line 160: * @return Token The found token, or null if nothing was found
Line 162: final protected function findTokenBackwards($token_types, $stop_list = null)
Line 164: if (! is_array($token_types)) $token_types = array($token_types);
Line 169: if (empty($this->tokens[$pos])) break;
Line 171: $tok = $this->tokens[$pos];
Line 176: if (in_array($tok->getType(), $token_types)) {
Line 177: $this->token_pos = $pos;
Line 189: * Gets the position of the last token found using one of the search functions
Line 191: final protected function getTokenPos()
Line 193: return $this->token_pos;

/processor/c_analyser.php   Highlighted file source

Line 31: * Analyses the C tokens, and creates a set of ParserItem objects.
Line 44: * Should create ParserItem objects that represent the provided tokens
Line 48: public function process($tokens, $parser_file)
Line 50: $this->setTokens($tokens);
Line 54: while ($token = $this->getToken()) {
Line 55: switch ($token->getType()) {
Line 56: case TOKEN_COMMENT:
Line 57: case TOKEN_C_PREPROCESSOR:
Line 60: case TOKEN_DOCBLOCK:
Line 61: $parser_file->applyComment($token->getValue());
Line 72: while ($function_open_bracket = $this->findTokenForwards(TOKEN_OPEN_NORMAL_BRACKET)) {
Line 73: $open_bracket_pos = $this->getTokenPos();
Line 80: $name = $this->findTokenBackwards(TOKEN_IDENTIFIER, array(TOKEN_CLOSE_CURLY_BRACKET, TOKEN_SEMICOLON));
Line 82: $this->setPos($this->getTokenPos());
Line 87: $return = $this->findTokenBackwards(array(TOKEN_IDENTIFIER, TOKEN_ASTERIX, TOKEN_CONST), array(TOKEN_CLOSE_CURLY_BRACKET, TOKEN_SEMICOLON));
Line 88: $this->setPos($this->getTokenPos());
Line 92: $return = $this->findTokenBackwards(array(TOKEN_IDENTIFIER, TOKEN_ASTERIX, TOKEN_CONST), array(TOKEN_CLOSE_CURLY_BRACKET, TOKEN_SEMICOLON));
Line 93: $this->setPos($this->getTokenPos());
Line 107: $reserved = $this->findTokenBackwards(array(TOKEN_RESERVED_WORD), array(TOKEN_CLOSE_CURLY_BRACKET, TOKEN_SEMICOLON));
Line 109: $this->setPos($this->getTokenPos());
Line 117: $reserved = $this->findTokenBackwards(array(TOKEN_RESERVED_WORD), array(TOKEN_CLOSE_CURLY_BRACKET, TOKEN_SEMICOLON));
Line 118: $this->setPos($this->getTokenPos());
Line 124: $docblock = $this->findTokenBackwards(TOKEN_DOCBLOCK, array(TOKEN_CLOSE_CURLY_BRACKET, TOKEN_SEMICOLON));
Line 132: TOKEN_OPEN_NORMAL_BRACKET,
Line 133: TOKEN_CLOSE_NORMAL_BRACKET,
Line 134: TOKEN_IDENTIFIER,
Line 135: TOKEN_COMMA,
Line 136: TOKEN_ASTERIX,
Line 137: TOKEN_CONST,
Line 139: $token = $function_open_bracket;
Line 141: $arg_tokens = array();
Line 142: while ($token) {
Line 143: switch ($token->getType()) {
Line 144: case TOKEN_OPEN_NORMAL_BRACKET:
Line 148: case TOKEN_CLOSE_NORMAL_BRACKET:
Line 152: case TOKEN_IDENTIFIER:
Line 153: case TOKEN_ASTERIX:
Line 154: case TOKEN_CONST:
Line 155: $arg_tokens[] = $token->getValue();
Line 158: case TOKEN_COMMA:
Line 161: $arg->name = array_pop($arg_tokens);
Line 163: if (count($arg_tokens) == 0) {
Line 166: $arg->type = trim(implode(' ', $arg_tokens));
Line 170: $arg_tokens = array();
Line 176: $token = $this->findTokenForwards($find_types);
Line 177: $this->setPos($this->getTokenPos());
Line 181: if (count($arg_tokens) > 0) {
Line 184: $arg->name = array_pop($arg_tokens);
Line 186: if (count($arg_tokens) == 0) {
Line 189: $arg->type = trim(implode(' ', $arg_tokens));
Line 196: $token = $this->findTokenForwards(array(TOKEN_OPEN_CURLY_BRACKET, TOKEN_SEMICOLON));
Line 197: $this->setPos($this->getTokenPos());
Line 198: while ($token) {
Line 199: if ($token->getType() == TOKEN_OPEN_CURLY_BRACKET) $depth++;
Line 200: if ($token->getType() == TOKEN_CLOSE_CURLY_BRACKET) $depth--;
Line 204: $token = $this->findTokenForwards(array(TOKEN_OPEN_CURLY_BRACKET, TOKEN_CLOSE_CURLY_BRACKET));
Line 205: $this->setPos($this->getTokenPos());

/processor/c_lexer.php   Highlighted file source

Line 31: * Tokenises a C file.
Line 37: '(' => TOKEN_OPEN_NORMAL_BRACKET,
Line 38: ')' => TOKEN_CLOSE_NORMAL_BRACKET,
Line 39: '{' => TOKEN_OPEN_CURLY_BRACKET,
Line 40: '}' => TOKEN_CLOSE_CURLY_BRACKET,
Line 41: '[' => TOKEN_OPEN_SQUARE_BRACKET,
Line 42: ']' => TOKEN_CLOSE_SQUARE_BRACKET,
Line 43: '=' => TOKEN_EQUALS,
Line 44: '.' => TOKEN_PERIOD,
Line 45: ',' => TOKEN_COMMA,
Line 46: ';' => TOKEN_SEMICOLON,
Line 47: '*' => TOKEN_ASTERIX
Line 56: private $token_words = array(
Line 57: 'const' => TOKEN_CONST,
Line 71: * Should return an array of zero or more Token objects
Line 77: $tokens = array();
Line 92: // Firstly, look for single character tokens
Line 94: foreach ($this->single_characters as $char => $token_type) {
Line 96: $tokens[] = new Token($token_type, $char);
Line 103: // Now use regular expressions to find various other tokens
Line 108: $tokens[] = new Token(TOKEN_C_PREPROCESSOR, $matches[0][0]);
Line 116: $tokens[] = new Token(TOKEN_DOCBLOCK, $matches[0][0]);
Line 124: $tokens[] = new Token(TOKEN_COMMENT, $matches[0][0]);
Line 132: $tokens[] = new Token(TOKEN_COMMENT, rtrim($matches[0][0]));
Line 140: $tokens[] = new Token(TOKEN_STRING, $matches[0][0]);
Line 148: $tokens[] = new Token(TOKEN_STRING, $matches[0][0]);
Line 157: $tokens[] = new Token(TOKEN_RESERVED_WORD, $word);
Line 167: $tokens[] = new Token(TOKEN_RESERVED_VALUE, $value);
Line 174: // Search for token words - reserved words with meaning
Line 175: foreach ($this->token_words as $word => $token_type) {
Line 177: $tokens[] = new Token($token_type, $word);
Line 191: $tokens[] = new Token(TOKEN_NUMBER, $matches[0][0]);
Line 200: $tokens[] = new Token(TOKEN_IDENTIFIER, $matches[0][0]);
Line 212: return $tokens;

/processor/c_parser.php   Highlighted file source

Line 58: $tokens = $this->lexer->process($source);
Line 59: if ($tokens === null) return null;
Line 62: //echo '<pre>Tokens for file ', $filename, "\n";
Line 63: //foreach ($tokens as $i => $t) echo "<b>{$i}</b> {$t->getTypeName()} <i>{$t->getValue()}</i>\n";
Line 70: $result = $this->analyser->process($tokens, $file);

/processor/javascript_analyser.php   Highlighted file source

Line 31: * Analyses the javascript tokens, and creates a set of ParserItem objects.
Line 44: * Should create ParserItem objects that represent the provided tokens
Line 48: public function process($tokens, $parser_file)
Line 50: $this->setTokens($tokens);
Line 54: while ($token = $this->getToken()) {
Line 55: switch ($token->getType()) {
Line 56: case TOKEN_COMMENT:
Line 59: case TOKEN_DOCBLOCK:
Line 60: $parser_file->applyComment($token->getValue());
Line 71: while ($function = $this->findTokenForwards(TOKEN_FUNCTION)) {
Line 72: $this->setPos($this->getTokenPos());
Line 78: $name = $this->findTokenForwards(TOKEN_IDENTIFIER);
Line 84: $docblock = $this->findTokenBackwards(TOKEN_DOCBLOCK, array(TOKEN_CLOSE_CURLY_BRACKET));
Line 92: TOKEN_OPEN_NORMAL_BRACKET,
Line 93: TOKEN_CLOSE_NORMAL_BRACKET,
Line 94: TOKEN_IDENTIFIER
Line 96: $token = $this->findTokenForwards(TOKEN_OPEN_NORMAL_BRACKET);
Line 97: $this->setPos($this->getTokenPos());
Line 98: while ($token) {
Line 99: switch ($token->getType()) {
Line 100: case TOKEN_OPEN_NORMAL_BRACKET:
Line 104: case TOKEN_CLOSE_NORMAL_BRACKET:
Line 108: case TOKEN_IDENTIFIER:
Line 110: $arg->name = $token->getValue();
Line 118: $token = $this->findTokenForwards($find_types);
Line 119: $this->setPos($this->getTokenPos());
Line 124: $token = $this->findTokenForwards(TOKEN_OPEN_CURLY_BRACKET);
Line 125: $this->setPos($this->getTokenPos());
Line 126: while ($token) {
Line 127: if ($token->getType() == TOKEN_OPEN_CURLY_BRACKET) $depth++;
Line 128: if ($token->getType() == TOKEN_CLOSE_CURLY_BRACKET) $depth--;
Line 132: $token = $this->findTokenForwards(array(TOKEN_OPEN_CURLY_BRACKET, TOKEN_CLOSE_CURLY_BRACKET));
Line 133: $this->setPos($this->getTokenPos());

/processor/javascript_lexer.php   Highlighted file source

Line 31: * Tokenises a javascript file.
Line 37: '(' => TOKEN_OPEN_NORMAL_BRACKET,
Line 38: ')' => TOKEN_CLOSE_NORMAL_BRACKET,
Line 39: '{' => TOKEN_OPEN_CURLY_BRACKET,
Line 40: '}' => TOKEN_CLOSE_CURLY_BRACKET,
Line 41: '[' => TOKEN_OPEN_SQUARE_BRACKET,
Line 42: ']' => TOKEN_CLOSE_SQUARE_BRACKET,
Line 43: '=' => TOKEN_EQUALS,
Line 44: '.' => TOKEN_PERIOD,
Line 45: ',' => TOKEN_COMMA,
Line 46: ';' => TOKEN_SEMICOLON
Line 70: * Should return an array of zero or more Token objects
Line 76: $tokens = array();
Line 79: Token::setCurrLineNum(1);
Line 83: Token::setIncrLineNum();
Line 89: // Firstly, look for single character tokens
Line 91: foreach ($this->single_characters as $char => $token_type) {
Line 93: $tokens[] = new Token($token_type, $char);
Line 99: // Now use regular expressions to find various other tokens
Line 104: $tokens[] = new Token(TOKEN_DOCBLOCK, $matches[0][0]);
Line 106: Token::setIncrLineNum(preg_match_all('/\n|\r|\n\r/', $matches[0][0], $junk));
Line 112: $tokens[] = new Token(TOKEN_COMMENT, $matches[0][0]);
Line 114: Token::setIncrLineNum(preg_match_all('/\n|\r|\n\r/', $matches[0][0], $junk));
Line 120: $tokens[] = new Token(TOKEN_COMMENT, rtrim($matches[0][0]));
Line 122: Token::setIncrLineNum();
Line 128: $tokens[] = new Token(TOKEN_STRING, $matches[0][0]);
Line 130: Token::setIncrLineNum(preg_match_all('/\n|\r|\n\r/', $matches[0][0], $junk));
Line 136: $tokens[] = new Token(TOKEN_STRING, $matches[0][0]);
Line 138: Token::setIncrLineNum(preg_match_all('/\n|\r|\n\r/', $matches[0][0], $junk));
Line 146: // Some reserved words get a specific token - basiclly anything that is understood by the analyser
Line 147: // everything else just gets the generic 'reserved word' token.
Line 150: $tokens[] = new Token(TOKEN_FUNCTION);
Line 154: $tokens[] = new Token(TOKEN_RESERVED_WORD, $word);
Line 166: $tokens[] = new Token(TOKEN_RESERVED_VALUE, $value);
Line 179: $tokens[] = new Token(TOKEN_NUMBER, $matches[0][0]);
Line 187: $tokens[] = new Token(TOKEN_IDENTIFIER, $matches[0][0]);
Line 195: return $tokens;

/processor/javascript_parser.php   Highlighted file source

Line 68: $tokens = $this->lexer->process($source);
Line 69: if ($tokens === null) return null;
Line 72: //echo '<pre>Tokens for file ', $filename, "\n";
Line 73: //foreach ($tokens as $i => $t) echo "<b>{$i}</b> {$t->getTypeName()} <i>{$t->getValue()}</i>\n";
Line 80: $result = $this->analyser->process($tokens, $file);

/processor/token.php   Highlighted file source

Line 23: * This class is used to represent a token that has been tokenised using a Lexer. (e.g. the JavascriptLexer)
Line 24: * These tokens are used to create various ParserItems, by passing them to an Analyser (e.g. the JavascriptAnalyser)
Line 26: class Token
Line 43: * Gets the type of this token
Line 52: * Gets the value of this token
Line 61: * Set the "current" line number. New tokens have a line number set to this figure.
Line 72: * Increment the "current" line number. New tokens have a line number set to this figure.
Line 91: * that this token referres to.
Line 98: if (strncmp($name, 'TOKEN_', 6) === 0 and $val == $this->type) {

A total of 222 lines in 8 files were found
Powered by Pelzini, version 0.9.0 Documentation is made available under the GNU Free Documentation License 1.2.
Generated: Thursday, 15th March, 2018 at 02:05 pm