- MFH: Added linenumbers to array returned by token_get_all()

NEWS

@@ -38,6 +38,7 @@ PHP                                                                        NEWS
 - Added php_pdo_sqlite_external.dll, a version of the PDO SQLite driver that
   links against an external sqlite3.dll.  This provides Windows users to upgrade
   their sqlite3 version outside of the PHP release cycle.  (Wez, Edin)
+- Added linenumbers to array returned by token_get_all(). (Johannes)
 - Fixed possible multi bytes issues in openssl csr parser (Pierre)
 - Fixed shmop_open() with IPC_CREAT|IPC_EXCL flags on Windows. 
   (Vladimir Kamaev, Tony).

ext/tokenizer/tests/003.phpt

@@ -23,20 +23,24 @@ array(0) {
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "0"
+    [2]=>
+    int(1)
   }
 }
 array(1) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "-1"
+    [2]=>
+    int(1)
   }
 }
 Done

ext/tokenizer/tests/bug26463.phpt

@@ -18,119 +18,147 @@ var_dump(token_get_all($str));
 --EXPECTF--
 array(19) {
   [0]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<?php
 "
+    [2]=>
+    int(1)
   }
   [1]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$x"
+    [2]=>
+    int(2)
   }
   [2]=>
   string(1) "="
   [3]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(6) "<<<DD
 "
+    [2]=>
+    int(2)
   }
   [4]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(3)
   }
   [5]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "DD"
+    [2]=>
+    int(4)
   }
   [6]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(4)
   }
   [7]=>
   string(1) "."
   [8]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) """"
+    [2]=>
+    int(5)
   }
   [9]=>
   string(1) ";"
   [10]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(5)
   }
   [11]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "$a"
+    [2]=>
+    int(6)
   }
   [12]=>
   string(1) "="
   [13]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(8) "<<<DDDD
 "
+    [2]=>
+    int(6)
   }
   [14]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(13) "jhdsjkfhjdsh
 "
+    [2]=>
+    int(7)
   }
   [15]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(4) "DDDD"
+    [2]=>
+    int(8)
   }
   [16]=>
   string(1) ";"
   [17]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(1) "
 "
+    [2]=>
+    int(8)
   }
   [18]=>
-  array(2) {
+  array(3) {
     [0]=>
     int(%d)
     [1]=>
     string(2) "?>"
+    [2]=>
+    int(9)
   }
 }

ext/tokenizer/tokenizer.c

@@ -272,6 +272,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
 	zval *keyword;
 	int token_type;
 	zend_bool destroy;
+	int token_line = 1;
 
 	array_init(return_value);
 
@@ -301,6 +302,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			} else {
 				add_next_index_stringl(keyword, zendtext, zendleng, 1);
 			}
+			add_next_index_long(keyword, token_line);
 			add_next_index_zval(return_value, keyword);
 		} else {
 			add_next_index_stringl(return_value, zendtext, zendleng, 1);
@@ -309,6 +311,8 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			zval_dtor(&token);
 		}
 		ZVAL_NULL(&token);
+
+		token_line = CG(zend_lineno);
 	}
 }