[ClassLoader] Fix storing not-found classes in APC cache

This PR was merged into the 2.3 branch.

Discussion
----------

Fix performance (PHP5) and memory (PHP7) issues when using token_get_all

| Q             | A
| ------------- | ---
| Bug fix?      | yes
| New feature?  | no
| BC breaks?    | no
| Deprecations? | no
| Tests pass?   | yes
| Fixed tickets | #16868
| License       | MIT
| Doc PR        | -

Although it's not the case anymore on PHP 7, on PHP 5, `is_array` checks are much slower than `isset` checks.

Also from @peteward in #17384:
>  New PHP7 memory manager will not release small buckets to OS automatically in cases exposed by `token_get_all()`. This function call addition specifically for PHP7 will reclaim this memory to keep the footprint down of long processe

> See above ticket and suggested actions by PHP internals team for long-running tasks (https://bugs.php.net/70098) - I think `cache:clear/warmup` on a heavy app justifies this.

> We're running on cloud-based hosting platforms under memory limitations (Platform.sh). When memory is exceeded we're into swap and the cache clearing process goes from seconds to minutes for the initial deployment, which really slows our development workflow and also causes holding page delays.

Commits
-------

e555aad Add gc_mem_caches() call for PHP7 after itoken_get_all() as new memory manager will not release small buckets to OS automatically
d1f72d8 Fix perf and mem issue when using token_get_all

ApcClassLoader.php

@@ -67,8 +67,8 @@ class ApcClassLoader
      */
     public function __construct($prefix, $decorated)
     {
-        if (!extension_loaded('apc')) {
-            throw new \RuntimeException('Unable to use ApcClassLoader as APC is not enabled.');
+        if (!function_exists('apcu_fetch')) {
+            throw new \RuntimeException('Unable to use ApcClassLoader as APC is not installed.');
         }
 
         if (!method_exists($decorated, 'findFile')) {
@@ -122,8 +122,10 @@ class ApcClassLoader
      */
     public function findFile($class)
     {
-        if (false === $file = apc_fetch($this->prefix.$class)) {
-            apc_store($this->prefix.$class, $file = $this->decorated->findFile($class));
+        $file = apcu_fetch($this->prefix.$class, $success);
+
+        if (!$success) {
+            apcu_store($this->prefix.$class, $file = $this->decorated->findFile($class) ?: null);
         }
 
         return $file;

ApcUniversalClassLoader.php

@@ -71,7 +71,7 @@ class ApcUniversalClassLoader extends UniversalClassLoader
      */
     public function __construct($prefix)
     {
-        if (!extension_loaded('apc')) {
+        if (!function_exists('apcu_fetch')) {
             throw new \RuntimeException('Unable to use ApcUniversalClassLoader as APC is not enabled.');
         }
 
@@ -87,8 +87,10 @@ class ApcUniversalClassLoader extends UniversalClassLoader
      */
     public function findFile($class)
     {
-        if (false === $file = apc_fetch($this->prefix.$class)) {
-            apc_store($this->prefix.$class, $file = parent::findFile($class));
+        $file = apcu_fetch($this->prefix.$class, $success);
+
+        if (!$success) {
+            apcu_store($this->prefix.$class, $file = parent::findFile($class) ?: null);
         }
 
         return $file;

ClassCollectionLoader.php

@@ -116,8 +116,8 @@ class ClassCollectionLoader
         }
 
         // cache the core classes
-        if (!is_dir(dirname($cache))) {
-            mkdir(dirname($cache), 0777, true);
+        if (!is_dir($cacheDir) && !@mkdir($cacheDir, 0777, true) && !is_dir($cacheDir)) {
+            throw new \RuntimeException(sprintf('Class Collection Loader was not able to create directory "%s"', $cacheDir));
         }
         self::writeCacheFile($cache, '<?php '.$content);
 
@@ -137,8 +137,8 @@ class ClassCollectionLoader
     public static function fixNamespaceDeclarations($source)
     {
         if (!function_exists('token_get_all') || !self::$useTokenizer) {
-            if (preg_match('/namespace(.*?)\s*;/', $source)) {
-                $source = preg_replace('/namespace(.*?)\s*;/', "namespace$1\n{", $source)."}\n";
+            if (preg_match('/(^|\s)namespace(.*?)\s*;/', $source)) {
+                $source = preg_replace('/(^|\s)namespace(.*?)\s*;/', "$1namespace$2\n{", $source)."}\n";
             }
 
             return $source;
@@ -149,8 +149,9 @@ class ClassCollectionLoader
         $inNamespace = false;
         $tokens = token_get_all($source);
 
-        for (reset($tokens); false !== $token = current($tokens); next($tokens)) {
-            if (is_string($token)) {
+        for ($i = 0; isset($tokens[$i]); ++$i) {
+            $token = $tokens[$i];
+            if (!isset($token[1]) || 'b"' === $token) {
                 $rawChunk .= $token;
             } elseif (in_array($token[0], array(T_COMMENT, T_DOC_COMMENT))) {
                 // strip comments
@@ -162,12 +163,12 @@ class ClassCollectionLoader
                 $rawChunk .= $token[1];
 
                 // namespace name and whitespaces
-                while (($t = next($tokens)) && is_array($t) && in_array($t[0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
-                    $rawChunk .= $t[1];
+                while (isset($tokens[++$i][1]) && in_array($tokens[$i][0], array(T_WHITESPACE, T_NS_SEPARATOR, T_STRING))) {
+                    $rawChunk .= $tokens[$i][1];
                 }
-                if ('{' === $t) {
+                if ('{' === $tokens[$i]) {
                     $inNamespace = false;
-                    prev($tokens);
+                    --$i;
                 } else {
                     $rawChunk = rtrim($rawChunk)."\n{";
                     $inNamespace = true;
@@ -175,8 +176,8 @@ class ClassCollectionLoader
             } elseif (T_START_HEREDOC === $token[0]) {
                 $output .= self::compressCode($rawChunk).$token[1];
                 do {
-                    $token = next($tokens);
-                    $output .= is_string($token) ? $token : $token[1];
+                    $token = $tokens[++$i];
+                    $output .= isset($token[1]) && 'b"' !== $token ? $token[1] : $token;
                 } while ($token[0] !== T_END_HEREDOC);
                 $output .= "\n";
                 $rawChunk = '';
@@ -192,7 +193,15 @@ class ClassCollectionLoader
             $rawChunk .= "}\n";
         }
 
-        return $output.self::compressCode($rawChunk);
+        $output .= self::compressCode($rawChunk);
+
+        if (PHP_VERSION_ID >= 70000) {
+            // PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
+            unset($tokens, $rawChunk);
+            gc_mem_caches();
+        }
+
+        return $output;
     }
 
     /**

ClassLoader.php

@@ -97,7 +97,7 @@ class ClassLoader
                     $paths
                 ));
             } elseif (!in_array($paths, $this->prefixes[$prefix])) {
-                 $this->prefixes[$prefix][] = $paths;
+                $this->prefixes[$prefix][] = $paths;
             }
         } else {
             $this->prefixes[$prefix] = array_unique((array) $paths);

ClassMapGenerator.php

@@ -72,6 +72,11 @@ class ClassMapGenerator
 
             $classes = self::findClasses($path);
 
+            if (PHP_VERSION_ID >= 70000) {
+                // PHP 7 memory manager will not release after token_get_all(), see https://bugs.php.net/70098
+                gc_mem_caches();
+            }
+
             foreach ($classes as $class) {
                 $map[$class] = $path;
             }
@@ -95,10 +100,10 @@ class ClassMapGenerator
         $classes = array();
 
         $namespace = '';
-        for ($i = 0, $max = count($tokens); $i < $max; ++$i) {
+        for ($i = 0; isset($tokens[$i]); ++$i) {
             $token = $tokens[$i];
 
-            if (is_string($token)) {
+            if (!isset($token[1])) {
                 continue;
             }
 
@@ -108,9 +113,9 @@ class ClassMapGenerator
                 case T_NAMESPACE:
                     $namespace = '';
                     // If there is a namespace, extract it
-                    while (($t = $tokens[++$i]) && is_array($t)) {
-                        if (in_array($t[0], array(T_STRING, T_NS_SEPARATOR))) {
-                            $namespace .= $t[1];
+                    while (isset($tokens[++$i][1])) {
+                        if (in_array($tokens[$i][0], array(T_STRING, T_NS_SEPARATOR))) {
+                            $namespace .= $tokens[$i][1];
                         }
                     }
                     $namespace .= '\\';
@@ -121,7 +126,7 @@ class ClassMapGenerator
                     // Skip usage of ::class constant
                     $isClassConstant = false;
                     for ($j = $i - 1; $j > 0; --$j) {
-                        if (is_string($tokens[$j])) {
+                        if (!isset($tokens[$j][1])) {
                             break;
                         }
 
@@ -134,14 +139,15 @@ class ClassMapGenerator
                     }
 
                     if ($isClassConstant) {
-                        continue;
+                        break;
                     }
 
                     // Find the classname
-                    while (($t = $tokens[++$i]) && is_array($t)) {
+                    while (isset($tokens[++$i][1])) {
+                        $t = $tokens[$i];
                         if (T_STRING === $t[0]) {
                             $class .= $t[1];
-                        } elseif ($class !== '' && T_WHITESPACE == $t[0]) {
+                        } elseif ('' !== $class && T_WHITESPACE === $t[0]) {
                             break;
                         }
                     }

DebugClassLoader.php

@@ -74,7 +74,7 @@ class DebugClassLoader
      */
     public function findFile($class)
     {
-        return $this->classFinder->findFile($class);
+        return $this->classFinder->findFile($class) ?: null;
     }
 
     /**

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2004-2015 Fabien Potencier
+Copyright (c) 2004-2016 Fabien Potencier
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,85 +1,14 @@
 ClassLoader Component
 =====================
 
-ClassLoader loads your project classes automatically if they follow some
-standard PHP conventions.
-
-The ClassLoader object is able to autoload classes that implement the PSR-0
-standard or the PEAR naming convention.
-
-First, register the autoloader:
-
-```php
-require_once __DIR__.'/src/Symfony/Component/ClassLoader/ClassLoader.php';
-
-use Symfony\Component\ClassLoader\ClassLoader;
-
-$loader = new ClassLoader();
-$loader->register();
-```
-
-Then, register some namespaces with the `addPrefix()` method:
-
-```php
-$loader->addPrefix('Symfony', __DIR__.'/src');
-$loader->addPrefix('Monolog', __DIR__.'/vendor/monolog/src');
-```
-
-The `addPrefix()` method takes a namespace prefix and a path where to
-look for the classes as arguments.
-
-You can also register a sub-namespaces:
-
-```php
-$loader->addPrefix('Doctrine\\Common', __DIR__.'/vendor/doctrine-common/lib');
-```
-
-The order of registration is significant and the first registered namespace
-takes precedence over later registered one.
-
-You can also register more than one path for a given namespace:
-
-```php
-$loader->addPrefix('Symfony', array(__DIR__.'/src', __DIR__.'/symfony/src'));
-```
-
-Alternatively, you can use the `addPrefixes()` method to register more
-than one namespace at once:
-
-```php
-$loader->addPrefixes(array(
-    'Symfony' => array(__DIR__.'/src', __DIR__.'/symfony/src'),
-    'Doctrine\\Common' => __DIR__.'/vendor/doctrine-common/lib',
-    'Doctrine' => __DIR__.'/vendor/doctrine/lib',
-    'Monolog' => __DIR__.'/vendor/monolog/src',
-));
-```
-
-For better performance, you can use the APC class loader:
-
-```php
-require_once __DIR__.'/src/Symfony/Component/ClassLoader/ClassLoader.php';
-require_once __DIR__.'/src/Symfony/Component/ClassLoader/ApcClassLoader.php';
-
-use Symfony\Component\ClassLoader\ClassLoader;
-use Symfony\Component\ClassLoader\ApcClassLoader;
-
-$loader = new ClassLoader();
-$loader->addPrefix('Symfony', __DIR__.'/src');
-
-$loader = new ApcClassLoader('apc.prefix.', $loader);
-$loader->register();
-```
-
-Furthermore, the component provides tools to aggregate classes into a single
-file, which is especially useful to improve performance on servers that do not
-provide byte caches.
+The ClassLoader component provides tools to autoload your classes and cache
+their locations for performance.
 
 Resources
 ---------
 
-You can run the unit tests with the following command:
-
-    $ cd path/to/Symfony/Component/ClassLoader/
-    $ composer install
-    $ phpunit
+  * [Documentation](https://symfony.com/doc/current/components/class_loader/index.html)
+  * [Contributing](https://symfony.com/doc/current/contributing/index.html)
+  * [Report issues](https://github.com/symfony/symfony/issues) and
+    [send Pull Requests](https://github.com/symfony/symfony/pulls)
+    in the [main Symfony repository](https://github.com/symfony/symfony)

Tests/ApcUniversalClassLoaderTest.php

@@ -13,15 +13,12 @@ namespace Symfony\Component\ClassLoader\Tests;
 
 use Symfony\Component\ClassLoader\ApcUniversalClassLoader;
 
-/**
- * @requires extension apc
- */
 class ApcUniversalClassLoaderTest extends \PHPUnit_Framework_TestCase
 {
     protected function setUp()
     {
         if (ini_get('apc.enabled') && ini_get('apc.enable_cli')) {
-            apc_clear_cache('user');
+            apcu_clear_cache();
         } else {
             $this->markTestSkipped('APC is not enabled.');
         }
@@ -30,7 +27,7 @@ class ApcUniversalClassLoaderTest extends \PHPUnit_Framework_TestCase
     protected function tearDown()
     {
         if (ini_get('apc.enabled') && ini_get('apc.enable_cli')) {
-            apc_clear_cache('user');
+            apcu_clear_cache();
         }
     }
 
@@ -39,7 +36,7 @@ class ApcUniversalClassLoaderTest extends \PHPUnit_Framework_TestCase
         $loader = new ApcUniversalClassLoader('test.prefix.');
         $loader->registerNamespace('Apc\Namespaced', __DIR__.DIRECTORY_SEPARATOR.'Fixtures');
 
-        $this->assertEquals($loader->findFile('\Apc\Namespaced\FooBar'), apc_fetch('test.prefix.\Apc\Namespaced\FooBar'), '__construct() takes a prefix as its first argument');
+        $this->assertEquals($loader->findFile('\Apc\Namespaced\FooBar'), apcu_fetch('test.prefix.\Apc\Namespaced\FooBar'), '__construct() takes a prefix as its first argument');
     }
 
    /**

Tests/ClassCollectionLoaderTest.php

@@ -205,7 +205,7 @@ class ClassCollectionLoaderTest extends \PHPUnit_Framework_TestCase
             array("namespace   Bar ;\nclass Foo {}\n", "namespace   Bar\n{\nclass Foo {}\n}\n"),
             array("namespace Foo\Bar;\nclass Foo {}\n", "namespace Foo\Bar\n{\nclass Foo {}\n}\n"),
             array("namespace Foo\Bar\Bar\n{\nclass Foo {}\n}\n", "namespace Foo\Bar\Bar\n{\nclass Foo {}\n}\n"),
-            array("namespace\n{\nclass Foo {}\n}\n", "namespace\n{\nclass Foo {}\n}\n"),
+            array("\nnamespace\n{\nclass Foo {}\n\$namespace=123;}\n", "\nnamespace\n{\nclass Foo {}\n\$namespace=123;}\n"),
         );
     }
 
@@ -241,26 +241,26 @@ class ClassCollectionLoaderTest extends \PHPUnit_Framework_TestCase
 
         spl_autoload_unregister($r);
 
-        $this->assertEquals(<<<EOF
+        $this->assertEquals(<<<'EOF'
 namespace Namespaced
 {
 class WithComments
 {
-public static \$loaded = true;
+public static $loaded = true;
 }
-\$string ='string should not be   modified {\$string}';
-\$heredoc = (<<<HD
+$string ='string should not be   modified {$string}';
+$heredoc = (<<<HD
 
 
-Heredoc should not be   modified {\$string}
+Heredoc should not be   modified {$string}
 
 
 HD
 );
-\$nowdoc =<<<'ND'
+$nowdoc =<<<'ND'
 
 
-Nowdoc should not be   modified {\$string}
+Nowdoc should not be   modified {$string}
 
 
 ND
@@ -270,7 +270,7 @@ namespace
 {
 class Pearlike_WithComments
 {
-public static \$loaded = true;
+public static $loaded = true;
 }
 }
 EOF

Tests/ClassMapGeneratorTest.php

@@ -22,7 +22,7 @@ class ClassMapGeneratorTest extends \PHPUnit_Framework_TestCase
 
     public function prepare_workspace()
     {
-        $this->workspace = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR).DIRECTORY_SEPARATOR.time().mt_rand(0, 1000);
+        $this->workspace = sys_get_temp_dir().'/'.microtime(true).'.'.mt_rand();
         mkdir($this->workspace, 0777, true);
         $this->workspace = realpath($this->workspace);
     }

WinCacheClassLoader.php

@@ -123,8 +123,10 @@ class WinCacheClassLoader
      */
     public function findFile($class)
     {
-        if (false === $file = wincache_ucache_get($this->prefix.$class)) {
-            wincache_ucache_set($this->prefix.$class, $file = $this->decorated->findFile($class), 0);
+        $file = wincache_ucache_get($this->prefix.$class, $success);
+
+        if (!$success) {
+            wincache_ucache_set($this->prefix.$class, $file = $this->decorated->findFile($class) ?: null, 0);
         }
 
         return $file;

XcacheClassLoader.php

@@ -126,7 +126,7 @@ class XcacheClassLoader
         if (xcache_isset($this->prefix.$class)) {
             $file = xcache_get($this->prefix.$class);
         } else {
-            $file = $this->decorated->findFile($class);
+            $file = $this->decorated->findFile($class) ?: null;
             xcache_set($this->prefix.$class, $file);
         }
 

composer.json

@@ -17,7 +17,8 @@
     ],
     "minimum-stability": "dev",
     "require": {
-        "php": ">=5.3.3"
+        "php": ">=5.3.3",
+        "symfony/polyfill-apcu": "~1.1"
     },
     "require-dev": {
         "symfony/finder": "~2.0,>=2.0.5"

phpunit.xml.dist

@@ -9,6 +9,7 @@
     <php>
         <ini name="error_reporting" value="-1" />
     </php>
+
     <testsuites>
         <testsuite name="Symfony ClassLoader Component Test Suite">
             <directory>./Tests/</directory>