Issue #2526150 by catch, Wim Leers, Denchev, hussainweb, dawehner, mpdonadio,...

Issue #2526150 by catch, Wim Leers, Denchev, hussainweb, dawehner, mpdonadio, borisson_, fgm, olli, alexpott, vaplas, flocondetoile, Berdir, Fabianx, cilefen: Database cache bins allow unlimited growth: cache DB tables of gigabytes!

core/core.api.php

@@ -606,6 +606,30 @@
  *  $settings['cache']['default'] = 'cache.custom';
  * @endcode
  *
+ * For cache bins that are stored in the database, the number of rows is limited
+ * to 5000 by default. This can be changed for all database cache bins. For
+ * example, to instead limit the number of rows to 50000:
+ * @code
+ * $settings['database_cache_max_rows']['default'] = 50000;
+ * @endcode
+ *
+ * Or per bin (in this example we allow infinite entries):
+ * @code
+ * $settings['database_cache_max_rows']['bins']['dynamic_page_cache'] = -1;
+ * @endcode
+ *
+ * For monitoring reasons it might be useful to figure out the amount of data
+ * stored in tables. The following SQL snippet can be used for that:
+ * @code
+ * SELECT table_name AS `Table`, table_rows AS 'Num. of Rows',
+ * ROUND(((data_length + index_length) / 1024 / 1024), 2) `Size in MB` FROM
+ * information_schema.TABLES WHERE table_schema = '***DATABASE_NAME***' AND
+ * table_name LIKE 'cache_%'  ORDER BY (data_length + index_length) DESC
+ * LIMIT 10;
+ * @encode
+ *
+ * @see \Drupal\Core\Cache\DatabaseBackend
+ *
  * Finally, you can chain multiple cache backends together, see
  * \Drupal\Core\Cache\ChainedFastBackend and \Drupal\Core\Cache\BackendChain.
  *

core/core.services.yml

@@ -192,7 +192,7 @@ services:
       - [setContainer, ['@service_container']]
   cache.backend.database:
     class: Drupal\Core\Cache\DatabaseBackendFactory
-    arguments: ['@database', '@cache_tags.invalidator.checksum']
+    arguments: ['@database', '@cache_tags.invalidator.checksum', '@settings']
   cache.backend.apcu:
     class: Drupal\Core\Cache\ApcuBackendFactory
     arguments: ['@app.root', '@site.path', '@cache_tags.invalidator.checksum']

core/lib/Drupal/Core/Cache/DatabaseBackend.php

@@ -16,6 +16,30 @@
  */
 class DatabaseBackend implements CacheBackendInterface {
 
+  /**
+   * The default maximum number of rows that this cache bin table can store.
+   *
+   * This maximum is introduced to ensure that the database is not filled with
+   * hundred of thousand of cache entries with gigabytes in size.
+   *
+   * Read about how to change it in the @link cache Cache API topic. @endlink
+   */
+  const DEFAULT_MAX_ROWS = 5000;
+
+  /**
+   * -1 means infinite allows numbers of rows for the cache backend.
+   */
+  const MAXIMUM_NONE = -1;
+
+  /**
+   * The maximum number of rows that this cache bin table is allowed to store.
+   *
+   * @see ::MAXIMUM_NONE
+   *
+   * @var int
+   */
+  protected $maxRows;
+
   /**
    * @var string
    */
@@ -45,14 +69,18 @@ class DatabaseBackend implements CacheBackendInterface {
    *   The cache tags checksum provider.
    * @param string $bin
    *   The cache bin for which the object is created.
+   * @param int $max_rows
+   *   (optional) The maximum number of rows that are allowed in this cache bin
+   *   table.
    */
-  public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, $bin) {
+  public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, $bin, $max_rows = NULL) {
     // All cache tables should be prefixed with 'cache_'.
     $bin = 'cache_' . $bin;
 
     $this->bin = $bin;
     $this->connection = $connection;
     $this->checksumProvider = $checksum_provider;
+    $this->maxRows = $max_rows === NULL ? static::DEFAULT_MAX_ROWS : $max_rows;
   }
 
   /**
@@ -326,6 +354,22 @@ public function invalidateAll() {
    */
   public function garbageCollection() {
     try {
+      // Bounded size cache bin, using FIFO.
+      if ($this->maxRows !== static::MAXIMUM_NONE) {
+        $first_invalid_create_time = $this->connection->select($this->bin)
+          ->fields($this->bin, ['created'])
+          ->orderBy("{$this->bin}.created", 'DESC')
+          ->range($this->maxRows, $this->maxRows + 1)
+          ->execute()
+          ->fetchField();
+
+        if ($first_invalid_create_time) {
+          $this->connection->delete($this->bin)
+            ->condition('created', $first_invalid_create_time, '<=')
+            ->execute();
+        }
+      }
+
       $this->connection->delete($this->bin)
         ->condition('expire', Cache::PERMANENT, '<>')
         ->condition('expire', REQUEST_TIME, '<')
@@ -472,10 +516,20 @@ public function schemaDefinition() {
       ],
       'indexes' => [
         'expire' => ['expire'],
+        'created' => ['created'],
       ],
       'primary key' => ['cid'],
     ];
     return $schema;
   }
 
+  /**
+   * The maximum number of rows that this cache bin table is allowed to store.
+   *
+   * @return int
+   */
+  public function getMaxRows() {
+    return $this->maxRows;
+  }
+
 }

core/lib/Drupal/Core/Cache/DatabaseBackendFactory.php

@@ -3,6 +3,7 @@
 namespace Drupal\Core\Cache;
 
 use Drupal\Core\Database\Connection;
+use Drupal\Core\Site\Settings;
 
 class DatabaseBackendFactory implements CacheFactoryInterface {
 
@@ -20,6 +21,13 @@ class DatabaseBackendFactory implements CacheFactoryInterface {
    */
   protected $checksumProvider;
 
+  /**
+   * The settings array.
+   *
+   * @var \Drupal\Core\Site\Settings
+   */
+  protected $settings;
+
   /**
    * Constructs the DatabaseBackendFactory object.
    *
@@ -27,10 +35,15 @@ class DatabaseBackendFactory implements CacheFactoryInterface {
    *   Database connection
    * @param \Drupal\Core\Cache\CacheTagsChecksumInterface $checksum_provider
    *   The cache tags checksum provider.
+   * @param \Drupal\Core\Site\Settings $settings
+   *   (optional) The settings array.
+   *
+   * @throws \BadMethodCallException
    */
-  public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider) {
+  public function __construct(Connection $connection, CacheTagsChecksumInterface $checksum_provider, Settings $settings = NULL) {
     $this->connection = $connection;
     $this->checksumProvider = $checksum_provider;
+    $this->settings = $settings ?: Settings::getInstance();
   }
 
   /**
@@ -43,7 +56,35 @@ public function __construct(Connection $connection, CacheTagsChecksumInterface $
    *   The cache backend object for the specified cache bin.
    */
   public function get($bin) {
-    return new DatabaseBackend($this->connection, $this->checksumProvider, $bin);
+    $max_rows = $this->getMaxRowsForBin($bin);
+    return new DatabaseBackend($this->connection, $this->checksumProvider, $bin, $max_rows);
+  }
+
+  /**
+   * Gets the max rows for the specified cache bin.
+   *
+   * @param string $bin
+   *   The cache bin for which the object is created.
+   *
+   * @return int
+   *   The maximum number of rows for the given bin. Defaults to
+   *   DatabaseBackend::DEFAULT_MAX_ROWS.
+   */
+  protected function getMaxRowsForBin($bin) {
+    $max_rows_settings = $this->settings->get('database_cache_max_rows');
+    // First, look for a cache bin specific setting.
+    if (isset($max_rows_settings['bins'][$bin])) {
+      $max_rows  = $max_rows_settings['bins'][$bin];
+    }
+    // Second, use configured default backend.
+    elseif (isset($max_rows_settings['default'])) {
+      $max_rows = $max_rows_settings['default'];
+    }
+    else {
+      // Fall back to the default max rows if nothing else is configured.
+      $max_rows = DatabaseBackend::DEFAULT_MAX_ROWS;
+    }
+    return $max_rows;
   }
 
 }

core/lib/Drupal/Core/DrupalKernel.php

@@ -7,6 +7,7 @@
 use Drupal\Component\FileCache\FileCacheFactory;
 use Drupal\Component\Utility\Unicode;
 use Drupal\Component\Utility\UrlHelper;
+use Drupal\Core\Cache\DatabaseBackend;
 use Drupal\Core\Config\BootstrapConfigStorageFactory;
 use Drupal\Core\Config\NullStorage;
 use Drupal\Core\Database\Database;
@@ -77,7 +78,7 @@ class DrupalKernel implements DrupalKernelInterface, TerminableInterface {
       ],
       'cache.container' => [
         'class' => 'Drupal\Core\Cache\DatabaseBackend',
-        'arguments' => ['@database', '@cache_tags_provider.container', 'container'],
+        'arguments' => ['@database', '@cache_tags_provider.container', 'container', DatabaseBackend::MAXIMUM_NONE],
       ],
       'cache_tags_provider.container' => [
         'class' => 'Drupal\Core\Cache\DatabaseCacheTagsChecksum',

core/modules/system/system.install

@@ -10,6 +10,7 @@
 use Drupal\Component\FileSystem\FileSystem;
 use Drupal\Component\Utility\OpCodeCache;
 use Drupal\Component\Utility\Unicode;
+use Drupal\Core\Cache\Cache;
 use Drupal\Core\Path\AliasStorage;
 use Drupal\Core\Url;
 use Drupal\Core\Database\Database;
@@ -2025,3 +2026,19 @@ function system_update_8402() {
     }
   }
 }
+
+/**
+ * Delete all cache_* tables. They are recreated on demand with the new schema.
+ */
+function system_update_8403() {
+  foreach (Cache::getBins() as $bin => $cache_backend) {
+    // Try to delete the table regardless of which cache backend is handling it.
+    // This is to ensure the new schema is used if the configuration for the
+    // backend class is changed after the update hook runs.
+    $table_name = "cache_$bin";
+    $schema = Database::getConnection()->schema();
+    if ($schema->tableExists($table_name)) {
+      $schema->dropTable($table_name);
+    }
+  }
+}

core/tests/Drupal/KernelTests/Core/Cache/ChainedFastBackendTest.php

@@ -20,7 +20,7 @@ class ChainedFastBackendTest extends GenericCacheBackendUnitTestBase {
    *   A new ChainedFastBackend object.
    */
   protected function createCacheBackend($bin) {
-    $consistent_backend = new DatabaseBackend(\Drupal::service('database'), \Drupal::service('cache_tags.invalidator.checksum'), $bin);
+    $consistent_backend = new DatabaseBackend(\Drupal::service('database'), \Drupal::service('cache_tags.invalidator.checksum'), $bin, 100);
     $fast_backend = new PhpBackend($bin, \Drupal::service('cache_tags.invalidator.checksum'));
     $backend = new ChainedFastBackend($consistent_backend, $fast_backend, $bin);
     // Explicitly register the cache bin as it can not work through the

core/tests/Drupal/KernelTests/Core/Cache/DatabaseBackendTest.php

@@ -11,6 +11,13 @@
  */
 class DatabaseBackendTest extends GenericCacheBackendUnitTestBase {
 
+  /**
+   * The max rows to use for test bins.
+   *
+   * @var int
+   */
+  protected static $maxRows = 100;
+
   /**
    * Modules to enable.
    *
@@ -25,7 +32,7 @@ class DatabaseBackendTest extends GenericCacheBackendUnitTestBase {
    *   A new DatabaseBackend object.
    */
   protected function createCacheBackend($bin) {
-    return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin);
+    return new DatabaseBackend($this->container->get('database'), $this->container->get('cache_tags.invalidator.checksum'), $bin, static::$maxRows);
   }
 
   /**
@@ -48,4 +55,52 @@ public function testSetGet() {
     $this->assertSame($cached_value_short, $backend->get($cid_short)->data, "Backend contains the correct value for short, non-ASCII cache id.");
   }
 
+  /**
+   * Tests the row count limiting of cache bin database tables.
+   */
+  public function testGarbageCollection() {
+    $backend = $this->getCacheBackend();
+    $max_rows = static::$maxRows;
+
+    $this->assertSame(0, (int) $this->getNumRows());
+
+    // Fill to just the limit.
+    for ($i = 0; $i < $max_rows; $i++) {
+      // Ensure that each cache item created happens in a different millisecond,
+      // by waiting 1 ms (1000 microseconds). The garbage collection might
+      // otherwise keep less than exactly 100 records (which is acceptable for
+      // real-world cases, but not for this test).
+      usleep(1000);
+      $backend->set("test$i", $i);
+    }
+    $this->assertSame($max_rows, $this->getNumRows());
+
+    // Garbage collection has no effect.
+    $backend->garbageCollection();
+    $this->assertSame($max_rows, $this->getNumRows());
+
+    // Go one row beyond the limit.
+    $backend->set('test' . ($max_rows + 1), $max_rows + 1);
+    $this->assertSame($max_rows + 1, $this->getNumRows());
+
+    // Garbage collection removes one row: the oldest.
+    $backend->garbageCollection();
+    $this->assertSame($max_rows, $this->getNumRows());
+    $this->assertFalse($backend->get('test0'));
+  }
+
+  /**
+   * Gets the number of rows in the test cache bin database table.
+   *
+   * @return int
+   *   The number of rows in the test cache bin database table.
+   */
+  protected function getNumRows() {
+    $table = 'cache_' . $this->testBin;
+    $connection = $this->container->get('database');
+    $query = $connection->select($table);
+    $query->addExpression('COUNT(cid)', 'cid');
+    return (int) $query->execute()->fetchField();
+  }
+
 }

core/tests/Drupal/KernelTests/Core/Command/DbDumpTest.php

@@ -70,7 +70,8 @@ public function register(ContainerBuilder $container) {
     parent::register($container);
     $container->register('cache_factory', 'Drupal\Core\Cache\DatabaseBackendFactory')
       ->addArgument(new Reference('database'))
-      ->addArgument(new Reference('cache_tags.invalidator.checksum'));
+      ->addArgument(new Reference('cache_tags.invalidator.checksum'))
+      ->addArgument(new Reference('settings'));
   }
 
   /**

core/tests/Drupal/Tests/Core/Cache/DatabaseBackendFactoryTest.php

+<?php
+
+namespace Drupal\Tests\Core\Cache;
+
+use Drupal\Core\Cache\CacheTagsChecksumInterface;
+use Drupal\Core\Cache\DatabaseBackend;
+use Drupal\Core\Cache\DatabaseBackendFactory;
+use Drupal\Core\Database\Connection;
+use Drupal\Core\Site\Settings;
+use Drupal\Tests\UnitTestCase;
+
+/**
+ * @coversDefaultClass \Drupal\Core\Cache\DatabaseBackendFactory
+ * @group Cache
+ */
+class DatabaseBackendFactoryTest extends UnitTestCase {
+
+  /**
+   * @covers ::__construct
+   * @covers ::get
+   * @dataProvider getProvider
+   */
+  public function testGet(array $settings, $expected_max_rows_foo, $expected_max_rows_bar) {
+    $database_backend_factory = new DatabaseBackendFactory(
+      $this->prophesize(Connection::class)->reveal(),
+      $this->prophesize(CacheTagsChecksumInterface::class)->reveal(),
+      new Settings($settings)
+    );
+
+    $this->assertSame($expected_max_rows_foo, $database_backend_factory->get('foo')->getMaxRows());
+    $this->assertSame($expected_max_rows_bar, $database_backend_factory->get('bar')->getMaxRows());
+  }
+
+  public function getProvider() {
+    return [
+      'default' => [
+        [],
+        DatabaseBackend::DEFAULT_MAX_ROWS,
+        DatabaseBackend::DEFAULT_MAX_ROWS,
+      ],
+      'default overridden' => [
+        [
+          'database_cache_max_rows' => [
+            'default' => 99,
+          ],
+        ],
+        99,
+        99,
+      ],
+      'default + foo bin overridden' => [
+        [
+          'database_cache_max_rows' => [
+            'bins' => [
+              'foo' => 13,
+            ],
+          ],
+        ],
+        13,
+        DatabaseBackend::DEFAULT_MAX_ROWS,
+      ],
+      'default + bar bin overridden' => [
+        [
+          'database_cache_max_rows' => [
+            'bins' => [
+              'bar' => 13,
+            ],
+          ],
+        ],
+        DatabaseBackend::DEFAULT_MAX_ROWS,
+        13,
+      ],
+      'default overridden + bar bin overridden' => [
+        [
+          'database_cache_max_rows' => [
+            'default' => 99,
+            'bins' => [
+              'bar' => 13,
+            ],
+          ],
+        ],
+        99,
+        13,
+      ],
+      'default + both bins overridden' => [
+        [
+          'database_cache_max_rows' => [
+            'bins' => [
+              'foo' => 13,
+              'bar' => 31,
+            ],
+          ],
+        ],
+        13,
+        31,
+      ],
+      'default overridden + both bins overridden' => [
+        [
+          'database_cache_max_rows' => [
+            'default' => 99,
+            'bins' => [
+              'foo' => 13,
+              'bar' => 31,
+            ],
+          ],
+        ],
+        13,
+        31,
+      ],
+    ];
+  }
+
+}