From 7f560d96d4b0cf20c6ee18806b842961adae293f Mon Sep 17 00:00:00 2001 From: Marcin Kurczewski Date: Sun, 12 Oct 2014 12:44:21 +0200 Subject: [PATCH] Improved migration script performance --- scripts/migrate-szuru1.php | 68 +++++++++++++------ .../SnapshotEntityConverter.php | 9 ++- src/Services/HistoryService.php | 8 ++- src/Upgrades/Upgrade22.php | 36 ++++++++++ src/di.php | 1 + 5 files changed, 95 insertions(+), 27 deletions(-) create mode 100644 src/Upgrades/Upgrade22.php diff --git a/scripts/migrate-szuru1.php b/scripts/migrate-szuru1.php index 451dad60..66816a43 100644 --- a/scripts/migrate-szuru1.php +++ b/scripts/migrate-szuru1.php @@ -124,27 +124,18 @@ abstract class SourcePdoTask extends PdoTask } } -class RemoveAllTablesTask extends Task +class EmptyRemainingTablesTask extends Task { protected function getDescription() { - return 'truncating tables in target database'; + return 'truncating remaining tables in target database'; } protected function run() { $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); $targetPdo->exec('DELETE FROM globals'); - $targetPdo->exec('DELETE FROM postRelations'); - $targetPdo->exec('DELETE FROM postTags'); - $targetPdo->exec('DELETE FROM scores'); - $targetPdo->exec('DELETE FROM favorites'); - $targetPdo->exec('DELETE FROM snapshots'); - $targetPdo->exec('DELETE FROM comments'); - $targetPdo->exec('DELETE FROM posts'); - $targetPdo->exec('DELETE FROM users'); $targetPdo->exec('DELETE FROM tokens'); - $targetPdo->exec('DELETE FROM tags'); } } @@ -170,18 +161,39 @@ class RemoveAllThumbnailsTask extends Task } } -class RemoveAllPostContentTask extends Task +class RemoveUnusedPostContentTask extends Task { + private $sourceDir; + + public function __construct($publicHtmlDir) + { + $this->sourceDir = $publicHtmlDir . DIRECTORY_SEPARATOR . 'files'; + } + protected function getDescription() { - return 'removing post content in target dir'; + return 'removing unused post content in target dir'; } protected function run() { $publicFileDao = Injector::get(PublicFileDao::class); $dir = $publicFileDao->getFullPath('posts'); - removeRecursively($dir); + + $files = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST); + + foreach ($files as $fileinfo) + { + if (!$fileinfo->isFile()) + continue; + $targetFile = $fileinfo->getRealPath(); + $sourceFile = $this->sourceDir . DIRECTORY_SEPARATOR . basename($targetFile); + if (!file_exists($sourceFile)) + unlink($targetFile); + } + } } @@ -211,7 +223,8 @@ class CopyPostContentTask extends Task function ($sourcePath) use ($targetDir) { $targetPath = $targetDir . DIRECTORY_SEPARATOR . basename($sourcePath); - copy($sourcePath, $targetPath); + if (!file_exists($targetPath)) + copy($sourcePath, $targetPath); }, 100); } @@ -258,6 +271,9 @@ class CopyUsersTask extends SourcePdoTask protected function run() { + $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM users'); + $userDao = Injector::get(UserDao::class); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM user'), function($arr) use ($userDao) { @@ -318,6 +334,9 @@ class CopyPostsTask extends SourcePdoTask protected function run() { + $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM posts'); + $postDao = Injector::get(PostDao::class); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM post'), function($arr) use ($postDao) { @@ -377,6 +396,8 @@ class CopyCommentsTask extends SourcePdoTask protected function run() { + $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM comments'); $commentDao = Injector::get(CommentDao::class); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM comment'), function($arr) use ($commentDao) { @@ -400,6 +421,8 @@ class CopyTagsTask extends SourcePdoTask protected function run() { + $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM tags'); $tagDao = Injector::get(TagDao::class); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM tag'), function($arr) use ($tagDao) { @@ -422,6 +445,7 @@ class CopyPostRelationsTask extends SourcePdoTask protected function run() { $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM postRelations'); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM crossref'), function($arr) use ($targetPdo) { $targetPdo->exec( @@ -442,6 +466,7 @@ class CopyPostFavoritesTask extends SourcePdoTask protected function run() { $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM favorites'); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM favoritee'), function($arr) use ($targetPdo) { $targetPdo->exec( @@ -462,6 +487,7 @@ class CopyPostScoresTask extends SourcePdoTask protected function run() { $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM scores'); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM post_score'), function($arr) use ($targetPdo) { $targetPdo->exec( @@ -483,6 +509,7 @@ class CopyPostTagRelationsTask extends SourcePdoTask protected function run() { $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM postTags'); $this->commitInChunks($this->sourcePdo->query('SELECT * FROM post_tag'), function($arr) use ($targetPdo) { $targetPdo->exec( @@ -502,11 +529,14 @@ class PreparePostHistoryTask extends PdoTask protected function run() { + $targetPdo = Injector::get(DatabaseConnection::class)->getPDO(); + $targetPdo->exec('DELETE FROM snapshots'); $postDao = Injector::get(PostDao::class); $historyService = Injector::get(HistoryService::class); $this->commitInChunks($postDao->findAll(), function($post) use ($postDao, $historyService) { - $historyService->saveSnapshot($historyService->getPostChangeSnapshot($post)); + $snapshot = $historyService->getPostChangeSnapshot($post); + $historyService->saveSnapshot($snapshot); }); } } @@ -559,10 +589,9 @@ class DownloadYoutubeThumbnailsTask extends PdoTask $tasks = [ - new RemoveAllTablesTask(), - new RemoveAllThumbnailsTask(), - new RemoveAllPostContentTask(), new CopyPostContentTask($sourcePublicHtmlDirectory), + new RemoveUnusedPostContentTask($sourcePublicHtmlDirectory), + new RemoveAllThumbnailsTask(), new CopyPostThumbSourceTask($sourcePublicHtmlDirectory), new CopyUsersTask($sourcePdo), new CopyPostsTask($sourcePdo), @@ -575,6 +604,7 @@ $tasks = new PreparePostHistoryTask(), new ExportTagsTask(), new DownloadYoutubeThumbnailsTask(), + new EmptyRemainingTablesTask(), ]; foreach ($tasks as $task) diff --git a/src/Dao/EntityConverters/SnapshotEntityConverter.php b/src/Dao/EntityConverters/SnapshotEntityConverter.php index c8e160c6..1b6f9794 100644 --- a/src/Dao/EntityConverters/SnapshotEntityConverter.php +++ b/src/Dao/EntityConverters/SnapshotEntityConverter.php @@ -15,8 +15,8 @@ class SnapshotEntityConverter extends AbstractEntityConverter implements IEntity 'primaryKey' => $entity->getPrimaryKey(), 'userId' => $entity->getUserId(), 'operation' => $entity->getOperation(), - 'data' => json_encode($entity->getData()), - 'dataDifference' => json_encode($entity->getDataDifference()), + 'data' => serialize($entity->getData()), + 'dataDifference' => serialize($entity->getDataDifference()), ]; } @@ -28,9 +28,8 @@ class SnapshotEntityConverter extends AbstractEntityConverter implements IEntity $entity->setPrimaryKey($array['primaryKey']); $entity->setUserId($array['userId']); $entity->setOperation($array['operation']); - $entity->setData(json_decode($array['data'], true)); - $entity->setDataDifference(json_decode($array['dataDifference'], true)); + $entity->setData(unserialize($array['data'])); + $entity->setDataDifference(unserialize($array['dataDifference'])); return $entity; } } - diff --git a/src/Services/HistoryService.php b/src/Services/HistoryService.php index 3abb5aa8..5d9e70d9 100644 --- a/src/Services/HistoryService.php +++ b/src/Services/HistoryService.php @@ -73,7 +73,7 @@ class HistoryService if ($otherSnapshots) { $lastSnapshot = array_shift($otherSnapshots); - if ($lastSnapshot->getData() === $snapshot->getData()) + if (md5(json_encode($lastSnapshot->getData())) === md5(json_encode($snapshot->getData()))) return $lastSnapshot; $dataDifference = $this->getSnapshotDataDifference($snapshot->getData(), $lastSnapshot->getData()); @@ -102,12 +102,14 @@ class HistoryService public function getPostChangeSnapshot(Post $post) { - $featuredPostParam = $this->globalParamDao->findByKey(GlobalParam::KEY_FEATURED_POST); + static $featuredPostParam = null; + if ($featuredPostParam === null) + $featuredPostParam = $this->globalParamDao->findByKey(GlobalParam::KEY_FEATURED_POST); $isFeatured = ($featuredPostParam and intval($featuredPostParam->getValue()) === $post->getId()); $flags = []; if ($post->getFlags() & Post::FLAG_LOOP) - $flags []= 'loop'; + $flags[] = 'loop'; $data = [ diff --git a/src/Upgrades/Upgrade22.php b/src/Upgrades/Upgrade22.php new file mode 100644 index 00000000..44410d71 --- /dev/null +++ b/src/Upgrades/Upgrade22.php @@ -0,0 +1,36 @@ +getPDO(); + $driver = $databaseConnection->getDriver(); + + $pdo->exec(' + CREATE TABLE snapshots2 ( + id INTEGER PRIMARY KEY ' . ($driver === 'mysql' ? 'AUTO_INCREMENT' : 'AUTOINCREMENT') . ', + time DATETIME NOT NULL, + type INTEGER NOT NULL, + primaryKey INTEGER NOT NULL, + operation INTEGER NOT NULL, + userId INTEGER DEFAULT NULL, + data BLOB, + dataDifference BLOB)'); + + $pdo->exec(' + INSERT INTO snapshots2 + (id, time, type, primaryKey, operation, userId, data, dataDifference) + SELECT + id, time, type, primaryKey, operation, userId, data, dataDifference + FROM snapshots'); + + $pdo->exec('DROP TABLE snapshots'); + $pdo->exec('ALTER TABLE snapshots2 RENAME TO snapshots'); + + $pdo->exec('CREATE INDEX idx_snapshots_time ON snapshots(time)'); + $pdo->exec('CREATE INDEX idx_snapshots_typePK ON snapshots(type, primaryKey)'); + } +} diff --git a/src/di.php b/src/di.php index 29187a74..fd7767c1 100644 --- a/src/di.php +++ b/src/di.php @@ -37,6 +37,7 @@ return [ $container->get(\Szurubooru\Upgrades\Upgrade19::class), $container->get(\Szurubooru\Upgrades\Upgrade20::class), $container->get(\Szurubooru\Upgrades\Upgrade21::class), + $container->get(\Szurubooru\Upgrades\Upgrade22::class), ]; }),