ulysia/szurubooru
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improved migration script performance

Browse source
This commit is contained in:
Marcin Kurczewski 2014-10-12 12:44:21 +02:00
parent 3ae01ee4e8
commit 7f560d96d4
5 changed files with 95 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
scripts/migrate-szuru1.php
View file

@ -124,27 +124,18 @@ abstract class SourcePdoTask extends PdoTask
}
}
class RemoveAllTablesTask extends Task
class EmptyRemainingTablesTask extends Task
{
protected function getDescription()
{
return 'truncating tables in target database';
return 'truncating remaining tables in target database';
}
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM globals');
$targetPdo->exec('DELETE FROM postRelations');
$targetPdo->exec('DELETE FROM postTags');
$targetPdo->exec('DELETE FROM scores');
$targetPdo->exec('DELETE FROM favorites');
$targetPdo->exec('DELETE FROM snapshots');
$targetPdo->exec('DELETE FROM comments');
$targetPdo->exec('DELETE FROM posts');
$targetPdo->exec('DELETE FROM users');
$targetPdo->exec('DELETE FROM tokens');
$targetPdo->exec('DELETE FROM tags');
}
}
@ -170,18 +161,39 @@ class RemoveAllThumbnailsTask extends Task
}
}
class RemoveAllPostContentTask extends Task
class RemoveUnusedPostContentTask extends Task
{
private $sourceDir;
public function __construct($publicHtmlDir)
{
$this->sourceDir = $publicHtmlDir . DIRECTORY_SEPARATOR . 'files';
}
protected function getDescription()
{
return 'removing post content in target dir';
return 'removing unused post content in target dir';
}
protected function run()
{
$publicFileDao = Injector::get(PublicFileDao::class);
$dir = $publicFileDao->getFullPath('posts');
removeRecursively($dir);
$files = new \RecursiveIteratorIterator(
new \RecursiveDirectoryIterator($dir, \RecursiveDirectoryIterator::SKIP_DOTS),
\RecursiveIteratorIterator::CHILD_FIRST);
foreach ($files as $fileinfo)
{
if (!$fileinfo->isFile())
continue;
$targetFile = $fileinfo->getRealPath();
$sourceFile = $this->sourceDir . DIRECTORY_SEPARATOR . basename($targetFile);
if (!file_exists($sourceFile))
unlink($targetFile);
}
}
}
@ -211,7 +223,8 @@ class CopyPostContentTask extends Task
function ($sourcePath) use ($targetDir)
{
$targetPath = $targetDir . DIRECTORY_SEPARATOR . basename($sourcePath);
copy($sourcePath, $targetPath);
if (!file_exists($targetPath))
copy($sourcePath, $targetPath);
},
100);
}
@ -258,6 +271,9 @@ class CopyUsersTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM users');
$userDao = Injector::get(UserDao::class);
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM user'), function($arr) use ($userDao)
{
@ -318,6 +334,9 @@ class CopyPostsTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM posts');
$postDao = Injector::get(PostDao::class);
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM post'), function($arr) use ($postDao)
{
@ -377,6 +396,8 @@ class CopyCommentsTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM comments');
$commentDao = Injector::get(CommentDao::class);
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM comment'), function($arr) use ($commentDao)
{
@ -400,6 +421,8 @@ class CopyTagsTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM tags');
$tagDao = Injector::get(TagDao::class);
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM tag'), function($arr) use ($tagDao)
{
@ -422,6 +445,7 @@ class CopyPostRelationsTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM postRelations');
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM crossref'), function($arr) use ($targetPdo)
{
$targetPdo->exec(
@ -442,6 +466,7 @@ class CopyPostFavoritesTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM favorites');
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM favoritee'), function($arr) use ($targetPdo)
{
$targetPdo->exec(
@ -462,6 +487,7 @@ class CopyPostScoresTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM scores');
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM post_score'), function($arr) use ($targetPdo)
{
$targetPdo->exec(
@ -483,6 +509,7 @@ class CopyPostTagRelationsTask extends SourcePdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM postTags');
$this->commitInChunks($this->sourcePdo->query('SELECT * FROM post_tag'), function($arr) use ($targetPdo)
{
$targetPdo->exec(
@ -502,11 +529,14 @@ class PreparePostHistoryTask extends PdoTask
protected function run()
{
$targetPdo = Injector::get(DatabaseConnection::class)->getPDO();
$targetPdo->exec('DELETE FROM snapshots');
$postDao = Injector::get(PostDao::class);
$historyService = Injector::get(HistoryService::class);
$this->commitInChunks($postDao->findAll(), function($post) use ($postDao, $historyService)
{
$historyService->saveSnapshot($historyService->getPostChangeSnapshot($post));
$snapshot = $historyService->getPostChangeSnapshot($post);
$historyService->saveSnapshot($snapshot);
});
}
}
@ -559,10 +589,9 @@ class DownloadYoutubeThumbnailsTask extends PdoTask
$tasks =
[
new RemoveAllTablesTask(),
new RemoveAllThumbnailsTask(),
new RemoveAllPostContentTask(),
new CopyPostContentTask($sourcePublicHtmlDirectory),
new RemoveUnusedPostContentTask($sourcePublicHtmlDirectory),
new RemoveAllThumbnailsTask(),
new CopyPostThumbSourceTask($sourcePublicHtmlDirectory),
new CopyUsersTask($sourcePdo),
new CopyPostsTask($sourcePdo),
@ -575,6 +604,7 @@ $tasks =
new PreparePostHistoryTask(),
new ExportTagsTask(),
new DownloadYoutubeThumbnailsTask(),
new EmptyRemainingTablesTask(),
];
foreach ($tasks as $task)

9
src/Dao/EntityConverters/SnapshotEntityConverter.php
View file

@ -15,8 +15,8 @@ class SnapshotEntityConverter extends AbstractEntityConverter implements IEntity
'primaryKey' => $entity->getPrimaryKey(),
'userId' => $entity->getUserId(),
'operation' => $entity->getOperation(),
'data' => json_encode($entity->getData()),
'dataDifference' => json_encode($entity->getDataDifference()),
'data' => serialize($entity->getData()),
'dataDifference' => serialize($entity->getDataDifference()),
];
}
@ -28,9 +28,8 @@ class SnapshotEntityConverter extends AbstractEntityConverter implements IEntity
$entity->setPrimaryKey($array['primaryKey']);
$entity->setUserId($array['userId']);
$entity->setOperation($array['operation']);
$entity->setData(json_decode($array['data'], true));
$entity->setDataDifference(json_decode($array['dataDifference'], true));
$entity->setData(unserialize($array['data']));
$entity->setDataDifference(unserialize($array['dataDifference']));
return $entity;
}
}

8
src/Services/HistoryService.php
View file

@ -73,7 +73,7 @@ class HistoryService
if ($otherSnapshots)
{
$lastSnapshot = array_shift($otherSnapshots);
if ($lastSnapshot->getData() === $snapshot->getData())
if (md5(json_encode($lastSnapshot->getData())) === md5(json_encode($snapshot->getData())))
return $lastSnapshot;
$dataDifference = $this->getSnapshotDataDifference($snapshot->getData(), $lastSnapshot->getData());
@ -102,12 +102,14 @@ class HistoryService
public function getPostChangeSnapshot(Post $post)
{
$featuredPostParam = $this->globalParamDao->findByKey(GlobalParam::KEY_FEATURED_POST);
static $featuredPostParam = null;
if ($featuredPostParam === null)
$featuredPostParam = $this->globalParamDao->findByKey(GlobalParam::KEY_FEATURED_POST);
$isFeatured = ($featuredPostParam and intval($featuredPostParam->getValue()) === $post->getId());
$flags = [];
if ($post->getFlags() & Post::FLAG_LOOP)
$flags []= 'loop';
$flags[] = 'loop';
$data =
[

36
src/Upgrades/Upgrade22.php Normal file
View file

@ -0,0 +1,36 @@
<?php
namespace Szurubooru\Upgrades;
use Szurubooru\DatabaseConnection;
class Upgrade22 implements IUpgrade
{
public function run(DatabaseConnection $databaseConnection)
{
$pdo = $databaseConnection->getPDO();
$driver = $databaseConnection->getDriver();
$pdo->exec('
CREATE TABLE snapshots2 (
id INTEGER PRIMARY KEY ' . ($driver === 'mysql' ? 'AUTO_INCREMENT' : 'AUTOINCREMENT') . ',
time DATETIME NOT NULL,
type INTEGER NOT NULL,
primaryKey INTEGER NOT NULL,
operation INTEGER NOT NULL,
userId INTEGER DEFAULT NULL,
data BLOB,
dataDifference BLOB)');
$pdo->exec('
INSERT INTO snapshots2
(id, time, type, primaryKey, operation, userId, data, dataDifference)
SELECT
id, time, type, primaryKey, operation, userId, data, dataDifference
FROM snapshots');
$pdo->exec('DROP TABLE snapshots');
$pdo->exec('ALTER TABLE snapshots2 RENAME TO snapshots');
$pdo->exec('CREATE INDEX idx_snapshots_time ON snapshots(time)');
$pdo->exec('CREATE INDEX idx_snapshots_typePK ON snapshots(type, primaryKey)');
}
}

1
src/di.php
View file

@ -37,6 +37,7 @@ return [
$container->get(\Szurubooru\Upgrades\Upgrade19::class),
$container->get(\Szurubooru\Upgrades\Upgrade20::class),
$container->get(\Szurubooru\Upgrades\Upgrade21::class),
$container->get(\Szurubooru\Upgrades\Upgrade22::class),
];
}),