Source of file FTFileMakerTask.php
Size: 16,222 Bytes - Last Modified: 2021-12-23T10:31:28+00:00
/var/www/docs.ssmods.com/process/src/code/tasks/FTFileMakerTask.php
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501 | <?php use SilverStripe\Assets\File; use SilverStripe\Assets\Folder; use SilverStripe\Assets\Image; use SilverStripe\Dev\BuildTask; use SilverStripe\ORM\DB; use GuzzleHttp\Client; use GuzzleHttp\Promise; use SilverStripe\Security\Member; use SilverStripe\Security\Security; use SilverStripe\Core\Path; use SilverStripe\Core\Manifest\ModuleResourceLoader; /** * Creates sample folder and file structure, useful to test performance, * UI behaviour on deeply nested structures etc. * * Downloads around 20MB of data from a public web location on first run, * in order to create reasonable fixture data (and keep it out of the module repo). * The related AWS S3 bucket is managed by SilverStripe Ltd. * * Protip: In case you want to test thumbnail generation, you can * recursively delete any generated ones through the following bash command in `assets/`: * `find . -name '*Resampled*' -print0 | xargs -0 rm` * * Parameters: * - reset=1: Optionally truncate ALL files and folders in the database, plus delete * the entire `assets/` directory. * * * Configuration * * app/_config/frameworktest.yml * * The following yml config make 1040 files / 520mb: * * To make 100K records, increase the base folderCountByDepth from 1 to 100 and run the task overnight * * ``` * FTFileMakerTask: * documentsOnly: true * doSetFolderPermissions: true * doSetOldCreationDate: false * doRandomlyPublish: false * depth: 3 * folderCountByDepth: * - 1 * - 9 * - 7 * - 0 * - 0 * fileCountByDepth: * - 50 * - 25 * - 20 * - 0 * - 0 * * Flush and run: * /dev/tasks/FTFileMakerTask?flush&reset=1 * * @todo Automatically retrieve file listing from S3 * @todo Handle HTTP errors from S3 */ class FTFileMakerTask extends BuildTask { /** * If set to TRUE skip thumbnail generation * @var bool * @config */ private static $documentsOnly = false; /** * Vary the permission on the folders * @var bool * @config */ private static $doSetFolderPermissions = true; /** * Put some files in wrong store to test logic meant to correct this kind of problem.. * @var bool * @config */ private static $doPutProtectedFilesInPublicStore = false; /** * Set the date of some files to an old date. * @var bool * @config */ private static $doSetOldCreationDate = true; /** * Publish some files. * @var bool * @config */ private static $doRandomlyPublish = true; /** * How deep should or folder hierachy be. * @var int * @config */ private static $depth = 2; /** * When true, watermark images for unique image binary per Image record * @var bool * @config */ private static $uniqueImages = true; /** * Number of folders to create certain hierarchy. * @var int[] * @config */ private static $folderCountByDepth = [ 0 => 2, 1 => 2, 2 => 2, 3 => 2, 4 => 2, ]; /** * Number of files to create at various depths in the hierachy * @var int[] * @config */ private static $fileCountByDepth = [ 0 => 100, 1 => 30, 2 => 5, 3 => 5, 4 => 5, ]; protected $fixtureFileBaseUrl = "https://s3-ap-southeast-2.amazonaws.com/silverstripe-frameworktest-assets/"; protected $defaultImageFileName = 'image-huge-tall.jpg'; protected $fixtureFileNames = [ 'archive.zip', 'animated.gif', 'document.docx', 'document.pdf', 'image-huge-tall.jpg', 'image-huge-wide.jpg', 'image-large.jpg', 'image-large.png', 'image-large.gif', 'image-medium.jpg', 'image-small.jpg', 'image-tiny.jpg', 'image-medium.bmp', 'spreadsheet.xlsx', 'video.m4v' ]; protected $fixtureFileTypes = [ 'archive.zip' => 'SilverStripe\Assets\File', 'animated.gif' => 'SilverStripe\Assets\Image', 'document.docx' => 'SilverStripe\Assets\File', 'document.pdf' => 'SilverStripe\Assets\File', 'image-huge-tall.jpg' => 'SilverStripe\Assets\Image', 'image-huge-wide.jpg' => 'SilverStripe\Assets\Image', 'image-large.jpg' => 'SilverStripe\Assets\Image', 'image-large.png' => 'SilverStripe\Assets\Image', 'image-large.gif' => 'SilverStripe\Assets\Image', 'image-medium.jpg' => 'SilverStripe\Assets\Image', 'image-small.jpg' => 'SilverStripe\Assets\Image', 'image-tiny.jpg' => 'SilverStripe\Assets\Image', 'image-medium.bmp' => 'SilverStripe\Assets\File', 'spreadsheet.xlsx' => 'SilverStripe\Assets\File', 'video.m4v' => 'SilverStripe\Assets\File', ]; protected $lineBreak = "\n<br>"; /** @var Member */ protected $anonymousMember = null; /** * Allow override of the fileCountByDepth * @var array */ protected $fileCounts = []; /** * Allow override of the folderCountByDepth * @var array */ protected $folderCounts = []; public function run($request) { set_time_limit(0); // used to test canView() permissions $this->anonymousMember = Member::get()->find('Email', 'frameworktestuser'); if (!$this->anonymousMember) { $this->anonymousMember = Member::create(); $this->anonymousMember->Email = 'frameworktestuser'; $this->anonymousMember->write(); } if (!$this->anonymousMember->inGroup('content-authors')) { $this->anonymousMember->addToGroupByCode('content-authors'); } Security::setCurrentUser($this->anonymousMember); if (php_sapi_name() == "cli") { $this->lineBreak = "\n"; } if ($request->getVar('reset')) { $this->reset(); } $fileCounts = $request->getVar('fileCounts'); if ($fileCounts) { $counts = explode(',', $fileCounts); $this->fileCounts = array_map(function ($int) { return (int) trim($int); }, $counts); } else { $this->fileCounts = self::config()->get('fileCountByDepth'); } $folderCounts = $request->getVar('folderCounts'); if ($folderCounts) { $counts = explode(',', $folderCounts); $this->folderCounts = array_map(function ($int) { return (int) trim($int); }, $counts); } else { $this->folderCounts = self::config()->get('folderCountByDepth'); } echo "Downloading fixtures" . $this->lineBreak; $fixtureFilePaths = $this->downloadFixtureFiles(); if (!self::config()->get('documentsOnly')) { echo "Generate thumbnails" . $this->lineBreak; $this->generateThumbnails($fixtureFilePaths); } echo "Generate files" . $this->lineBreak; $this->generateFiles($fixtureFilePaths); if (!self::config()->get('doPutProtectedFilesInPublicStore')) { echo "Incorrectly putting protected files into public asset store on purpose" . $this->lineBreak; $this->putProtectedFilesInPublicAssetStore(); } } protected function reset() { echo "Resetting assets" . $this->lineBreak; DB::query('TRUNCATE "File"'); DB::query('TRUNCATE "File_Live"'); DB::query('TRUNCATE "File_Versions"'); if (file_exists(ASSETS_PATH) && ASSETS_PATH && ASSETS_PATH !== '/') { exec("rm -rf " . ASSETS_PATH); } } protected function downloadFixtureFiles() { $client = new Client(['base_uri' => $this->fixtureFileBaseUrl]); $fixtureFileNames = $this->fixtureFileNames; if (self::config()->get('documentsOnly')) { $fixtureFileNames = array_filter($fixtureFileNames, function($v) { return (bool) preg_match('%\.(docx|xlsx|pdf)$%', $v); }); } // Initiate each request but do not block $promises = []; $paths = []; foreach ($fixtureFileNames as $filename) { $path = TEMP_FOLDER . '/' . $filename; $paths[$filename] = $path; $url = "{$this->fixtureFileBaseUrl}/{$filename}"; if (!file_exists($path)) { $promises[$filename] = $client->getAsync($filename, [ 'sink' => $path ]); echo "Downloading $url" . $this->lineBreak; } } // Wait on all of the requests to complete. Throws a ConnectException // if any of the requests fail Promise\unwrap($promises); return $paths; } /** * Creates thumbnails of sample images * * @param array $fixtureFilePaths */ protected function generateThumbnails($fixtureFilePaths) { $folder = Folder::find_or_make('testfolder-thumbnail'); $fileName = $this->defaultImageFileName; foreach(['draft', 'published'] as $state) { $file = new Image([ 'ParentID' => $folder->ID, 'Title' => "{$fileName} {$state}", 'Name' => $fileName, ]); $file->File->setFromLocalFile($fixtureFilePaths[$fileName], $folder->getFilename() . $fileName); $file->write(); if ($state === 'published') { $file->publishFile(); } $file->Pad(60, 60)->CropHeight(30); } } protected function generateFiles($fixtureFilePaths, $depth = 0, $prefix = "0", $parentID = 0) { $folderCount = $this->folderCounts[$depth]; $fileCount = $this->fileCounts[$depth]; $doSetFolderPermissions = (bool) self::config()->get('doSetFolderPermissions'); $doSetOldCreationDate = (bool) self::config()->get('doSetOldCreationDate'); $doRandomlyPublish = (bool) self::config()->get('doRandomlyPublish'); $uniqueImages = (bool) self::config()->get('uniqueImages'); $watermarkPath = ModuleResourceLoader::singleton()->resolvePath( 'silverstripe/frameworktest: images/silverstripe.png' ); $absWatermarkPath = Path::join(BASE_PATH, $watermarkPath); for ($i = 1; $i <= $folderCount; $i++) { $folder = new Folder([ 'ParentID' => $parentID, 'Title' => "testfolder-{$prefix}{$i}", 'Name' => "testfolder-{$prefix}{$i}", ]); if ($doSetFolderPermissions) { if ($i === 1) { // the first folder should always be public to ensure there's some public folders $folder->CanViewType = 'Inherit'; } elseif ($i === $folderCount) { // the last folder should always be protected to ensure there's some protected folders $folder->CanViewType = 'OnlyTheseUsers'; } else { // all the other folder have a 33% chance of being a protected folder $folder->CanViewType = rand(0, 2) === 0 ? 'OnlyTheseUsers' : 'Inherit'; } } $folder->write(); for ($j = 1; $j <= $fileCount; $j++) { $randomFileName = array_keys($fixtureFilePaths)[rand(0, count($fixtureFilePaths) - 1)]; $randomFilePath = $fixtureFilePaths[$randomFileName]; $fileName = pathinfo($randomFilePath, PATHINFO_FILENAME) . "-{$prefix}-{$j}" . "." . pathinfo($randomFilePath, PATHINFO_EXTENSION); // Add a random prefix to avoid all types of files showing up on a single screen page $fileName = substr(md5($fileName), 0, 5) . '-' . $fileName; $class = $this->fixtureFileTypes[$randomFileName]; // If we're uniquifying images, copy the path and watermark it. if ($class === Image::class && $uniqueImages) { $copyPath = Path::join(dirname($randomFilePath), $fileName); copy($randomFilePath, $copyPath); $newPath = $this->watermarkImage($absWatermarkPath, $copyPath); if ($newPath) { $randomFilePath = $newPath; } } $file = new $class([ 'ParentID' => $folder->ID, 'Title' => $fileName, 'Name' => $fileName, ]); $file->File->setFromLocalFile($randomFilePath, $folder->getFilename() . $fileName); // Randomly set old created date (for testing) if ($doSetOldCreationDate) { if (rand(0, 10) === 0) { $file->Created = '2010-01-01 00:00:00'; $file->Title = '[old] ' . $file->Title; } } $file->write(); if ($doRandomlyPublish) { if (rand(0, 1) === 0) { $file->publishFile(); } } else { // publish files that should be viewable if ($file->canView($this->anonymousMember)) { $url = $file->getAbsoluteURL(); $file->publishFile(); } } } if ($depth < self::config()->get('depth') - 1) { $this->generateFiles($fixtureFilePaths, $depth + 1, "{$prefix}-{$i}", $folder->ID); } } } protected function putProtectedFilesInPublicAssetStore() { /** @var File $file */ foreach (File::get()->exclude(['ClassName' => Folder::class]) as $file) { // file is already in public asset store if ($file->canView($this->anonymousMember)) { continue; } // randomly move 50% of the files that should be in the protected store to the public store if (rand(0, 1) == 0) { continue; } // this will move the file into the public asset store, even it it should be protected // i.e. the parent folder CanViewType = OnlyTheseUsers $file->publishFile(); $url = $file->getAbsoluteURL(); } } /** * @param string $stampPath * @param string $targetPath * @return null */ protected function watermarkImage(string $stampPath, string $targetPath): ?string { // Load the stamp and the photo to apply the watermark to $ext = strtolower(pathinfo($targetPath, PATHINFO_EXTENSION)); $functions = null; if (in_array($ext, ['jpeg', 'jpg'])) { $functions = ['imagecreatefromjpeg', 'imagejpeg']; } elseif ($ext === 'png') { $functions = ['imagecreatefrompng', 'imagepng']; } if (!$functions) { return null; } $stamp = imagecreatefrompng($stampPath); $targetImage = call_user_func($functions[0], $targetPath); // Set the margins for the stamp and get the height/width of the stamp image $targetX = imagesx($targetImage); $targetY = imagesy($targetImage); $stampX = imagesx($stamp); $stampY = imagesy($stamp); $marge_right = rand($stampX, $targetX - $stampX); $marge_bottom = rand($stampY, $targetY - $stampY); // Copy the stamp image onto our photo using the margin offsets and the photo // width to calculate positioning of the stamp. imagecopy( $targetImage, $stamp, $targetX - $stampX - $marge_right, $targetY - $stampY - $marge_bottom, 0, 0, $stampX, $stampY ); call_user_func($functions[1], $targetImage, $targetPath); return $targetPath; } } |