Source of file Searchable.php
Size: 21,251 Bytes - Last Modified: 2021-12-24T06:51:11+00:00
/var/www/docs.ssmods.com/process/src/src/Searchable.php
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713 | <?php namespace Heyday\Elastica; use BadMethodCallException; use Elastica\Document; use Elastica\Type\Mapping; use Exception; use Heyday\Elastica\Jobs\ReindexAfterWriteJob; use Psr\Log\LoggerInterface; use SilverStripe\Assets\File; use SilverStripe\CMS\Model\SiteTree; use SilverStripe\Dev\Deprecation; use SilverStripe\ORM\DataExtension; use SilverStripe\ORM\DataList; use SilverStripe\ORM\DataObject; use SilverStripe\ORM\DataObjectSchema; use SilverStripe\Versioned\Versioned; use Symbiote\QueuedJobs\Services\QueuedJobService; use function is_numeric; /** * Adds elastic search integration to a data object. * * @property DataObject|Searchable $owner */ class Searchable extends DataExtension { public static $published_field = 'SS_Published'; /** * @config * @var array */ public static $mappings = array( 'PrimaryKey' => 'integer', 'ForeignKey' => 'integer', 'DBClassName' => 'string', 'DBDatetime' => 'date', 'Boolean' => 'boolean', 'Decimal' => 'double', 'Double' => 'double', 'Enum' => 'string', 'Float' => 'float', 'HTMLText' => 'string', 'HTMLVarchar' => 'string', 'Int' => 'integer', 'Datetime' => 'date', 'Text' => 'string', 'Varchar' => 'string', 'Year' => 'integer', 'File' => 'attachment', 'Date' => 'date' ); /** * @config * @var array */ private static $exclude_relations = array(); /** * @var ElasticaService */ private $service; /** * @var LoggerInterface */ protected $logger = null; /** * @var bool */ private $queued = false; /** * @param boolean $queued */ public function setQueued($queued) { $this->queued = $queued; } /** * Check if queued jobs for reindexing is enabled * * @return bool */ protected function getUseQueuedJobs() { return $this->queued && class_exists(QueuedJobService::class); } /** * @param ElasticaService $service * @param LoggerInterface $logger */ public function __construct(ElasticaService $service, LoggerInterface $logger = null) { $this->service = $service; $this->logger = $logger; parent::__construct(); } /** * Returns an array of fields to be indexed. Additional configuration can be attached to these fields. * * Format: array('FieldName' => array('type' => 'string')); * * FieldName can be a field in the database or a method name * * @return array */ public function indexedFields() { $fields = $this->owner->config()->get('indexed_fields'); $normalised = []; foreach ($fields as $fieldName => $params) { // Normalise field into name, specs (array) if (is_array($params) && is_numeric($fieldName)) { // Field name => specs are nested $fieldName = key($params); $params = array_shift($params); } elseif (is_numeric($fieldName)) { // Field name only is specified as value $fieldName = $params; $params = []; } $normalised[$fieldName] = $params; } return $normalised; } /** * Return an array of dependant class names. These are classes that need to be reindexed when an instance of the * extended class is updated or when a relationship to it changes. * @return array */ public function dependentClasses() { return $this->owner->config()->get('dependent_classes'); } /** * @return string */ public function getElasticaType() { return get_class($this->owner); } /** * Replacing the SS3 inheritedDatabaseFields() method * @return array */ public function inheritedDatabaseFields() { return $this->owner->getSchema()->fieldSpecs($this->owner->getClassName()); } /** * Gets an array of elastic field definitions. * This is also where we set the type of field ($spec['type']) and the analyzer for the field ($spec['analyzer']) * if needed. First we go through all the regular fields belonging to pages, then to the dataobjects related to * those pages * * @return array */ protected function getElasticaFields() { return array_merge( [ self::$published_field => [ 'type' => 'boolean' ] ], $this->getSearchableFields() ); } /** * Get the searchable fields for the owner data object * * @return array */ public function getSearchableFields() { $result = []; foreach ($this->owner->indexedFields() as $fieldName => $params) { // Check nested relation class $relationClass = isset($params['relationClass']) ? $params['relationClass'] : $this->owner->getRelationClass($fieldName); unset($params['relationClass']); // Don't send to elasticsearch // Build nested field from relation if ($relationClass) { // Relations can add multiple fields, so merge them all here $nestedFields = $this->getSearchableFieldsForRelation($fieldName, $params, $relationClass); $result = array_merge($result, $nestedFields); continue; } // Get extra params $params = $this->getExtraFieldParams($fieldName, $params); // Add field $result[$fieldName] = $params; } return $result; } /** * Get the searchable fields for the relationships of the owner data object * Note we currently only go one layer down eg the property of the document can be Relation_RelationField * * @return array * @deprecated */ protected function getReferenceSearchableFields() { Deprecation::notice('2.0.0', 'Use getSearchableFields instead'); return $this->getSearchableFields(); } /** * Clean up the data type name * @param string $dataType * @return string */ protected function stripDataTypeParameters($dataType) { return strtok($dataType, '('); } /** * @param string $dateString * @return string|null */ protected function formatDate($dateString) { if (empty($dateString)) { return null; } return date('Y-m-d\TH:i:s', strtotime($dateString)); } /** * Coerce strings into integers * * @param mixed $intString * @return int|null */ protected function formatInt($intString) { if (is_null($intString)) { return null; } return (int)$intString; } /** * Coerce strings into floats * * @param mixed $floatString * @return float|null */ protected function formatFloat($floatString) { if (is_null($floatString)) { return null; } return (float)$floatString; } /** * @return bool|Mapping */ public function getElasticaMapping() { //Only get the mapping for non supporting types. if (!$this->owner->config()->get('supporting_type')) { $fields = $this->getElasticaFields(); if (count($fields)) { $mapping = new Mapping(); $mapping->setProperties($fields); return $mapping; } } return false; } /** * @param Document $document */ protected function setPublishedStatus($document) { $isLive = true; if ($this->owner->hasExtension(Versioned::class)) { if ($this->owner instanceof SiteTree) { $isLive = $this->owner->isPublished(); } } $document->set(self::$published_field, (bool)$isLive); } /** * Assigns value to the fields indexed from getElasticaFields() * * @return Document */ public function getElasticaDocument() { $document = new Document($this->owner->ID); // Set published state $this->setPublishedStatus($document); // Add all nested field values foreach ($this->getSearchableFieldValues() as $field => $value) { $document->set($field, $value); } return $document; } /** * Get values for all searchable fields as an array. * Similr to getSearchableFields() but returns field values instead of spec * * @return array */ public function getSearchableFieldValues() { $fieldValues = []; foreach ($this->owner->indexedFields() as $fieldName => $params) { // Check nested relation class $relationClass = isset($params['relationClass']) ? $params['relationClass'] : $this->owner->getRelationClass($fieldName); unset($params['relationClass']); // Don't send to elasticsearch // Build nested field from relation if ($relationClass) { // Relations can add multiple fields, so merge them all here $nestedFieldValues = $this->getSearchableFieldValuesForRelation($fieldName, $params, $relationClass); $fieldValues = array_merge($fieldValues, $nestedFieldValues); continue; } // Check field exists on parent if ($this->owner->hasField($fieldName)) { $params = $this->getExtraFieldParams($fieldName, $params); $fieldValue = $this->formatValue($params, $this->owner->$fieldName); $fieldValues[$fieldName] = $fieldValue; } } return $fieldValues; } /** * Updates the record in the search index, or removes it as necessary. * @throws Exception */ public function onAfterWrite() { if ($this->getUseQueuedJobs()) { $this->queueReindex(); } else { $this->reIndex(); } } /** * reIndex related content * * @param string $stage * @throws Exception */ public function reIndex($stage = Versioned::LIVE) { $versionToIndex = $this->owner; $currentStage = Versioned::get_stage(); if ($stage !== $currentStage) { $versionToIndex = Versioned::get_by_stage($this->owner->ClassName, $stage)->byID($this->owner->ID); } if (is_null($versionToIndex)) { return; } if (!$versionToIndex->hasField('ShowInSearch') || $versionToIndex->ShowInSearch) { $this->service->index($versionToIndex); } else { $this->service->remove($versionToIndex); } $this->updateDependentClasses(); } /** * Batch update all documents attached to the index for this record * * @param callable $callback * @param int $documentsProcessed * @return mixed * @throws Exception */ public function batchIndex(callable $callback, &$documentsProcessed = 0) { return $this->service->batch($callback, $documentsProcessed); } /** * Removes the record from the search index. * @throws Exception */ public function onBeforeDelete() { $this->service->remove($this->owner); $this->updateDependentClasses(); } /** * Update dependent classes after the extended object has been removed from a ManyManyList * @throws Exception */ public function onAfterManyManyRelationRemove() { if ($this->getUseQueuedJobs()) { $this->queueReindex(); } else { $this->updateDependentClasses(); } } /** * Update dependent classes after the extended object has been added to a ManyManyList * @throws Exception */ public function onAfterManyManyRelationAdd() { if ($this->getUseQueuedJobs()) { $this->queueReindex(); } else { $this->updateDependentClasses(); } } /** * Updates the records of all instances of dependent classes. * @throws Exception */ protected function updateDependentClasses() { $classes = $this->dependentClasses(); if ($classes) { foreach ($classes as $class) { $list = DataList::create($class); foreach ($list as $object) { if ($object instanceof DataObject && $object->hasExtension(Searchable::class)) { if (!$object->hasField('ShowInSearch') || $object->ShowInSearch) { $this->service->index($object); } else { $this->service->remove($object); } } } } } } /** * Serialise a file attachment * * @param File $file * @return array Value for 'attachment' type */ protected function createAttachment(File $file) { $value = base64_encode($file->getStream()); $mimeType = $file->getMimeType(); return [ '_content_type' => $mimeType, '_name' => $file->Name, '_content' => $value, ]; } /** * Build searchable spec for a given field * * @param string $fieldName * @param array $params Spec params * @param string $className * @return array */ protected function getSearchableFieldsForRelation($fieldName, $params, $className) { // Detect attachment; Skip relational check if (isset($params['type']) && $params['type'] === 'attachment') { return [$fieldName => $params]; }; // Skip if this relation class has no elasticsearch content /** @var DataObject|Searchable $related */ $related = DataObject::singleton($className); if (!$related->hasExtension(Searchable::class)) { return []; } // Get nested fields $nestedFields = $related->getSearchableFields(); // Determine if merging into parent as either a multilevel object (default) // or nested objects (requires 'nested' param to be set) if (isset($params['type']) && $params['type'] === 'nested') { // Set nested fields // https://www.elastic.co/guide/en/elasticsearch/guide/current/nested-mapping.html // https://www.elastic.co/guide/en/elasticsearch/reference/5.6/nested.html return [ $fieldName => array_merge( $params, ['properties' => $nestedFields] ) ]; } // If not nested default to multilevel object $newFields = []; foreach ($nestedFields as $relatedFieldName => $relatedParams) { // Flatten each field as a sub_name. E.g. Book_Title $nestedName = "{$fieldName}_{$relatedFieldName}"; $newFields[$nestedName] = $relatedParams; } return $newFields; } /** * Get all fields from a relation on a parent object * * @param string $fieldName * @param array $params Spec params * @param string $className * @return array */ protected function getSearchableFieldValuesForRelation($fieldName, $params, $className) { // Detect attachment if (isset($params['type']) && $params['type'] === 'attachment') { /** @var File $file */ $file = $this->owner->$fieldName(); if (!$file instanceof File || !$file->exists()) { return []; } return [$fieldName => $this->createAttachment($file)]; } // Skip if this relation class has no elasticsearch content /** @var DataObject|Searchable $relatedSingleton */ $relatedSingleton = DataObject::singleton($className); if (!$relatedSingleton->hasExtension(Searchable::class)) { return []; } // Get item from parent $relatedList = $this->owner->$fieldName(); if (!$relatedList) { return []; } // Handle unary relations /** @var DataObject|Searchable $relatedItem */ $relatedItem = null; // Handle unary sets $isUnary = $relatedList instanceof DataObject; if ($isUnary) { $relatedItem = $relatedList; $relatedList = [$relatedItem]; } // Determine if merging into parent as either a multilevel object (default) // or nested objects (requires 'nested' param to be set) // Note: Unary relations are treated as a single-length list if (isset($params['type']) && $params['type'] === 'nested') { $relationValues = []; /** @var DataObject|Searchable $relationListItem */ foreach ($relatedList as $relationListItem) { $relationValues[] = $relationListItem->getSearchableFieldValues(); } return [$fieldName => $relationValues]; } // If not nested default to multilevel object // Handle unary-multilevel // I.e. Relation_Field = 'value' if ($isUnary) { // We will return multiple values, one for each sub-column $fieldValues = []; foreach ($relatedItem->getSearchableFieldValues() as $relatedFieldName => $relatedFieldValue) { $nestedName = "{$fieldName}_{$relatedFieldName}"; $fieldValues[$nestedName] = $relatedItem->IsInDB() ? $relatedFieldValue : null; } return $fieldValues; } // Handle non-unary-multilevel // I.e. Relation_Field = ['value1', 'value2'] $fieldValues = []; // Bootstrap set with empty arrays for each top level key // This also ensures we set empty data if $relatedList is empty foreach ($relatedSingleton->getSearchableFields() as $relatedFieldName => $spec) { $nestedName = "{$fieldName}_{$relatedFieldName}"; $fieldValues[$nestedName] = []; } // Add all documents to the list foreach ($relatedList as $relatedListItem) { foreach ($relatedListItem->getSearchableFieldValues() as $relatedFieldName => $relatedFieldValue) { $nestedName = "{$fieldName}_{$relatedFieldName}"; $fieldValues[$nestedName][] = $relatedFieldValue; } } return $fieldValues; } /** * @param $fieldValue * @return string */ protected function formatBoolean($fieldValue) { return boolval($fieldValue) ? 'true' : 'false'; } /** * Format a scalar value for the index document * * @param array $params Spec params * @param mixed $fieldValue * @return mixed */ protected function formatValue($params, $fieldValue) { $type = isset($params['type']) ? $params['type'] : null; switch ($type) { case 'boolean': return $this->formatBoolean($fieldValue); case 'date': return $this->formatDate($fieldValue); case 'integer': return $this->formatInt($fieldValue); case 'float': return $this->formatFloat($fieldValue); default: return $fieldValue; } } /** * Get extra params for a field from the parent document * * @param string $fieldName * @param array $params * @return array */ protected function getExtraFieldParams($fieldName, $params) { // Skip if type is already define if (isset($params['type'])) { return $params; } // Guess type from $db spec $fields = DataObjectSchema::singleton()->fieldSpecs($this->owner); if (array_key_exists($fieldName, $fields)) { // Strip and check data type mapping $dataType = $this->stripDataTypeParameters($fields[$fieldName]); if (array_key_exists($dataType, self::$mappings)) { $params['type'] = self::$mappings[$dataType]; } } return $params; } /** * Trigger a queuedjob to update this item. * Require queuedjobs to be setup. */ protected function queueReindex() { if (!$this->getUseQueuedJobs()) { throw new BadMethodCallException("Queued is disabled or queuedjobs module is not installed"); } $reindex = new ReindexAfterWriteJob($this->owner->ID, $this->owner->ClassName); QueuedJobService::singleton()->queueJob($reindex); } } |