From 94ae4e0604fdeba1581ee2ad598f7ef61c3f098f Mon Sep 17 00:00:00 2001 From: Xiao Hu Tai Date: Tue, 30 Oct 2018 14:04:42 +0100 Subject: [PATCH] Add `Bolt\Storage\Query` and `Bolt\Twig\TwigRecordsView` from Bolt 3 This is commit c3ead6d48cc2431d72ae7f81947df7fa5fefa92b in https://github.com/bolt/bolt Copied on 2018-10-30 14:00 Once Query works, it could be refactored from `Bolt\Storage\Query` to `Bolt\Query` --- src/Storage/Query/Adapter/PostgresSearch.php | 81 +++ src/Storage/Query/ContentQueryInterface.php | 32 ++ src/Storage/Query/ContentQueryParser.php | 489 ++++++++++++++++++ .../Query/Directive/GetQueryDirective.php | 20 + .../Query/Directive/HydrateDirective.php | 20 + .../Query/Directive/LimitDirective.php | 20 + .../Query/Directive/OffsetDirective.php | 22 + .../Query/Directive/OrderDirective.php | 68 +++ .../Query/Directive/PagingDirective.php | 20 + .../Query/Directive/PrintQueryDirective.php | 19 + .../Query/Directive/ReturnSingleDirective.php | 20 + src/Storage/Query/Filter.php | 118 +++++ src/Storage/Query/FrontendQueryScope.php | 82 +++ .../Query/Handler/FirstQueryHandler.php | 26 + .../Query/Handler/IdentifiedSelectHandler.php | 30 ++ .../Query/Handler/LatestQueryHandler.php | 26 + .../Query/Handler/NativeSearchHandler.php | 52 ++ .../Query/Handler/RandomQueryHandler.php | 31 ++ .../Query/Handler/SearchQueryHandler.php | 70 +++ .../Query/Handler/SelectQueryHandler.php | 114 ++++ src/Storage/Query/Query.php | 112 ++++ src/Storage/Query/QueryInterface.php | 26 + src/Storage/Query/QueryParameterParser.php | 347 +++++++++++++ src/Storage/Query/QueryResultset.php | 105 ++++ src/Storage/Query/QueryScopeInterface.php | 14 + src/Storage/Query/SearchConfig.php | 210 ++++++++ src/Storage/Query/SearchQuery.php | 156 ++++++ src/Storage/Query/SearchQueryResultset.php | 83 +++ src/Storage/Query/SearchWeighter.php | 214 ++++++++ src/Storage/Query/SelectQuery.php | 279 ++++++++++ src/Storage/Query/TaxonomyQuery.php | 179 +++++++ src/Storage/Query/TaxonomyQueryResultset.php | 40 ++ src/Twig/TwigRecordsView.php | 167 ++++++ 33 files changed, 3292 insertions(+) create mode 100644 src/Storage/Query/Adapter/PostgresSearch.php create mode 100644 src/Storage/Query/ContentQueryInterface.php create mode 100644 src/Storage/Query/ContentQueryParser.php create mode 100644 src/Storage/Query/Directive/GetQueryDirective.php create mode 100644 src/Storage/Query/Directive/HydrateDirective.php create mode 100644 src/Storage/Query/Directive/LimitDirective.php create mode 100644 src/Storage/Query/Directive/OffsetDirective.php create mode 100644 src/Storage/Query/Directive/OrderDirective.php create mode 100644 src/Storage/Query/Directive/PagingDirective.php create mode 100644 src/Storage/Query/Directive/PrintQueryDirective.php create mode 100644 src/Storage/Query/Directive/ReturnSingleDirective.php create mode 100644 src/Storage/Query/Filter.php create mode 100644 src/Storage/Query/FrontendQueryScope.php create mode 100644 src/Storage/Query/Handler/FirstQueryHandler.php create mode 100644 src/Storage/Query/Handler/IdentifiedSelectHandler.php create mode 100644 src/Storage/Query/Handler/LatestQueryHandler.php create mode 100644 src/Storage/Query/Handler/NativeSearchHandler.php create mode 100644 src/Storage/Query/Handler/RandomQueryHandler.php create mode 100644 src/Storage/Query/Handler/SearchQueryHandler.php create mode 100644 src/Storage/Query/Handler/SelectQueryHandler.php create mode 100644 src/Storage/Query/Query.php create mode 100644 src/Storage/Query/QueryInterface.php create mode 100644 src/Storage/Query/QueryParameterParser.php create mode 100644 src/Storage/Query/QueryResultset.php create mode 100644 src/Storage/Query/QueryScopeInterface.php create mode 100644 src/Storage/Query/SearchConfig.php create mode 100644 src/Storage/Query/SearchQuery.php create mode 100644 src/Storage/Query/SearchQueryResultset.php create mode 100644 src/Storage/Query/SearchWeighter.php create mode 100644 src/Storage/Query/SelectQuery.php create mode 100644 src/Storage/Query/TaxonomyQuery.php create mode 100644 src/Storage/Query/TaxonomyQueryResultset.php create mode 100644 src/Twig/TwigRecordsView.php diff --git a/src/Storage/Query/Adapter/PostgresSearch.php b/src/Storage/Query/Adapter/PostgresSearch.php new file mode 100644 index 00000000..0aa57701 --- /dev/null +++ b/src/Storage/Query/Adapter/PostgresSearch.php @@ -0,0 +1,81 @@ +qb = $qb; + $this->config = $config; + $this->searchWords = $searchWords; + } + + public function setContentType($type) + { + $this->contentType = $type; + } + + public function getQuery() + { + $words = implode('&', $this->searchWords); + $sub = clone $this->qb; + $this->qb->addSelect("ts_rank(bsearch.document, to_tsquery('" . $words . "')) as score"); + $sub->select('*'); + $select = []; + + $fieldsToSearch = $this->config->getConfig($this->contentType); + $joins = $this->config->getJoins($this->contentType); + $fieldsToSearch = array_diff_key($fieldsToSearch, array_flip($joins)); + + $from = $this->qb->getQueryPart('from'); + if (isset($from[0]['alias'])) { + $alias = $from[0]['alias']; + } else { + $alias = $from[0]['table']; + } + foreach ($fieldsToSearch as $fieldName => $config) { + $weight = $this->getWeight($config['weight']); + $select[] = "setweight(to_tsvector($alias.$fieldName), '$weight')"; + } + $sub->select('*, ' . implode(' || ', $select) . ' AS document'); + $sub->groupBy("$alias.id"); + + $this->qb->from('(' . $sub->getSQL() . ')', 'bsearch'); + + $this->qb->where("bsearch.document @@ to_tsquery('" . $words . "')"); + $this->qb->orderBy('score', 'DESC'); + + return $this->qb; + } + + public function getWeight($score) + { + switch (true) { + case $score >= 75: + return 'A'; + + case $score >= 50: + return 'B'; + + case $score >= 25: + return 'C'; + + case $score < 25: + return 'D'; + } + + return 'A'; + } +} diff --git a/src/Storage/Query/ContentQueryInterface.php b/src/Storage/Query/ContentQueryInterface.php new file mode 100644 index 00000000..7ca8b732 --- /dev/null +++ b/src/Storage/Query/ContentQueryInterface.php @@ -0,0 +1,32 @@ + + */ +class ContentQueryParser +{ + /** @var EntityManager */ + protected $em; + /** @var string */ + protected $query; + /** @var array */ + protected $params = []; + /** @var array */ + protected $contentTypes = []; + /** @var string */ + protected $operation; + /** @var string */ + protected $identifier; + /** @var array */ + protected $operations = ['search', 'latest', 'first', 'random', 'nativesearch']; + /** @var array */ + protected $directives = []; + /** @var callable[] */ + protected $directiveHandlers = []; + /** @var callable[] */ + protected $handlers = []; + /** @var QueryInterface[] */ + protected $services = []; + /** @var QueryScopeInterface */ + protected $scope; + + /** + * Constructor. + * + * @param EntityManager $em + * @param QueryInterface $queryHandler + */ + public function __construct(EntityManager $em, QueryInterface $queryHandler = null) + { + $this->em = $em; + + if ($queryHandler !== null) { + $this->addService('select', $queryHandler); + } + + $this->setupDefaults(); + } + + /** + * Internal method to initialise the default handlers. + */ + protected function setupDefaults() + { + $this->addHandler('select', new SelectQueryHandler()); + $this->addHandler('search', new SearchQueryHandler()); + $this->addHandler('random', new RandomQueryHandler()); + $this->addHandler('first', new FirstQueryHandler()); + $this->addHandler('latest', new LatestQueryHandler()); + $this->addHandler('nativesearch', new NativeSearchHandler()); + $this->addHandler('namedselect', new IdentifiedSelectHandler()); + + $this->addDirectiveHandler('getquery', new GetQueryDirective()); + $this->addDirectiveHandler('hydrate', new HydrateDirective()); + $this->addDirectiveHandler('limit', new LimitDirective()); + $this->addDirectiveHandler('order', new OrderDirective()); + $this->addDirectiveHandler('page', new OffsetDirective()); + $this->addDirectiveHandler('paging', new PagingDirective()); + $this->addDirectiveHandler('printquery', new PrintQueryDirective()); + $this->addDirectiveHandler('returnsingle', new ReturnSingleDirective()); + } + + /** + * Sets the input query. + * + * @param string $query + */ + public function setQuery($query) + { + $this->query = $query; + } + + /** + * Sets the input parameters to handle. + * + * @param array $params + */ + public function setParameters(array $params) + { + $this->params = $params; + } + + /** + * Sets a single input parameter. + * + * @param string $param + * @param mixed $value + */ + public function setParameter($param, $value) + { + $this->params[$param] = $value; + } + + /** + * Parse a query. + */ + public function parse() + { + $this->parseContent(); + $this->parseOperation(); + $this->parseDirectives(); + } + + /** + * Parses the content area of the query string. + */ + protected function parseContent() + { + $contentString = strtok($this->query, '/'); + + $content = []; + $delim = '(),'; + $tok = strtok($contentString, $delim); + while ($tok !== false) { + $content[] = $tok; + $tok = strtok($delim); + } + + $this->contentTypes = $content; + } + + /** + * Internal method that takes the 'query' part of the input and + * parses it into one of the various operations supported. + * + * A simple select operation will just contain the ContentType eg 'pages' + * but additional operations can be triggered using the '/' separator. + * + * @internal + */ + protected function parseOperation() + { + $operation = 'select'; + + $queryParts = explode('/', $this->query); + array_shift($queryParts); + + if (!count($queryParts)) { + $this->operation = $operation; + + return; + } + + if (in_array($queryParts[0], $this->operations, true)) { + $operation = array_shift($queryParts); + if (count($queryParts) && is_numeric($queryParts[0])) { + $this->params['limit'] = array_shift($queryParts); + } + $this->identifier = implode(',', $queryParts); + } else { + $this->identifier = implode(',', $queryParts); + } + + if (!empty($this->identifier)) { + $operation = 'namedselect'; + } + + $this->operation = $operation; + } + + /** + * Directives are all of the other parameters supported by Bolt that do not + * relate to an actual filter query. Some examples include 'printquery', 'limit', + * 'order' or 'returnsingle'. + * + * All these need to parsed and taken out of the params that are sent to the query. + */ + protected function parseDirectives() + { + $this->directives = []; + + if (!$this->params) { + return; + } + + foreach ($this->params as $key => $value) { + if ($this->hasDirectiveHandler($key)) { + $this->directives[$key] = $value; + unset($this->params[$key]); + } + } + } + + /** + * This runs the callbacks attached to each directive command. + * + * @param QueryInterface $query + * @param array $skipDirective + */ + public function runDirectives(QueryInterface $query, array $skipDirective = []) + { + foreach ($this->directives as $key => $value) { + if (in_array($key, $skipDirective, true)) { + continue; + } + if (!$this->hasDirectiveHandler($key)) { + continue; + } + if (is_callable($this->getDirectiveHandler($key))) { + call_user_func($this->getDirectiveHandler($key), $query, $value, $this->directives); + } + } + } + + public function setScope(QueryScopeInterface $scope) + { + $this->scope = $scope; + } + + public function runScopes(ContentQueryInterface $query) + { + if ($this->scope !== null) { + $this->scope->onQueryExecute($query); + } + } + + /** + * Gets the object EntityManager. + * + * @return EntityManager + */ + public function getEntityManager() + { + return $this->em; + } + + /** + * Returns the parsed content types. + * + * @return array + */ + public function getContentTypes() + { + return $this->contentTypes; + } + + /** + * Returns the parsed operation. + * + * @return string + */ + public function getOperation() + { + return $this->operation; + } + + /** + * Returns the parsed identifier. + * + * @return string + */ + public function getIdentifier() + { + return $this->identifier; + } + + /** + * Returns a directive from the parsed list. + * + * @param string $key + * + * @return string|null + */ + public function getDirective($key) + { + if (array_key_exists($key, $this->directives)) { + return $this->directives[$key]; + } + + return null; + } + + /** + * Sets a directive for the named key. + * + * @param string $key + * @param string|bool $value + */ + public function setDirective($key, $value) + { + $this->directives[$key] = $value; + } + + /** + * Returns the handler for the named directive. + * + * @param string $check + * + * @return callable + */ + public function getDirectiveHandler($check) + { + return $this->directiveHandlers[$check]; + } + + /** + * Returns boolean for existence of handler. + * + * @param string $check + * + * @return bool + */ + public function hasDirectiveHandler($check) + { + return array_key_exists($check, $this->directiveHandlers); + } + + /** + * Adds a handler for the named directive. + * + * @param string $key + * @param callable|null $callback + */ + public function addDirectiveHandler($key, callable $callback = null) + { + if (!array_key_exists($key, $this->directiveHandlers)) { + $this->directiveHandlers[$key] = $callback; + } + } + + /** + * Adds a handler AND operation for the named operation. + * + * @param string $operation + * @param callable $callback + */ + public function addHandler($operation, callable $callback) + { + $this->handlers[$operation] = $callback; + $this->addOperation($operation); + } + + /** + * Returns a handler for the named operation. + * + * @param string $operation + * + * @return callable + */ + public function getHandler($operation) + { + return $this->handlers[$operation]; + } + + /** + * Adds a service for the named operation. + * + * @param string $operation + * @param QueryInterface $service + */ + public function addService($operation, $service) + { + $this->services[$operation] = $service; + } + + /** + * Returns a service for the named operation. + * + * @param string $operation + * + * @return QueryInterface + */ + public function getService($operation) + { + return $this->services[$operation]; + } + + /** + * Returns the current parameters. + * + * @return array + */ + public function getParameters() + { + return $this->params; + } + + /** + * Helper method to check if parameters are set for a specific key. + * + * @param string $param + * + * @return bool + */ + public function hasParameter($param) + { + return array_key_exists($param, $this->params); + } + + /** + * Returns a single named parameter. + * + * @param string $param + * + * @return array + */ + public function getParameter($param) + { + return $this->params[$param]; + } + + /** + * Runs the query and fetches the results. + * + * @return QueryResultset|Content|null + */ + public function fetch() + { + $this->parse(); + $parseEvent = new QueryEvent($this); + $this->getEntityManager()->getEventManager()->dispatch(QueryEvents::PARSE, $parseEvent); + + $result = call_user_func($this->handlers[$this->getOperation()], $this); + $executeEvent = new QueryEvent($this, $result); + $this->getEntityManager()->getEventManager()->dispatch(QueryEvents::EXECUTE, $executeEvent); + + return $result; + } + + /** + * Getter to return the currently registered operations. + * + * @return array + */ + public function getOperations() + { + return $this->operations; + } + + /** + * Adds a new operation to the list supported. + * + * @param string $operation name of operation to parse for + */ + public function addOperation($operation) + { + if (!in_array($operation, $this->operations, true)) { + $this->operations[] = $operation; + } + } + + /** + * Removes an operation from the list supported. + * + * @param string $operation name of operation to remove + */ + public function removeOperation($operation) + { + if (in_array($operation, $this->operations, true)) { + $key = array_search($operation, $this->operations, true); + unset($this->operations[$key]); + } + } +} diff --git a/src/Storage/Query/Directive/GetQueryDirective.php b/src/Storage/Query/Directive/GetQueryDirective.php new file mode 100644 index 00000000..35170bea --- /dev/null +++ b/src/Storage/Query/Directive/GetQueryDirective.php @@ -0,0 +1,20 @@ +getQueryBuilder()->setMaxResults($limit); + } +} diff --git a/src/Storage/Query/Directive/OffsetDirective.php b/src/Storage/Query/Directive/OffsetDirective.php new file mode 100644 index 00000000..b9ea397e --- /dev/null +++ b/src/Storage/Query/Directive/OffsetDirective.php @@ -0,0 +1,22 @@ +getQueryBuilder()->setFirstResult(($page - 1) * $limit); + } +} diff --git a/src/Storage/Query/Directive/OrderDirective.php b/src/Storage/Query/Directive/OrderDirective.php new file mode 100644 index 00000000..a15d828d --- /dev/null +++ b/src/Storage/Query/Directive/OrderDirective.php @@ -0,0 +1,68 @@ +'-datepublish'] + */ +class OrderDirective +{ + /** + * @param QueryInterface $query + * @param string $order + */ + public function __invoke(QueryInterface $query, $order) + { + if (!$order) { + return; + } + + // remove default order + $query->getQueryBuilder()->resetQueryPart('orderBy'); + + $separatedOrders = $this->getOrderBys($order); + foreach ($separatedOrders as $order) { + $order = trim($order); + if (strpos($order, '-') === 0) { + $direction = 'DESC'; + $order = substr($order, 1); + } elseif (strpos($order, ' DESC') !== false) { + $direction = 'DESC'; + $order = str_replace(' DESC', '', $order); + } else { + $direction = null; + } + $query->getQueryBuilder()->addOrderBy($order, $direction); + } + } + + /** + * @param $order + * + * @return array + */ + protected function getOrderBys($order) + { + $separatedOrders = [$order]; + + if ($this->isMultiOrderQuery($order)) { + $separatedOrders = explode(',', $order); + } + + return $separatedOrders; + } + + /** + * @param $order + * + * @return bool + */ + protected function isMultiOrderQuery($order) + { + return strpos($order, ',') !== false; + } +} diff --git a/src/Storage/Query/Directive/PagingDirective.php b/src/Storage/Query/Directive/PagingDirective.php new file mode 100644 index 00000000..5d6d5e4d --- /dev/null +++ b/src/Storage/Query/Directive/PagingDirective.php @@ -0,0 +1,20 @@ +getQueryBuilder()->setMaxResults(1); + $query->setSingleFetchMode(true); + } +} diff --git a/src/Storage/Query/Filter.php b/src/Storage/Query/Filter.php new file mode 100644 index 00000000..289bdd31 --- /dev/null +++ b/src/Storage/Query/Filter.php @@ -0,0 +1,118 @@ + + */ +class Filter +{ + protected $key; + /** @var CompositeExpression */ + protected $expression; + /** @var array */ + protected $parameters = []; + + /** + * Sets the key that this filter affects. + * + * @param string $key + */ + public function setKey($key) + { + $this->key = $key; + } + + /** + * Getter for key. + * + * @return string + */ + public function getKey() + { + return $this->key; + } + + /** + * Gets the compiled expression as a string. This will look + * something like `(alias.key = :placeholder)`. + * + * @return string + */ + public function getExpression() + { + return $this->expression->__toString(); + } + + /** + * Allows replacing the expression object with a modified one. + * + * @param CompositeExpression $expression + */ + public function setExpression(CompositeExpression $expression) + { + $this->expression = $expression; + } + + /** + * Returns the actual object of the expression. This is generally + * only needed for on the fly modification, to get the compiled + * expression use getExpression(). + * + * @return CompositeExpression + */ + public function getExpressionObject() + { + return $this->expression; + } + + /** + * Returns the array of parameters attached to this filter. These are + * normally used to replace placeholders at compile time. + * + * @return array + */ + public function getParameters() + { + return $this->parameters; + } + + /** + * Setter method to replace parameters. + * + * @param array $parameters + */ + public function setParameters(array $parameters) + { + $this->parameters = $parameters; + } + + /** + * Helper method to check if parameters are set for a specific key. + * + * @param string $param + * + * @return bool + */ + public function hasParameter($param) + { + return array_key_exists($param, $this->parameters); + } + + /** + * Allows setting a parameter for a single key. + * + * @param string $param + * @param mixed $value + */ + public function setParameter($param, $value) + { + $this->parameters[$param] = $value; + } +} diff --git a/src/Storage/Query/FrontendQueryScope.php b/src/Storage/Query/FrontendQueryScope.php new file mode 100644 index 00000000..36b383c3 --- /dev/null +++ b/src/Storage/Query/FrontendQueryScope.php @@ -0,0 +1,82 @@ +config = $config; + $this->parseContenttypes(); + } + + /** + * Get the default order setting for a given content type. + * + * @param string $contentType + * + * @return array|false + */ + public function getOrder($contentType) + { + if (isset($this->orderBys[$contentType])) { + return $this->orderBys[$contentType]; + } + + return false; + } + + /** + * Iterates over the main config and sets up what the default ordering should be + */ + protected function parseContenttypes() + { + $contentTypes = $this->config->get('contenttypes'); + foreach ($contentTypes as $type => $values) { + $sort = $values['sort'] ?: '-datepublish'; + $this->orderBys[$type] = $sort; + if (isset($values['singular_slug'])) { + $this->orderBys[$values['singular_slug']] = $sort; + } + } + } + + /** + * @param ContentQueryInterface $query + */ + public function onQueryExecute(ContentQueryInterface $query) + { + $ct = $query->getContentType(); + + // Setup default ordering of queries on a per-contenttype basis + if (empty($query->getQueryBuilder()->getQueryPart('orderBy')) && isset($this->orderBys[$ct])) { + $handler = new OrderDirective(); + $handler($query, $this->orderBys[$ct]); + } + + // Setup status to only published unless otherwise specified + $status = $query->getParameter('status'); + if (!$status) { + $query->setParameter('status', 'published'); + } + } +} diff --git a/src/Storage/Query/Handler/FirstQueryHandler.php b/src/Storage/Query/Handler/FirstQueryHandler.php new file mode 100644 index 00000000..a2ec2200 --- /dev/null +++ b/src/Storage/Query/Handler/FirstQueryHandler.php @@ -0,0 +1,26 @@ +setDirective('order', 'id'); + + return call_user_func($contentQuery->getHandler('select'), $contentQuery); + } +} diff --git a/src/Storage/Query/Handler/IdentifiedSelectHandler.php b/src/Storage/Query/Handler/IdentifiedSelectHandler.php new file mode 100644 index 00000000..a47046bb --- /dev/null +++ b/src/Storage/Query/Handler/IdentifiedSelectHandler.php @@ -0,0 +1,30 @@ +getIdentifier())) { + $contentQuery->setParameter('id', $contentQuery->getIdentifier()); + } else { + $contentQuery->setParameter('slug', $contentQuery->getIdentifier()); + } + if (count($contentQuery->getContentTypes()) === 1) { + $contentQuery->setDirective('returnsingle', true); + } + + return call_user_func($contentQuery->getHandler('select'), $contentQuery); + } +} diff --git a/src/Storage/Query/Handler/LatestQueryHandler.php b/src/Storage/Query/Handler/LatestQueryHandler.php new file mode 100644 index 00000000..7adc4117 --- /dev/null +++ b/src/Storage/Query/Handler/LatestQueryHandler.php @@ -0,0 +1,26 @@ +setDirective('order', '-id'); + + return call_user_func($contentQuery->getHandler('select'), $contentQuery); + } +} diff --git a/src/Storage/Query/Handler/NativeSearchHandler.php b/src/Storage/Query/Handler/NativeSearchHandler.php new file mode 100644 index 00000000..e3e7bcfc --- /dev/null +++ b/src/Storage/Query/Handler/NativeSearchHandler.php @@ -0,0 +1,52 @@ +getEntityManager()->createQueryBuilder()->getConnection()->getParams(); + if (strpos($params['driver'], 'postgres') !== false) { + return $this->postgresSearch($contentQuery); + } + + return call_user_func($contentQuery->getHandler('search'), $contentQuery); + } + + /** + * @param ContentQueryParser $contentQuery + * + * @return SearchQueryResultset + */ + public function postgresSearch(ContentQueryParser $contentQuery) + { + $set = new SearchQueryResultset(); + + foreach ($contentQuery->getContentTypes() as $contentType) { + $repo = $contentQuery->getEntityManager()->getRepository($contentType); + $query = $repo->createQueryBuilder($contentType); + $config = $contentQuery->getService('search_config'); + $search = $contentQuery->getParameter('filter'); + $adapter = new PostgresSearch($query, $config, explode(' ', $search)); + $adapter->setContentType($contentType); + $result = $repo->findWith($adapter->getQuery()); + $set->add($result, $contentType); + } + + return $set; + } +} diff --git a/src/Storage/Query/Handler/RandomQueryHandler.php b/src/Storage/Query/Handler/RandomQueryHandler.php new file mode 100644 index 00000000..a242d771 --- /dev/null +++ b/src/Storage/Query/Handler/RandomQueryHandler.php @@ -0,0 +1,31 @@ +getEntityManager()->createQueryBuilder()->getConnection()->getParams(); + if (strpos($params['driver'], 'mysql') !== false) { + $contentQuery->setDirective('order', 'RAND()'); + } else { + $contentQuery->setDirective('order', 'RANDOM()'); + } + + return call_user_func($contentQuery->getHandler('select'), $contentQuery); + } +} diff --git a/src/Storage/Query/Handler/SearchQueryHandler.php b/src/Storage/Query/Handler/SearchQueryHandler.php new file mode 100644 index 00000000..8f26210f --- /dev/null +++ b/src/Storage/Query/Handler/SearchQueryHandler.php @@ -0,0 +1,70 @@ +getService('search'); + /** @var SearchQuery $query */ + $query = clone $cleanSearchQuery; + + foreach ($contentQuery->getContentTypes() as $contentType) { + $repo = $contentQuery->getEntityManager()->getRepository($contentType); + $query->setQueryBuilder($repo->createQueryBuilder('_' . $contentType)); + $query->setContentType($contentType); + + $searchParam = $contentQuery->getParameter('filter'); + $query->setParameters($contentQuery->getParameters()); + $query->setSearch($searchParam); + + $contentQuery->runDirectives($query); + $contentQuery->runScopes($query); + + $result = $repo->queryWith($query); + if ($result) { + if (count($result) > 0) { + /** @var SearchWeighter $weighter */ + $weighter = $contentQuery->getService('search_weighter'); + $weighter->setContentType($contentType); + $weighter->setResults($result); + $weighter->setSearchWords($query->getSearchWords()); + + $scores = $weighter->weight(); + $set->setOriginalQuery($contentType, $query->getQueryBuilder()); + $set->add($result, $contentType, $scores); + } else { + $set->setOriginalQuery($contentType, $query->getQueryBuilder()); + $set->add($result, $contentType); + } + } + } + + if ($query->getSingleFetchMode()) { + if ($set->count() === 0) { + return false; + } + + return $set->current(); + } + + return $set; + } +} diff --git a/src/Storage/Query/Handler/SelectQueryHandler.php b/src/Storage/Query/Handler/SelectQueryHandler.php new file mode 100644 index 00000000..19c91705 --- /dev/null +++ b/src/Storage/Query/Handler/SelectQueryHandler.php @@ -0,0 +1,114 @@ +getService('select'); + $query->setSingleFetchMode(false); + + foreach ($contentQuery->getContentTypes() as $contentType) { + $contentType = str_replace('-', '_', $contentType); + $repo = $contentQuery->getEntityManager()->getRepository($contentType); + $query->setQueryBuilder($repo->createQueryBuilder('_' . $contentType)); + $query->setContentType($contentType); + + /** Run the parameters through the whitelister. If we get a false back from this method it's because there + * is no need to continue with the query. + */ + $params = $this->whitelistParameters($contentQuery->getParameters(), $repo); + if (!$params && count($contentQuery->getParameters())) { + continue; + } + + /** Continue and run the query add the results to the set */ + $query->setParameters($params); + $contentQuery->runScopes($query); + $contentQuery->runDirectives($query); + + $result = $repo->queryWith($query); + if ($result) { + $set->setOriginalQuery($contentType, $query->getQueryBuilder()); + $set->add($result, $contentType); + } + } + + if ($query->getSingleFetchMode()) { + if ($set->count() === 0) { + return false; + } + + return $set->current(); + } + + return $set; + } + + /** + * This block is added to deal with the possibility that a requested filter is not an allowable option on the + * database table. If the requested field filter is not a valid field on this table then we completely skip + * the query because no results will be expected if the field does not exist. The exception to this is if the field + * is part of an OR query then we remove the missing field from the stack but still allow the other fields through. + * + * @param array $queryParams + * @param Repository $repo + * + * @return bool|array $cleanParams + */ + public function whitelistParameters(array $queryParams, Repository $repo) + { + $metadata = $repo->getClassMetadata(); + $allowedParams = array_keys($metadata->getFieldMappings()); + $cleanParams = []; + foreach ($queryParams as $fieldSelect => $valueSelect) { + $stack = []; + + if (is_string($valueSelect)) { + $stack = preg_split('/ *(\|\|\|) */', $fieldSelect); + $valueStack = preg_split('/ *(\|\|\|) */', $valueSelect); + } + + if (count($stack) > 1) { + $allowedKeys = []; + $allowedVals = []; + foreach ($stack as $i => $stackItem) { + if (in_array($stackItem, $allowedParams)) { + $allowedKeys[] = $stackItem; + $allowedVals[] = $valueStack[$i]; + } + } + + if (!count($allowedKeys)) { + return false; + } + $allowed = implode(' ||| ', $allowedKeys); + $cleanParams[$allowed] = implode(' ||| ', $allowedVals); + } else { + if (!in_array($fieldSelect, $allowedParams)) { + return false; + } + $cleanParams[$fieldSelect] = $valueSelect; + } + } + + return $cleanParams; + } +} diff --git a/src/Storage/Query/Query.php b/src/Storage/Query/Query.php new file mode 100644 index 00000000..166ff01f --- /dev/null +++ b/src/Storage/Query/Query.php @@ -0,0 +1,112 @@ +parser = $parser; + $this->recordsView = $recordsView; + } + + /** + * @param string $name + * @param QueryScopeInterface $scope + */ + public function addScope($name, QueryScopeInterface $scope) + { + $this->scopes[$name] = $scope; + } + + /** + * @param string $name + * + * @return QueryScopeInterface|null + */ + public function getScope($name) + { + if (array_key_exists($name, $this->scopes)) { + return $this->scopes[$name]; + } + + return null; + } + + /** + * getContent based on a 'human readable query'. + * + * Used by the twig command {% setcontent %} but also directly. + * For reference refer to @link https://docs.bolt.cm/templating/content-fetching + * + * @param string $textQuery + * @param array|string $parameters + * + * @return QueryResultset|Content|null + */ + public function getContent($textQuery, array $parameters = []) + { + $this->parser->setQuery($textQuery); + $this->parser->setParameters($parameters); + + return $this->parser->fetch(); + } + + /** + * @param string $scopeName + * @param string $textQuery + * @param array $parameters + * + * @return QueryResultset|null + */ + public function getContentByScope($scopeName, $textQuery, array $parameters = []) + { + if ($scope = $this->getScope($scopeName)) { + $this->parser->setQuery($textQuery); + $this->parser->setParameters($parameters); + $this->parser->setScope($scope); + + return $this->parser->fetch(); + } + + return null; + } + + /** + * Helper to be called from Twig that is passed via a TwigRecordsView rather than the raw records. + * + * @param $textQuery + * @param array $parameters + * + * @return QueryResultset|null + */ + public function getContentForTwig($textQuery, array $parameters = []) + { + // fix BC break + if (func_num_args() === 3) { + $whereparameters = func_get_arg(2); + if (is_array($whereparameters) && !empty($whereparameters)) { + $parameters = array_merge($parameters, $whereparameters); + } + } + return $this->recordsView->createView( + $this->getContentByScope('frontend', $textQuery, $parameters) + ); + } +} diff --git a/src/Storage/Query/QueryInterface.php b/src/Storage/Query/QueryInterface.php new file mode 100644 index 00000000..486e7853 --- /dev/null +++ b/src/Storage/Query/QueryInterface.php @@ -0,0 +1,26 @@ + + */ +class QueryParameterParser +{ + /** @var string */ + public $alias; + + /** @var string */ + protected $key; + /** @var mixed */ + protected $value; + /** @var ExpressionBuilder */ + protected $expr; + /** @var array */ + protected $valueMatchers = []; + /** @var Filter[] */ + protected $filterHandlers = []; + + /** + * Constructor. + * + * @param ExpressionBuilder $expr + */ + public function __construct(ExpressionBuilder $expr = null) + { + $this->expr = $expr; + $this->setupDefaults(); + } + + public function setupDefaults() + { + $word = "[\p{L}\p{N}_]+"; + + // @codingStandardsIgnoreStart + $this->addValueMatcher("<($word)", ['value' => '$1', 'operator' => 'lt']); + $this->addValueMatcher("<=($word)", ['value' => '$1', 'operator' => 'lte']); + $this->addValueMatcher(">=($word)", ['value' => '$1', 'operator' => 'gte']); + $this->addValueMatcher(">($word)", ['value' => '$1', 'operator' => 'gt']); + $this->addValueMatcher('!$', ['value' => '', 'operator' => 'isNotNull']); + $this->addValueMatcher("!($word)", ['value' => '$1', 'operator' => 'neq']); + $this->addValueMatcher('!\[([\p{L}\p{N} ,]+)\]', ['value' => function ($val) { return explode(',', $val); }, 'operator' => 'notIn']); + $this->addValueMatcher('\[([\p{L}\p{N} ,]+)\]', ['value' => function ($val) { return explode(',', $val); }, 'operator' => 'in']); + $this->addValueMatcher("(%$word|$word%|%$word%)", ['value' => '$1', 'operator' => 'like']); + $this->addValueMatcher("($word)", ['value' => '$1', 'operator' => 'eq']); + // @codingStandardsIgnoreEnd + + $this->addFilterHandler([$this, 'defaultFilterHandler']); + $this->addFilterHandler([$this, 'multipleValueHandler']); + $this->addFilterHandler([$this, 'multipleKeyAndValueHandler']); + $this->addFilterHandler([$this, 'incorrectQueryHandler']); + } + + /** + * Sets the select alias to be used in sql queries. + * + * @param string $alias + */ + public function setAlias($alias) + { + $this->alias = $alias . '.'; + } + + /** + * Runs the keys/values through the relevant parsers. + * + * @param string $key + * @param mixed $value + * + * @throws QueryParseException + * @throws QueryParseException + * + * @return Filter|null + */ + public function getFilter($key, $value = null) + { + if (!$this->expr instanceof ExpressionBuilder) { + throw new QueryParseException('Cannot call method without an Expression Builder parameter set', 1); + } + + /** @var callable $callback */ + foreach ($this->filterHandlers as $callback) { + $result = $callback($key, $value, $this->expr); + if ($result instanceof Filter) { + return $result; + } + } + + return null; + } + + /** + * Handles some errors in key/value string formatting. + * + * @param string $key + * @param string $value + * @param ExpressionBuilder $expr + * + * @throws QueryParseException + * + * @return null + */ + public function incorrectQueryHandler($key, $value, $expr) + { + if (!is_string($value)) { + return null; + } + if (strpos($value, '&&') && strpos($value, '||')) { + throw new QueryParseException('Mixed && and || operators are not supported', 1); + } + } + + /** + * This handler processes 'triple pipe' queries as implemented in Bolt + * It looks for three pipes in the key and value and creates an OR composite + * expression for example: 'username|||email':'fred|||pete'. + * + * @param string $key + * @param string $value + * @param ExpressionBuilder $expr + * + * @throws QueryParseException + * + * @return Filter|null + */ + public function multipleKeyAndValueHandler($key, $value, $expr) + { + if (!strpos($key, '|||')) { + return null; + } + + $keys = preg_split('/ *(\|\|\|) */', $key); + $inputKeys = $keys; + $values = preg_split('/ *(\|\|\|) */', $value); + $values = array_pad($values, count($keys), end($values)); + + $filterParams = []; + $parts = []; + $count = 1; + + while (($key = array_shift($keys)) && ($val = array_shift($values))) { + $multipleValue = $this->multipleValueHandler($key, $val, $this->expr); + if ($multipleValue) { + $filter = $multipleValue->getExpression(); + $filterParams += $multipleValue->getParameters(); + } else { + $val = $this->parseValue($val); + $placeholder = $key . '_' . $count; + $filterParams[$placeholder] = $val['value']; + $exprMethod = $val['operator']; + $filter = $this->expr->$exprMethod($this->alias . $key, ':' . $placeholder); + } + + $parts[] = $filter; + ++$count; + } + + $filter = new Filter(); + $filter->setKey($inputKeys); + $filter->setExpression(call_user_func_array([$expr, 'orX'], $parts)); + $filter->setParameters($filterParams); + + return $filter; + } + + /** + * This handler processes multiple value queries as defined in the Bolt 'Fetching Content' + * documentation. It allows a value to be parsed to and AND/OR expression. + * + * For example, this handler will correctly parse values like: + * 'username': 'fred||bob' + * 'id': '<5 && !1' + * + * @param string $key + * @param string $value + * @param ExpressionBuilder $expr + * + * @throws QueryParseException + * + * @return Filter|null + */ + public function multipleValueHandler($key, $value, $expr) + { + if (!is_string($value)) { + return null; + } + if (strpos($value, '&&') === false && strpos($value, '||') === false) { + return null; + } + + $values = preg_split('/ *(&&|\|\|) */', $value, -1, PREG_SPLIT_DELIM_CAPTURE); + $op = $values[1]; + + if ($op === '&&') { + $comparison = 'andX'; + } elseif ($op === '||') { + $comparison = 'orX'; + } + + $values = array_diff($values, ['&&', '||']); + + $filterParams = []; + $parts = []; + $count = 1; + + while ($val = array_shift($values)) { + $val = $this->parseValue($val); + $placeholder = $key . '_' . $count; + $filterParams[$placeholder] = $val['value']; + $exprMethod = $val['operator']; + $parts[] = $this->expr->$exprMethod($this->alias . $key, ':' . $placeholder); + ++$count; + } + + $filter = new Filter(); + $filter->setKey($key); + $filter->setExpression(call_user_func_array([$expr, $comparison], $parts)); + $filter->setParameters($filterParams); + + return $filter; + } + + /** + * The default handler is the last to be run and handler simple value parsing. + * + * @param string $key + * @param string|array $value + * @param ExpressionBuilder $expr + * + * @throws QueryParseException + * + * @return Filter + */ + public function defaultFilterHandler($key, $value, $expr) + { + $filter = new Filter(); + $filter->setKey($key); + + if (is_array($value)) { + $count = 1; + + $composite = $expr->andX(); + + foreach ($value as $paramName => $valueItem) { + $val = $this->parseValue($valueItem); + $placeholder = sprintf('%s_%s_%s', $key, $paramName, $count); + $exprMethod = $val['operator']; + $composite->add($expr->$exprMethod($this->alias . $key, ':' . $placeholder)); + $filter->setParameter($placeholder, $val['value']); + + $count ++; + } + $filter->setExpression($composite); + + return $filter; + } + + $val = $this->parseValue($value); + $placeholder = $key . '_1'; + $exprMethod = $val['operator']; + + $filter->setExpression($expr->andX($expr->$exprMethod($this->alias . $key, ':' . $placeholder))); + $filter->setParameters([$placeholder => $val['value']]); + + return $filter; + } + + /** + * This method uses the defined value matchers to parse a passed in value. + * + * The following component parts will be returned in the array: + * [ + * 'value' => + * 'operator' => + * 'matched' => + * ] + * + * @param string $value Value to process + * + * @throws QueryParseException + * + * @return array Parsed values + */ + public function parseValue($value) + { + foreach ($this->valueMatchers as $matcher) { + $regex = sprintf('/%s/u', $matcher['token']); + $values = $matcher['params']; + if (preg_match($regex, $value)) { + if (is_callable($values['value'])) { + preg_match($regex, $value, $output); + $values['value'] = $values['value']($output[1]); + } else { + $values['value'] = preg_replace($regex, $values['value'], $value); + } + $values['matched'] = $matcher['token']; + + return $values; + } + } + + throw new QueryParseException(sprintf('No matching value found for "%s"', $value)); + } + + /** + * The goal of this class is to turn any key:value into a Filter class. + * Adding a handler here will push the new filter callback onto the top + * of the Queue along with the built in defaults. + * + * Note: the callback should either return nothing or an instance of + * \Bolt\Storage\Query\Filter + * + * @param callable $handler + */ + public function addFilterHandler(callable $handler) + { + array_unshift($this->filterHandlers, $handler); + } + + /** + * Adds an additional token to parse for value parameters. + * + * This gives the ability to define additional value -> operator matches + * + * @param string $token Regex pattern to match against + * @param array $params Options to provide to the matched param + * @param bool $priority If set item will be prepended to start of list + */ + public function addValueMatcher($token, array $params = [], $priority = null) + { + if ($priority) { + array_unshift($this->valueMatchers, ['token' => $token, 'params' => $params]); + } else { + $this->valueMatchers[] = ['token' => $token, 'params' => $params]; + } + } +} diff --git a/src/Storage/Query/QueryResultset.php b/src/Storage/Query/QueryResultset.php new file mode 100644 index 00000000..81129d8a --- /dev/null +++ b/src/Storage/Query/QueryResultset.php @@ -0,0 +1,105 @@ +results[$type] = $results; + } else { + $this->results = array_merge($this->results, $results); + } + + $this->append(new ArrayIterator($results)); + } + + /** + * Allows retrieval of a set or results, if a label has been used to + * store results then passing the label as a parameter returns just + * that set of results. + * + * @param string $label + * + * @return ArrayIterator + */ + public function get($label = null) + { + if ($label && array_key_exists($label, $this->results)) { + return $this->results[$label]; + } + $results = []; + foreach ($this->results as $v) { + if (is_array($v)) { + $results = array_merge($results, $v); + } else { + $results[] = $v; + } + } + + return $results; + } + + /** + * Returns the total count. + * + * @return int + */ + public function count() + { + return count($this->get()); + } + + /** + * @param $type + * @param $originalQuery + */ + public function setOriginalQuery($type, $originalQuery) + { + $this->originalQueries[$type] = $originalQuery; + } + + /** + * @param null $type + * + * @return QueryBuilder + */ + public function getOriginalQuery($type = null) + { + if ($type !== null) { + return $this->originalQueries[$type]; + } + + return reset($this->originalQueries); + } + + /** + * @return QueryBuilder[] + */ + public function getOriginalQueries() + { + return $this->originalQueries; + } +} diff --git a/src/Storage/Query/QueryScopeInterface.php b/src/Storage/Query/QueryScopeInterface.php new file mode 100644 index 00000000..80a97c9a --- /dev/null +++ b/src/Storage/Query/QueryScopeInterface.php @@ -0,0 +1,14 @@ + + */ +interface QueryScopeInterface +{ + public function onQueryExecute(ContentQueryInterface $query); +} diff --git a/src/Storage/Query/SearchConfig.php b/src/Storage/Query/SearchConfig.php new file mode 100644 index 00000000..c4993d4d --- /dev/null +++ b/src/Storage/Query/SearchConfig.php @@ -0,0 +1,210 @@ +config = $config; + $this->parseContenttypes(); + } + + /** + * Get the config of all fields for a given content type. + * + * @param string $contentType + * + * @return array|false + */ + public function getConfig($contentType) + { + if (array_key_exists($contentType, $this->searchableTypes)) { + return $this->searchableTypes[$contentType]; + } + + if ($this->canSearchInvisible() && array_key_exists($contentType, $this->invisibleTypes)) { + return $this->invisibleTypes[$contentType]; + } + + return false; + } + + + /** + * Get the config of one given field for a given content type. + * + * @param string $contentType + * @param string $field + * + * @return array|false + */ + public function getFieldConfig($contentType, $field) + { + if (isset($this->searchableTypes[$contentType][$field])) { + return $this->searchableTypes[$contentType][$field]; + } + + if ($this->canSearchInvisible() && isset($this->invisibleTypes[$contentType][$field])) { + return $this->invisibleTypes[$contentType][$field]; + } + + return false; + } + + /** + * Iterates over the main config and delegates weighting to both + * searchable columns and searchable taxonomies. + */ + protected function parseContenttypes() + { + $contentTypes = $this->config->get('contenttypes'); + + foreach ($contentTypes as $type => $values) { + $this->getSearchableColumns($type); + if (isset($values['taxonomy'])) { + $this->parseTaxonomies($type, $values['taxonomy']); + } + } + } + + /** + * Iterates the taxonomies for a given ContentType, then assigns a + * weighting based on type. + * + * @param string $contentType + * @param array $taxonomies + */ + protected function parseTaxonomies($contentType, $taxonomies) + { + foreach ((array) $taxonomies as $taxonomy) { + $taxonomyConfig = $this->config->get('taxonomy/' . $taxonomy); + if (isset($taxonomyConfig['searchweight'])) { + $weight = $taxonomyConfig['searchweight']; + } elseif (isset($taxonomyConfig['behaves_like']) && $taxonomyConfig['behaves_like'] === 'tags') { + $weight = 75; + } else { + $weight = 50; + } + if (!$this->isInvisible($contentType)) { + $this->searchableTypes[$contentType][$taxonomy] = ['weight' => $weight]; + } else { + $this->invisibleTypes[$contentType][$taxonomy] = ['weight' => $weight]; + } + $this->joins[$contentType][] = $taxonomy; + } + } + + /** + * Helper method to return the join search columns for a ContentType + * weighting based on type. + * + * @param string $contentType + * + * @return array + */ + public function getJoins($contentType) + { + return $this->joins[$contentType]; + } + + /** + * Determine what columns are searchable for a given ContentType. + * + * @param string $type + */ + protected function getSearchableColumns($type) + { + $fields = $this->config->get('contenttypes/' . $type . '/fields'); + + foreach ($fields as $field => $options) { + if (in_array($options['type'], ['text', 'textarea', 'html', 'markdown']) || + (isset($options['searchable']) && $options['searchable'] === true)) { + if (isset($options['searchweight'])) { + $weight = (int) $options['searchweight']; + } elseif (isset($fields['slug']['uses']) && in_array($field, (array)$fields['slug']['uses'], true)) { + $weight = 100; + } else { + $weight = 50; + } + + if (!$this->isInvisible($type)) { + $this->searchableTypes[$type][$field] = ['weight' => $weight]; + } else { + $this->invisibleTypes[$type][$field] = ['weight' => $weight]; + } + } + } + } + + /** + * Does some checks to see whether a ContentType should appear in search results. + * This is based on ContentType options. + * + * @param string $contentType + * + * @return bool + */ + protected function isInvisible($contentType) + { + $info = $this->config->get('contenttypes/' . $contentType); + if ($info) { + if (array_key_exists('viewless', $info) && $info['viewless'] === true) { + return true; + } + if (array_key_exists('searchable', $info) && $info['searchable'] === false) { + return true; + } + } + + return false; + } + + /** + * @return bool + */ + public function canSearchInvisible() + { + return $this->searchInvisible; + } + + /** + * @param bool $searchInvisible + */ + public function enableSearchInvisible($searchInvisible) + { + $this->searchInvisible = $searchInvisible; + } + + /** + * Return an array of searchable contenttypes + * @return array + */ + public function getSearchableTypes() + { + return $this->searchableTypes; + } + +} diff --git a/src/Storage/Query/SearchQuery.php b/src/Storage/Query/SearchQuery.php new file mode 100644 index 00000000..0efff273 --- /dev/null +++ b/src/Storage/Query/SearchQuery.php @@ -0,0 +1,156 @@ + + */ +class SearchQuery extends SelectQuery +{ + /** @var string */ + protected $search; + /** @var SearchConfig */ + protected $config; + + /** + * Constructor. + * + * @param QueryBuilder $qb + * @param QueryParameterParser $parser + * @param SearchConfig $config + */ + public function __construct(QueryBuilder $qb, QueryParameterParser $parser, SearchConfig $config) + { + parent::__construct($qb, $parser); + $this->config = $config; + } + + /** + * This method sets the search filter which then triggers the process method. + * + * @param string $search full search query + * + * @throws QueryParseException + */ + public function setSearch($search) + { + $this->search = $search; + $this->processFilters(); + } + + /** + * Sets the overall parameters on the query. This may include others + * than the search query itself which gets set to the 'filter' param. + * + * @param array $params + */ + public function setParameters(array $params) + { + $this->params = $params; + } + + /** + * Gets the individual elements of the search query as an array. + * + * @return array + */ + public function getSearchWords() + { + return explode(' ', $this->search); + } + + /** + * This is an internal helper method to get the search words prepared to + * be passed to the expression builder. + * + * @return string + */ + protected function getSearchParameter() + { + if (strpos($this->search, '+')) { + $words = preg_split('/[\s\+]+/', $this->search); + + return '%' . implode('% && %', $words) . '%'; + } + $words = explode(' ', $this->search); + + return '%' . implode('% || %', $words) . '%'; + } + + /** + * This overrides the SelectQuery default to do some extra preparation for a search query. + * Firstly it builds separate filters for the search query and then it removes the filter + * from the params and the others will then get processed normally by the parent. + * + * @throws QueryParseException + */ + protected function processFilters() + { + $params = $this->params; + + if (!$this->contentType) { + throw new QueryParseException('You have attempted to run a search query without specifying a ContentType', 1); + } + + if (isset($params['invisible']) && $params['invisible'] === true) { + $this->config->enableSearchInvisible(true); + } + + if (!$config = $this->config->getConfig($this->contentType)) { + throw new QueryParseException('You have attempted to run a search query on an unknown ContentType or one that is not searchable', 1); + } + + unset($params['filter'], $params['invisible']); + + foreach ($config as $field => $options) { + $params[$field] = $this->getSearchParameter(); + } + + $this->params = $params; + + parent::processFilters(); + } + + /** + * Creates a composite expression that adds all the attached + * filters individual expressions into a combined one. + * + * @return CompositeExpression|null + */ + public function getWhereExpression() + { + if (!count($this->filters)) { + return null; + } + + $wrapExpr = $this->qb->expr()->andX(); + $config = $this->config->getConfig($this->contentType); + $searchExpr = $this->qb->expr()->orX(); + $searchKeys = array_keys($config); + + /** @var Filter $filter */ + foreach ($this->filters as $filter) { + if (in_array($filter->getKey(), $searchKeys, true)) { + $searchExpr->add($filter->getExpression()); + } else { + $wrapExpr->add($filter->getExpression()); + } + } + $wrapExpr->add($searchExpr); + + return $wrapExpr; + } +} diff --git a/src/Storage/Query/SearchQueryResultset.php b/src/Storage/Query/SearchQueryResultset.php new file mode 100644 index 00000000..b64ae059 --- /dev/null +++ b/src/Storage/Query/SearchQueryResultset.php @@ -0,0 +1,83 @@ +results[$type] = $results; + $this->scores[$type] = $scores; + $this->sortSingle($type); + } else { + $this->results = array_merge($this->results, $results); + } + + $this->append(new ArrayIterator($results)); + } + + /** + * @param string $label + */ + public function sortSingle($label) + { + $results = $this->get($label); + $scores = $this->scores[$label]; + arsort($scores); + $sorted = []; + + foreach ($scores as $k => $v) { + $sorted[] = $results[$k]; + } + + $this->results[$label] = $sorted; + } + + /** + * @return array + */ + public function getSortedResults() + { + $results = []; + foreach ($this->results as $type => $records) { + $scores = $this->scores[$type]; + + foreach ($records as $i => $record) { + $results[] = [ + 'record' => $record, + 'score' => $scores[$i], + ]; + } + } + + usort($results, function ($a, $b) { + if ($a['score'] == $b['score']) { + return 0; + } + return ($a['score'] < $b['score']) ? -1 : 1; + }); + + $results = array_map(function ($item) { + return $item['record']; + }, $results); + + return $results; + } +} diff --git a/src/Storage/Query/SearchWeighter.php b/src/Storage/Query/SearchWeighter.php new file mode 100644 index 00000000..70a03333 --- /dev/null +++ b/src/Storage/Query/SearchWeighter.php @@ -0,0 +1,214 @@ +config = $config; + } + + /** + * Sets an iterable group of results, this normally comes directly + * from the database query. + * + * @param QueryResultset|array $results + */ + public function setResults(array $results) + { + $this->results = $results; + } + + /** + * Sets the ContentType that we are weighting, that is, what type the results + * array is. That allows us to map against the configuration to see which fields + * to scan for relevant text. + * + * @param string $type + */ + public function setContentType($type) + { + $this->contentType = $type; + } + + /** + * Sets the words that we want to query against. Normally this comes from the + * filter in a search, exploded into an array so the words are separated. + * + * @param array $words + */ + public function setSearchWords(array $words) + { + $this->searchWords = $words; + } + + /** + * This is the public method that gets a score for a the set of results. + * + * @return array An array of scores for each of the corresponding results + */ + public function weight() + { + $scores = []; + foreach ($this->results as $result) { + $scores[] = $this->getResultScore($result); + } + + return $scores; + } + + /** + * Helper method to fetch the fields for an individual ContentType. + * + * @return array|false + */ + protected function getContentFields() + { + return $this->config->getConfig($this->contentType); + } + + /** + * This is a simple version of the Vector Space Model. + * + * @see https://en.wikipedia.org/wiki/Vector_space_model + * + * The goal is to determine a relavance score for a corpus of values + * based on both the existence of a word or words but also based on + * how important the words are. + * + * For example, when querying results against a search of 'lorem ipsum'; + * a result with the title 'Lorem Ipsum' should score higher + * than a result with the title 'An article about robots and lorem ipsum' + * + * The ratio of the appearance of the query words to the overall size of + * the document is used to produce a better score. + * + * @param object $result A single result to score + * + * @return array An array consisting of a count / dictionary of words + */ + protected function buildResultIndex($result) + { + $corpus = []; + + foreach ($this->getContentFields() as $field => $weightings) { + $textualContent = $result->{$field}; + + // This is to handle taxonomies that need to be converted from an array + // into a string of words. + if (is_array($textualContent)) { + $textualContent = implode(' ', $textualContent); + } + + $textualContent = strip_tags($textualContent); + $textualContent = preg_replace('/[^\w\s]/', '', $textualContent); + $textualContent = mb_strtolower($textualContent); + $corpus[$field] = $textualContent; + } + + $dictionary = []; + $count = []; + + foreach ($corpus as $id => $doc) { + $terms = explode(' ', $doc); + $count[$id] = count($terms); + + foreach ($terms as $term) { + if (!isset($dictionary[$term])) { + $dictionary[$term] = ['frequency' => 0, 'postings' => []]; + } + if (!isset($dictionary[$term]['postings'][$id])) { + ++$dictionary[$term]['frequency']; + $dictionary[$term]['postings'][$id] = ['frequency' => 0]; + } + + ++$dictionary[$term]['postings'][$id]['frequency']; + } + } + + return ['count' => $count, 'dictionary' => $dictionary]; + } + + /** + * This method uses the index built in the method above to do some quick + * score calculations for each word of the query, versus each word of the + * index dictionary. + * + * @param object $result + * + * @return float + */ + protected function getResultScore($result) + { + $output = []; + + $corpus = $this->buildResultIndex($result); + $count = count($corpus['count']); + + // This block iterates the search query words and checks both their + // existence and frequency in the index. + // + // The score is passed through log(x, 2) to reduce the smooth the difference. + // + foreach ($this->searchWords as $word) { + $word = mb_strtolower($word); + if (isset($corpus['dictionary'][$word])) { + $entry = $corpus['dictionary'][$word]; + + foreach ($entry['postings'] as $field => $posting) { + //get term frequency–inverse document frequency + $score = $posting['frequency'] * log($count + 1 / $entry['frequency'] + 1, 2); + + if (isset($output[$field])) { + $output[$field] += $score; + } else { + $output[$field] = $score; + } + } + } + } + + // length normalise, we do this to stop smaller amounts of text having + // a disproportionate effect on the score. + foreach ($output as $field => $score) { + $output[$field] = $score / $corpus['count'][$field]; + } + + // Finally this weights by using the field specific weighting value that + // is set inside `contenttypes.yml` This uses a weighting factor from 0 to + // 100 that alters the score accordingly. + $weights = $this->getContentFields(); + foreach ($output as $field => &$score) { + if (isset($weights[$field]['weight'])) { + $multiplier = $weights[$field]['weight'] / 100; + + if ($multiplier > 0) { + $score *= $multiplier; + } + } + } + + return array_sum($output); + } +} diff --git a/src/Storage/Query/SelectQuery.php b/src/Storage/Query/SelectQuery.php new file mode 100644 index 00000000..6b39744c --- /dev/null +++ b/src/Storage/Query/SelectQuery.php @@ -0,0 +1,279 @@ + + */ +class SelectQuery implements ContentQueryInterface +{ + /** @var QueryBuilder */ + protected $qb; + /** @var QueryParameterParser */ + protected $parser; + /** @var string */ + protected $contentType; + /** @var array */ + protected $params; + /** @var Filter[] */ + protected $filters = []; + protected $replacements = []; + /** @var bool */ + protected $singleFetchMode = false; + + /** + * Constructor. + * + * @param QueryBuilder $qb + * @param QueryParameterParser $parser + */ + public function __construct(QueryBuilder $qb, QueryParameterParser $parser) + { + $this->qb = $qb; + $this->parser = $parser; + } + + /** + * Sets the ContentType that this query will run against. + * + * @param string $contentType + */ + public function setContentType($contentType) + { + $this->contentType = $contentType; + } + + /** + * Gets the ContentType that this query will run against. + * + * @return string + */ + public function getContentType() + { + return $this->contentType; + } + + /** + * Sets the parameters that will filter / alter the query. + * + * @param array $params + */ + public function setParameters(array $params) + { + $this->params = array_filter($params); + $this->processFilters(); + } + + /** + * Getter to allow access to a set parameter. + * + * @param $name + * + * @return array|null + */ + public function getParameter($name) + { + if (array_key_exists($name, $this->params)) { + return $this->params[$name]; + } + + return null; + } + + /** + * Setter to allow writing to a named parameter. + * + * @param string $name + * @param mixed $value + */ + public function setParameter($name, $value) + { + $this->params[$name] = $value; + $this->processFilters(); + } + + /** + * Creates a composite expression that adds all the attached + * filters individual expressions into a combined one. + * + * @return CompositeExpression + */ + public function getWhereExpression() + { + if (!count($this->filters)) { + return null; + } + + $expr = $this->qb->expr()->andX(); + foreach ($this->filters as $filter) { + $expr = $expr->add($filter->getExpression()); + } + + return $expr; + } + + /** + * Returns all the parameters for the query. + * + * @return array + */ + public function getWhereParameters() + { + $params = []; + foreach ($this->filters as $filter) { + $params = array_merge($params, $filter->getParameters()); + } + + return $params; + } + + /** + * Gets all the parameters for a specific field name. + * + * @param string $fieldName + * + * @return array array of key=>value parameters + */ + public function getWhereParametersFor($fieldName) + { + return array_intersect_key( + $this->getWhereParameters(), + array_flip(preg_grep('/^' . $fieldName . '_\d+$/', array_keys($this->getWhereParameters()))) + ); + } + + /** + * Sets all the parameters for a specific field name. + * + * @param string $key + * @param mixed $value + */ + public function setWhereParameter($key, $value) + { + foreach ($this->filters as $filter) { + if ($filter->hasParameter($key)) { + $filter->setParameter($key, $value); + } + } + } + + /** + * @param Filter $filter + */ + public function addFilter(Filter $filter) + { + $this->filters[] = $filter; + } + + /** + * Returns all the filters attached to the query. + * + * @return Filter[] + */ + public function getFilters() + { + return $this->filters; + } + + /** + * Part of the QueryInterface this turns all the input into a Doctrine + * QueryBuilder object and is usually run just before query execution. + * That allows modifications to be made to any of the parameters up until + * query execution time. + * + * @return QueryBuilder + */ + public function build() + { + $query = $this->qb; + if ($this->getWhereExpression()) { + $query->where($this->getWhereExpression()); + } + foreach ($this->getWhereParameters() as $key => $param) { + $query->setParameter($key, $param, is_array($param) ? Connection::PARAM_STR_ARRAY : null); + } + + return $query; + } + + /** + * Allows public access to the QueryBuilder object. + * + * @return QueryBuilder + */ + public function getQueryBuilder() + { + return $this->qb; + } + + /** + * Allows replacing the default QueryBuilder. + * + * @param QueryBuilder $qb + */ + public function setQueryBuilder(QueryBuilder $qb) + { + $this->qb = $qb; + } + + /** + * Returns whether the query is in single fetch mode. + * + * @return bool + */ + public function getSingleFetchMode() + { + return $this->singleFetchMode; + } + + /** + * Turns single fetch mode on or off. + * + * @param bool $value + */ + public function setSingleFetchMode($value) + { + $this->singleFetchMode = (bool) $value; + } + + /** + * @return string String representation of query + */ + public function __toString() + { + $query = $this->build(); + + return $query->getSQL(); + } + + /** + * Internal method that runs the individual key/value input through + * the QueryParameterParser. This allows complicated expressions to + * be turned into simple sql expressions. + * + * @throws \Bolt\Exception\QueryParseException + */ + protected function processFilters() + { + $this->filters = []; + foreach ($this->params as $key => $value) { + $this->parser->setAlias('_' . $this->contentType); + $filter = $this->parser->getFilter($key, $value); + if ($filter) { + $this->addFilter($filter); + } + } + } +} diff --git a/src/Storage/Query/TaxonomyQuery.php b/src/Storage/Query/TaxonomyQuery.php new file mode 100644 index 00000000..e9e95cec --- /dev/null +++ b/src/Storage/Query/TaxonomyQuery.php @@ -0,0 +1,179 @@ + + */ +class TaxonomyQuery implements QueryInterface +{ + /** @var QueryBuilder */ + protected $qb; + /** @var array */ + protected $params; + /** @var array */ + protected $contentTypes; + /** @var array */ + protected $taxonomyTypes; + /**@var Pimple */ + private $schema; + + /** + * Constructor. + * + * @param QueryBuilder $qb + * @param Pimple $schema + */ + public function __construct(QueryBuilder $qb, Pimple $schema) + { + $this->qb = $qb; + $this->schema = $schema; + } + + /** + * Sets the parameters that will filter / alter the query. + * + * @param array $params + */ + public function setParameters(array $params) + { + $this->params = array_filter($params); + } + + /** + * Getter to allow access to a set parameter. + * + * @param $name + * + * @return array|null + */ + public function getParameter($name) + { + if (array_key_exists($name, $this->params)) { + return $this->params[$name]; + } + + return null; + } + + /** + * Setter to allow writing to a named parameter. + * + * @param string $name + * @param mixed $value + */ + public function setParameter($name, $value) + { + $this->params[$name] = $value; + } + + /** + * Setter to specify which content types to search on + * + * @param array $contentTypes + */ + public function setContentTypes(array $contentTypes) + { + $this->contentTypes = $contentTypes; + } + + /** + * Setter to specify which taxonomy types to search on + * + * @param array $taxonomyTypes + */ + public function setTaxonomyTypes(array $taxonomyTypes) + { + $this->taxonomyTypes = $taxonomyTypes; + } + + /** + * Part of the QueryInterface this turns all the input into a Doctrine + * QueryBuilder object and is usually run just before query execution. + * That allows modifications to be made to any of the parameters up until + * query execution time. + * + * @return QueryBuilder + */ + public function build() + { + $query = $this->qb; + $this->buildJoin(); + $this->buildWhere(); + + return $query; + } + + /** + * Allows public access to the QueryBuilder object. + * + * @return QueryBuilder + */ + public function getQueryBuilder() + { + return $this->qb; + } + + /** + * Allows replacing the default QueryBuilder. + * + * @param QueryBuilder $qb + */ + public function setQueryBuilder(QueryBuilder $qb) + { + $this->qb = $qb; + } + + /** + * @return string String representation of query + */ + public function __toString() + { + $query = $this->build(); + + return $query->getSQL(); + } + + protected function buildJoin() + { + $subQuery = '(SELECT '; + $fragments = []; + foreach ($this->contentTypes as $content) { + /** @var ContentType $table */ + $table = $this->schema[$content]; + $tableName = $table->getTableName(); + $fragments[] = "id,status, '$content' AS tablename FROM " . $tableName; + } + $subQuery .= join(' UNION SELECT ', $fragments); + $subQuery .= ')'; + + $this->qb->from($subQuery, 'content'); + $this->qb->addSelect('content.*'); + } + + protected function buildWhere() + { + $params = []; + $i = 0; + $where = $this->qb->expr()->andX(); + foreach ($this->taxonomyTypes as $name => $slug) { + $where->add($this->qb->expr()->eq('taxonomy.taxonomytype', ':taxonomytype_' . $i)); + $where->add($this->qb->expr()->eq('taxonomy.slug', ':slug_' . $i)); + $params['taxonomytype_' . $i] = $name; + $params['slug_' . $i] = $slug; + $i++; + } + $this->qb->where($where)->setParameters($params); + $this->qb->andWhere("content.status='published'"); + $this->qb->andWhere('taxonomy.contenttype=content.tablename'); + $this->qb->andWhere('taxonomy.content_id=content.id'); + } +} diff --git a/src/Storage/Query/TaxonomyQueryResultset.php b/src/Storage/Query/TaxonomyQueryResultset.php new file mode 100644 index 00000000..56a5cdac --- /dev/null +++ b/src/Storage/Query/TaxonomyQueryResultset.php @@ -0,0 +1,40 @@ +results as $proxy) { + $collection->add(new EntityProxy($proxy['contenttype'], $proxy['id'], $this->getEntityManager())); + } + + return $collection; + } + + public function setEntityManager(EntityManager $em) + { + $this->em = $em; + } + + /** + * @return EntityManager + */ + public function getEntityManager() + { + return $this->em; + } +} diff --git a/src/Twig/TwigRecordsView.php b/src/Twig/TwigRecordsView.php new file mode 100644 index 00000000..0588ffa6 --- /dev/null +++ b/src/Twig/TwigRecordsView.php @@ -0,0 +1,167 @@ + + */ +class TwigRecordsView +{ + /** @var MetadataDriver */ + protected $metadata; + /** @var array $transformers */ + protected $transformers; + + /** + * Constructor. + * + * @param MetadataDriver $metadata + */ + public function __construct(MetadataDriver $metadata) + { + $this->metadata = $metadata; + $this->setupDefaults(); + } + + /** + * Here we register some transformers that prepare the fields to be passed to + * twig. For repeaters and blocks this is recursive. + */ + protected function setupDefaults() + { + $this->addTransformer('html', function ($value) { + return new Markup($value, 'UTF-8'); + }); + $this->addTransformer('text', function ($value) { + return new Markup($value, 'UTF-8'); + }); + $this->addTransformer('textarea', function ($value) { + return new Markup($value, 'UTF-8'); + }); + $this->addTransformer('markdown', function ($value) { + $markdown = new ParsedownExtra(); + $value = $markdown->text($value); + + return new Markup($value, 'UTF-8'); + }); + + $this->addTransformer('repeater', function ($value) { + /** @var FieldCollection $collection */ + foreach ($value as $collection) { + foreach ($collection as $field) { + $field->setValue($this->transform($field->getValue(), $field->getFieldType())); + } + } + + return $value; + }); + + $this->addTransformer('block', function ($value) { + return $this->transform($value, 'repeater'); + }); + } + + /** + * This loads the relevant record or records and activates the class. + * + * @param Content|QueryResultset $records + * + * @return Content|QueryResultset + */ + public function createView($records) + { + if ($records instanceof Content) { + $this->processSingleRecord($records); + } elseif ($records instanceof QueryResultset) { + $this->processRecords($records); + } + + return $records; + } + + /** + * @param $record + */ + protected function processSingleRecord($record) + { + $values = $record->getValues(); + if (is_array($values)) { + foreach ($values as $field => $value) { + $type = $this->metadata->getFieldMetadata((string) $record->getContenttype(), $field); + $boltType = $type['data']['type']; + $record->set($field, $this->transform($value, $boltType, $type)); + } + } + } + + /** + * @param $records + */ + protected function processRecords($records) + { + foreach ($records as $record) { + $this->processSingleRecord($record); + } + } + + /** + * Adds a transformer callback to the field type $label + * + * @param $label + * @param callable $callback + */ + public function addTransformer($label, callable $callback) + { + $this->transformers[$label] = $callback; + } + + /** + * Checks if a transformer is registered for $label + * + * @param $label + * + * @return bool + */ + public function hasTransformer($label) + { + return array_key_exists($label, $this->transformers); + } + + /** + * @param $label + * + * @return array|mixed + */ + public function getTransformer($label) + { + if ($this->hasTransformer($label)) { + return $this->transformers[$label]; + } + } + + /** + * @param $value + * @param $label + * @param array $fieldData + * + * @return mixed + */ + protected function transform($value, $label, array $fieldData = []) + { + if ($this->hasTransformer($label)) { + return $this->transformers[$label]($value, $fieldData); + } + + return $value; + } +}