Skip to content

Commit

Permalink
Feature: highlight fulltext search results
Browse files Browse the repository at this point in the history 
How it works:

  1. when a fulltext search is made, Shaarli looks for the first
occurence position of every term matching the search. No change here,
but we store these positions in an array, in Bookmark's additionalContent.
  2. when formatting bookmarks (through BookmarkFormatter
implementation):
    1. first we insert specific tokens at every search result positions
    2. we format the content (escape HTML, apply markdown, etc.)
    3. as a last step, we replace our token with displayable span
elements

Cons: this tightens coupling between search filters and formatters
Pros: it was absolutely necessary not to perform the
search twice. this solution has close to no impact on performances.

Fixes shaarli#205
  • Loading branch information
@ArthurHoaro
ArthurHoaro committed Oct 16, 2020
1 parent 64cac25 commit 1d83281
Show file tree
Hide file tree
Showing 7 changed files with 349 additions and 35 deletions.
46 changes: 46 additions & 0 deletions application/bookmark/Bookmark.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ class Bookmark
/** @var bool True if the bookmark can only be seen while logged in */
protected $private;

/** @var mixed[] Available to store any additional content for a bookmark. Currently used for search highlight. */
protected $additionalContent = [];

/**
* Initialize a link from array data. Especially useful to create a Bookmark from former link storage format.
*
Expand Down Expand Up @@ -95,6 +98,8 @@ public function fromArray(array $data): Bookmark
* - the URL with the permalink
* - the title with the URL
*
* Also make sure that we do not save search highlights in the datastore.
*
* @throws InvalidBookmarkException
*/
public function validate(): void
Expand All @@ -112,6 +117,9 @@ public function validate(): void
if (empty($this->title)) {
$this->title = $this->url;
}
if (array_key_exists('search_highlight', $this->additionalContent)) {
unset($this->additionalContent['search_highlight']);
}
}

/**
Expand Down Expand Up @@ -435,6 +443,44 @@ public function setTagsString(?string $tags): Bookmark
return $this;
}

/**
* Get entire additionalContent array.
*
* @return mixed[]
*/
public function getAdditionalContent(): array
{
return $this->additionalContent;
}

/**
* Set a single entry in additionalContent, by key.
*
* @param string $key
* @param mixed|null $value Any type of value can be set.
*
* @return $this
*/
public function addAdditionalContentEntry(string $key, $value): self
{
$this->additionalContent[$key] = $value;

return $this;
}

/**
* Get a single entry in additionalContent, by key.
*
* @param string $key
* @param mixed|null $default
*
* @return mixed|null can be any type or even null.
*/
public function getAdditionalContentEntry(string $key, $default = null)
{
return array_key_exists($key, $this->additionalContent) ? $this->additionalContent[$key] : $default;
}

/**
* Rename a tag in tags list.
*
Expand Down
111 changes: 93 additions & 18 deletions application/bookmark/BookmarkFilter.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ private function filterFulltext(string $searchterms, string $visibility = 'all')
return $this->noFilter($visibility);
}

$filtered = array();
$filtered = [];
$search = mb_convert_case(html_entity_decode($searchterms), MB_CASE_LOWER, 'UTF-8');
$exactRegex = '/"([^"]+)"/';
// Retrieve exact search terms.
Expand All @@ -213,8 +213,8 @@ private function filterFulltext(string $searchterms, string $visibility = 'all')
$explodedSearchAnd = array_values(array_filter($explodedSearchAnd));

// Filter excluding terms and update andSearch.
$excludeSearch = array();
$andSearch = array();
$excludeSearch = [];
$andSearch = [];
foreach ($explodedSearchAnd as $needle) {
if ($needle[0] == '-' && strlen($needle) > 1) {
$excludeSearch[] = substr($needle, 1);
Expand All @@ -234,33 +234,38 @@ private function filterFulltext(string $searchterms, string $visibility = 'all')
}
}

// Concatenate link fields to search across fields.
// Adds a '\' separator for exact search terms.
$content = mb_convert_case($link->getTitle(), MB_CASE_LOWER, 'UTF-8') .'\\';
$content .= mb_convert_case($link->getDescription(), MB_CASE_LOWER, 'UTF-8') .'\\';
$content .= mb_convert_case($link->getUrl(), MB_CASE_LOWER, 'UTF-8') .'\\';
$content .= mb_convert_case($link->getTagsString(), MB_CASE_LOWER, 'UTF-8') .'\\';
$lengths = [];
$content = $this->buildFullTextSearchableLink($link, $lengths);

// Be optimistic
$found = true;
$foundPositions = [];

// First, we look for exact term search
for ($i = 0; $i < count($exactSearch) && $found; $i++) {
$found = strpos($content, $exactSearch[$i]) !== false;
}

// Iterate over keywords, if keyword is not found,
// Then iterate over keywords, if keyword is not found,
// no need to check for the others. We want all or nothing.
for ($i = 0; $i < count($andSearch) && $found; $i++) {
$found = strpos($content, $andSearch[$i]) !== false;
foreach ([$exactSearch, $andSearch] as $search) {
for ($i = 0; $i < count($search) && $found !== false; $i++) {
$found = mb_strpos($content, $search[$i]);
if ($found === false) {
break;
}

$foundPositions[] = ['start' => $found, 'end' => $found + mb_strlen($search[$i])];
}
}

// Exclude terms.
for ($i = 0; $i < count($excludeSearch) && $found; $i++) {
for ($i = 0; $i < count($excludeSearch) && $found !== false; $i++) {
$found = strpos($content, $excludeSearch[$i]) === false;
}

if ($found) {
if ($found !== false) {
$link->addAdditionalContentEntry(
'search_highlight',
$this->postProcessFoundPositions($lengths, $foundPositions)
);

$filtered[$id] = $link;
}
}
Expand Down Expand Up @@ -477,4 +482,74 @@ public static function tagsStrToArray(string $tags, bool $casesensitive): array

return preg_split('/\s+/', $tagsOut, -1, PREG_SPLIT_NO_EMPTY);
}

/**
* This method finalize the content of the foundPositions array,
* by associated all search results to their associated bookmark field,
* making sure that there is no overlapping results, etc.
*
* @param array $fieldLengths Start and end positions of every bookmark fields in the aggregated bookmark content.
* @param array $foundPositions Positions where the search results were found in the aggregated content.
*
* @return array Updated $foundPositions, by bookmark field.
*/
protected function postProcessFoundPositions(array $fieldLengths, array $foundPositions): array
{
// Sort results by starting position ASC.
usort($foundPositions, function (array $entryA, array $entryB): int {
return $entryA['start'] > $entryB['start'] ? 1 : -1;
});

$out = [];
$currentMax = -1;
foreach ($foundPositions as $foundPosition) {
// we do not allow overlapping highlights
if ($foundPosition['start'] < $currentMax) {
continue;
}

$currentMax = $foundPosition['end'];
foreach ($fieldLengths as $part => $length) {
if ($foundPosition['start'] < $length['start'] || $foundPosition['start'] > $length['end']) {
continue;
}

$out[$part][] = [
'start' => $foundPosition['start'] - $length['start'],
'end' => $foundPosition['end'] - $length['start'],
];
break;
}
}

return $out;
}

/**
* Concatenate link fields to search across fields. Adds a '\' separator for exact search terms.
* Also populate $length array with starting and ending positions of every bookmark field
* inside concatenated content.
*
* @param Bookmark $link
* @param array $lengths (by reference)
*
* @return string Lowercase concatenated fields content.
*/
protected function buildFullTextSearchableLink(Bookmark $link, array &$lengths): string
{
$content = mb_convert_case($link->getTitle(), MB_CASE_LOWER, 'UTF-8') .'\\';
$content .= mb_convert_case($link->getDescription(), MB_CASE_LOWER, 'UTF-8') .'\\';
$content .= mb_convert_case($link->getUrl(), MB_CASE_LOWER, 'UTF-8') .'\\';
$content .= mb_convert_case($link->getTagsString(), MB_CASE_LOWER, 'UTF-8') .'\\';

$lengths['title'] = ['start' => 0, 'end' => mb_strlen($link->getTitle())];
$nextField = $lengths['title']['end'] + 1;
$lengths['description'] = ['start' => $nextField, 'end' => $nextField + mb_strlen($link->getDescription())];
$nextField = $lengths['description']['end'] + 1;
$lengths['url'] = ['start' => $nextField, 'end' => $nextField + mb_strlen($link->getUrl())];
$nextField = $lengths['url']['end'] + 1;
$lengths['tags'] = ['start' => $nextField, 'end' => $nextField + mb_strlen($link->getTagsString())];

return $content;
}
}
132 changes: 127 additions & 5 deletions application/formatter/BookmarkDefaultFormatter.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,42 @@
*/
class BookmarkDefaultFormatter extends BookmarkFormatter
{
const SEARCH_HIGHLIGHT_OPEN = '|@@HIGHLIGHT';
const SEARCH_HIGHLIGHT_CLOSE = 'HIGHLIGHT@@|';

/**
* @inheritdoc
*/
public function formatTitle($bookmark)
protected function formatTitle($bookmark)
{
return escape($bookmark->getTitle());
}

/**
* @inheritdoc
*/
public function formatDescription($bookmark)
protected function formatTitleHtml($bookmark)
{
$title = $this->tokenizeSearchHighlightField(
$bookmark->getTitle() ?? '',
$bookmark->getAdditionalContentEntry('search_highlight')['title'] ?? []
);

return $this->replaceTokens(escape($title));
}

/**
* @inheritdoc
*/
protected function formatDescription($bookmark)
{
$indexUrl = ! empty($this->contextData['index_url']) ? $this->contextData['index_url'] : '';
return format_description(escape($bookmark->getDescription()), $indexUrl);
$description = $this->tokenizeSearchHighlightField(
$bookmark->getDescription() ?? '',
$bookmark->getAdditionalContentEntry('search_highlight')['description'] ?? []
);

return $this->replaceTokens(format_description(escape($description), $indexUrl));
}

/**
Expand All @@ -40,15 +61,35 @@ protected function formatTagList($bookmark)
/**
* @inheritdoc
*/
public function formatTagString($bookmark)
protected function formatTagListHtml($bookmark)
{
if (empty($bookmark->getAdditionalContentEntry('search_highlight')['tags'])) {
return $this->formatTagList($bookmark);
}

$tags = $this->tokenizeSearchHighlightField(
$bookmark->getTagsString(),
$bookmark->getAdditionalContentEntry('search_highlight')['tags']
);
$tags = $this->filterTagList(explode(' ', $tags));
$tags = escape($tags);
$tags = $this->replaceTokensArray($tags);

return $tags;
}

/**
* @inheritdoc
*/
protected function formatTagString($bookmark)
{
return implode(' ', $this->formatTagList($bookmark));
}

/**
* @inheritdoc
*/
public function formatUrl($bookmark)
protected function formatUrl($bookmark)
{
if ($bookmark->isNote() && isset($this->contextData['index_url'])) {
return rtrim($this->contextData['index_url'], '/') . '/' . escape(ltrim($bookmark->getUrl(), '/'));
Expand Down Expand Up @@ -77,11 +118,92 @@ protected function formatRealUrl($bookmark)
return escape($bookmark->getUrl());
}

/**
* @inheritdoc
*/
protected function formatUrlHtml($bookmark)
{
$url = $this->tokenizeSearchHighlightField(
$bookmark->getUrl() ?? '',
$bookmark->getAdditionalContentEntry('search_highlight')['url'] ?? []
);

return $this->replaceTokens(escape($url));
}

/**
* @inheritdoc
*/
protected function formatThumbnail($bookmark)
{
return escape($bookmark->getThumbnail());
}

/**
* Insert search highlight token in provided field content based on a list of search result positions
*
* @param string $fieldContent
* @param array|null $positions List of of search results with 'start' and 'end' positions.
*
* @return string Updated $fieldContent.
*/
protected function tokenizeSearchHighlightField(string $fieldContent, ?array $positions): string
{
if (empty($positions)) {
return $fieldContent;
}

$insertedTokens = 0;
$tokenLength = strlen(static::SEARCH_HIGHLIGHT_OPEN);
foreach ($positions as $position) {
$position = [
'start' => $position['start'] + ($insertedTokens * $tokenLength),
'end' => $position['end'] + ($insertedTokens * $tokenLength),
];

$content = mb_substr($fieldContent, 0, $position['start']);
$content .= static::SEARCH_HIGHLIGHT_OPEN;
$content .= mb_substr($fieldContent, $position['start'], $position['end'] - $position['start']);
$content .= static::SEARCH_HIGHLIGHT_CLOSE;
$content .= mb_substr($fieldContent, $position['end']);

$fieldContent = $content;

$insertedTokens += 2;
}

return $fieldContent;
}

/**
* Replace search highlight tokens with HTML highlighted span.
*
* @param string $fieldContent
*
* @return string updated content.
*/
protected function replaceTokens(string $fieldContent): string
{
return str_replace(
[static::SEARCH_HIGHLIGHT_OPEN, static::SEARCH_HIGHLIGHT_CLOSE],
['<span class="search-highlight">', '</span>'],
$fieldContent
);
}

/**
* Apply replaceTokens to an array of content strings.
*
* @param string[] $fieldContents
*
* @return array
*/
protected function replaceTokensArray(array $fieldContents): array
{
foreach ($fieldContents as &$entry) {
$entry = $this->replaceTokens($entry);
}

return $fieldContents;
}
}
Loading

0 comments on commit 1d83281

Please sign in to comment.