From 3e1d61516c40880f0a4c2ec5a82f7ef5b40770b6 Mon Sep 17 00:00:00 2001 From: darrell-k Date: Tue, 10 Sep 2024 20:16:39 +0100 Subject: [PATCH 1/3] speed up works search, livesearch queries and library view checks --- Slim/Control/Queries.pm | 37 ++++++++++++++-------------- Slim/Plugin/FullTextSearch/Plugin.pm | 9 ++++++- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/Slim/Control/Queries.pm b/Slim/Control/Queries.pm index 6974be41a5e..06556ab515d 100644 --- a/Slim/Control/Queries.pm +++ b/Slim/Control/Queries.pm @@ -510,16 +510,18 @@ sub albumsQuery { push @{$p}, $year; } - if (defined $fromSearch) { + if (defined $fromSearch && !defined $search) { # If we've got here from a search, we don't want to show the album unless it matches all the user's search criteria. # This matters for a Works search: we've shown the user a Work because it matches their criteria, but it is possible # that not all albums containing the Work match the all the criteria. (In this context, a work is a group of albums.) if ( Slim::Schema->canFulltextSearch ) { - $fromSearch =~ s/ /* /g; - $fromSearch .= '*'; - $fromSearch = "type:album " . $fromSearch; - push @{$w}, "EXISTS (select * FROM fulltext WHERE SUBSTR(fulltext.id, 33)=albums.id AND fulltext MATCH ?)"; - push @{$p}, $fromSearch; + Slim::Plugin::FullTextSearch::Plugin->createHelperTable({ + name => 'albumsSearch', + search => $fromSearch, + type => 'album', + createPrimaryKey => 1, + }); + $sql .= "JOIN albumsSearch ON albums.id = albumsSearch.id "; } else { my $strings = Slim::Utils::Text::searchStringSplit($fromSearch); if ( ref $strings->[0] eq 'ARRAY' ) { @@ -3185,6 +3187,7 @@ sub searchQuery { name => 'quickSearch', search => $search, type => $type, + createPrimaryKey => 1, checkLargeResultset => sub { my $isLarge = shift; return ($isLarge && $isLarge > ($index + $quantity)) ? ('ORDER BY fulltextweight DESC LIMIT ' . $isLarge) : ''; @@ -3199,28 +3202,25 @@ sub searchQuery { } if ( $libraryID ) { + my $exists; if ( $type eq 'contributor') { - $sql .= 'JOIN contributor_track ON contributor_track.contributor = me.id '; - $sql .= 'JOIN library_track ON library_track.track = contributor_track.track '; + $exists = 'EXISTS (SELECT * FROM contributor_track, library_track WHERE library_track.track = contributor_track.track AND contributor_track.contributor = me.id'; } elsif ( $type eq 'work') { - $sql .= 'JOIN tracks ON tracks.work = me.id '; - $sql .= 'JOIN library_track ON library_track.track = tracks.id '; + $exists = 'EXISTS (SELECT * FROM tracks, library_track WHERE library_track.track = tracks.id AND tracks.work = me.id'; } elsif ( $type eq 'album' ) { - $sql .= 'JOIN tracks ON tracks.album = me.id '; - $sql .= 'JOIN library_track ON library_track.track = tracks.id '; + $exists = 'EXISTS (SELECT * FROM tracks, library_track WHERE library_track.track = tracks.id AND tracks.album = me.id'; } elsif ( $type eq 'genre' ) { - $sql .= 'JOIN genre_track ON genre_track.genre = me.id '; - $sql .= 'JOIN library_track ON library_track.track = genre_track.track '; + $exists = 'EXISTS (SELECT * FROM genre_track, library_track WHERE library_track.track = genre_track.track AND genre_track.genre = me.id'; } elsif ( $type eq 'track' ) { - $sql .= 'JOIN library_track ON library_track.track = me.id '; + $exists = 'EXISTS (SELECT * FROM library_track WHERE library_track.track = me.id'; } - push @{$w}, 'library_track.library = ?'; + push @{$w}, "$exists AND library_track.library = ?)"; push @{$p}, $libraryID; } @@ -4672,6 +4672,7 @@ sub worksQuery { name => 'worksSearch', search => $search, type => 'work', + createPrimaryKey => 1, }); $sql .= "JOIN worksSearch ON works.id = worksSearch.id "; @@ -4679,6 +4680,7 @@ sub worksQuery { name => 'albumsSearch', search => $search, type => 'album', + createPrimaryKey => 1, }); $sql .= "JOIN albumsSearch ON albums.id = albumsSearch.id "; } else { @@ -4722,8 +4724,7 @@ sub worksQuery { } if (defined $libraryID) { - $sql .= 'JOIN library_track ON library_track.track = tracks.id '; - push @{$w}, 'library_track.library = ?'; + push @{$w}, 'EXISTS (SELECT * FROM library_album WHERE library_album.album = albums.id AND library_album.library = ?)'; push @{$p}, $libraryID; } diff --git a/Slim/Plugin/FullTextSearch/Plugin.pm b/Slim/Plugin/FullTextSearch/Plugin.pm index 95b4789af07..f1a218637ed 100644 --- a/Slim/Plugin/FullTextSearch/Plugin.pm +++ b/Slim/Plugin/FullTextSearch/Plugin.pm @@ -266,6 +266,7 @@ sub createHelperTable { my $name = $args->{name}; my $type = $args->{type}; + my $createPrimaryKey = $args->{createPrimaryKey} || 0; my ($tokens, $isLarge); my $orderOrLimit = ''; @@ -287,7 +288,13 @@ sub createHelperTable { $orderOrLimit = 'LIMIT 0' if !$tokens; # The first 32 bytes of the ID are either an MD5 of the ID, or some buster to make it "non searchable" - remove that prefix - my $searchSQL = "CREATE $temp TABLE $name AS SELECT SUBSTR(fulltext.id, 33) AS id, FULLTEXTWEIGHT(matchinfo(fulltext)) AS fulltextweight FROM fulltext WHERE fulltext MATCH 'type:$type $tokens' $orderOrLimit"; + my $searchSQL = "SELECT SUBSTR(fulltext.id, 33) AS id, FULLTEXTWEIGHT(matchinfo(fulltext)) AS fulltextweight FROM fulltext WHERE fulltext MATCH 'type:$type $tokens' $orderOrLimit"; + if ($createPrimaryKey) { + $dbh->do("CREATE $temp TABLE $name (id integer primary key, fulltextweight integer)"); + $searchSQL = "INSERT INTO $name $searchSQL"; + } else { + $searchSQL = "CREATE $temp TABLE $name AS $searchSQL"; + } if ( main::DEBUGLOG ) { my $log2 = $sqllog->is_debug ? $sqllog : $log; From c95e89ec48d239699ccdf666445f12f702a7d97d Mon Sep 17 00:00:00 2001 From: darrell-k Date: Tue, 10 Sep 2024 23:24:25 +0100 Subject: [PATCH 2/3] revert to joins for livesearch library views --- Slim/Control/Queries.pm | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Slim/Control/Queries.pm b/Slim/Control/Queries.pm index 06556ab515d..ce7de93ea3b 100644 --- a/Slim/Control/Queries.pm +++ b/Slim/Control/Queries.pm @@ -3202,25 +3202,28 @@ sub searchQuery { } if ( $libraryID ) { - my $exists; if ( $type eq 'contributor') { - $exists = 'EXISTS (SELECT * FROM contributor_track, library_track WHERE library_track.track = contributor_track.track AND contributor_track.contributor = me.id'; + $sql .= 'JOIN contributor_track ON contributor_track.contributor = me.id '; + $sql .= 'JOIN library_track ON library_track.track = contributor_track.track '; } elsif ( $type eq 'work') { - $exists = 'EXISTS (SELECT * FROM tracks, library_track WHERE library_track.track = tracks.id AND tracks.work = me.id'; + $sql .= 'JOIN tracks ON tracks.work = me.id '; + $sql .= 'JOIN library_track ON library_track.track = tracks.id '; } elsif ( $type eq 'album' ) { - $exists = 'EXISTS (SELECT * FROM tracks, library_track WHERE library_track.track = tracks.id AND tracks.album = me.id'; + $sql .= 'JOIN tracks ON tracks.album = me.id '; + $sql .= 'JOIN library_track ON library_track.track = tracks.id '; } elsif ( $type eq 'genre' ) { - $exists = 'EXISTS (SELECT * FROM genre_track, library_track WHERE library_track.track = genre_track.track AND genre_track.genre = me.id'; + $sql .= 'JOIN genre_track ON genre_track.genre = me.id '; + $sql .= 'JOIN library_track ON library_track.track = genre_track.track '; } elsif ( $type eq 'track' ) { - $exists = 'EXISTS (SELECT * FROM library_track WHERE library_track.track = me.id'; + $sql .= 'JOIN library_track ON library_track.track = me.id '; } - push @{$w}, "$exists AND library_track.library = ?)"; + push @{$w}, 'library_track.library = ?'; push @{$p}, $libraryID; } From cff5900f5220d596fdf7b0aa770cb96cf85be95d Mon Sep 17 00:00:00 2001 From: darrell-k Date: Wed, 11 Sep 2024 19:21:13 +0100 Subject: [PATCH 3/3] further improvements --- Slim/Control/Queries.pm | 73 +++++++++++++++------------- Slim/Plugin/FullTextSearch/Plugin.pm | 10 +--- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/Slim/Control/Queries.pm b/Slim/Control/Queries.pm index ce7de93ea3b..ed78b9b43b6 100644 --- a/Slim/Control/Queries.pm +++ b/Slim/Control/Queries.pm @@ -519,7 +519,6 @@ sub albumsQuery { name => 'albumsSearch', search => $fromSearch, type => 'album', - createPrimaryKey => 1, }); $sql .= "JOIN albumsSearch ON albums.id = albumsSearch.id "; } else { @@ -3187,7 +3186,6 @@ sub searchQuery { name => 'quickSearch', search => $search, type => $type, - createPrimaryKey => 1, checkLargeResultset => sub { my $isLarge = shift; return ($isLarge && $isLarge > ($index + $quantity)) ? ('ORDER BY fulltextweight DESC LIMIT ' . $isLarge) : ''; @@ -4656,7 +4654,7 @@ sub worksQuery { # get them all by default my $where = {}; - my $w = ["tracks.work IS NOT NULL"]; + my $w = []; my $p = []; my $columns = "works.title, works.id, composer.name, composer.id, composer.namesort, works.titlesort, GROUP_CONCAT(DISTINCT albums.artwork), GROUP_CONCAT(DISTINCT albums.id)"; @@ -4670,22 +4668,24 @@ sub worksQuery { JOIN albums ON tracks.album = albums.id '; if (specified($search)) { + if ( Slim::Schema->canFulltextSearch ) { Slim::Plugin::FullTextSearch::Plugin->createHelperTable({ name => 'worksSearch', search => $search, type => 'work', - createPrimaryKey => 1, }); - $sql .= "JOIN worksSearch ON works.id = worksSearch.id "; - Slim::Plugin::FullTextSearch::Plugin->createHelperTable({ name => 'albumsSearch', search => $search, type => 'album', - createPrimaryKey => 1, }); - $sql .= "JOIN albumsSearch ON albums.id = albumsSearch.id "; + $sql = 'SELECT %s FROM workssearch + join works on works.id=workssearch.id + join tracks on tracks.work=workssearch.id + join albums on albums.id=tracks.album + join albumsSearch on albumsSearch.id=albums.id + join contributors composer on composer.id=works.composer '; } else { my $strings = Slim::Utils::Text::searchStringSplit($search); if ( ref $strings->[0] eq 'ARRAY' ) { @@ -4701,43 +4701,48 @@ sub worksQuery { push @{$p}, @{$strings}; } } - } - if ( defined $workID && $workID != -1 ) { - push @{$w}, "works.id = ?"; - push @{$p}, $workID; - } + } else { - if ( defined $year ) { - push @{$w}, "tracks.year = ?"; - push @{$p}, $year; - } + push @{$w}, "tracks.work IS NOT NULL"; - if ( defined $roleID ) { - my @roles = split(',', $roleID); - if (scalar @roles) { - push @{$p}, map { Slim::Schema::Contributor->typeToRole($_) } @roles; - push @{$w}, 'contributor_track.role IN (' . join(', ', map {'?'} @roles) . ')'; + if ( defined $workID && $workID != -1 ) { + push @{$w}, "works.id = ?"; + push @{$p}, $workID; + } + + if ( defined $year ) { + push @{$w}, "tracks.year = ?"; + push @{$p}, $year; + } + + if ( defined $roleID ) { + my @roles = split(',', $roleID); + if (scalar @roles) { + push @{$p}, map { Slim::Schema::Contributor->typeToRole($_) } @roles; + push @{$w}, 'contributor_track.role IN (' . join(', ', map {'?'} @roles) . ')'; + } + } + + if (defined $artistID) { + push @{$w}, "contributors.id = ?"; + push @{$p}, $artistID; + } + + if (defined $genreID) { + my @genreIDs = split(/,/, $genreID); + $sql .= 'JOIN genre_track ON genre_track.track = tracks.id '; + push @{$w}, 'genre_track.genre IN (' . join(', ', map {'?'} @genreIDs) . ')'; + push @{$p}, @genreIDs; } - } - if (defined $artistID) { - push @{$w}, "contributors.id = ?"; - push @{$p}, $artistID; } if (defined $libraryID) { - push @{$w}, 'EXISTS (SELECT * FROM library_album WHERE library_album.album = albums.id AND library_album.library = ?)'; + push @{$w}, 'EXISTS (SELECT 1 FROM library_album WHERE library_album.album = albums.id AND library_album.library = ?)'; push @{$p}, $libraryID; } - if (defined $genreID) { - my @genreIDs = split(/,/, $genreID); - $sql .= 'JOIN genre_track ON genre_track.track = tracks.id '; - push @{$w}, 'genre_track.genre IN (' . join(', ', map {'?'} @genreIDs) . ')'; - push @{$p}, @genreIDs; - } - if ( @{$w} ) { $sql .= 'WHERE '; $sql .= join( ' AND ', @{$w} ); diff --git a/Slim/Plugin/FullTextSearch/Plugin.pm b/Slim/Plugin/FullTextSearch/Plugin.pm index f1a218637ed..420c6501b51 100644 --- a/Slim/Plugin/FullTextSearch/Plugin.pm +++ b/Slim/Plugin/FullTextSearch/Plugin.pm @@ -266,7 +266,6 @@ sub createHelperTable { my $name = $args->{name}; my $type = $args->{type}; - my $createPrimaryKey = $args->{createPrimaryKey} || 0; my ($tokens, $isLarge); my $orderOrLimit = ''; @@ -288,19 +287,14 @@ sub createHelperTable { $orderOrLimit = 'LIMIT 0' if !$tokens; # The first 32 bytes of the ID are either an MD5 of the ID, or some buster to make it "non searchable" - remove that prefix - my $searchSQL = "SELECT SUBSTR(fulltext.id, 33) AS id, FULLTEXTWEIGHT(matchinfo(fulltext)) AS fulltextweight FROM fulltext WHERE fulltext MATCH 'type:$type $tokens' $orderOrLimit"; - if ($createPrimaryKey) { - $dbh->do("CREATE $temp TABLE $name (id integer primary key, fulltextweight integer)"); - $searchSQL = "INSERT INTO $name $searchSQL"; - } else { - $searchSQL = "CREATE $temp TABLE $name AS $searchSQL"; - } + my $searchSQL = "INSERT INTO $name SELECT SUBSTR(fulltext.id, 33) AS id, FULLTEXTWEIGHT(matchinfo(fulltext)) AS fulltextweight FROM fulltext WHERE fulltext MATCH 'type:$type $tokens' $orderOrLimit"; if ( main::DEBUGLOG ) { my $log2 = $sqllog->is_debug ? $sqllog : $log; $log2->is_debug && $log2->debug( "Fulltext search query ($type): $searchSQL" ); } + $dbh->do("CREATE $temp TABLE $name (id INTEGER PRIMARY KEY, fulltextweight INTEGER)"); $dbh->do($searchSQL); }