Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Quick search improvements #2772

Open
wants to merge 36 commits into
base: Develop
Choose a base branch
from
Open
Changes from 2 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
5233f78
comments
Apr 25, 2023
4b2e7b8
Changed the quick search behavior so that it allows title and author …
Apr 26, 2023
d13d465
proof of concept fuzzy matching
Apr 26, 2023
ada0cc4
fuzzy matching for all but cc
Apr 29, 2023
8e8c9a1
fuzzy matching for all categories but cc
Apr 29, 2023
4be089d
Merge remote-tracking branch 'origin/master'
Apr 29, 2023
61f1e20
fuzzy matching for all categories but cc
Apr 29, 2023
7a43e07
Merge branch 'master' of https://github.com/quarz12/calibre-web
Apr 29, 2023
f497cc0
removed todo
Apr 29, 2023
2e3c93b
removed debugging help
Apr 29, 2023
6b93829
Merge remote-tracking branch 'origin/master'
Apr 29, 2023
af40fee
I accidentally used pycharms auto-add-to-requirements feature which r…
May 3, 2023
9fc0d54
idea for weighted sorting
May 8, 2023
4ba3b4e
typing for query
quarz12 May 9, 2023
932c796
fixed an issue where the lowering and stripping of the search term wa…
May 10, 2023
97c94f2
moved sorting back to original place
May 10, 2023
45d8d63
sort using only authorsort and title
quarz12 May 11, 2023
c115fe9
use partial token set ratio instead
quarz12 May 11, 2023
086527f
test at home
quarz12 May 11, 2023
ad5313e
new idea
May 15, 2023
e45619f
progress building string of book
quarz12 May 17, 2023
a936a33
updated Books string repr
May 27, 2023
cb5e66f
add partial token set ratio to db
May 28, 2023
025a888
rolled back string repr of book, moved that part to a new method
May 28, 2023
caf6079
moved author filter to the rest of the filters, ignore words smaller …
May 28, 2023
56d4a3d
Merge branch 'master' into sorting
quarz12 May 28, 2023
c018ef7
Merge pull request #2 from quarz12/sorting
quarz12 May 28, 2023
b96d02c
now return empty list if all words of query are < 3 letters, only com…
May 30, 2023
5e0430e
message when query returns 0 results
May 31, 2023
896e8fd
minor cleanup, removed unused code
Jun 3, 2023
4b36261
removed unwanted refactoring
quarz12 Apr 13, 2024
2ac1566
Merge pull request #3 from quarz12/sorting
quarz12 Apr 13, 2024
518c56e
removed some more unwanted refactoring
quarz12 Apr 13, 2024
a295651
Merge branch 'master' of https://github.com/quarz12/calibre-web
quarz12 Apr 13, 2024
cca178f
Merge branch 'master' of https://github.com/quarz12/calibre-web
quarz12 Apr 13, 2024
b37554f
Merge branch 'master' of https://github.com/quarz12/calibre-web
quarz12 Apr 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 17 additions & 22 deletions cps/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import os
import re
import json
import traceback
from datetime import datetime
from urllib.parse import quote
import unidecode
Expand All @@ -33,7 +32,6 @@
from sqlalchemy.orm.collections import InstrumentedList
from sqlalchemy.ext.declarative import DeclarativeMeta
from sqlalchemy.exc import OperationalError

try:
# Compatibility with sqlalchemy 2.0
from sqlalchemy.orm import declarative_base
Expand All @@ -42,7 +40,6 @@
from sqlalchemy.pool import StaticPool
from sqlalchemy.sql.expression import and_, true, false, text, func, or_
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy import desc
from flask_login import current_user
from flask_babel import gettext as _
from flask_babel import get_locale
Expand All @@ -52,7 +49,7 @@
from .pagination import Pagination

from weakref import WeakSet
from thefuzz.fuzz import partial_ratio, partial_token_set_ratio, partial_token_sort_ratio, ratio
from thefuzz.fuzz import partial_token_sort_ratio, ratio

# %-level, 100 means exact match, 75 allows exactly 1 wrong character in a 4 letter word
FUZZY_SEARCH_ACCURACY = 75
Expand Down Expand Up @@ -384,8 +381,8 @@ def __init__(self, title, sort, author_sort, timestamp, pubdate, series_index, l

def __repr__(self):
return "<Books('{0},{1}{2}{3}{4}{5}{6}{7}{8}')>".format(self.title, self.sort, self.author_sort,
self.timestamp, self.pubdate, self.series_index,
self.last_modified, self.path, self.has_cover)
self.timestamp, self.pubdate, self.series_index,
self.last_modified, self.path, self.has_cover)

def __str__(self):
return "{0} {1} {2} {3} {4}".format(self.title, " ".join([tag.name for tag in self.tags]),
Expand Down Expand Up @@ -439,15 +436,13 @@ def to_json(self, value, extra, sequence):
content['category_sort'] = "value"
content['is_csp'] = False
content['is_editable'] = self.editable
content['rec_index'] = sequence + 22 # toDo why ??
content['rec_index'] = sequence + 22 # toDo why ??
if isinstance(value, datetime):
content['#value#'] = {"__class__": "datetime.datetime",
"__value__": value.strftime("%Y-%m-%dT%H:%M:%S+00:00")}
content['#value#'] = {"__class__": "datetime.datetime", "__value__": value.strftime("%Y-%m-%dT%H:%M:%S+00:00")}
else:
content['#value#'] = value
content['#extra#'] = extra
content['is_multiple2'] = {} if not self.is_multiple else {"cache_to_list": "|", "ui_to_list": ",",
"list_to_ui": ", "}
content['is_multiple2'] = {} if not self.is_multiple else {"cache_to_list": "|", "ui_to_list": ",", "list_to_ui": ", "}
return json.dumps(content, ensure_ascii=False)


Expand All @@ -468,7 +463,7 @@ def default(self, o):
el = list()
# ele = None
for ele in data:
if hasattr(ele, 'value'): # converter for custom_column values
if hasattr(ele, 'value'): # converter for custom_column values
el.append(str(ele.value))
elif ele.get:
el.append(ele.get())
Expand Down Expand Up @@ -507,6 +502,7 @@ def __init__(self, expire_on_commit=True, init=False):
if init:
self.init_db(expire_on_commit)


def init_db(self, expire_on_commit=True):
if self._init:
self.init_session(expire_on_commit)
Expand Down Expand Up @@ -678,13 +674,13 @@ def get_book_read_archived(self, book_id, read_column, allow_show_archived=False
if not read_column:
bd = (self.session.query(Books, ub.ReadBook.read_status, ub.ArchivedBook.is_archived).select_from(Books)
.join(ub.ReadBook, and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == book_id),
isouter=True))
isouter=True))
else:
try:
read_column = cc_classes[read_column]
bd = (self.session.query(Books, read_column.value, ub.ArchivedBook.is_archived).select_from(Books)
.join(read_column, read_column.book == book_id,
isouter=True))
isouter=True))
except (KeyError, AttributeError, IndexError):
log.error("Custom Column No.{} does not exist in calibre database".format(read_column))
# Skip linking read column and return None instead of read status
Expand Down Expand Up @@ -737,11 +733,11 @@ def common_filters(self, allow_show_archived=False, return_all_languages=False):
pos_cc_list = current_user.allowed_column_value.split(',')
pos_content_cc_filter = true() if pos_cc_list == [''] else \
getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
neg_cc_list = current_user.denied_column_value.split(',')
neg_content_cc_filter = false() if neg_cc_list == [''] else \
getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
except (KeyError, AttributeError, IndexError):
pos_content_cc_filter = false()
neg_content_cc_filter = true()
Expand Down Expand Up @@ -821,18 +817,18 @@ def fill_indexpage_with_archived_books(self, page, database, pagesize, db_filter
element = 0
while indx:
if indx >= 3:
query = query.outerjoin(join[element], join[element + 1]).outerjoin(join[element + 2])
query = query.outerjoin(join[element], join[element+1]).outerjoin(join[element+2])
indx -= 3
element += 3
elif indx == 2:
query = query.outerjoin(join[element], join[element + 1])
query = query.outerjoin(join[element], join[element+1])
indx -= 2
element += 2
elif indx == 1:
query = query.outerjoin(join[element])
indx -= 1
element += 1
query = query.filter(db_filter) \
query = query.filter(db_filter)\
.filter(self.common_filters(allow_show_archived))
entries = list()
pagination = list()
Expand Down Expand Up @@ -1008,8 +1004,8 @@ def speaking_language(self, languages=None, return_all_languages=False, with_cou
return sorted(tags, key=lambda x: x[0].name.lower(), reverse=reverse_order)
else:
if not languages:
languages = self.session.query(Languages) \
.join(books_languages_link) \
languages = self.session.query(Languages)\
.join(books_languages_link)\
.join(Books) \
.filter(self.common_filters(return_all_languages=return_all_languages)) \
.group_by(text('books_languages_link.lang_code')).all()
Expand Down Expand Up @@ -1104,7 +1100,6 @@ def __init__(self, name, cat_id, rating=None):
self.rating = rating
self.count = 1


'''class Count:
count = None

Expand Down