diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index ea099589c7..55526b7818 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1474,6 +1474,21 @@ class DB(object): with f: return True, f.read(), stat.st_mtime + def compress_covers(self, path_map, jpeg_quality, progress_callback): + cpath_map = {} + if not progress_callback: + progress_callback = lambda book_id, old_sz, new_sz: None + for book_id, path in path_map.items(): + path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg')) + try: + sz = os.path.getsize(path) + except OSError: + progress_callback(book_id, 0, 'ENOENT') + else: + cpath_map[book_id] = (path, sz) + from calibre.db.covers import compress_covers + compress_covers(cpath_map, jpeg_quality, progress_callback) + def set_cover(self, book_id, path, data, no_processing=False): path = os.path.abspath(os.path.join(self.library_path, path)) if not os.path.exists(path): diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ccee2deca8..e8897855a5 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -763,6 +763,25 @@ class Cache(object): return self.backend.copy_cover_to(path, dest, use_hardlink=use_hardlink, report_file_size=report_file_size) + @write_api + def compress_covers(self, book_ids, jpeg_quality=100, progress_callback=None): + ''' + Compress the cover images for the specified books. A compression quality of 100 + will perform lossless compression, otherwise lossy compression. + + The progress callback will be called with the book_id and the old and new sizes + for each book that has been processed. If an error occurs, the news size will + be a string with th error details. + ''' + jpeg_quality = max(10, min(jpeg_quality, 100)) + path_map = {} + for book_id in book_ids: + try: + path_map[book_id] = self._field_for('path', book_id).replace('/', os.sep) + except AttributeError: + continue + self.backend.compress_covers(path_map, jpeg_quality, progress_callback) + @read_api def copy_format_to(self, book_id, fmt, dest, use_hardlink=False, report_file_size=None): ''' diff --git a/src/calibre/db/covers.py b/src/calibre/db/covers.py new file mode 100644 index 0000000000..6101e690f1 --- /dev/null +++ b/src/calibre/db/covers.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2021, Kovid Goyal + +import os +from queue import Queue +from threading import Thread + +from calibre import detect_ncpus +from calibre.utils.img import encode_jpeg, optimize_jpeg + + +def worker(input_queue, output_queue, jpeg_quality): + while True: + task = input_queue.get() + if task is None: + break + book_id, path = task + try: + if jpeg_quality >= 100: + stderr = optimize_jpeg(path) + else: + stderr = encode_jpeg(path, jpeg_quality) + except Exception: + import traceback + stderr = traceback.format_exc() + if stderr: + output_queue.put((book_id, stderr)) + else: + try: + sz = os.path.getsize(path) + except OSError as err: + sz = str(err) + output_queue.put((book_id, sz)) + + +def compress_covers(path_map, jpeg_quality, progress_callback): + input_queue = Queue() + output_queue = Queue() + num_workers = detect_ncpus() + sz_map = {} + for book_id, (path, sz) in path_map.items(): + input_queue.put((book_id, path)) + sz_map[book_id] = sz + workers = [ + Thread(target=worker, args=(input_queue, output_queue, jpeg_quality), daemon=True, name=f'CCover-{i}') + for i in range(num_workers) + ] + [w.start() for w in workers] + pending = set(path_map) + while pending: + book_id, new_sz = output_queue.get() + pending.remove(book_id) + progress_callback(book_id, sz_map[book_id], new_sz) + for w in workers: + input_queue.put(None) + for w in workers: + w.join() diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 0a27bb1351..d127ab989e 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -3,34 +3,36 @@ # License: GPLv3 Copyright: 2008, Kovid Goyal -import regex, numbers +import numbers +import regex from collections import defaultdict, namedtuple from io import BytesIO +from PyQt5.Qt import ( + QApplication, QComboBox, QCompleter, QCoreApplication, QDateTime, QDialog, + QDialogButtonBox, QFont, QGridLayout, QInputDialog, QLabel, QLineEdit, + QProgressBar, QSize, Qt, QVBoxLayout, pyqtSignal +) from threading import Thread -from PyQt5.Qt import ( - QCompleter, QCoreApplication, QDateTime, QDialog, QDialogButtonBox, QFont, QProgressBar, QComboBox, - QGridLayout, QInputDialog, QLabel, QLineEdit, QSize, Qt, QVBoxLayout, pyqtSignal, QApplication -) - -from calibre import prints +from calibre import human_readable, prints from calibre.constants import DEBUG from calibre.db import _get_next_series_num_for_list from calibre.ebooks.metadata import authors_to_string, string_to_authors, title_sort from calibre.ebooks.metadata.book.formatter import SafeFormat from calibre.ebooks.metadata.opf2 import OPF from calibre.gui2 import ( - UNDEFINED_QDATETIME, FunctionDispatcher, error_dialog, gprefs, question_dialog + UNDEFINED_QDATETIME, FunctionDispatcher, error_dialog, gprefs, info_dialog, + question_dialog ) from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.gui2.dialogs.template_line_editor import TemplateLineEditor +from calibre.gui2.widgets import LineEditECM from calibre.utils.config import JSONConfig, dynamic, prefs, tweaks -from calibre.utils.date import qt_to_dt, internal_iso_format_string +from calibre.utils.date import internal_iso_format_string, qt_to_dt from calibre.utils.icu import capitalize, sort_key from calibre.utils.titlecase import titlecase -from calibre.gui2.widgets import LineEditECM from polyglot.builtins import ( error_message, filter, iteritems, itervalues, native_string_type, unicode_type ) @@ -39,7 +41,7 @@ Settings = namedtuple('Settings', 'remove_all remove add au aus do_aus rating pub do_series do_autonumber ' 'do_swap_ta do_remove_conv do_auto_author series do_series_restart series_start_value series_increment ' 'do_title_case cover_action clear_series clear_pub pubdate adddate do_title_sort languages clear_languages ' - 'restore_original comments generate_cover_settings read_file_metadata casing_algorithm') + 'restore_original comments generate_cover_settings read_file_metadata casing_algorithm do_compress_cover compress_cover_quality') null = object() @@ -68,6 +70,7 @@ class MyBlockingBusy(QDialog): # {{{ self._layout = l = QVBoxLayout() self.setLayout(l) + self.cover_sizes = {'old': 0, 'new': 0} # Every Path that will be taken in do_all options = [ args.cover_action == 'fromfmt' or args.read_file_metadata, @@ -82,7 +85,7 @@ class MyBlockingBusy(QDialog): # {{{ is not None, args.do_series, bool(args.series) and args.do_autonumber, args.comments is not null, args.do_remove_conv, args.clear_languages, args.remove_all, - bool(do_sr) + bool(do_sr), args.do_compress_cover ] self.selected_options = sum(options) if DEBUG: @@ -312,7 +315,9 @@ class MyBlockingBusy(QDialog): # {{{ elif args.cover_action == 'trim': self.progress_next_step_range.emit(len(self.ids)) - from calibre.utils.img import remove_borders_from_image, image_to_data, image_from_data + from calibre.utils.img import ( + image_from_data, image_to_data, remove_borders_from_image + ) for book_id in self.ids: cdata = cache.cover(book_id) if cdata: @@ -437,6 +442,18 @@ class MyBlockingBusy(QDialog): # {{{ self.db.bulk_modify_tags(self.ids, add=args.add, remove=args.remove) self.progress_finished_cur_step.emit() + if args.do_compress_cover: + self.progress_next_step_range.emit(len(self.ids)) + + def pc(book_id, old_sz, new_sz): + if isinstance(new_sz, int): + self.cover_sizes['old'] += old_sz + self.cover_sizes['new'] += new_sz + self.progress_update.emit(1) + + self.db.new_api.compress_covers(self.ids, args.compress_cover_quality, pc) + self.progress_finished_cur_step.emit() + if self.do_sr: self.progress_next_step_range.emit(len(self.ids)) for book_id in self.ids: @@ -1182,6 +1199,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): do_auto_author = self.auto_author_sort.isChecked() do_title_case = self.change_title_to_title_case.isChecked() do_title_sort = self.update_title_sort.isChecked() + do_compress_cover = self.compress_cover_images.isChecked() + compress_cover_quality = self.compress_quality.value() read_file_metadata = self.read_file_metadata.isChecked() clear_languages = self.clear_languages.isChecked() restore_original = self.restore_original.isChecked() @@ -1211,7 +1230,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): do_title_case, cover_action, clear_series, clear_pub, pubdate, adddate, do_title_sort, languages, clear_languages, restore_original, self.comments, self.generate_cover_settings, - read_file_metadata, self.casing_map[self.casing_algorithm.currentIndex()]) + read_file_metadata, self.casing_map[self.casing_algorithm.currentIndex()], do_compress_cover, compress_cover_quality) if DEBUG: print('Running bulk metadata operation with settings:') print(args) @@ -1239,6 +1258,14 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): dynamic['s_r_search_mode'] = self.search_mode.currentIndex() gprefs.set('bulk-mde-casing-algorithm', args.casing_algorithm) self.db.clean() + if args.do_compress_cover: + total_old, total_new = bb.cover_sizes['old'], bb.cover_sizes['new'] + percent = (total_old - total_new) / total_old + info_dialog(self, _('Covers compressed'), _( + 'Covers were compressed by {percent:.1%} from a total size of' + ' {old} to {new}.').format( + percent=percent, old=human_readable(total_old), new=human_readable(total_new)) + ).exec_() return QDialog.accept(self) def series_changed(self, *args): diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index b093fde110..0e7c34a3ad 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -6,7 +6,7 @@ 0 0 - 957 + 955 740 @@ -33,65 +33,532 @@ &Basic metadata - + 0 0 - 935 + 933 660 - - - - - true - - - - - - - A&pply date - - - - - - - - - This will cause the author sort field to be automatically updated - based on the authors field for each selected book. Note that if - you use the control above to set authors in bulk, the author sort - field is updated anyway, regardless of the value of this checkbox. - + + + + + - A&utomatically set author sort + &Author(s): + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + authors - - + + + + true + + + + + + + + + This will cause the author sort field to be automatically updated + based on the authors field for each selected book. Note that if + you use the control above to set authors in bulk, the author sort + field is updated anyway, regardless of the value of this checkbox. + + + A&utomatically set author sort + + + + + + + S&wap title and author + + + + + + + - S&wap title and author + Author s&ort: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + author_sort + + + + + + + Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles. + + + + + + + &Rating: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + rating + + + + + + + + + + &Apply rating + + + + + + + &Publisher: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + publisher + + + + + + + true + + + + + + + If checked, the publisher will be cleared + + + &Clear pub + + + + + + + Add ta&gs: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + tags + + + + + + + Tags categorize the book. This is particularly useful while searching. <br><br>They can be any words or phrases, separated by commas. + + + + + + + Open Tag editor + + + Open Tag editor + + + + :/images/chapters.png:/images/chapters.png + + + + + + + &Remove tags: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + remove_tags + + + + + + + Comma separated list of tags to remove from the books. + + + + + + + Check this box to remove all tags from the books. + + + Remove &all + + + + + + + Ser&ies: + + + Qt::PlainText + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + series + + + + + + + + 0 + 0 + + + + List of known series. You can add new series. + + + List of known series. You can add new series. + + + true + + + QComboBox::InsertAlphabetically + + + QComboBox::AdjustToMinimumContentsLengthWithIcon + + + 40 + + + + + + + If checked, the series will be cleared + + + &Clear series + + + + + + + + + false + + + If not checked, the series number for the books will be set to 1. +If checked, selected books will be automatically numbered, in the order +you selected them. So if you selected Book A and then Book B, +Book A will have series number 1 and Book B series number 2. + + + &Automatically number books in this series + + + + + + + false + + + Series will normally be renumbered from the highest number in the database +for that series. Checking this box will tell calibre to start numbering +from the value in the box + + + &Force numbers to start with: + + + + + + + false + + + 0.000000000000000 + + + 99000000.000000000000000 + + + 1.000000000000000 + + + + + + + false + + + The amount by which to increment the series number for successive books. Only applicable when using force series numbers. + + + + + + + 0.000000000000000 + + + 99999.000000000000000 + + + 1.000000000000000 + + + + + + + + + &Date: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + adddate + + + + + + + + + d MMM yyyy + + + true + + + + + + + ... + + + + :/images/trash.png:/images/trash.png + + + + + + + + + A&pply date + + + + + + + P&ublished: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + pubdate + + + + + + + + + MMM yyyy + + + true + + + + + + + Clear published date + + + ... + + + + :/images/trash.png:/images/trash.png + + + + + + + + + A&pply date + + + + + + + &Languages: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + languages + + + + + + + + + + Remove &all - - + + + + Qt::Vertical + + + QSizePolicy::Fixed + + + + 17 + 13 + + + + + + + + + + Remove stored conversion settings for the selected books. + +Future conversion of these books will use the default settings. + + + Remove &stored conversion settings for the selected books + + + + + + + When doing a same format to same format conversion, +for e.g., EPUB to EPUB, calibre saves the original EPUB + as ORIGINAL_EPUB. This option tells calibre to restore + the EPUB from ORIGINAL_EPUB. Useful if you did a bulk + conversion of a large number of books and something went wrong. + + + Restore pre conversion &originals, if available + + + + + + + Force the title to be in title case. If both this and swap authors are checked, +title and author are swapped before the title case is set + + + Change title &case to: + + + + + + + + 150 + 0 + + + + + + + + Update title sort based on the current title. This will be applied only after other changes to title. + + + Update &title sort + + + + + + + - Update title sort based on the current title. This will be applied only after other changes to title. + Set the metadata in calibre from the metadata in the e-book files associated with each book. Note that this does not change the cover, for that, use the separate option below. - Update &title sort + Set &metadata (except cover) from the e-book files - + Change &cover @@ -144,26 +611,43 @@ as that of the first selected book. - + - + + + + 0 + 0 + + - Force the title to be in title case. If both this and swap authors are checked, -title and author are swapped before the title case is set + <p>Compress cover images, if the quality is set to 100 compression is lossless, otherwise it is lossy - Change title &case to: + Co&mpress cover images: - - - - 150 - 0 - + + + + 0 + 0 + + + + <p>Cover image compression quality. If the quality is set to 100 compression is lossless, otherwise it is lossy + + + 10 + + + 100 + + + 100 @@ -182,126 +666,7 @@ title and author are swapped before the title case is set - - - - &Date: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - adddate - - - - - - - If checked, the publisher will be cleared - - - &Clear pub - - - - - - - P&ublished: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - pubdate - - - - - - - - - false - - - If not checked, the series number for the books will be set to 1. -If checked, selected books will be automatically numbered, in the order -you selected them. So if you selected Book A and then Book B, -Book A will have series number 1 and Book B series number 2. - - - &Automatically number books in this series - - - - - - - false - - - Series will normally be renumbered from the highest number in the database -for that series. Checking this box will tell calibre to start numbering -from the value in the box - - - &Force numbers to start with: - - - - - - - false - - - 0.000000000000000 - - - 99000000.000000000000000 - - - 1.000000000000000 - - - - - - - false - - - The amount by which to increment the series number for successive books. Only applicable when using force series numbers. - - - + - - - 0.000000000000000 - - - 99999.000000000000000 - - - 1.000000000000000 - - - - - - - - - If checked, the series will be cleared - - - &Clear series - - - - + @@ -341,342 +706,19 @@ from the value in the box - - - - &Remove tags: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - remove_tags - - - - - - - Remove &all - - - - - - - Add ta&gs: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - tags - - - - - - - Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles. - - - - - - - &Apply rating - - - - - - - - - MMM yyyy - - - true - - - - - - - Clear published date - - - ... - - - - :/images/trash.png:/images/trash.png - - - - - - - - - - - - &Author(s): - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - authors - - - - - - - - - d MMM yyyy - - - true - - - - - - - ... - - - - :/images/trash.png:/images/trash.png - - - - - - - - - &Languages: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - languages - - - - - - - &Publisher: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - publisher - - - - + Qt::Vertical - 20 - 40 + 17 + 17 - - - - Comma separated list of tags to remove from the books. - - - - - - - Open Tag editor - - - Open Tag editor - - - - :/images/chapters.png:/images/chapters.png - - - - - - - - - - Qt::Vertical - - - QSizePolicy::Fixed - - - - 20 - 15 - - - - - - - - Tags categorize the book. This is particularly useful while searching. <br><br>They can be any words or phrases, separated by commas. - - - - - - - Author s&ort: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - author_sort - - - - - - - - 0 - 0 - - - - List of known series. You can add new series. - - - List of known series. You can add new series. - - - true - - - QComboBox::InsertAlphabetically - - - QComboBox::AdjustToMinimumContentsLengthWithIcon - - - 40 - - - - - - - A&pply date - - - - - - - Set the metadata in calibre from the metadata in the e-book files associated with each book. Note that this does not change the cover, for that, use the separate option below. - - - Set &metadata (except cover) from the e-book files - - - - - - - Ser&ies: - - - Qt::PlainText - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - series - - - - - - - - - Remove stored conversion settings for the selected books. - -Future conversion of these books will use the default settings. - - - Remove &stored conversion settings for the selected books - - - - - - - When doing a same format to same format conversion, -for e.g., EPUB to EPUB, calibre saves the original EPUB - as ORIGINAL_EPUB. This option tells calibre to restore - the EPUB from ORIGINAL_EPUB. Useful if you did a bulk - conversion of a large number of books and something went wrong. - - - Restore pre conversion &originals, if available - - - - - - - - - Check this box to remove all tags from the books. - - - Remove &all - - - - - - - true - - - - - - - &Rating: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - rating - - - @@ -716,7 +758,7 @@ for e.g., EPUB to EPUB, calibre saves the original EPUB 0 0 - 804 + 777 388 @@ -1195,7 +1237,7 @@ not multiple and the destination field is multiple 0 0 - 203 + 745 70 @@ -1355,10 +1397,8 @@ is completed. This can be slow on large libraries. clear_languages remove_conversion_settings restore_original - read_file_metadata change_title_to_title_case casing_algorithm - update_title_sort cover_generate cover_remove cover_trim