From 0fdbe5a753fad2d241057897809157a06caa2462 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 29 Jul 2013 22:07:12 +0530 Subject: [PATCH] Device drivers: Explicitly fsync() all files when writing to devices --- src/calibre/__init__.py | 4 ++++ src/calibre/devices/android/driver.py | 3 +++ src/calibre/devices/cli.py | 3 ++- src/calibre/devices/cybook/driver.py | 2 ++ src/calibre/devices/hanvon/driver.py | 2 ++ src/calibre/devices/kindle/apnx.py | 3 ++- src/calibre/devices/kindle/driver.py | 3 ++- src/calibre/devices/kobo/driver.py | 12 +++++++----- src/calibre/devices/misc.py | 7 +++++-- src/calibre/devices/nook/driver.py | 2 ++ src/calibre/devices/prs505/driver.py | 2 ++ src/calibre/devices/prs505/sony_cache.py | 5 ++++- src/calibre/devices/prst1/driver.py | 2 ++ src/calibre/devices/udisks.py | 4 ++-- src/calibre/devices/usbms/cli.py | 6 ++++-- src/calibre/devices/usbms/driver.py | 5 ++++- 16 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 5d938ecc55..eaf5102aa1 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -710,3 +710,7 @@ def ipython(user_ns=None): from calibre.utils.ipython import ipython ipython(user_ns=user_ns) +def fsync(fileobj): + fileobj.flush() + os.fsync(fileobj.fileno()) + diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 055fd32f88..28f55751e2 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -8,6 +8,7 @@ import os import cStringIO +from calibre import fsync from calibre.devices.usbms.driver import USBMS HTC_BCDS = [0x100, 0x0222, 0x0224, 0x0226, 0x227, 0x228, 0x229, 0x0231, 0x9999] @@ -400,6 +401,7 @@ class WEBOS(USBMS): with open(os.path.join(path, 'coverCache', filename + '-medium.jpg'), 'wb') as coverfile: coverfile.write(coverdata) + fsync(coverfile) coverdata = getattr(metadata, 'thumbnail', None) if coverdata and coverdata[2]: @@ -423,6 +425,7 @@ class WEBOS(USBMS): with open(os.path.join(path, 'coverCache', filename + '-small.jpg'), 'wb') as coverfile: coverfile.write(coverdata) + fsync(coverfile) diff --git a/src/calibre/devices/cli.py b/src/calibre/devices/cli.py index e1a5375b6b..6defc27541 100755 --- a/src/calibre/devices/cli.py +++ b/src/calibre/devices/cli.py @@ -9,7 +9,7 @@ For usage information run the script. import StringIO, sys, time, os from optparse import OptionParser -from calibre import __version__, __appname__, human_readable +from calibre import __version__, __appname__, human_readable, fsync from calibre.devices.errors import PathError from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked from calibre.customize.ui import device_plugins @@ -293,6 +293,7 @@ def main(): parser.print_help() return 1 dev.get_file(path, outfile) + fsync(outfile) outfile.close() elif args[1].startswith("dev:"): try: diff --git a/src/calibre/devices/cybook/driver.py b/src/calibre/devices/cybook/driver.py index 0dd88263f0..64bd37b196 100644 --- a/src/calibre/devices/cybook/driver.py +++ b/src/calibre/devices/cybook/driver.py @@ -11,6 +11,7 @@ Device driver for Bookeen's Cybook Gen 3 and Opus and Orizon import os import re +from calibre import fsync from calibre.constants import isunix from calibre.devices.usbms.driver import USBMS import calibre.devices.cybook.t2b as t2b @@ -50,6 +51,7 @@ class CYBOOK(USBMS): coverdata = None with open('%s_6090.t2b' % os.path.join(path, filename), 'wb') as t2bfile: t2b.write_t2b(t2bfile, coverdata) + fsync(t2bfile) @classmethod def can_handle(cls, device_info, debug=False): diff --git a/src/calibre/devices/hanvon/driver.py b/src/calibre/devices/hanvon/driver.py index c967f2c54c..ea13e5a6d0 100644 --- a/src/calibre/devices/hanvon/driver.py +++ b/src/calibre/devices/hanvon/driver.py @@ -9,6 +9,7 @@ Device driver for Hanvon devices ''' import re, os +from calibre import fsync from calibre.devices.usbms.driver import USBMS def is_alex(device_info): @@ -123,6 +124,7 @@ class ALEX(N516): os.makedirs(cdir) with open(cpath, 'wb') as coverfile: coverfile.write(cover) + fsync(coverfile) def delete_books(self, paths, end_session=True): for i, path in enumerate(paths): diff --git a/src/calibre/devices/kindle/apnx.py b/src/calibre/devices/kindle/apnx.py index d6fec9ffd1..faad3c6cd7 100644 --- a/src/calibre/devices/kindle/apnx.py +++ b/src/calibre/devices/kindle/apnx.py @@ -14,7 +14,7 @@ from calibre.ebooks.mobi.reader.mobi6 import MobiReader from calibre.ebooks.pdb.header import PdbHeaderReader from calibre.ebooks.mobi.reader.headers import MetadataHeader from calibre.utils.logging import default_log -from calibre import prints +from calibre import prints, fsync from calibre.constants import DEBUG class APNXBuilder(object): @@ -80,6 +80,7 @@ class APNXBuilder(object): # Write the APNX. with open(apnx_path, 'wb') as apnxf: apnxf.write(apnx) + fsync(apnxf) def generate_apnx(self, pages, apnx_meta): apnx = '' diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 461176b95c..4e083fbe04 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -12,7 +12,7 @@ import datetime, os, re, sys, json, hashlib from calibre.devices.kindle.bookmark import Bookmark from calibre.devices.usbms.driver import USBMS -from calibre import strftime +from calibre import strftime, fsync ''' Notes on collections: @@ -410,6 +410,7 @@ class KINDLE2(KINDLE): uuid=mh.exth.uuid, cdetype=mh.exth.cdetype)) with open(thumbfile, 'wb') as f: f.write(coverdata[2]) + fsync(f) def upload_apnx(self, path, filename, metadata, filepath): from calibre.devices.kindle.apnx import APNXBuilder diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 278798160e..feedab0127 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -23,7 +23,7 @@ from calibre.devices.kobo.books import Book from calibre.devices.kobo.books import ImageWrapper from calibre.devices.mime import mime_type_ext from calibre.devices.usbms.driver import USBMS, debug_print -from calibre import prints +from calibre import prints, fsync from calibre.ptempfile import PersistentTemporaryFile from calibre.constants import DEBUG from calibre.utils.config_base import prefs @@ -974,6 +974,7 @@ class KOBO(USBMS): with open(fpath, 'wb') as f: f.write(data) + fsync(f) else: debug_print("ImageID could not be retreived from the database") @@ -1621,7 +1622,7 @@ class KOBOTOUCH(KOBO): debug_print("KoboTouch:books - shelf list:", self.bookshelvelist) opts = self.settings() - + columns = 'Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ImageID, ReadStatus' if self.dbversion >= 16: columns += ', ___ExpirationStatus, FavouritesIndex, Accessibility' @@ -1635,7 +1636,7 @@ class KOBOTOUCH(KOBO): columns += ", Series, SeriesNumber, ___UserID, ExternalId" else: columns += ', null as Series, null as SeriesNumber, ___UserID, null as ExternalId' - + where_clause = '' if self.supports_kobo_archive(): where_clause = (" where BookID is Null " \ @@ -1670,13 +1671,13 @@ class KOBOTOUCH(KOBO): else: where_clause = ' where BookID is Null' - # Note: The card condition should not need the contentId test for the SD card. But the ExternalId does not get set for sideloaded kepubs on the SD card. + # Note: The card condition should not need the contentId test for the SD card. But the ExternalId does not get set for sideloaded kepubs on the SD card. card_condition = '' if self.has_externalid(): card_condition = " AND (externalId IS NOT NULL AND externalId <> '' OR contentId LIKE 'file:///mnt/sd/%')" if oncard == 'carda' else " AND (externalId IS NULL OR externalId = '') AND contentId NOT LIKE 'file:///mnt/sd/%'" else: card_condition = " AND contentId LIKE 'file:///mnt/sd/%'" if oncard == 'carda' else " AND contentId NOT LIKE'file:///mnt/sd/%'" - + query = 'SELECT ' + columns + ' FROM content ' + where_clause + card_condition debug_print("KoboTouch:books - query=", query) @@ -2283,6 +2284,7 @@ class KOBOTOUCH(KOBO): with open(fpath, 'wb') as f: f.write(data) + fsync(f) except Exception as e: err = str(e) debug_print("KoboTouch:_upload_cover - Exception string: %s"%err) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 13ae870fd1..3338a98eac 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -9,8 +9,7 @@ __docformat__ = 'restructuredtext en' import os from calibre.devices.usbms.driver import USBMS -from calibre import prints -prints +from calibre import fsync class PALMPRE(USBMS): @@ -100,6 +99,7 @@ class PDNOVEL(USBMS): if coverdata and coverdata[2]: with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile: coverfile.write(coverdata[2]) + fsync(coverfile) class PDNOVEL_KOBO(PDNOVEL): name = 'Pandigital Kobo device interface' @@ -118,6 +118,7 @@ class PDNOVEL_KOBO(PDNOVEL): os.makedirs(dirpath) with open(os.path.join(dirpath, filename+'.jpg'), 'wb') as coverfile: coverfile.write(coverdata[2]) + fsync(coverfile) class VELOCITYMICRO(USBMS): @@ -190,6 +191,7 @@ class LUMIREAD(USBMS): os.makedirs(pdir) with open(cfilepath+'.jpg', 'wb') as f: f.write(metadata.thumbnail[-1]) + fsync(f) class ALURATEK_COLOR(USBMS): @@ -334,6 +336,7 @@ class NEXTBOOK(USBMS): os.makedirs(thumbnail_dir) with open(os.path.join(thumbnail_dir, filename+'.jpg'), 'wb') as f: f.write(metadata.thumbnail[-1]) + fsync(f) ''' class MOOVYBOOK(USBMS): diff --git a/src/calibre/devices/nook/driver.py b/src/calibre/devices/nook/driver.py index 09924a8204..e24950359c 100644 --- a/src/calibre/devices/nook/driver.py +++ b/src/calibre/devices/nook/driver.py @@ -12,6 +12,7 @@ import os import cStringIO +from calibre import fsync from calibre.constants import isosx from calibre.devices.usbms.driver import USBMS @@ -76,6 +77,7 @@ class NOOK(USBMS): with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile: coverfile.write(coverdata) + fsync(coverfile) def sanitize_path_components(self, components): return [x.replace('#', '_') for x in components] diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index a1c7c78d18..10cb7df22c 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -8,6 +8,7 @@ Device driver for the SONY devices import os, time, re +from calibre import fsync from calibre.devices.usbms.driver import USBMS, debug_print from calibre.devices.prs505 import MEDIA_XML, MEDIA_EXT, CACHE_XML, CACHE_EXT, \ MEDIA_THUMBNAIL, CACHE_THUMBNAIL @@ -142,6 +143,7 @@ class PRS505(USBMS): '''.encode('utf8')) + fsync(f) return True except: import traceback diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index 979940229a..86a1b50582 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -9,7 +9,7 @@ import os, time from base64 import b64decode from datetime import date -from calibre import prints, guess_type, isbytestring +from calibre import prints, guess_type, isbytestring, fsync from calibre.devices.errors import DeviceError from calibre.devices.usbms.driver import debug_print from calibre.constants import DEBUG, preferred_encoding @@ -122,6 +122,7 @@ class XMLCache(object): try: with open(path, 'wb') as f: f.write(EMPTY_EXT_CACHE) + fsync(f) except: pass if os.access(path, os.W_OK): @@ -726,6 +727,7 @@ class XMLCache(object): '') with open(path, 'wb') as f: f.write(raw) + fsync(f) for i, path in self.ext_paths.items(): try: @@ -737,6 +739,7 @@ class XMLCache(object): '') with open(path, 'wb') as f: f.write(raw) + fsync(f) # }}} diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 9c76eb096f..d2482d51a0 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -15,6 +15,7 @@ import os, time, re from contextlib import closing from datetime import date +from calibre import fsync from calibre.devices.errors import DeviceError from calibre.devices.usbms.driver import USBMS, debug_print from calibre.devices.usbms.device import USBDevice @@ -761,6 +762,7 @@ class PRST1(USBMS): with open(thumbnail_file_path, 'wb') as f: f.write(book.thumbnail[-1]) + fsync(f) query = 'UPDATE books SET thumbnail = ? WHERE _id = ?' t = (thumbnail_path, book.bookId,) diff --git a/src/calibre/devices/udisks.py b/src/calibre/devices/udisks.py index b9ab4be498..8156fb227b 100644 --- a/src/calibre/devices/udisks.py +++ b/src/calibre/devices/udisks.py @@ -46,7 +46,7 @@ class UDisks(object): try: return unicode(d.FilesystemMount('', ['auth_no_user_interaction', 'rw', 'noexec', 'nosuid', - 'sync', 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()])) + 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()])) except: # May be already mounted, check mp = node_mountpoint(str(device_node_path)) @@ -123,7 +123,7 @@ class UDisks2(object): def mount(self, device_node_path): d = self.device(device_node_path) mount_options = ['rw', 'noexec', 'nosuid', - 'sync', 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()] + 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()] try: return unicode(d.Mount( { diff --git a/src/calibre/devices/usbms/cli.py b/src/calibre/devices/usbms/cli.py index 4ff9efef8b..6482a5b12b 100644 --- a/src/calibre/devices/usbms/cli.py +++ b/src/calibre/devices/usbms/cli.py @@ -6,6 +6,7 @@ __docformat__ = 'restructuredtext en' import os, shutil, time +from calibre import fsync from calibre.devices.errors import PathError from calibre.utils.filenames import case_preserving_open_file @@ -58,14 +59,15 @@ class CLI(object): dest.seek(0) dest.truncate() shutil.copyfileobj(infile, dest) + fsync(dest) #if not check_transfer(infile, dest): raise Exception('Transfer failed') if close: infile.close() return actual_path def munge_path(self, path): - if path.startswith('/') and not (path.startswith(self._main_prefix) or \ - (self._card_a_prefix and path.startswith(self._card_a_prefix)) or \ + if path.startswith('/') and not (path.startswith(self._main_prefix) or + (self._card_a_prefix and path.startswith(self._card_a_prefix)) or (self._card_b_prefix and path.startswith(self._card_b_prefix))): path = self._main_prefix + path[1:] elif path.startswith('carda:'): diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index f367e549e2..9dbb718fc0 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -14,7 +14,7 @@ import os, time, json, shutil from itertools import cycle from calibre.constants import numeric_version -from calibre import prints, isbytestring +from calibre import prints, isbytestring, fsync from calibre.constants import filesystem_encoding, DEBUG from calibre.devices.usbms.cli import CLI from calibre.devices.usbms.device import Device @@ -85,10 +85,12 @@ class USBMS(CLI, Device): location_code, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) + fsync(f) else: driveinfo = self._update_driveinfo_record({}, prefix, location_code, name) with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f: f.write(json.dumps(driveinfo, default=to_json)) + fsync(f) return driveinfo def get_device_information(self, end_session=True): @@ -388,6 +390,7 @@ class USBMS(CLI, Device): os.makedirs(self.normalize_path(prefix)) with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f: json_codec.encode_to_file(f, booklists[listid]) + fsync(f) write_prefix(self._main_prefix, 0) write_prefix(self._card_a_prefix, 1) write_prefix(self._card_b_prefix, 2)