Device drivers: Explicitly fsync() all files when writing to devices

src/calibre/__init__.py

@@ -710,3 +710,7 @@ def ipython(user_ns=None):
     from calibre.utils.ipython import ipython
     ipython(user_ns=user_ns)
 
+def fsync(fileobj):
+    fileobj.flush()
+    os.fsync(fileobj.fileno())
+

src/calibre/devices/android/driver.py

@@ -8,6 +8,7 @@ import os
 
 import cStringIO
 
+from calibre import fsync
 from calibre.devices.usbms.driver import USBMS
 
 HTC_BCDS = [0x100, 0x0222, 0x0224, 0x0226, 0x227, 0x228, 0x229, 0x0231, 0x9999]
@@ -400,6 +401,7 @@ class WEBOS(USBMS):
 
         with open(os.path.join(path, 'coverCache', filename + '-medium.jpg'), 'wb') as coverfile:
             coverfile.write(coverdata)
+            fsync(coverfile)
 
         coverdata = getattr(metadata, 'thumbnail', None)
         if coverdata and coverdata[2]:
@@ -423,6 +425,7 @@ class WEBOS(USBMS):
         with open(os.path.join(path, 'coverCache', filename +
             '-small.jpg'), 'wb') as coverfile:
             coverfile.write(coverdata)
+            fsync(coverfile)
 
 
 

src/calibre/devices/cli.py

@@ -9,7 +9,7 @@ For usage information run the script.
 import StringIO, sys, time, os
 from optparse import OptionParser
 
-from calibre import __version__, __appname__, human_readable
+from calibre import __version__, __appname__, human_readable, fsync
 from calibre.devices.errors import PathError
 from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked
 from calibre.customize.ui import device_plugins
@@ -293,6 +293,7 @@ def main():
                     parser.print_help()
                     return 1
                 dev.get_file(path, outfile)
+                fsync(outfile)
                 outfile.close()
             elif args[1].startswith("dev:"):
                 try:

src/calibre/devices/cybook/driver.py

@@ -11,6 +11,7 @@ Device driver for Bookeen's Cybook Gen 3 and Opus and Orizon
 import os
 import re
 
+from calibre import fsync
 from calibre.constants import isunix
 from calibre.devices.usbms.driver import USBMS
 import calibre.devices.cybook.t2b as t2b
@@ -50,6 +51,7 @@ class CYBOOK(USBMS):
             coverdata = None
         with open('%s_6090.t2b' % os.path.join(path, filename), 'wb') as t2bfile:
             t2b.write_t2b(t2bfile, coverdata)
+            fsync(t2bfile)
 
     @classmethod
     def can_handle(cls, device_info, debug=False):

src/calibre/devices/hanvon/driver.py

@@ -9,6 +9,7 @@ Device driver for Hanvon devices
 '''
 import re, os
 
+from calibre import fsync
 from calibre.devices.usbms.driver import USBMS
 
 def is_alex(device_info):
@@ -123,6 +124,7 @@ class ALEX(N516):
             os.makedirs(cdir)
         with open(cpath, 'wb') as coverfile:
             coverfile.write(cover)
+            fsync(coverfile)
 
     def delete_books(self, paths, end_session=True):
         for i, path in enumerate(paths):

src/calibre/devices/kindle/apnx.py

@@ -14,7 +14,7 @@ from calibre.ebooks.mobi.reader.mobi6 import MobiReader
 from calibre.ebooks.pdb.header import PdbHeaderReader
 from calibre.ebooks.mobi.reader.headers import MetadataHeader
 from calibre.utils.logging import default_log
-from calibre import prints
+from calibre import prints, fsync
 from calibre.constants import DEBUG
 
 class APNXBuilder(object):
@@ -80,6 +80,7 @@ class APNXBuilder(object):
         # Write the APNX.
         with open(apnx_path, 'wb') as apnxf:
             apnxf.write(apnx)
+            fsync(apnxf)
 
     def generate_apnx(self, pages, apnx_meta):
         apnx = ''

src/calibre/devices/kindle/driver.py

@@ -12,7 +12,7 @@ import datetime, os, re, sys, json, hashlib
 
 from calibre.devices.kindle.bookmark import Bookmark
 from calibre.devices.usbms.driver import USBMS
-from calibre import strftime
+from calibre import strftime, fsync
 
 '''
 Notes on collections:
@@ -410,6 +410,7 @@ class KINDLE2(KINDLE):
                     uuid=mh.exth.uuid, cdetype=mh.exth.cdetype))
         with open(thumbfile, 'wb') as f:
             f.write(coverdata[2])
+            fsync(f)
 
     def upload_apnx(self, path, filename, metadata, filepath):
         from calibre.devices.kindle.apnx import APNXBuilder

src/calibre/devices/kobo/driver.py

@@ -23,7 +23,7 @@ from calibre.devices.kobo.books import Book
 from calibre.devices.kobo.books import ImageWrapper
 from calibre.devices.mime import mime_type_ext
 from calibre.devices.usbms.driver import USBMS, debug_print
-from calibre import prints
+from calibre import prints, fsync
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.constants import DEBUG
 from calibre.utils.config_base import prefs
@@ -974,6 +974,7 @@ class KOBO(USBMS):
 
                             with open(fpath, 'wb') as f:
                                 f.write(data)
+                                fsync(f)
 
                 else:
                     debug_print("ImageID could not be retreived from the database")
@@ -1621,7 +1622,7 @@ class KOBOTOUCH(KOBO):
             debug_print("KoboTouch:books - shelf list:", self.bookshelvelist)
 
             opts = self.settings()
-            
+
             columns = 'Title, Attribution, DateCreated, ContentID, MimeType, ContentType, ImageID, ReadStatus'
             if self.dbversion >= 16:
                 columns += ', ___ExpirationStatus, FavouritesIndex, Accessibility'
@@ -1635,7 +1636,7 @@ class KOBOTOUCH(KOBO):
                 columns += ", Series, SeriesNumber, ___UserID, ExternalId"
             else:
                 columns += ', null as Series, null as SeriesNumber, ___UserID, null as ExternalId'
-            
+
             where_clause = ''
             if self.supports_kobo_archive():
                 where_clause = (" where BookID is Null " \
@@ -1670,13 +1671,13 @@ class KOBOTOUCH(KOBO):
             else:
                 where_clause = ' where BookID is Null'
 
-            # Note: The card condition should not need the contentId test for the SD card. But the ExternalId does not get set for sideloaded kepubs on the SD card.  
+            # Note: The card condition should not need the contentId test for the SD card. But the ExternalId does not get set for sideloaded kepubs on the SD card.
             card_condition = ''
             if self.has_externalid():
                 card_condition = " AND (externalId IS NOT NULL AND externalId <> '' OR contentId LIKE 'file:///mnt/sd/%')" if oncard == 'carda' else " AND (externalId IS NULL OR externalId = '') AND contentId NOT LIKE 'file:///mnt/sd/%'"
             else:
                 card_condition = " AND contentId LIKE 'file:///mnt/sd/%'" if oncard == 'carda' else " AND contentId NOT LIKE'file:///mnt/sd/%'"
-                
+
 
             query = 'SELECT ' + columns + ' FROM content ' + where_clause + card_condition
             debug_print("KoboTouch:books - query=", query)
@@ -2283,6 +2284,7 @@ class KOBOTOUCH(KOBO):
 
                                 with open(fpath, 'wb') as f:
                                     f.write(data)
+                                    fsync(f)
                 except Exception as e:
                     err = str(e)
                     debug_print("KoboTouch:_upload_cover - Exception string: %s"%err)

src/calibre/devices/misc.py

@@ -9,8 +9,7 @@ __docformat__ = 'restructuredtext en'
 import os
 
 from calibre.devices.usbms.driver import USBMS
-from calibre import prints
-prints
+from calibre import fsync
 
 class PALMPRE(USBMS):
 
@@ -100,6 +99,7 @@ class PDNOVEL(USBMS):
         if coverdata and coverdata[2]:
             with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile:
                 coverfile.write(coverdata[2])
+                fsync(coverfile)
 
 class PDNOVEL_KOBO(PDNOVEL):
     name = 'Pandigital Kobo device interface'
@@ -118,6 +118,7 @@ class PDNOVEL_KOBO(PDNOVEL):
                 os.makedirs(dirpath)
             with open(os.path.join(dirpath, filename+'.jpg'), 'wb') as coverfile:
                 coverfile.write(coverdata[2])
+                fsync(coverfile)
 
 
 class VELOCITYMICRO(USBMS):
@@ -190,6 +191,7 @@ class LUMIREAD(USBMS):
                 os.makedirs(pdir)
             with open(cfilepath+'.jpg', 'wb') as f:
                 f.write(metadata.thumbnail[-1])
+                fsync(f)
 
 class ALURATEK_COLOR(USBMS):
 
@@ -334,6 +336,7 @@ class NEXTBOOK(USBMS):
                 os.makedirs(thumbnail_dir)
             with open(os.path.join(thumbnail_dir, filename+'.jpg'), 'wb') as f:
                 f.write(metadata.thumbnail[-1])
+                fsync(f)
     '''
 
 class MOOVYBOOK(USBMS):

src/calibre/devices/nook/driver.py

@@ -12,6 +12,7 @@ import os
 
 import cStringIO
 
+from calibre import fsync
 from calibre.constants import isosx
 from calibre.devices.usbms.driver import USBMS
 
@@ -76,6 +77,7 @@ class NOOK(USBMS):
 
         with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile:
             coverfile.write(coverdata)
+            fsync(coverfile)
 
     def sanitize_path_components(self, components):
         return [x.replace('#', '_') for x in components]

src/calibre/devices/prs505/driver.py

@@ -8,6 +8,7 @@ Device driver for the SONY devices
 
 import os, time, re
 
+from calibre import fsync
 from calibre.devices.usbms.driver import USBMS, debug_print
 from calibre.devices.prs505 import MEDIA_XML, MEDIA_EXT, CACHE_XML, CACHE_EXT, \
             MEDIA_THUMBNAIL, CACHE_THUMBNAIL
@@ -142,6 +143,7 @@ class PRS505(USBMS):
                             <cache xmlns="http://www.kinoma.com/FskCache/1">
                             </cache>
                             '''.encode('utf8'))
+                        fsync(f)
                 return True
             except:
                 import traceback

src/calibre/devices/prs505/sony_cache.py

@@ -9,7 +9,7 @@ import os, time
 from base64 import b64decode
 from datetime import date
 
-from calibre import prints, guess_type, isbytestring
+from calibre import prints, guess_type, isbytestring, fsync
 from calibre.devices.errors import DeviceError
 from calibre.devices.usbms.driver import debug_print
 from calibre.constants import DEBUG, preferred_encoding
@@ -122,6 +122,7 @@ class XMLCache(object):
                 try:
                     with open(path, 'wb') as f:
                         f.write(EMPTY_EXT_CACHE)
+                        fsync(f)
                 except:
                     pass
             if os.access(path, os.W_OK):
@@ -726,6 +727,7 @@ class XMLCache(object):
                     '<?xml version="1.0" encoding="UTF-8"?>')
             with open(path, 'wb') as f:
                 f.write(raw)
+                fsync(f)
 
         for i, path in self.ext_paths.items():
             try:
@@ -737,6 +739,7 @@ class XMLCache(object):
                     '<?xml version="1.0" encoding="UTF-8"?>')
             with open(path, 'wb') as f:
                 f.write(raw)
+                fsync(f)
 
     # }}}
 

src/calibre/devices/prst1/driver.py

@@ -15,6 +15,7 @@ import os, time, re
 from contextlib import closing
 from datetime import date
 
+from calibre import fsync
 from calibre.devices.errors import DeviceError
 from calibre.devices.usbms.driver import USBMS, debug_print
 from calibre.devices.usbms.device import USBDevice
@@ -761,6 +762,7 @@ class PRST1(USBMS):
 
         with open(thumbnail_file_path, 'wb') as f:
             f.write(book.thumbnail[-1])
+            fsync(f)
 
         query = 'UPDATE books SET thumbnail = ? WHERE _id = ?'
         t = (thumbnail_path, book.bookId,)

src/calibre/devices/udisks.py

@@ -46,7 +46,7 @@ class UDisks(object):
         try:
             return unicode(d.FilesystemMount('',
                 ['auth_no_user_interaction', 'rw', 'noexec', 'nosuid',
-                'sync', 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()]))
+                 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()]))
         except:
             # May be already mounted, check
             mp = node_mountpoint(str(device_node_path))
@@ -123,7 +123,7 @@ class UDisks2(object):
     def mount(self, device_node_path):
         d = self.device(device_node_path)
         mount_options = ['rw', 'noexec', 'nosuid',
-                'sync', 'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()]
+                'nodev', 'uid=%d'%os.geteuid(), 'gid=%d'%os.getegid()]
         try:
             return unicode(d.Mount(
                 {

src/calibre/devices/usbms/cli.py

@@ -6,6 +6,7 @@ __docformat__ = 'restructuredtext en'
 
 import os, shutil, time
 
+from calibre import fsync
 from calibre.devices.errors import PathError
 from calibre.utils.filenames import case_preserving_open_file
 
@@ -58,14 +59,15 @@ class CLI(object):
                 dest.seek(0)
                 dest.truncate()
                 shutil.copyfileobj(infile, dest)
+            fsync(dest)
             #if not check_transfer(infile, dest): raise Exception('Transfer failed')
         if close:
             infile.close()
         return actual_path
 
     def munge_path(self, path):
-        if path.startswith('/') and not (path.startswith(self._main_prefix) or \
-            (self._card_a_prefix and path.startswith(self._card_a_prefix)) or \
+        if path.startswith('/') and not (path.startswith(self._main_prefix) or
+            (self._card_a_prefix and path.startswith(self._card_a_prefix)) or
             (self._card_b_prefix and path.startswith(self._card_b_prefix))):
             path = self._main_prefix + path[1:]
         elif path.startswith('carda:'):

src/calibre/devices/usbms/driver.py

@@ -14,7 +14,7 @@ import os, time, json, shutil
 from itertools import cycle
 
 from calibre.constants import numeric_version
-from calibre import prints, isbytestring
+from calibre import prints, isbytestring, fsync
 from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
 from calibre.devices.usbms.device import Device
@@ -85,10 +85,12 @@ class USBMS(CLI, Device):
                                                           location_code, name)
             with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f:
                 f.write(json.dumps(driveinfo, default=to_json))
+                fsync(f)
         else:
             driveinfo = self._update_driveinfo_record({}, prefix, location_code, name)
             with open(os.path.join(prefix, self.DRIVEINFO), 'wb') as f:
                 f.write(json.dumps(driveinfo, default=to_json))
+                fsync(f)
         return driveinfo
 
     def get_device_information(self, end_session=True):
@@ -388,6 +390,7 @@ class USBMS(CLI, Device):
                     os.makedirs(self.normalize_path(prefix))
                 with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f:
                     json_codec.encode_to_file(f, booklists[listid])
+                    fsync(f)
         write_prefix(self._main_prefix, 0)
         write_prefix(self._card_a_prefix, 1)
         write_prefix(self._card_b_prefix, 2)