Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-08-11 09:13:57 -04:00
Code Issues Packages Projects Releases Wiki Activity

sync to trunk.

Browse Source
This commit is contained in:
John Schember 2012-09-15 21:08:40 -04:00
commit a74ee35f2b
308 changed files with 188454 additions and 112912 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
Changelog.yaml
View File

@ -19,6 +19,178 @@
# new recipes:
# - title:
- version: 0.8.69
date: 2012-09-14
new features:
- title: "E-book viewer: Add a button to the toolbar to switch themes easily"
tickets: [1047992]
- title: "When downloading metadata for many books, if some of them fail, add an option to the downloaded message to show the failed books in the main book list, so that they can be individually processed easily"
- title: "Remember last used window size of the conversion dialogs."
tickets: [1049265]
- title: "Kindle driver: Turn on sending of azw3 files to kindles by default, since the KK now has azw3 support"
- title: "Conversion: Add support for CSS pseudo classes :hover, :link, :visited, :first-line, :focus, :active, :first-letter"
- title: "Wireless device driver: Make the default save template not use folders"
bug fixes:
- title: "Fix a regression in th previous release that broke sending of books to the second SD card in SONY readers"
tickets: [1047992]
- title: "Fix a memory leak when scanning for devices in windows"
- title: "Ebook-viewer: When displaying mathematics, reflow equations that dont fit on a single line"
- title: "Catalogs: Do not mark the AZW3 catalog as a periodical, as most Kindle devices cannot handle AZW3 periodicals"
- title: "Content server: When using a custom IP address to listen on via Preferences->Tweaks advertise that IP address via BonJour."
- title: "Fix ebook catalog generation on linux systems where the encoding is not UTF-8."
tickets: [1048404]
improved recipes:
- De Volksrant
- Metro UK
- Countryfile
- Die Zeit (subscription)
- Birmingham post
new recipes:
- title: History Today
author: Rick Shang
- version: 0.8.68
date: 2012-09-07
new features:
- title: "Drivers for the Nokia N9, Viewsonic 7e, Prestigio PER3274B and Coby Kyros 7035 "
tickets: [1046794,1046544]
- title: "Add a tutorial on creating catalogs to the User Manual and a link to it in the create catalogs dialog"
- title: "Wireless device connections: Add an option to force calibre to listen on a particular IP address. Access it by customizing the plugin in Preferences->Plugins"
- title: "Android driver: Add an extra customization option to configure the directory to which ebooks are sent on the storage cards."
tickets: [1045045]
- title: "Add an option under Preferences->Look & Feel->Book Details to hide the cover in the book details panel"
- title: "The Calibre Companion Android app that allows wireless connection of Android device to calibre is out of beta. See https://play.google.com/store/apps/details?id=com.multipie.calibreandroid"
bug fixes:
- title: "Fix sorting by author not working in the device view in calibre when connected to iTunes"
tickets: [1044619]
- title: "Fix using the 'configure this device' menu action not validating settings"
- title: "Device drivers: Ignore corrupted entries in metadata.calibre, instead of raising an error"
- title: "PDF Output: Do not error out when generating an outline which points to pages that have been removed."
tickets: [1044799]
- title: "PDF Output: Fix incorrect page numbers being generated in the outline when converting some books"
- title: "PDF Output: Reduce memory consumption when writing out the PDF file, by using a stream"
- title: "EPUB metadata: When there are multiple <dc:date> tags use the one with the earliest date as the published date"
improved recipes:
- Wall Street journal (subscription version)
- Houston Chronicle
- Various Romanian news sources
- Business Week Magazine
- Arcamax
- version: 0.8.67
date: 2012-08-31
new features:
- title: "PDF Output: Generate a PDF Outline based on the Table of Contents of the input document"
- title: "Conversion: Add an option under Structure Detection to set the 'Start reading at' metadata with an XPath expression."
tickets: [1043233]
- title: "Speed up changing the title and author of files with books larger than 3MB by avoiding an unnecessary extra copy."
- title: "Wireless device driver: Make detecting and connecting to devices easier on networks where mdns is disabled"
- title: "PDF Output: Allow choosing the default font family and size when generating PDF files (under PDF Options) in the conversion dialog"
- title: "Metadata dialog: Comments editor: Allow specifying the name of a link when using the insert link button."
tickets: [1042683]
- title: "Remove the unmaintained pdfmanipulate command line utility. There are many other tools that provide similar functionality, for example, pdftk and podofo"
bug fixes:
- title: "Catalogs: Fix regression that broke sorting of non series titles before series titles"
- title: "PDF Output: Do not create duplicate embedded fonts in the PDF for every individual HTML file in the input document"
- title: "Fix regression that broke DnD of files having a # character in their names to the book details panel"
- title: "PDF Output: Allow generating PDF files with more than 512 pages on windows."
tickets: [1041614]
- title: "Fix minor bug in handling of the completion popups when using the next/previous buttons in the edit metadata dialog"
ticket: [1041389]
improved recipes:
- Coding Horror
- TIME Magazine
new recipes:
- title: Cumhuriyet Yzarlar
author: Sethi Eksi
- title: Arcadia
author: Masahiro Hasegawa
- title: Business Week Magazine and Chronicle of Higher Education
author: Rick Shang
- title: CIPER Chile
author: Darko Miletic
- version: 0.8.66
date: 2012-08-24
new features:
- title: "E-book viewer: Support the display of mathematics in e-books. Supports both embedded TeX and MathML"
description: "The calibre ebook viewer can now display embedded mathematics (symbols, equations, fractions, matrices, etc.) in EPUB and HTML ebooks. For details, see: http://manual.calibre-ebook.com/typesetting_math.html"
type: major
- title: "Drivers for SONY PRS-T2, Freelander PD10 and Coolreader Tablet"
tickets: [1039103]
- title: "Wireless device connections: Use a streamed mode for improved networking performance leading to much less time spent sending metadata to/from the device. Also make it easier to specify a fixed port directly in the dialog used to start the connection."
- title: "Get books: Add ebooksgratuitis.com"
bug fixes:
- title: "PDF Output: Handle input epub documents with filenames starting with a dot. Also do not hang if there is an unhandled error."
tickets: [1040603]
- title: "Get Books: Update B&N plugin to handle changes to the B&N website"
- title: "Content server: Fix regression that caused the port being advertised via BonJour to be incorrect if the user changed the port for the server."
tickets: [1037912]
improved recipes:
- Variety
- The Times UK
new recipes:
- title: Le Monde subscription version
author: Remi Vanicat
- title: Brecha Digital
author: Darko Miletic
- version: 0.8.65
date: 2012-08-17

12
format_docs/pdb/apnx.txt
View File

@ -9,7 +9,7 @@ the file are big-endian.
Layout
------
bytes content comments
bytes content comments
4 00010001 Format identifier. Value of 65537 little-endian.
4 start of next The offset after ending location of the first header.
@ -25,7 +25,7 @@ Starts next sequence
2 unknown Always 32
N second header String containing the page mapping header
4*N padding The first number given in the page mapping header indicates the number of 0 bytes.
4*N page list
4*N page list
Content Header
@ -44,6 +44,14 @@ Example:
{"contentGuid":"d8c14b0","asin":"B000JML5VM","cdeType":"EBOK","fileRevisionId":"1296874359405"}
In devices with KF8 support, we're seeing an extended content header (which seems to be required by some FW versions for KF8 files, like FW 3.4):
format Mobi version. MOBI_8 for KF8, MOBI_7 for legacy mobi files.
acr Palm DB name
Example:
{"contentGuid":"f2fc7597","asin":"B003M68YKM","cdeType":"EBOK","format":"MOBI_8","fileRevisionId":"1342776186889","acr":"CR!1F5WDHWWVN4Y78MA87Z13H9K6RKE"}
Page Mapping Header
-------------------

237
imgsrc/mimetypes/azw2.svg Normal file
View File

@ -0,0 +1,237 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="128"
height="128"
id="svg2606"
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="azw2.svg"
inkscape:export-filename="/home/niluje/Patchland/calibre/imgsrc/mimetypes/azw2.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1920"
inkscape:window-height="1025"
id="namedview45"
showgrid="false"
inkscape:zoom="1.84375"
inkscape:cx="-11.118644"
inkscape:cy="42.305085"
inkscape:window-x="-2"
inkscape:window-y="-3"
inkscape:window-maximized="1"
inkscape:current-layer="svg2606" />
<defs
id="defs2608">
<linearGradient
id="linearGradient10207">
<stop
id="stop10209"
style="stop-color:#a2a2a2;stop-opacity:1"
offset="0" />
<stop
id="stop10211"
style="stop-color:#ffffff;stop-opacity:1"
offset="1" />
</linearGradient>
<linearGradient
x1="96"
y1="104"
x2="88.000198"
y2="96.000198"
id="XMLID_12_"
gradientUnits="userSpaceOnUse">
<stop
id="stop83"
style="stop-color:#888a85;stop-opacity:1"
offset="0" />
<stop
id="stop85"
style="stop-color:#8c8e89;stop-opacity:1"
offset="0.0072" />
<stop
id="stop87"
style="stop-color:#abaca9;stop-opacity:1"
offset="0.0673" />
<stop
id="stop89"
style="stop-color:#c5c6c4;stop-opacity:1"
offset="0.1347" />
<stop
id="stop91"
style="stop-color:#dbdbda;stop-opacity:1"
offset="0.2652576" />
<stop
id="stop93"
style="stop-color:#ebebeb;stop-opacity:1"
offset="0.37646064" />
<stop
id="stop95"
style="stop-color:#f7f7f6;stop-opacity:1"
offset="0.48740286" />
<stop
id="stop97"
style="stop-color:#fdfdfd;stop-opacity:1"
offset="0.6324091" />
<stop
id="stop99"
style="stop-color:#ffffff;stop-opacity:1"
offset="1" />
</linearGradient>
<radialGradient
cx="102"
cy="112.3047"
r="139.55859"
id="XMLID_8_"
gradientUnits="userSpaceOnUse">
<stop
id="stop41"
style="stop-color:#b7b8b9;stop-opacity:1"
offset="0" />
<stop
id="stop47"
style="stop-color:#ececec;stop-opacity:1"
offset="0.18851049" />
<stop
id="stop49"
style="stop-color:#fafafa;stop-opacity:1"
offset="0.25718147" />
<stop
id="stop51"
style="stop-color:#ffffff;stop-opacity:1"
offset="0.30111277" />
<stop
id="stop53"
style="stop-color:#fafafa;stop-opacity:1"
offset="0.53130001" />
<stop
id="stop55"
style="stop-color:#ebecec;stop-opacity:1"
offset="0.84490001" />
<stop
id="stop57"
style="stop-color:#e1e2e3;stop-opacity:1"
offset="1" />
</radialGradient>
<filter
x="-0.19200002"
y="-0.19199999"
width="1.3839999"
height="1.3839999"
color-interpolation-filters="sRGB"
id="filter6697">
<feGaussianBlur
id="feGaussianBlur6699"
stdDeviation="1.9447689" />
</filter>
<clipPath
id="clipPath7084">
<path
d="m 72,88 -32,32 -8,0 0,-40 40,0 0,8 z"
id="path7086"
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none" />
</clipPath>
<radialGradient
cx="102"
cy="112.3047"
r="139.55859"
id="radialGradient9437"
xlink:href="#XMLID_8_"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(1,0,0,0.9996653,2e-6,0.00301608)" />
<linearGradient
x1="98.617439"
y1="106.41443"
x2="91.228737"
y2="99.254974"
id="linearGradient10213"
xlink:href="#linearGradient10207"
gradientUnits="userSpaceOnUse" />
<filter
color-interpolation-filters="sRGB"
id="filter2770">
<feGaussianBlur
id="feGaussianBlur2772"
stdDeviation="2.0786429" />
</filter>
</defs>
<metadata
id="metadata2611">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g
id="layer1">
<path
d="m 16,8 0,112 c 0,0 63.15625,0 63.15625,0 l 0.03125,0 c 3e-6,0 11.90625,-9.90625 17.40625,-15.40625 C 102.09375,99.09375 112,87.1875 112,87.1875 L 112,87.15625 112,8 16,8 z"
transform="matrix(1.0416667,0,0,1.0267857,-2.6666667,-1.2142891)"
id="path7865"
style="opacity:0.5;fill:#000000;fill-opacity:1;filter:url(#filter2770)" />
<path
d="M 16.000001,8 16,120 c 0,0 63.146418,0 63.146418,0 L 112,87.14642 112,8 16.000001,8 z"
id="path34"
style="fill:#ffffff;fill-opacity:1" />
<path
d="m 18.000002,9.0000034 c -0.551,0 -1,0.44885 -1,0.999665 l 0,107.9638516 c 0,0.55181 0.449,0.99966 1,0.99966 l 59.171997,0 c 0.263,0 2.76268,0.11813 2.948681,-0.0688 L 110.707,88.094202 C 110.894,87.907264 111,85.40942 111,85.146508 l 0,-75.1468396 c 0,-0.550815 -0.448,-0.999665 -1,-0.999665 l -91.999998,0 z"
id="path59"
style="fill:url(#radialGradient9437);fill-opacity:1" />
<path
d="m 41.879531,115.98249 c 0,0 24.309609,-24.309614 24.309609,-24.309614 0,0 -9.35314,2.913124 -19.60314,2.913124 0,10.25 -4.706469,21.39649 -4.706469,21.39649 z"
transform="translate(40,0)"
clip-path="url(#clipPath7084)"
id="path5540"
style="opacity:0.4;fill:#000000;fill-opacity:1;filter:url(#filter6697)" />
<path
d="m 79.172,120 c 0,0 11.914,-9.914 17.414,-15.414 5.5,-5.5 15.414,-17.414 15.414,-17.414 0,0 -13.75,8.828 -24,8.828 0,10.25 -8.828,24 -8.828,24 z"
id="path14523"
style="fill:url(#linearGradient10213);fill-opacity:1" />
<text
x="63.980469"
y="32.160156"
id="text3772"
xml:space="preserve"
style="font-size:24px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:DejaVu Sans;-inkscape-font-specification:DejaVu Sans"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan3021"
x="63.980469"
y="32.160156">Kindlet</tspan></text>
<path
d="m 68.227,60.477999 c 0,2.157 0.052,3.954 -1.035,5.874 -0.88,1.561 -2.279,2.517 -3.833,2.517 -2.121,0 -3.366,-1.62 -3.366,-4.015 C 59.993,60.14 64.225,59.283 68.226,59.283 v 1.194999 z m 5.579,13.496 c -0.365,0.332 -0.896,0.352 -1.307,0.132 -1.838,-1.528 -2.167,-2.231 -3.174,-3.69 -3.035,3.094 -5.188,4.023 -9.123,4.023 -4.663,0 -8.284,-2.876 -8.284,-8.629 0,-4.49 2.433,-7.543999 5.899,-9.044999 3.005,-1.317 7.202,-1.556 10.41,-1.914 v -0.723 c 0,-1.313 0.104,-2.875 -0.671,-4.012 -0.674,-1.021 -1.968,-1.437 -3.106,-1.437 -2.111,0 -3.99,1.078 -4.45,3.321 -0.097,0.498 -0.46,0.991 -0.962,1.017 l -5.364,-0.581 c -0.456,-0.102 -0.958,-0.463 -0.828,-1.155 1.233,-6.511 7.109,-8.475 12.378,-8.475 2.693,0 6.215,0.719 8.335,2.757 2.692997,2.515 2.431997,5.869 2.431997,9.524 v 8.622999 c 0,2.596 1.081,3.732 2.091,5.128 0.354,0.503 0.434,1.103 -0.018,1.473 -1.131,0.949 -3.138997,2.693 -4.243997,3.676 l -0.014,-0.013 z"
id="path4047"
style="fill-rule:evenodd" />
<path
d="m 99.325111,79.885297 c -8.716894,6.432161 -21.357191,9.853809 -32.243032,9.853809 -15.251419,0 -28.989193,-5.636741 -39.38419,-15.021522 -0.815557,-0.738364 -0.08726,-1.746902 0.89191,-1.173831 C 39.807066,80.071565 53.675732,84 68,84 c 9.664184,0 20.284886,-2.004491 30.058986,-6.151079 1.474215,-0.623415 2.709284,0.97246 1.266125,2.036376 z"
id="path3858"
style="fill:#ff9201;fill-rule:evenodd" />
<path
d="m 104,76 c -1.11342,-1.426386 -7.371903,-0.676274 -10.179364,-0.337298 -0.853315,0.09817 -0.984206,-0.641874 -0.217315,-1.184739 4.990671,-3.505554 13.168059,-2.491141 14.119539,-1.318987 0.95736,1.187256 -0.25087,9.384779 -4.92858,13.293915 -0.71907,0.604117 -1.40373,0.286117 -1.08573,-0.510142 C 102.75988,83.311486 105.11761,77.427225 104,76 z"
id="path3860"
style="fill:#ff9201;fill-rule:evenodd" />
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 9.1 KiB

237
imgsrc/mimetypes/azw3.svg Normal file
View File

@ -0,0 +1,237 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="128"
height="128"
id="svg2606"
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="azw3.svg"
inkscape:export-filename="/home/niluje/Patchland/calibre/imgsrc/mimetypes/azw3.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1920"
inkscape:window-height="1025"
id="namedview45"
showgrid="false"
inkscape:zoom="1.84375"
inkscape:cx="-11.118644"
inkscape:cy="42.305085"
inkscape:window-x="-2"
inkscape:window-y="-3"
inkscape:window-maximized="1"
inkscape:current-layer="svg2606" />
<defs
id="defs2608">
<linearGradient
id="linearGradient10207">
<stop
id="stop10209"
style="stop-color:#a2a2a2;stop-opacity:1"
offset="0" />
<stop
id="stop10211"
style="stop-color:#ffffff;stop-opacity:1"
offset="1" />
</linearGradient>
<linearGradient
x1="96"
y1="104"
x2="88.000198"
y2="96.000198"
id="XMLID_12_"
gradientUnits="userSpaceOnUse">
<stop
id="stop83"
style="stop-color:#888a85;stop-opacity:1"
offset="0" />
<stop
id="stop85"
style="stop-color:#8c8e89;stop-opacity:1"
offset="0.0072" />
<stop
id="stop87"
style="stop-color:#abaca9;stop-opacity:1"
offset="0.0673" />
<stop
id="stop89"
style="stop-color:#c5c6c4;stop-opacity:1"
offset="0.1347" />
<stop
id="stop91"
style="stop-color:#dbdbda;stop-opacity:1"
offset="0.2652576" />
<stop
id="stop93"
style="stop-color:#ebebeb;stop-opacity:1"
offset="0.37646064" />
<stop
id="stop95"
style="stop-color:#f7f7f6;stop-opacity:1"
offset="0.48740286" />
<stop
id="stop97"
style="stop-color:#fdfdfd;stop-opacity:1"
offset="0.6324091" />
<stop
id="stop99"
style="stop-color:#ffffff;stop-opacity:1"
offset="1" />
</linearGradient>
<radialGradient
cx="102"
cy="112.3047"
r="139.55859"
id="XMLID_8_"
gradientUnits="userSpaceOnUse">
<stop
id="stop41"
style="stop-color:#b7b8b9;stop-opacity:1"
offset="0" />
<stop
id="stop47"
style="stop-color:#ececec;stop-opacity:1"
offset="0.18851049" />
<stop
id="stop49"
style="stop-color:#fafafa;stop-opacity:1"
offset="0.25718147" />
<stop
id="stop51"
style="stop-color:#ffffff;stop-opacity:1"
offset="0.30111277" />
<stop
id="stop53"
style="stop-color:#fafafa;stop-opacity:1"
offset="0.53130001" />
<stop
id="stop55"
style="stop-color:#ebecec;stop-opacity:1"
offset="0.84490001" />
<stop
id="stop57"
style="stop-color:#e1e2e3;stop-opacity:1"
offset="1" />
</radialGradient>
<filter
x="-0.19200002"
y="-0.19199999"
width="1.3839999"
height="1.3839999"
color-interpolation-filters="sRGB"
id="filter6697">
<feGaussianBlur
id="feGaussianBlur6699"
stdDeviation="1.9447689" />
</filter>
<clipPath
id="clipPath7084">
<path
d="m 72,88 -32,32 -8,0 0,-40 40,0 0,8 z"
id="path7086"
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none" />
</clipPath>
<radialGradient
cx="102"
cy="112.3047"
r="139.55859"
id="radialGradient9437"
xlink:href="#XMLID_8_"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(1,0,0,0.9996653,2e-6,0.00301608)" />
<linearGradient
x1="98.617439"
y1="106.41443"
x2="91.228737"
y2="99.254974"
id="linearGradient10213"
xlink:href="#linearGradient10207"
gradientUnits="userSpaceOnUse" />
<filter
color-interpolation-filters="sRGB"
id="filter2770">
<feGaussianBlur
id="feGaussianBlur2772"
stdDeviation="2.0786429" />
</filter>
</defs>
<metadata
id="metadata2611">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title />
</cc:Work>
</rdf:RDF>
</metadata>
<g
id="layer1">
<path
d="m 16,8 0,112 c 0,0 63.15625,0 63.15625,0 l 0.03125,0 c 3e-6,0 11.90625,-9.90625 17.40625,-15.40625 C 102.09375,99.09375 112,87.1875 112,87.1875 L 112,87.15625 112,8 16,8 z"
transform="matrix(1.0416667,0,0,1.0267857,-2.6666667,-1.2142891)"
id="path7865"
style="opacity:0.5;fill:#000000;fill-opacity:1;filter:url(#filter2770)" />
<path
d="M 16.000001,8 16,120 c 0,0 63.146418,0 63.146418,0 L 112,87.14642 112,8 16.000001,8 z"
id="path34"
style="fill:#ffffff;fill-opacity:1" />
<path
d="m 18.000002,9.0000034 c -0.551,0 -1,0.44885 -1,0.999665 l 0,107.9638516 c 0,0.55181 0.449,0.99966 1,0.99966 l 59.171997,0 c 0.263,0 2.76268,0.11813 2.948681,-0.0688 L 110.707,88.094202 C 110.894,87.907264 111,85.40942 111,85.146508 l 0,-75.1468396 c 0,-0.550815 -0.448,-0.999665 -1,-0.999665 l -91.999998,0 z"
id="path59"
style="fill:url(#radialGradient9437);fill-opacity:1" />
<path
d="m 41.879531,115.98249 c 0,0 24.309609,-24.309614 24.309609,-24.309614 0,0 -9.35314,2.913124 -19.60314,2.913124 0,10.25 -4.706469,21.39649 -4.706469,21.39649 z"
transform="translate(40,0)"
clip-path="url(#clipPath7084)"
id="path5540"
style="opacity:0.4;fill:#000000;fill-opacity:1;filter:url(#filter6697)" />
<path
d="m 79.172,120 c 0,0 11.914,-9.914 17.414,-15.414 5.5,-5.5 15.414,-17.414 15.414,-17.414 0,0 -13.75,8.828 -24,8.828 0,10.25 -8.828,24 -8.828,24 z"
id="path14523"
style="fill:url(#linearGradient10213);fill-opacity:1" />
<text
x="64.392578"
y="32.103516"
id="text3772"
xml:space="preserve"
style="font-size:28px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:DejaVu Sans;-inkscape-font-specification:DejaVu Sans"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan3026"
x="64.392578"
y="32.103516">KF8</tspan></text>
<path
d="m 68.227,60.477999 c 0,2.157 0.052,3.954 -1.035,5.874 -0.88,1.561 -2.279,2.517 -3.833,2.517 -2.121,0 -3.366,-1.62 -3.366,-4.015 C 59.993,60.14 64.225,59.283 68.226,59.283 v 1.194999 z m 5.579,13.496 c -0.365,0.332 -0.896,0.352 -1.307,0.132 -1.838,-1.528 -2.167,-2.231 -3.174,-3.69 -3.035,3.094 -5.188,4.023 -9.123,4.023 -4.663,0 -8.284,-2.876 -8.284,-8.629 0,-4.49 2.433,-7.543999 5.899,-9.044999 3.005,-1.317 7.202,-1.556 10.41,-1.914 v -0.723 c 0,-1.313 0.104,-2.875 -0.671,-4.012 -0.674,-1.021 -1.968,-1.437 -3.106,-1.437 -2.111,0 -3.99,1.078 -4.45,3.321 -0.097,0.498 -0.46,0.991 -0.962,1.017 l -5.364,-0.581 c -0.456,-0.102 -0.958,-0.463 -0.828,-1.155 1.233,-6.511 7.109,-8.475 12.378,-8.475 2.693,0 6.215,0.719 8.335,2.757 2.692997,2.515 2.431997,5.869 2.431997,9.524 v 8.622999 c 0,2.596 1.081,3.732 2.091,5.128 0.354,0.503 0.434,1.103 -0.018,1.473 -1.131,0.949 -3.138997,2.693 -4.243997,3.676 l -0.014,-0.013 z"
id="path4047"
style="fill-rule:evenodd" />
<path
d="m 99.325111,79.885297 c -8.716894,6.432161 -21.357191,9.853809 -32.243032,9.853809 -15.251419,0 -28.989193,-5.636741 -39.38419,-15.021522 -0.815557,-0.738364 -0.08726,-1.746902 0.89191,-1.173831 C 39.807066,80.071565 53.675732,84 68,84 c 9.664184,0 20.284886,-2.004491 30.058986,-6.151079 1.474215,-0.623415 2.709284,0.97246 1.266125,2.036376 z"
id="path3858"
style="fill:#ff9201;fill-rule:evenodd" />
<path
d="m 104,76 c -1.11342,-1.426386 -7.371903,-0.676274 -10.179364,-0.337298 -0.853315,0.09817 -0.984206,-0.641874 -0.217315,-1.184739 4.990671,-3.505554 13.168059,-2.491141 14.119539,-1.318987 0.95736,1.187256 -0.25087,9.384779 -4.92858,13.293915 -0.71907,0.604117 -1.40373,0.286117 -1.08573,-0.510142 C 102.75988,83.311486 105.11761,77.427225 104,76 z"
id="path3860"
style="fill:#ff9201;fill-rule:evenodd" />
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 9.1 KiB

42
imgsrc/mimetypes/mobi.svg
View File

@ -8,10 +8,37 @@
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="128"
height="128"
id="svg2606">
id="svg2606"
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="mobi.svg"
inkscape:export-filename="/home/niluje/Patchland/calibre/imgsrc/mimetypes/mobi.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1920"
inkscape:window-height="1025"
id="namedview45"
showgrid="false"
inkscape:zoom="1.84375"
inkscape:cx="64"
inkscape:cy="64"
inkscape:window-x="-2"
inkscape:window-y="-3"
inkscape:window-maximized="1"
inkscape:current-layer="svg2606" />
<defs
id="defs2608">
<linearGradient
@ -184,15 +211,16 @@
id="path14523"
style="fill:url(#linearGradient10213);fill-opacity:1" />
<text
x="32"
y="32"
x="64.902344"
y="32.103516"
id="text3772"
xml:space="preserve"
style="font-size:28px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;fill:#000000;fill-opacity:1;stroke:none;font-family:FreeSans;-inkscape-font-specification:FreeSans"><tspan
x="32"
y="32"
style="font-size:28px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:DejaVu Sans;-inkscape-font-specification:DejaVu Sans"
sodipodi:linespacing="125%"><tspan
x="64.902344"
y="32.103516"
id="tspan3774"
style="font-size:28px;fill:#000000;fill-opacity:1">mobi</tspan></text>
style="font-size:28px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;font-family:DejaVu Sans;-inkscape-font-specification:DejaVu Sans">mobi</tspan></text>
<path
d="m 68.227,60.477999 c 0,2.157 0.052,3.954 -1.035,5.874 -0.88,1.561 -2.279,2.517 -3.833,2.517 -2.121,0 -3.366,-1.62 -3.366,-4.015 C 59.993,60.14 64.225,59.283 68.226,59.283 v 1.194999 z m 5.579,13.496 c -0.365,0.332 -0.896,0.352 -1.307,0.132 -1.838,-1.528 -2.167,-2.231 -3.174,-3.69 -3.035,3.094 -5.188,4.023 -9.123,4.023 -4.663,0 -8.284,-2.876 -8.284,-8.629 0,-4.49 2.433,-7.543999 5.899,-9.044999 3.005,-1.317 7.202,-1.556 10.41,-1.914 v -0.723 c 0,-1.313 0.104,-2.875 -0.671,-4.012 -0.674,-1.021 -1.968,-1.437 -3.106,-1.437 -2.111,0 -3.99,1.078 -4.45,3.321 -0.097,0.498 -0.46,0.991 -0.962,1.017 l -5.364,-0.581 c -0.456,-0.102 -0.958,-0.463 -0.828,-1.155 1.233,-6.511 7.109,-8.475 12.378,-8.475 2.693,0 6.215,0.719 8.335,2.757 2.692997,2.515 2.431997,5.869 2.431997,9.524 v 8.622999 c 0,2.596 1.081,3.732 2.091,5.128 0.354,0.503 0.434,1.103 -0.018,1.473 -1.131,0.949 -3.138997,2.693 -4.243997,3.676 l -0.014,-0.013 z"
id="path4047"

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 8.1 KiB

After

Width:  |  Height:  |  Size: 9.3 KiB

237
imgsrc/mimetypes/tpz.svg Normal file
View File

@ -0,0 +1,237 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.0"
width="128"
height="128"
id="svg2606"
inkscape:version="0.48.3.1 r9886"
sodipodi:docname="tpz.svg"
inkscape:export-filename="/home/niluje/Patchland/calibre/imgsrc/mimetypes/tpz.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1920"
inkscape:window-height="1025"
id="namedview45"
showgrid="false"
inkscape:zoom="1.84375"
inkscape:cx="-11.118644"
inkscape:cy="42.305085"
inkscape:window-x="-2"
inkscape:window-y="-3"
inkscape:window-maximized="1"
inkscape:current-layer="svg2606" />
<defs
id="defs2608">
<linearGradient
id="linearGradient10207">
<stop
id="stop10209"
style="stop-color:#a2a2a2;stop-opacity:1"
offset="0" />
<stop
id="stop10211"
style="stop-color:#ffffff;stop-opacity:1"
offset="1" />
</linearGradient>
<linearGradient
x1="96"
y1="104"
x2="88.000198"
y2="96.000198"
id="XMLID_12_"
gradientUnits="userSpaceOnUse">
<stop
id="stop83"
style="stop-color:#888a85;stop-opacity:1"
offset="0" />
<stop
id="stop85"
style="stop-color:#8c8e89;stop-opacity:1"
offset="0.0072" />
<stop
id="stop87"
style="stop-color:#abaca9;stop-opacity:1"
offset="0.0673" />
<stop
id="stop89"
style="stop-color:#c5c6c4;stop-opacity:1"
offset="0.1347" />
<stop
id="stop91"
style="stop-color:#dbdbda;stop-opacity:1"
offset="0.2652576" />
<stop
id="stop93"
style="stop-color:#ebebeb;stop-opacity:1"
offset="0.37646064" />
<stop
id="stop95"
style="stop-color:#f7f7f6;stop-opacity:1"
offset="0.48740286" />
<stop
id="stop97"
style="stop-color:#fdfdfd;stop-opacity:1"
offset="0.6324091" />
<stop
id="stop99"
style="stop-color:#ffffff;stop-opacity:1"
offset="1" />
</linearGradient>
<radialGradient
cx="102"
cy="112.3047"
r="139.55859"
id="XMLID_8_"
gradientUnits="userSpaceOnUse">
<stop
id="stop41"
style="stop-color:#b7b8b9;stop-opacity:1"
offset="0" />
<stop
id="stop47"
style="stop-color:#ececec;stop-opacity:1"
offset="0.18851049" />
<stop
id="stop49"
style="stop-color:#fafafa;stop-opacity:1"
offset="0.25718147" />
<stop
id="stop51"
style="stop-color:#ffffff;stop-opacity:1"
offset="0.30111277" />
<stop
id="stop53"
style="stop-color:#fafafa;stop-opacity:1"
offset="0.53130001" />
<stop
id="stop55"
style="stop-color:#ebecec;stop-opacity:1"
offset="0.84490001" />
<stop
id="stop57"
style="stop-color:#e1e2e3;stop-opacity:1"
offset="1" />
</radialGradient>
<filter
x="-0.19200002"
y="-0.19199999"
width="1.3839999"
height="1.3839999"
color-interpolation-filters="sRGB"
id="filter6697">
<feGaussianBlur
id="feGaussianBlur6699"
stdDeviation="1.9447689" />
</filter>
<clipPath
id="clipPath7084">
<path
d="m 72,88 -32,32 -8,0 0,-40 40,0 0,8 z"
id="path7086"
style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none" />
</clipPath>
<radialGradient
cx="102"
cy="112.3047"
r="139.55859"
id="radialGradient9437"
xlink:href="#XMLID_8_"
gradientUnits="userSpaceOnUse"
gradientTransform="matrix(1,0,0,0.9996653,2e-6,0.00301608)" />
<linearGradient
x1="98.617439"
y1="106.41443"
x2="91.228737"
y2="99.254974"
id="linearGradient10213"
xlink:href="#linearGradient10207"
gradientUnits="userSpaceOnUse" />
<filter
color-interpolation-filters="sRGB"
id="filter2770">
<feGaussianBlur
id="feGaussianBlur2772"
stdDeviation="2.0786429" />
</filter>
</defs>
<metadata
id="metadata2611">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g
id="layer1">
<path
d="m 16,8 0,112 c 0,0 63.15625,0 63.15625,0 l 0.03125,0 c 3e-6,0 11.90625,-9.90625 17.40625,-15.40625 C 102.09375,99.09375 112,87.1875 112,87.1875 L 112,87.15625 112,8 16,8 z"
transform="matrix(1.0416667,0,0,1.0267857,-2.6666667,-1.2142891)"
id="path7865"
style="opacity:0.5;fill:#000000;fill-opacity:1;filter:url(#filter2770)" />
<path
d="M 16.000001,8 16,120 c 0,0 63.146418,0 63.146418,0 L 112,87.14642 112,8 16.000001,8 z"
id="path34"
style="fill:#ffffff;fill-opacity:1" />
<path
d="m 18.000002,9.0000034 c -0.551,0 -1,0.44885 -1,0.999665 l 0,107.9638516 c 0,0.55181 0.449,0.99966 1,0.99966 l 59.171997,0 c 0.263,0 2.76268,0.11813 2.948681,-0.0688 L 110.707,88.094202 C 110.894,87.907264 111,85.40942 111,85.146508 l 0,-75.1468396 c 0,-0.550815 -0.448,-0.999665 -1,-0.999665 l -91.999998,0 z"
id="path59"
style="fill:url(#radialGradient9437);fill-opacity:1" />
<path
d="m 41.879531,115.98249 c 0,0 24.309609,-24.309614 24.309609,-24.309614 0,0 -9.35314,2.913124 -19.60314,2.913124 0,10.25 -4.706469,21.39649 -4.706469,21.39649 z"
transform="translate(40,0)"
clip-path="url(#clipPath7084)"
id="path5540"
style="opacity:0.4;fill:#000000;fill-opacity:1;filter:url(#filter6697)" />
<path
d="m 79.172,120 c 0,0 11.914,-9.914 17.414,-15.414 5.5,-5.5 15.414,-17.414 15.414,-17.414 0,0 -13.75,8.828 -24,8.828 0,10.25 -8.828,24 -8.828,24 z"
id="path14523"
style="fill:url(#linearGradient10213);fill-opacity:1" />
<text
x="64.703125"
y="32.175781"
id="text3772"
xml:space="preserve"
style="font-size:28px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;text-align:center;line-height:125%;writing-mode:lr-tb;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:DejaVu Sans;-inkscape-font-specification:DejaVu Sans"
sodipodi:linespacing="125%"><tspan
sodipodi:role="line"
id="tspan3021"
x="64.703125"
y="32.175781">Topaz</tspan></text>
<path
d="m 68.227,60.477999 c 0,2.157 0.052,3.954 -1.035,5.874 -0.88,1.561 -2.279,2.517 -3.833,2.517 -2.121,0 -3.366,-1.62 -3.366,-4.015 C 59.993,60.14 64.225,59.283 68.226,59.283 v 1.194999 z m 5.579,13.496 c -0.365,0.332 -0.896,0.352 -1.307,0.132 -1.838,-1.528 -2.167,-2.231 -3.174,-3.69 -3.035,3.094 -5.188,4.023 -9.123,4.023 -4.663,0 -8.284,-2.876 -8.284,-8.629 0,-4.49 2.433,-7.543999 5.899,-9.044999 3.005,-1.317 7.202,-1.556 10.41,-1.914 v -0.723 c 0,-1.313 0.104,-2.875 -0.671,-4.012 -0.674,-1.021 -1.968,-1.437 -3.106,-1.437 -2.111,0 -3.99,1.078 -4.45,3.321 -0.097,0.498 -0.46,0.991 -0.962,1.017 l -5.364,-0.581 c -0.456,-0.102 -0.958,-0.463 -0.828,-1.155 1.233,-6.511 7.109,-8.475 12.378,-8.475 2.693,0 6.215,0.719 8.335,2.757 2.692997,2.515 2.431997,5.869 2.431997,9.524 v 8.622999 c 0,2.596 1.081,3.732 2.091,5.128 0.354,0.503 0.434,1.103 -0.018,1.473 -1.131,0.949 -3.138997,2.693 -4.243997,3.676 l -0.014,-0.013 z"
id="path4047"
style="fill-rule:evenodd" />
<path
d="m 99.325111,79.885297 c -8.716894,6.432161 -21.357191,9.853809 -32.243032,9.853809 -15.251419,0 -28.989193,-5.636741 -39.38419,-15.021522 -0.815557,-0.738364 -0.08726,-1.746902 0.89191,-1.173831 C 39.807066,80.071565 53.675732,84 68,84 c 9.664184,0 20.284886,-2.004491 30.058986,-6.151079 1.474215,-0.623415 2.709284,0.97246 1.266125,2.036376 z"
id="path3858"
style="fill:#ff9201;fill-rule:evenodd" />
<path
d="m 104,76 c -1.11342,-1.426386 -7.371903,-0.676274 -10.179364,-0.337298 -0.853315,0.09817 -0.984206,-0.641874 -0.217315,-1.184739 4.990671,-3.505554 13.168059,-2.491141 14.119539,-1.318987 0.95736,1.187256 -0.25087,9.384779 -4.92858,13.293915 -0.71907,0.604117 -1.40373,0.286117 -1.08573,-0.510142 C 102.75988,83.311486 105.11761,77.427225 104,76 z"
id="path3860"
style="fill:#ff9201;fill-rule:evenodd" />
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 9.1 KiB

140
manual/catalogs.rst Normal file
View File

@ -0,0 +1,140 @@
.. include:: global.rst
.. _catalog_tut:
Creating AZW3 • EPUB • MOBI Catalogs
=====================================
|app|'s Create catalog feature enables you to create a catalog of your library in a variety of formats. This help file describes cataloging options when generating a catalog in AZW3, EPUB and MOBI formats.
.. contents::
:depth: 1
:local:
Selecting books to catalog
-------------------------------
If you want *all* of your library cataloged, remove any search or filtering criteria in the main window. With a single book selected, all books in your library will be candidates for inclusion in the generated catalog. Individual books may be excluded by various criteria; see the :ref:`excluded_genres` section below for more information.
If you want only *some* of your library cataloged, you have two options:
* Create a multiple selection of the books you want cataloged. With more than one book selected in |app|'s main window, only the selected books will be cataloged.
* Use the Search field or the Tag Browser to filter the displayed books. Only the displayed books will be cataloged.
To begin catalog generation, select the menu item :guilabel:`Convert books > Create a catalog of the books in your calibre library`. You may also add a :guilabel:`Create Catalog` button to a toolbar in :guilabel:`Preferences > Interface > Toolbars` for easier access to the Generate catalog dialog.
.. image:: images/catalog_options.png
:alt: Catalog options
:align: center
In :guilabel:`Catalog options`, select **AZW3, EPUB or MOBI** as the Catalog format. In the :guilabel:`Catalog title` field, provide a name that will be used for the generated catalog. If a catalog of the same name and format already exists, it will be replaced with the newly-generated catalog.
.. image:: images/catalog_send_to_device.png
:alt: Catalog send to device
:align: center
Enabling :guilabel:`Send catalog to device automatically` will download the generated catalog to a connected device upon completion.
Included sections
-------------------
.. image:: images/included_sections.png
:alt: Included sections
:align: center
Sections enabled by a checkmark will be included in the generated catalog:
* :guilabel:`Authors` - all books, sorted by author, presented in a list format. Non-series books are listed before series books.
* :guilabel:`Titles` - all books, sorted by title, presented in a list format.
* :guilabel:`Series` - all books that are part of a series, sorted by series, presented in a list format.
* :guilabel:`Genres` - individual genres presented in a list, sorted by Author and Series.
* :guilabel:`Recently` Added - all books, sorted in reverse chronological order. List includes books added in the last 30 days, then a month-by-month listing of added books.
* :guilabel:`Descriptions` - detailed description page for each book, including a cover thumbnail and comments. Sorted by author, with non-series books listed before series books.
Prefixes
---------
.. image:: images/prefix_rules.png
:alt: Prefix rules
:align: center
Prefix rules allow you to add a prefix to book listings when certain criteria are met. For example, you might want to mark books you've read with a checkmark, or books on your wishlist with an X.
The checkbox in the first column enables the rule. :guilabel:`Name` is a rule name that you provide. :guilabel:`Field` is either :guilabel:`Tags` or a custom column from your library. :guilabel:`Value` is the content of :guilabel:`Field` to match. When a prefix rule is satisfied, the book will be marked with the selected :guilabel:`Prefix`.
Three prefix rules have been specified in the example above:
1. :guilabel:`Read book` specifies that a book with any date in a custom column named :guilabel:`Last read` will be prefixed with a checkmark symbol.
2. :guilabel:`Wishlist` item specifies that any book with a :guilabel:`Wishlist` tag will be prefixed with an X symbol.
3. :guilabel:`Library` books specifies that any book with a value of True (or Yes) in a custom column :guilabel:`Available in Library` will be prefixed with a double arrow symbol.
The first matching rule supplies the prefix. Disabled or incomplete rules are ignored.
Excluded books
-----------------
.. image:: images/excluded_books.png
:alt: Excluded books
:align: center
Exclusion rules allow you to specify books that will not be cataloged.
The checkbox in the first column enables the rule. :guilabel:`Name` is a rule name that you provide. :guilabel:`Field` is either :guilabel:`Tags` or a custom column in your library. :guilabel:`Value` is the content of :guilabel:`Field` to match. When an exclusion rule is satisfied, the book will be excluded from the generated catalog.
Two exclusion rules have been specified in the example above:
1. The :guilabel:`Catalogs` rule specifies that any book with a :guilabel:`Catalog` tag will be excluded from the generated catalog.
2. The :guilabel:`Archived` Books rule specifies that any book with a value of :guilabel:`Archived` in the custom column :guilabel:`Status` will be excluded from the generated catalog.
All rules are evaluated for every book. Disabled or incomplete rules are ignored.
.. _excluded_genres:
Excluded genres
---------------
.. image:: images/excluded_genres.png
:alt: Excluded genres
:align: center
When the catalog is generated, tags in your database are used as genres. For example, you may use the tags ``Fiction`` and ``Nonfiction``. These tags become genres in the generated catalog, with books listed under their respective genre lists based on their assigned tags. A book will be listed in every genre section for which it has a corresponding tag.
You may be using certain tags for other purposes, perhaps a + to indicate a read book, or a bracketed tag like ``[Amazon Freebie]`` to indicate a book's source. The :guilabel:`Excluded genres` regex allows you to specify tags that you don't want used as genres in the generated catalog. The default exclusion regex pattern ``\[.+\]\+`` excludes any tags of the form ``[tag]``, as well as excluding ``+``, the default tag for read books, from being used as genres in the generated catalog.
You can also use an exact tag name in a regex. For example, ``[Amazon Freebie]`` or ``[Project Gutenberg]``. If you want to list multiple exact tags for exclusion, put a pipe (vertical bar) character between them: ``[Amazon Freebie]|[Project Gutenberg]``.
:guilabel:`Results of regex` shows you which tags will be excluded when the catalog is built, based on the tags in your database and the regex pattern you enter. The results are updated as you modify the regex pattern.
Other options
--------------
.. image:: images/other_options.png
:alt: Other options
:align: center
:guilabel:`Catalog cover` specifies whether to generate a new cover or use an existing cover. It is possible to create a custom cover for your catalogs - see :ref:`Custom catalog covers` for more information. If you have created a custom cover that you want to reuse, select :guilabel:`Use existing cover`. Otherwise, select :guilabel:`Generate new cover`.
:guilabel:`Extra Description note` specifies a custom column's contents to be inserted into the Description page, next to the cover thumbnail. For example, you might want to display the date you last read a book using a :guilabel:`Last Read` custom column. For advanced use of the Description note feature, see `this post in the calibre forum <http://www.mobileread.com/forums/showpost.php?p=1335767&postcount=395>`_.
:guilabel:`Thumb width` specifies a width preference for cover thumbnails included with Descriptions pages. Thumbnails are cached to improve performance.To experiment with different widths, try generating a catalog with just a few books until you've determined your preferred width, then generate your full catalog. The first time a catalog is generated with a new thumbnail width, performance will be slower, but subsequent builds of the catalog will take advantage of the thumbnail cache.
:guilabel:`Merge with Comments` specifies a custom column whose content will be non-destructively merged with the Comments metadata during catalog generation. For example, you might have a custom column :guilabel:`Author Bio` that you'd like to append to the Comments metadata. You can choose to insert the custom column contents *before or after* the Comments section, and optionally separate the appended content with a horizontal rule separator. Eligible custom column types include ``text, comments, and composite``.
.. _Custom catalog covers:
Custom catalog covers
-----------------------
.. |cc| image:: images/custom_cover.png
|cc| With the `Generate Cover plugin <http://www.mobileread.com/forums/showthread.php?t=124219>`_ installed, you can create custom covers for your catalog.
To install the plugin, go to :guilabel:`Preferences > Advanced > Plugins > Get new plugins`.
Additional help resources
---------------------------
For more information on |app|'s Catalog feature, see the MobileRead forum sticky `Creating Catalogs - Start here <http://www.mobileread.com/forums/showthread.php?t=118556>`_, where you can find information on how to customize the catalog templates, and how to submit a bug report.
To ask questions or discuss calibre's Catalog feature with other users, visit the MobileRead forum `Calibre Catalogs <http://www.mobileread.com/forums/forumdisplay.php?f=238>`_.

4
manual/creating_plugins.rst
View File

@ -182,6 +182,10 @@ The plugin API
As you may have noticed above, a plugin in |app| is a class. There are different classes for the different types of plugins in |app|.
Details on each class, including the base class of all plugins can be found in :ref:`plugins`.
Your plugin is almost certainly going to use code from |app|. To learn
how to find various bits of functionality in the
|app| code base, read the section on the |app| :ref:`code_layout`.
Debugging plugins
-------------------

2
manual/develop.rst
View File

@ -30,6 +30,8 @@ a device driver plugin. You can browse the
for new conversion formats involves writing input/output format plugins. Another example of the modular design is the :ref:`recipe system <news>` for
fetching news. For more examples of plugins designed to add features to |app|, see the `plugin index <http://www.mobileread.com/forums/showthread.php?p=1362767#post1362767>`_.
.. _code_layout:
Code layout
^^^^^^^^^^^^^^

5
manual/gui.rst
View File

@ -103,10 +103,9 @@ The :guilabel:`Convert books` action has three variations, accessed by doing a r
3. **Create a catalog of the books in your calibre library**: Allows you to generate a complete listing of the books in your library, including all metadata,
in several formats such as XML, CSV, BiBTeX, EPUB and MOBI. The catalog will contain all the books currently showing in the library view.
This allows you to use the search features to limit the books to be catalogued. In addition, if you select multiple books using the mouse,
only those books will be added to the catalog. If you generate the catalog in an ebook format such as EPUB or MOBI,
only those books will be added to the catalog. If you generate the catalog in an ebook format such as EPUB, MOBI or AZW3,
the next time you connect your ebook reader the catalog will be automatically sent to the device.
For more information on how catalogs work, read the `catalog creation tutorial <http://www.mobileread.com/forums/showthread.php?p=755468#post755468>`_
at MobileRead.
For more information on how catalogs work, read the :ref:`catalog_tut`.
.. _view:

BIN
manual/images/catalog_options.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 33 KiB

BIN
manual/images/catalog_send_to_device.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

BIN
manual/images/custom_cover.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 100 KiB

BIN
manual/images/excluded_books.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 28 KiB

BIN
manual/images/excluded_genres.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 25 KiB

BIN
manual/images/included_sections.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 21 KiB

BIN
manual/images/other_options.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 40 KiB

BIN
manual/images/prefix_rules.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 38 KiB

4
manual/sub_groups.rst
View File

@ -108,10 +108,10 @@ After creating the saved search, you can use it as a restriction.
Useful Template Functions
-------------------------
You might want to use the genre information in a template, such as with save to disk or send to device. The question might then be "How do I get the outermost genre name or names?" An |app| template function, subitems, is provided to make doing this easier.
You might want to use the genre information in a template, such as with save to disk or send to device. The question might then be "How do I get the outermost genre name or names?" A |app| template function, subitems, is provided to make doing this easier.
For example, assume you want to add the outermost genre level to the save-to-disk template to make genre folders, as in "History/The Gathering Storm - Churchill, Winston". To do this, you must extract the first level of the hierarchy and add it to the front along with a slash to indicate that it should make a folder. The template below accomplishes this::
{#genre:subitems(0,1)||/}{title} - {authors}
See :ref:`The |app| template language <templatelangcalibre>` for more information templates and the subitem function.
See :ref:`The template language <templatelangcalibre>` for more information templates and the :func:`subitems` function.

1
manual/tutorials.rst
View File

@ -19,4 +19,5 @@ Here you will find tutorials to get you started using |app|'s more advanced feat
server
creating_plugins
typesetting_math
catalogs

2
manual/typesetting_math.rst
View File

@ -48,7 +48,7 @@ This snippet looks like the following screen shot in the |app| viewer.
.. figure:: images/lorentz.png
:align: center
:guilabel:`The Lorentz Equations`
:guilabel:`The Lorenz Equations`
The complete HTML file, with more equations and inline mathematics is
reproduced below. You can convert this HTML file to EPUB in |app| to end up

35
recipes/arcadia.recipe Normal file
View File

@ -0,0 +1,35 @@
# -*- coding: utf8 -*-
from calibre.web.feeds.recipes import BasicNewsRecipe
import re
class Arcadia_BBS(BasicNewsRecipe):
title = u'Arcadia'
__author__ = 'Masahiro Hasegawa'
language = 'ja'
encoding = 'utf8'
filter_regexps = [r'ad\.jp\.ap\.valuecommerce.com',]
timefmt = '[%Y/%m/%d]'
remove_tags_before = dict(name='a', attrs={'name':'kiji'})
sid_list = [] #some sotory id
def parse_index(self):
result = []
for sid in self.sid_list:
s_result = []
soup = self.index_to_soup(
'http://www.mai-net.net/bbs/sst/sst.php?act=dump&all=%d'
% sid)
sec = soup.findAll('a', attrs={'href':re.compile(r'.*?kiji')})
for s in sec[:-2]:
s_result.append(dict(title=s.string,
url="http://www.mai-net.net" + s['href'],
date=s.parent.parent.parent.findAll('td')[3].string[:-6],
description='', content=''))
result.append((s_result[0]['title'], s_result))
return result

91
recipes/arcamax.recipe
View File

@ -31,55 +31,53 @@ class Arcamax(BasicNewsRecipe):
, 'language' : language
}
keep_only_tags = [dict(name='div', attrs={'class':['comics-header']}),
dict(name='b', attrs={'class':['current']}),
dict(name='article', attrs={'class':['comic']}),
keep_only_tags = [dict(name='article', attrs={'class':['comic']}),
]
remove_tags = [dict(name='div', attrs={'id':['comicfull' ]}),
dict(name='div', attrs={'class':['calendar' ]}),
dict(name='nav', attrs={'class':['calendar-nav' ]}),
]
#remove_tags = [dict(name='div', attrs={'id':['comicfull' ]}),
#dict(name='div', attrs={'class':['calendar' ]}),
#dict(name='nav', attrs={'class':['calendar-nav' ]}),
#]
def parse_index(self):
feeds = []
for title, url in [
######## COMICS - GENERAL ########
#(u"9 Chickweed Lane", u"http://www.arcamax.com/ninechickweedlane"),
#(u"Agnes", u"http://www.arcamax.com/agnes"),
#(u"Andy Capp", u"http://www.arcamax.com/andycapp"),
(u"BC", u"http://www.arcamax.com/bc"),
#(u"Baby Blues", u"http://www.arcamax.com/babyblues"),
#(u"Beetle Bailey", u"http://www.arcamax.com/beetlebailey"),
(u"Blondie", u"http://www.arcamax.com/blondie"),
#u"Boondocks", u"http://www.arcamax.com/boondocks"),
#(u"Cathy", u"http://www.arcamax.com/cathy"),
#(u"Daddys Home", u"http://www.arcamax.com/daddyshome"),
(u"Dilbert", u"http://www.arcamax.com/dilbert"),
#(u"Dinette Set", u"http://www.arcamax.com/thedinetteset"),
(u"Dog Eat Doug", u"http://www.arcamax.com/dogeatdoug"),
(u"Doonesbury", u"http://www.arcamax.com/doonesbury"),
#(u"Dustin", u"http://www.arcamax.com/dustin"),
(u"Family Circus", u"http://www.arcamax.com/familycircus"),
(u"Garfield", u"http://www.arcamax.com/garfield"),
#(u"Get Fuzzy", u"http://www.arcamax.com/getfuzzy"),
#(u"Girls and Sports", u"http://www.arcamax.com/girlsandsports"),
#(u"Hagar the Horrible", u"http://www.arcamax.com/hagarthehorrible"),
#(u"Heathcliff", u"http://www.arcamax.com/heathcliff"),
#(u"Jerry King Cartoons", u"http://www.arcamax.com/humorcartoon"),
#(u"Luann", u"http://www.arcamax.com/luann"),
#(u"Momma", u"http://www.arcamax.com/momma"),
#(u"Mother Goose and Grimm", u"http://www.arcamax.com/mothergooseandgrimm"),
(u"Mutts", u"http://www.arcamax.com/mutts"),
#(u"Non Sequitur", u"http://www.arcamax.com/nonsequitur"),
#(u"Pearls Before Swine", u"http://www.arcamax.com/pearlsbeforeswine"),
#(u"Pickles", u"http://www.arcamax.com/pickles"),
#(u"Red and Rover", u"http://www.arcamax.com/redandrover"),
#(u"Rubes", u"http://www.arcamax.com/rubes"),
#(u"Rugrats", u"http://www.arcamax.com/rugrats"),
(u"Speed Bump", u"http://www.arcamax.com/speedbump"),
(u"Wizard of Id", u"http://www.arcamax.com/wizardofid"),
(u"Zits", u"http://www.arcamax.com/zits"),
#(u"9 Chickweed Lane", #u"http://www.arcamax.com/thefunnies/ninechickweedlane"),
#(u"Agnes", u"http://www.arcamax.com/thefunnies/agnes"),
#(u"Andy Capp", #u"http://www.arcamax.com/thefunnies/andycapp"),
(u"BC", u"http://www.arcamax.com/thefunnies/bc"),
#(u"Baby Blues", #u"http://www.arcamax.com/thefunnies/babyblues"),
#(u"Beetle Bailey", #u"http://www.arcamax.com/thefunnies/beetlebailey"),
(u"Blondie", u"http://www.arcamax.com/thefunnies/blondie"),
#u"Boondocks", u"http://www.arcamax.com/thefunnies/boondocks"),
#(u"Cathy", u"http://www.arcamax.com/thefunnies/cathy"),
#(u"Daddys Home", #u"http://www.arcamax.com/thefunnies/daddyshome"),
(u"Dilbert", u"http://www.arcamax.com/thefunnies/dilbert"),
#(u"Dinette Set", #u"http://www.arcamax.com/thefunnies/thedinetteset"),
(u"Dog Eat Doug", u"http://www.arcamax.com/thefunnies/dogeatdoug"),
(u"Doonesbury", u"http://www.arcamax.com/thefunnies/doonesbury"),
#(u"Dustin", u"http://www.arcamax.com/thefunnies/dustin"),
(u"Family Circus", u"http://www.arcamax.com/thefunnies/familycircus"),
(u"Garfield", u"http://www.arcamax.com/thefunnies/garfield"),
#(u"Get Fuzzy", #u"http://www.arcamax.com/thefunnies/getfuzzy"),
#(u"Girls and Sports", #u"http://www.arcamax.com/thefunnies/girlsandsports"),
#(u"Hagar the Horrible", #u"http://www.arcamax.com/thefunnies/hagarthehorrible"),
#(u"Heathcliff", #u"http://www.arcamax.com/thefunnies/heathcliff"),
#(u"Jerry King Cartoons", #u"http://www.arcamax.com/thefunnies/humorcartoon"),
#(u"Luann", u"http://www.arcamax.com/thefunnies/luann"),
#(u"Momma", u"http://www.arcamax.com/thefunnies/momma"),
#(u"Mother Goose and Grimm", #u"http://www.arcamax.com/thefunnies/mothergooseandgrimm"),
(u"Mutts", u"http://www.arcamax.com/thefunnies/mutts"),
#(u"Non Sequitur", #u"http://www.arcamax.com/thefunnies/nonsequitur"),
#(u"Pearls Before Swine", #u"http://www.arcamax.com/thefunnies/pearlsbeforeswine"),
#(u"Pickles", u"http://www.arcamax.com/thefunnies/pickles"),
#(u"Red and Rover", #u"http://www.arcamax.com/thefunnies/redandrover"),
#(u"Rubes", u"http://www.arcamax.com/thefunnies/rubes"),
#(u"Rugrats", u"http://www.arcamax.com/thefunnies/rugrats"),
(u"Speed Bump", u"http://www.arcamax.com/thefunnies/speedbump"),
(u"Wizard of Id", u"http://www.arcamax.com/thefunnies/wizardofid"),
(u"Zits", u"http://www.arcamax.com/thefunnies/zits"),
]:
articles = self.make_links(url)
if articles:
@ -93,11 +91,11 @@ class Arcamax(BasicNewsRecipe):
for page in pages:
page_soup = self.index_to_soup(url)
if page_soup:
title = self.tag_to_string(page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0])
title = self.tag_to_string(page_soup.find(name='div', attrs={'class':'columnheader'}).h1.contents[0])
page_url = url
# orig prev_page_url = 'http://www.arcamax.com' + page_soup.find('a', attrs={'class':'prev'}, text='Previous').parent['href']
prev_page_url = 'http://www.arcamax.com' + page_soup.find('span', text='Previous').parent.parent['href']
date = self.tag_to_string(page_soup.find(name='b', attrs={'class':['current']}))
prev_page_url = 'http://www.arcamax.com' + page_soup.find(name='a', attrs={'class':['prev']})['href']
date = self.tag_to_string(page_soup.find(name='span', attrs={'class':['cur']}))
current_articles.append({'title': title, 'url': page_url, 'description':'', 'date': date})
url = prev_page_url
current_articles.reverse()
@ -126,4 +124,5 @@ class Arcamax(BasicNewsRecipe):
img {max-width:100%; min-width:100%;}
p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
'''
'''

2
recipes/atlantic.recipe
View File

@ -39,7 +39,7 @@ class TheAtlantic(BasicNewsRecipe):
cover = soup.find('img', src=True, attrs={'class':'cover'})
if cover is not None:
self.cover_url = cover['src']
self.cover_url = cover['src'].replace(' ', '%20')
feeds = []
seen_titles = set([])

32
recipes/birmingham_post.recipe
View File

@ -1,14 +1,17 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1306097511(BasicNewsRecipe):
title = u'Birmingham post'
description = 'News for Birmingham UK'
timefmt = ''
description = 'Author D.Asbury. News for Birmingham UK'
#timefmt = ''
# last update 8/9/12
__author__ = 'Dave Asbury'
cover_url = 'http://1.bp.blogspot.com/_GwWyq5eGw9M/S9BHPHxW55I/AAAAAAAAB6Q/iGCWl0egGzg/s320/Birmingham+post+Lite+front.JPG'
cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/161987_9010212100_2035706408_n.jpg'
oldest_article = 2
max_articles_per_feed = 12
linearize_tables = True
remove_empty_feeds = True
remove_javascript = True
no_stylesheets = True
#auto_cleanup = True
language = 'en_GB'
@ -17,11 +20,12 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
keep_only_tags = [
dict(name='h1',attrs={'id' : 'article-headline'}),
dict(attrs={'id' : 'article-header'}),
#dict(name='h1',attrs={'id' : 'article-header'}),
dict(attrs={'class':['article-meta-author','article-meta-date','article main','art-o art-align-center otm-1 ']}),
dict(name='div',attrs={'class' : 'article-image full'}),
dict(attrs={'clas' : 'art-o art-align-center otm-1 '}),
dict(name='div',attrs={'class' : 'article main'}),
dict(name='div',attrs={'class' : 'article-image full'}),
dict(attrs={'clas' : 'art-o art-align-center otm-1 '}),
dict(name='div',attrs={'class' : 'article main'}),
#dict(name='p')
#dict(attrs={'id' : 'three-col'})
]
@ -37,11 +41,9 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
(u'Bloggs & Comments',u'http://www.birminghampost.net/comment/rss.xml')
]
extra_css = '''
body {font: sans-serif medium;}'
h1 {text-align : center; font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold;}
h2 {text-align : center;color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:15px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; }
span{ font-size:9.5px; font-weight:bold;font-style:italic}
p { text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
'''
extra_css = '''
h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;text-align:center;}
h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
'''

68
recipes/bwmagazine2.recipe Normal file
View File

@ -0,0 +1,68 @@
from calibre.web.feeds.recipes import BasicNewsRecipe
from collections import OrderedDict
class BusinessWeekMagazine(BasicNewsRecipe):
title = 'Business Week Magazine'
__author__ = 'Rick Shang'
description = 'A renowned business publication. Business news, trends and profiles of successful businesspeople.'
language = 'en'
category = 'news'
encoding = 'UTF-8'
keep_only_tags = [
dict(name='div', attrs={'id':'article_body_container'}),
]
remove_tags = [dict(name='ui'),dict(name='li')]
no_javascript = True
no_stylesheets = True
cover_url = 'http://images.businessweek.com/mz/covers/current_120x160.jpg'
def parse_index(self):
#Go to the issue
soup = self.index_to_soup('http://www.businessweek.com/magazine/news/articles/business_news.htm')
#Find date
mag=soup.find('h2',text='Magazine')
dates=self.tag_to_string(mag.findNext('h3'))
self.timefmt = u' [%s]'%dates
#Go to the main body
div0 = soup.find ('div', attrs={'class':'column left'})
section_title = ''
feeds = OrderedDict()
for div in div0.findAll('h4'):
articles = []
section_title = self.tag_to_string(div.findPrevious('h3')).strip()
title=self.tag_to_string(div.a).strip()
url=div.a['href']
soup0 = self.index_to_soup(url)
urlprint=soup0.find('li', attrs={'class':'print'}).a['href']
articles.append({'title':title, 'url':urlprint, 'description':'', 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
div1 = soup.find ('div', attrs={'class':'column center'})
section_title = ''
for div in div1.findAll('h5'):
articles = []
desc=self.tag_to_string(div.findNext('p')).strip()
section_title = self.tag_to_string(div.findPrevious('h3')).strip()
title=self.tag_to_string(div.a).strip()
url=div.a['href']
soup0 = self.index_to_soup(url)
urlprint=soup0.find('li', attrs={'class':'print'}).a['href']
articles.append({'title':title, 'url':urlprint, 'description':desc, 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
ans = [(key, val) for key, val in feeds.iteritems()]
return ans

16
recipes/catavencu.recipe
View File

@ -12,7 +12,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
class AcademiaCatavencu(BasicNewsRecipe):
title = u'Academia Ca\u0163avencu'
__author__ = u'Silviu Cotoar\u0103'
description = 'Tagma cum laude'
description = 'Academia Catavencu. Pamflete!'
publisher = u'Ca\u0163avencu'
oldest_article = 5
language = 'ro'
@ -21,7 +21,7 @@ class AcademiaCatavencu(BasicNewsRecipe):
use_embedded_content = False
category = 'Ziare'
encoding = 'utf-8'
cover_url = 'http://www.academiacatavencu.info/images/logo.png'
cover_url = 'http://www.inpolitics.ro/Uploads/Articles/academia_catavencu.jpg'
conversion_options = {
'comments' : description
@ -31,21 +31,21 @@ class AcademiaCatavencu(BasicNewsRecipe):
}
keep_only_tags = [
dict(name='h1', attrs={'class':'art_title'}),
dict(name='div', attrs={'class':'art_text'})
dict(name='h1', attrs={'class':'entry-title'}),
dict(name='div', attrs={'class':'entry-content'})
]
remove_tags = [
dict(name='div', attrs={'class':['desp_m']})
, dict(name='div', attrs={'id':['tags']})
dict(name='div', attrs={'class':['mr_social_sharing_wrapper']})
, dict(name='div', attrs={'id':['fb_share_1']})
]
remove_tags_after = [
dict(name='div', attrs={'class':['desp_m']})
dict(name='div', attrs={'id':['fb_share_1']})
]
feeds = [
(u'Feeds', u'http://www.academiacatavencu.info/rss.xml')
(u'Feeds', u'http://www.academiacatavencu.info/feed')
]
def preprocess_html(self, soup):

81
recipes/chronicle_higher_ed.recipe Normal file
View File

@ -0,0 +1,81 @@
from calibre.web.feeds.recipes import BasicNewsRecipe
from collections import OrderedDict
class Chronicle(BasicNewsRecipe):
title = 'The Chronicle of Higher Education'
__author__ = 'Rick Shang'
description = 'Weekly news and job-information source for college and university faculty members, administrators, and students.'
language = 'en'
category = 'news'
encoding = 'UTF-8'
keep_only_tags = [
dict(name='div', attrs={'class':'article'}),
]
remove_tags = [dict(name='div',attrs={'class':['related module1','maintitle']}),
dict(name='div', attrs={'id':['section-nav','icon-row']})]
no_javascript = True
no_stylesheets = True
needs_subscription = True
def get_browser(self):
br = BasicNewsRecipe.get_browser()
if self.username is not None and self.password is not None:
br.open('http://chronicle.com/myaccount/login')
br.select_form(nr=1)
br['username'] = self.username
br['password'] = self.password
br.submit()
return br
def parse_index(self):
#Go to the issue
soup0 = self.index_to_soup('http://chronicle.com/section/Archives/39/')
issue = soup0.find('ul',attrs={'class':'feature-promo-list'}).li
issueurl = "http://chronicle.com"+issue.a['href']
#Find date
dates = self.tag_to_string(issue.a).split(': ')[-1]
self.timefmt = u' [%s]'%dates
#Find cover
cover=soup0.find('div',attrs={'class':'promo'}).findNext('div')
self.cover_url="http://chronicle.com"+cover.find('img')['src']
#Go to the main body
soup = self.index_to_soup(issueurl)
div = soup.find ('div', attrs={'id':'article-body'})
feeds = OrderedDict()
section_title = ''
for post in div.findAll('li'):
articles = []
a=post.find('a', href=True)
if a is not None:
title=self.tag_to_string(a)
url="http://chronicle.com"+a['href'].strip()
sectiontitle=post.findPrevious('h3')
if sectiontitle is None:
sectiontitle=post.findPrevious('h4')
section_title=self.tag_to_string(sectiontitle)
desc=self.tag_to_string(post.find('p'))
articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
ans = [(key, val) for key, val in feeds.iteritems()]
return ans
def preprocess_html(self,soup):
#process all the images
for div in soup.findAll('div', attrs={'class':'tableauPlaceholder'}):
noscripts=div.find('noscript').a
div.replaceWith(noscripts)
for div0 in soup.findAll('div',text='Powered by Tableau'):
div0.extract()
return soup

58
recipes/ciperchile.recipe Normal file
View File

@ -0,0 +1,58 @@
__license__ = 'GPL v3'
__copyright__ = '2012, Darko Miletic <darko.miletic at gmail.com>'
'''
ciperchile.cl
'''
from calibre.web.feeds.news import BasicNewsRecipe
class CiperChile(BasicNewsRecipe):
title = 'CIPER Chile'
__author__ = 'Darko Miletic'
description = 'El Centro de Investigacion e Informacion Periodistica (CIPER) es una institucion independiente que desarrolla reportajes de investigacion de acuerdo a principios de maxima calidad e integridad profesional. Para lograr dicho objetivo, los profesionales de CIPER incorporan a las tecnicas propias del reporteo el uso sistematico de las leyes chilenas que norman el libre acceso a la informacion, de manera que los documentos que se obtengan por esta via esten puestos a disposicion del publico sin restricciones.'
publisher = 'CIPER'
category = 'news, politics, Chile'
oldest_article = 15
max_articles_per_feed = 200
no_stylesheets = True
encoding = 'utf8'
use_embedded_content = False
language = 'es_CL'
auto_cleanup = False
remove_empty_feeds = True
publication_type = 'blog'
masthead_url = 'http://ciperchile.cl/wp-content/themes/cipertheme/css/ui/ciper-logo.png'
extra_css = """
body{font-family: Arial,sans-serif}
.excerpt{font-family: Georgia,"Times New Roman",Times,serif; font-style: italic; font-size: 1.25em}
.author{font-family: Georgia,"Times New Roman",Times,serif; font-style: italic; font-size: small}
.date{font-family: Georgia,"Times New Roman",Times,serif; font-size: small; color: grey}
.epigrafe{font-size: small; color: grey}
img{margin-bottom: 0.4em; display:block}
"""
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
remove_tags = [
dict(name=['meta','link']),
dict(attrs={'class':['articleSharingTools','articleNav']})
]
remove_attributes=['lang']
remove_tags_before=dict(name='p', attrs={'class':'epigrafe'})
remove_tags_after=dict(name='div', attrs={'class':'articleBody'})
keep_only_tags = [dict(name='div', attrs={'class':'articleElements'})]
feeds = [
(u'Opinion del lector', u'http://ciperchile.cl/category/opinion-del-lector/feed/')
,(u'Reportajes de investigacion', u'http://ciperchile.cl/category/reportajes-de-investigacion/feed/')
,(u'Actualidad y Entrevistas', u'http://ciperchile.cl/category/actualidad-y-entrevistas/feed/')
,(u'Opinion', u'http://ciperchile.cl/category/opinion/feed/')
,(u'Accesso a la informacion', u'http://ciperchile.cl/category/acceso-a-la-informacion/feed/')
,(u'Libros', u'http://ciperchile.cl/category/libros/feed/')
,(u'Blog', u'http://ciperchile.cl/category/blog/feed/')
]

29
recipes/coding_horror.recipe
View File

@ -1,7 +1,5 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2009-2012, Darko Miletic <darko.miletic at gmail.com>'
'''
www.codinghorror.com/blog/
'''
@ -14,28 +12,25 @@ class CodingHorror(BasicNewsRecipe):
description = 'programming and human factors - Jeff Atwood'
category = 'blog, programming'
publisher = 'Jeff Atwood'
language = 'en'
author = 'Jeff Atwood'
language = 'en'
oldest_article = 30
max_articles_per_feed = 100
no_stylesheets = True
use_embedded_content = True
encoding = 'cp1252'
encoding = 'utf8'
auto_cleanup = True
html2lrf_options = [
'--comment' , description
, '--category' , category
, '--publisher', publisher
, '--author' , author
]
html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nauthors="' + author + '"'
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher': publisher
, 'language' : language
, 'authors' : publisher
}
remove_tags = [
dict(name=['object','link'])
,dict(name='div',attrs={'class':'feedflare'})
]
feeds = [(u'Articles', u'http://feeds2.feedburner.com/codinghorror' )]
feeds = [(u'Articles', u'http://feeds2.feedburner.com/codinghorror' )]

12
recipes/countryfile.recipe
View File

@ -1,12 +1,11 @@
from calibre import browser
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1325006965(BasicNewsRecipe):
title = u'Countryfile.com'
#cover_url = 'http://www.countryfile.com/sites/default/files/imagecache/160px_wide/cover/2_1.jpg'
__author__ = 'Dave Asbury'
description = 'The official website of Countryfile Magazine'
# last updated 15/4/12
# last updated 9/9//12
language = 'en_GB'
oldest_article = 30
max_articles_per_feed = 25
@ -17,13 +16,14 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
def get_cover_url(self):
soup = self.index_to_soup('http://www.countryfile.com/')
cov = soup.find(attrs={'class' : 'imagecache imagecache-160px_wide imagecache-linked imagecache-160px_wide_linked'})
#print '******** ',cov,' ***'
print '******** ',cov,' ***'
cov2 = str(cov)
cov2=cov2[124:-90]
#print '******** ',cov2,' ***'
cov2=cov2[140:223]
print '******** ',cov2,' ***'
#cov2='http://www.countryfile.com/sites/default/files/imagecache/160px_wide/cover/1b_0.jpg'
# try to get cover - if can't get known cover
br = browser()
br.set_handle_redirect(False)
try:
br.open_novisit(cov2)

59
recipes/cumhuriyet.recipe Normal file
View File

@ -0,0 +1,59 @@
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import unicode_literals
# based on a recipe by Darko Miletic
#
# Cumhuriyet Gazetesi'nin köşe yazıları okuyuculara cumhuriyet.com.tr
# adresi üzerinden ücretsiz olarak sunulmaktadır.
# Calibre yazılımıyla kullanılabilen bu reçete Cumhuriyet Gazetesi'nin
# günlük köşe yazılarını hızlıca derleyip e-okuyucunuzda kolayca okunabilir
# hale getirir. Yazıların yayınlanma saati sabah olduğu için reçeteyi
# 7:00-24:00 arasında çizelgelemeniz gerekmektedir.
__license__ = 'GPL v3'
__copyright__ = '2012, Sethi Eksi <sethi.eksi at gmail.com>'
'''
cumhuriyet.com.tr
'''
from calibre.web.feeds.news import BasicNewsRecipe
class Cumhuriyet_tr(BasicNewsRecipe):
title = 'Cumhuriyet - Yazarlar'
__author__ = 'Cumhuriyet Gazetesi Yazarları'
description = 'Günlük Cumhuriyet Gazetesi Köşe Yazıları'
publisher = 'Cumhuriyet'
category = 'news, politics, Turkey'
oldest_article = 1
max_articles_per_feed = 150
no_stylesheets = True
encoding = 'cp1254'
use_embedded_content = False
masthead_url = 'http://www.cumhuriyet.com.tr/home/cumhuriyet/sablon2000/img/cumlogobeyaz1.gif'
language = 'tr'
extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
.article_description,body{font-family: Arial,Verdana,Helvetica,sans1,sans-serif}
"""
conversion_options = {
'comment' : description
, 'tags' : category
, 'publisher' : publisher
, 'language' : language
}
remove_tags = [dict(name=['embed','iframe','object','link','base'])]
remove_tags_before = dict(attrs={'class':'c565'})
remove_tags_after = dict(attrs={'class':'c565'})
feeds = [
(u'Yazarlar' , u'http://www.cumhuriyet.com.tr/?kn=5&xl=rss')
]
def print_version(self, url):
articleid = url.rpartition('hn=')[2]
return 'http://www.cumhuriyet.com.tr/?hn=' + articleid
def preprocess_html(self, soup):
return self.adeify_images(soup)

112
recipes/dilemaveche.recipe
View File

@ -1,71 +1,51 @@
# -*- coding: utf-8 -*-
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = u'2011, Silviu Cotoar\u0103'
'''
dilemaveche.ro
'''
from calibre.web.feeds.news import BasicNewsRecipe
class DilemaVeche(BasicNewsRecipe):
title = u'Dilema Veche' # apare vinerea, mai pe dupa-masa,depinde de Luiza cred (care se semneaza ca fiind creatorul fiecarui articol in feed-ul RSS)
__author__ = 'song2' # inspirat din scriptul pentru Le Monde. Inspired from the Le Monde script
description = '"Sint vechi, domnule!" (I.L. Caragiale)'
publisher = 'Adevarul Holding'
oldest_article = 7
max_articles_per_feed = 200
encoding = 'utf8'
language = 'ro'
masthead_url = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
publication_type = 'magazine'
feeds = [
('Editoriale si opinii - Situatiunea', 'http://www.dilemaveche.ro/taxonomy/term/37/0/feed'),
('Editoriale si opinii - Pe ce lume traim', 'http://www.dilemaveche.ro/taxonomy/term/38/0/feed'),
('Editoriale si opinii - Bordeie si obiceie', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'),
('Editoriale si opinii - Talc Show', 'http://www.dilemaveche.ro/taxonomy/term/44/0/feed'),
('Tema saptamanii', 'http://www.dilemaveche.ro/taxonomy/term/19/0/feed'),
('La zi in cultura - Dilema va recomanda', 'http://www.dilemaveche.ro/taxonomy/term/58/0/feed'),
('La zi in cultura - Carte', 'http://www.dilemaveche.ro/taxonomy/term/14/0/feed'),
('La zi in cultura - Film', 'http://www.dilemaveche.ro/taxonomy/term/13/0/feed'),
('La zi in cultura - Muzica', 'http://www.dilemaveche.ro/taxonomy/term/1341/0/feed'),
('La zi in cultura - Arte performative', 'http://www.dilemaveche.ro/taxonomy/term/1342/0/feed'),
('La zi in cultura - Arte vizuale', 'http://www.dilemaveche.ro/taxonomy/term/1512/0/feed'),
('Societate - Ieri cu vedere spre azi', 'http://www.dilemaveche.ro/taxonomy/term/15/0/feed'),
('Societate - Din polul opus', 'http://www.dilemaveche.ro/taxonomy/term/41/0/feed'),
('Societate - Mass comedia', 'http://www.dilemaveche.ro/taxonomy/term/43/0/feed'),
('Societate - La singular si la plural', 'http://www.dilemaveche.ro/taxonomy/term/42/0/feed'),
('Oameni si idei - Educatie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'),
('Oameni si idei - Polemici si dezbateri', 'http://www.dilemaveche.ro/taxonomy/term/48/0/feed'),
('Oameni si idei - Stiinta si tehnologie', 'http://www.dilemaveche.ro/taxonomy/term/46/0/feed'),
('Dileme on-line', 'http://www.dilemaveche.ro/taxonomy/term/005/0/feed')
]
remove_tags_before = dict(name='div',attrs={'class':'spacer_10'})
remove_tags = [
dict(name='div', attrs={'class':'art_related_left'}),
dict(name='div', attrs={'class':'controale'}),
dict(name='div', attrs={'class':'simple_overlay'}),
]
remove_tags_after = [dict(id='facebookLike')]
remove_javascript = True
title = u'Dilema Veche'
__author__ = u'Silviu Cotoar\u0103'
description = 'Sint vechi, domnule! (I.L. Caragiale)'
publisher = u'Adev\u0103rul Holding'
oldest_article = 5
language = 'ro'
max_articles_per_feed = 100
no_stylesheets = True
remove_empty_feeds = True
extra_css = """
body{font-family: Georgia,Times,serif }
img{margin-bottom: 0.4em; display:block}
"""
def get_cover_url(self):
cover_url = None
soup = self.index_to_soup('http://dilemaveche.ro')
link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'})
if link_item and link_item.a:
cover_url = link_item.a['href']
br = BasicNewsRecipe.get_browser()
try:
br.open(cover_url)
except: #daca nu gaseste pdf-ul
self.log("\nPDF indisponibil")
link_item = soup.find('div',attrs={'class':'box_dr_pdf_picture'})
if link_item and link_item.img:
cover_url = link_item.img['src']
br = BasicNewsRecipe.get_browser()
try:
br.open(cover_url)
except: #daca nu gaseste nici imaginea mica mica
print('Mama lor de nenorociti! nu este nici pdf nici imagine')
cover_url ='http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
return cover_url
cover_margins = (10, 15, '#ffffff')
use_embedded_content = False
category = 'Ziare'
encoding = 'utf-8'
cover_url = 'http://dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
conversion_options = {
'comments' : description
,'tags' : category
,'language' : language
,'publisher' : publisher
}
keep_only_tags = [
dict(name='div', attrs={'class':'c_left_column'})
]
remove_tags = [
dict(name='div', attrs={'id':['adshop_widget_428x60']}) ,
dict(name='div', attrs={'id':['gallery']})
]
remove_tags_after = [
dict(name='div', attrs={'id':['adshop_widget_428x60']})
]
feeds = [
(u'Feeds', u'http://dilemaveche.ro/rss.xml')
]
def preprocess_html(self, soup):
return self.adeify_images(soup)

52
recipes/doghousediaries.recipe Normal file
View File

@ -0,0 +1,52 @@
__license__ = 'GPL v3'
__copyright__ = '2010-2012, NiLuJe <niluje at ak-team.com>'
'''
Fetch DoghouseDiaries.
'''
import re
from calibre.web.feeds.news import BasicNewsRecipe
class DoghouseDiaries(BasicNewsRecipe):
title = 'Doghouse Diaries'
description = 'A webcomic.'
__author__ = 'NiLuJe'
language = 'en'
use_embedded_content = False
# 14 comics per fetch (not really days... but we can't easily get the date of individual comics, short of parsing each one...)
oldest_article = 14
cover_url = 'http://www.thedoghousediaries.com/logos/logo3.png'
masthead_url = 'http://www.thedoghousediaries.com/logos/logo3.png'
keep_only_tags = [dict(name='img', attrs={'class': re.compile("comic-item*")}), dict(name='h1'), dict(name='div', attrs={'class':'entry'}), dict(name='p', id='alttext')]
remove_tags = [dict(name='div', attrs={'class':'pin-it-btn-wrapper'}), dict(name='span'), dict(name='div', id='wp_fb_like_button')]
remove_attributes = ['width', 'height']
no_stylesheets = True
# Turn image bubblehelp into a paragraph (NOTE: We run before the remove_tags cleanup, so we need to make sure we only parse the comic-item img, not the pinterest one pulled by the entry div)
preprocess_regexps = [
(re.compile(r'(<img.*src="http://thedoghousediaries.com/comics/.*title=")([^"]+)(".*>)'),
lambda m: '%s%s<p id="alttext"><strong>%s</strong></p>' % (m.group(1), m.group(3), m.group(2)))
]
def parse_index(self):
INDEX = 'http://www.thedoghousediaries.com/'
soup = self.index_to_soup(INDEX)
articles = []
# Since the feed sucks, and there's no real archive, we use the 'Quick Archive' thingie, but we can't get the date from here, so stop after 14 comics...
for item in soup.findAll('option', {}, True, None, self.oldest_article+1):
# Skip the quick archive itself
if ( item['value'] != '0' ):
articles.append({
'title': self.tag_to_string(item).encode('UTF-8'),
'url': item['value'],
'description': '',
'content': '',
})
return [('Doghouse Diaries', articles)]

15
recipes/financial_times_uk.recipe
View File

@ -10,7 +10,7 @@ from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
class FinancialTimes(BasicNewsRecipe):
title = 'Financial Times - UK printed edition'
title = 'Financial Times (UK)'
__author__ = 'Darko Miletic'
description = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy."
publisher = 'The Financial Times Ltd.'
@ -101,17 +101,19 @@ class FinancialTimes(BasicNewsRecipe):
def parse_index(self):
feeds = []
soup = self.index_to_soup(self.INDEX)
dates= self.tag_to_string(soup.find('div', attrs={'class':'btm-links'}).find('div'))
self.timefmt = ' [%s]'%dates
wide = soup.find('div',attrs={'class':'wide'})
if not wide:
return feeds
strest = wide.findAll('h3', attrs={'class':'section'})
if not strest:
return feeds
st = wide.find('h4',attrs={'class':'section-no-arrow'})
st = wide.findAll('h4',attrs={'class':'section-no-arrow'})
if st:
strest.insert(0,st)
st.extend(strest)
count = 0
for item in strest:
for item in st:
count = count + 1
if self.test and count > 2:
return feeds
@ -151,7 +153,7 @@ class FinancialTimes(BasicNewsRecipe):
def get_cover_url(self):
cdate = datetime.date.today()
if cdate.isoweekday() == 7:
cdate -= datetime.timedelta(days=1)
cdate -= datetime.timedelta(days=1)
return cdate.strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_LON.pdf')
def get_obfuscated_article(self, url):
@ -163,9 +165,8 @@ class FinancialTimes(BasicNewsRecipe):
count = 10
except:
print "Retrying download..."
count += 1
count += 1
self.temp_files.append(PersistentTemporaryFile('_fa.html'))
self.temp_files[-1].write(html)
self.temp_files[-1].close()
return self.temp_files[-1].name

87
recipes/history_today.recipe Normal file
View File

@ -0,0 +1,87 @@
import re
from calibre.web.feeds.recipes import BasicNewsRecipe
from collections import OrderedDict
class HistoryToday(BasicNewsRecipe):
title = 'History Today'
__author__ = 'Rick Shang'
description = 'UK-based magazine, publishing articles and book reviews covering all types and periods of history.'
language = 'en'
category = 'news'
encoding = 'UTF-8'
remove_tags = [dict(name='div',attrs={'class':['print-logo','print-site_name','print-breadcrumb']}),
dict(name='div', attrs={'id':['ht-tools','ht-tools2','ht-tags']})]
no_javascript = True
no_stylesheets = True
needs_subscription = True
def get_browser(self):
br = BasicNewsRecipe.get_browser()
if self.username is not None and self.password is not None:
br.open('http://www.historytoday.com/user/login')
br.select_form(nr=1)
br['name'] = self.username
br['pass'] = self.password
res = br.submit()
raw = res.read()
if 'Session limit exceeded' in raw:
br.select_form(nr=1)
control=br.find_control('sid').items[1]
sid = []
br['sid']=sid.join(control)
br.submit()
return br
def parse_index(self):
#Find date
soup0 = self.index_to_soup('http://www.historytoday.com/')
dates = self.tag_to_string(soup0.find('div',attrs={'id':'block-block-226'}).span)
self.timefmt = u' [%s]'%dates
#Go to issue
soup = self.index_to_soup('http://www.historytoday.com/contents')
cover = soup.find('div',attrs={'id':'content-area'}).find('img')['src']
self.cover_url=cover
#Go to the main body
div = soup.find ('div', attrs={'class':'region region-content-bottom'})
feeds = OrderedDict()
section_title = ''
for section in div.findAll('div', attrs={'id':re.compile("block\-views\-contents.*")}):
section_title = self.tag_to_string(section.find('h2',attrs={'class':'title'}))
sectionbody=section.find('div', attrs={'class':'view-content'})
for article in sectionbody.findAll('div',attrs={'class':re.compile("views\-row.*")}):
articles = []
subarticle = []
subarticle = article.findAll('div')
if len(subarticle) < 2:
continue
title=self.tag_to_string(subarticle[0])
originalurl="http://www.historytoday.com" + subarticle[0].span.a['href'].strip()
originalpage=self.index_to_soup(originalurl)
printurl=originalpage.find('div',attrs = {'id':'ht-tools'}).a['href'].strip()
url="http://www.historytoday.com" + printurl
desc=self.tag_to_string(subarticle[1])
articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
if articles:
if section_title not in feeds:
feeds[section_title] = []
feeds[section_title] += articles
ans = [(key, val) for key, val in feeds.iteritems()]
return ans
def cleanup(self):
self.browser.open('http://www.historytoday.com/logout')

10
recipes/houston_chronicle.recipe
View File

@ -7,17 +7,18 @@ class HoustonChronicle(BasicNewsRecipe):
title = u'The Houston Chronicle'
description = 'News from Houston, Texas'
__author__ = 'Kovid Goyal'
__author__ = 'Kovid Goyal'
language = 'en'
timefmt = ' [%a, %d %b, %Y]'
no_stylesheets = True
use_embedded_content = False
remove_attributes = ['style']
auto_cleanup = True
oldest_article = 2.0
oldest_article = 3.0
keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or
'hst-articletext' in x or 'hst-galleryitem' in x)}
#keep_only_tags = {'class':lambda x: x and ('hst-articletitle' in x or
#'hst-articletext' in x or 'hst-galleryitem' in x)}
remove_attributes = ['xmlns']
feeds = [
@ -37,3 +38,4 @@ class HoustonChronicle(BasicNewsRecipe):
]

BIN
recipes/icons/ciperchile.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.5 KiB

BIN
recipes/icons/stamgasten.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 639 B

2
recipes/le_monde.recipe
View File

@ -9,7 +9,7 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
class LeMonde(BasicNewsRecipe):
title = 'Le Monde'
__author__ = 'veezh'
description = 'Actualités'
description = u'Actualités'
oldest_article = 1
max_articles_per_feed = 100
no_stylesheets = True

132
recipes/le_monde_sub.recipe Normal file
View File

@ -0,0 +1,132 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2012, Rémi Vanicat <vanicat at debian.org>'
'''
Lemonde.fr: Version abonnée
'''
import os, zipfile, re, time
from calibre import strftime
from calibre.web.feeds.news import BasicNewsRecipe
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ptempfile import PersistentTemporaryFile
class LeMondeAbonne(BasicNewsRecipe):
title = u'Le Monde: Édition abonnés'
__author__ = u'Rémi Vanicat'
description = u'Actualités'
category = u'Actualités, France, Monde'
language = 'fr'
needs_subscription = True
no_stylesheets = True
extra_css = u'''
h1{font-size:130%;}
.ariane{font-size:xx-small;}
.source{font-size:xx-small;}
.href{font-size:xx-small;}
.LM_caption{color:#666666; font-size:x-small;}
.main-article-info{font-family:Arial,Helvetica,sans-serif;}
#full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
#match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
'''
zipurl_format = 'http://medias.lemonde.fr/abonnes/editionelectronique/%Y%m%d/html/%y%m%d.zip'
coverurl_format = '/img/%y%m%d01.jpg'
path_format = "%y%m%d"
login_url = 'http://www.lemonde.fr/web/journal_electronique/identification/1,56-0,45-0,0.html'
keep_only_tags = [ dict(name="div", attrs={ 'class': 'po-prti' }), dict(name=['h1']), dict(name='div', attrs={ 'class': 'photo' }), dict(name='div', attrs={ 'class': 'po-ti2' }), dict(name='div', attrs={ 'class': 'ar-txt' }), dict(name='div', attrs={ 'class': 'po_rtcol' }) ]
article_id_pattern = re.compile("[0-9]+\\.html")
article_url_format = 'http://www.lemonde.fr/journalelectronique/donnees/protege/%Y%m%d/html/'
def get_browser(self):
br = BasicNewsRecipe.get_browser()
if self.username is not None and self.password is not None:
br.open(self.login_url)
br.select_form(nr=0)
br['login'] = self.username
br['password'] = self.password
br.submit()
return br
decalage = 24 * 60 * 60 # today Monde has tomorow date
def get_cover_url(self):
url = time.strftime(self.coverurl_format, self.ltime)
return self.articles_path + url
def parse_index(self):
browser = self.get_browser()
second = time.time()
second += self.decalage
ltime = self.ltime = time.gmtime(second)
url = time.strftime(self.zipurl_format, ltime)
self.timefmt=strftime(" %A %d %B %Y", ltime)
response = browser.open(url)
tmp = PersistentTemporaryFile(suffix='.zip')
self.report_progress(0.1,_('downloading zip file'))
tmp.write(response.read())
tmp.close()
zfile = zipfile.ZipFile(tmp.name, 'r')
self.report_progress(0.1,_('extracting zip file'))
zfile.extractall(self.output_dir)
zfile.close()
path = os.path.join(self.output_dir, time.strftime(self.path_format, ltime), "data")
self.articles_path = path
files = os.listdir(path)
nb_index_files = len([ name for name in files if re.match("frame_gauche_[0-9]+.html", name) ])
flux = []
article_url = time.strftime(self.article_url_format, ltime)
for i in range(nb_index_files):
filename = os.path.join(path, "selection_%d.html" % (i + 1))
tmp = open(filename,'r')
soup=BeautifulSoup(tmp)
title=soup.find('span').contents[0]
tmp.close()
filename = os.path.join(path, "frame_gauche_%d.html" % (i + 1))
tmp = open(filename,'r')
soup = BeautifulSoup(tmp)
articles = []
for link in soup.findAll("a"):
article_file = link['href']
article_id=self.article_id_pattern.search(article_file).group()
article = {
'title': link.contents[0],
'url': article_url + article_id,
'descripion': '',
'content': ''
}
articles.append(article)
tmp.close()
flux.append((title, articles))
return flux
# Local Variables:
# mode: python
# End:

25
recipes/livemint.recipe
View File

@ -7,20 +7,29 @@ class LiveMint(BasicNewsRecipe):
#encoding = 'cp1252'
oldest_article = 1 #days
max_articles_per_feed = 25
use_embedded_content = True
use_embedded_content = False
no_stylesheets = True
auto_cleanup = True
feeds = [
('Latest News',
'http://www.livemint.com/StoryRss.aspx?LN=Latestnews'),
('Gallery',
'http://www.livemint.com/GalleryRssfeed.aspx'),
('Companies',
'http://www.livemint.com/rss/companies'),
('Consumer',
'http://www.livemint.com/rss/consumer'),
('Top Stories',
'http://www.livemint.com/StoryRss.aspx?ts=Topstories'),
('Banking',
'http://www.livemint.com/StoryRss.aspx?Id=104'),
'http://www.livemint.com/rss/homepage'),
('Opinion',
'http://www.livemint.com/rss/opinion'),
('Money',
'http://www.livemint.com/rss/money'),
('Industry',
'http://www.livemint.com/rss/industry'),
('Economy Politics',
'http://www.livemint.com/rss/economy_politics'),
('Lounge',
'http://www.livemint.com/rss/lounge'),
]

27
recipes/metro_uk.recipe
View File

@ -1,10 +1,10 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1306097511(BasicNewsRecipe):
title = u'Metro UK'
description = 'Author Dave Asbury : News as provide by The Metro -UK'
description = 'Author Dave Asbury : News from The Metro - UK'
#timefmt = ''
__author__ = 'Dave Asbury'
#last update 4/8/12
#last update 9/9/12
cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg'
no_stylesheets = True
oldest_article = 1
@ -17,23 +17,24 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
language = 'en_GB'
masthead_url = 'http://e-edition.metro.co.uk/images/metro_logo.gif'
extra_css = '''
h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:1.6em;}
h1{font-family:Arial,Helvetica,sans-serif; font-weight:900;font-size:1.6em;}
h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:1.2em;}
p{font-family:Arial,Helvetica,sans-serif;font-size:1.0em;}
body{font-family:Helvetica,Arial,sans-serif;font-size:1.0em;}
'''
'''
keep_only_tags = [
#dict(name='h1'),
#dict(name='h2'),
#dict(name='div', attrs={'class' : ['row','article','img-cnt figure','clrd']})
#dict(name='h3'),
#dict(attrs={'class' : 'BText'}),
]
#dict(name='h1'),
#dict(name='h2'),
#dict(name='div', attrs={'class' : ['row','article','img-cnt figure','clrd']})
#dict(name='h3'),
#dict(attrs={'class' : 'BText'}),
]
remove_tags = [
dict(name='div',attrs={'class' : 'art-fd fd-gr1-b clrd'}),
dict(name='span',attrs={'class' : 'share'}),
dict(name='li'),
dict(attrs={'class' : ['twitter-share-button','header-forms','hdr-lnks','close','art-rgt','fd-gr1-b clrd google-article','news m12 clrd clr-b p5t shareBtm','item-ds csl-3-img news','c-1of3 c-last','c-1of1','pd','item-ds csl-3-img sport']}),
dict(attrs={'id' : ['','sky-left','sky-right','ftr-nav','and-ftr','notificationList','logo','miniLogo','comments-news','metro_extras']})
dict(name='li'),
dict(attrs={'class' : ['twitter-share-button','header-forms','hdr-lnks','close','art-rgt','fd-gr1-b clrd google-article','news m12 clrd clr-b p5t shareBtm','item-ds csl-3-img news','c-1of3 c-last','c-1of1','pd','item-ds csl-3-img sport']}),
dict(attrs={'id' : ['','sky-left','sky-right','ftr-nav','and-ftr','notificationList','logo','miniLogo','comments-news','metro_extras']})
]
remove_tags_before = dict(name='h1')
#remove_tags_after = dict(attrs={'id':['topic-buttons']})

8
recipes/scmp.recipe
View File

@ -39,10 +39,10 @@ class SCMP(BasicNewsRecipe):
#br.set_debug_responses(True)
#br.set_debug_redirects(True)
if self.username is not None and self.password is not None:
br.open('http://www.scmp.com/portal/site/SCMP/')
br.select_form(name='loginForm')
br['Login' ] = self.username
br['Password'] = self.password
br.open('http://www.scmp.com/')
br.select_form(nr=1)
br['name'] = self.username
br['pass'] = self.password
br.submit()
return br

16
recipes/stamgasten.recipe Normal file
View File

@ -0,0 +1,16 @@
class AdvancedUserRecipe1347706704(BasicNewsRecipe):
title = u'Stamgasten'
__author__ = u'DrMerry'
description = u'Stamgasten de populaire strip van Toon van Driel (http://www.toonvandriel.nl)'
language = u'nl'
oldest_article = 7
max_articles_per_feed = 100
auto_cleanup = False
cover_url = 'http://shop.toonvandriel.nl/productimg.php?type=canvas&id=15&size=large'
no_stylesheets = True
remove_javascript = True
remove_empty_feeds = True
remove_tags_before = dict(id='title')
remove_tags_after = dict(attrs={'class':'entry-content rich-content'})
feeds = [(u'Stamgasten', u'http://toonvandriel.nl/feed/')]

11
recipes/sunday_times.recipe
View File

@ -1,6 +1,6 @@
__license__ = 'GPL v3'
__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2010-2012, Darko Miletic <darko.miletic at gmail.com>'
'''
www.thesundaytimes.co.uk
'''
@ -43,13 +43,14 @@ class TimesOnline(BasicNewsRecipe):
def get_browser(self):
br = BasicNewsRecipe.get_browser()
br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thesundaytimes.co.uk/sto/')
br.open('http://www.thesundaytimes.co.uk/sto/')
if self.username is not None and self.password is not None:
data = urllib.urlencode({ 'userName':self.username
data = urllib.urlencode({
'gotoUrl' :self.INDEX
,'username':self.username
,'password':self.password
,'keepMeLoggedIn':'false'
})
br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
br.open('https://acs.thetimes.co.uk/user/login',data)
return br
remove_tags = [

54
recipes/time_magazine.recipe
View File

@ -2,6 +2,7 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
'''
time.com
'''
@ -11,28 +12,23 @@ from calibre.web.feeds.news import BasicNewsRecipe
from lxml import html
class Time(BasicNewsRecipe):
#recipe_disabled = ('This recipe has been disabled as TIME no longer'
# ' publish complete articles on the web.')
title = u'Time'
__author__ = 'Kovid Goyal'
__author__ = 'Kovid Goyal, Rick Shang'
description = ('Weekly US magazine.')
encoding = 'utf-8'
no_stylesheets = True
language = 'en'
remove_javascript = True
#needs_subscription = 'optional'
needs_subscription = True
keep_only_tags = [
{
'class':['artHd', 'articleContent',
'entry-title','entry-meta', 'entry-content', 'thumbnail']
'class':['tout1', 'entry-content', 'external-gallery-img', 'image-meta']
},
]
remove_tags = [
{'class':['content-tools', 'quigo', 'see',
'first-tier-social-tools', 'navigation', 'enlarge lightbox']},
{'id':['share-tools']},
{'rel':'lightbox'},
{'class':['thumbnail', 'button']},
]
recursions = 10
@ -43,17 +39,25 @@ class Time(BasicNewsRecipe):
def get_browser(self):
br = BasicNewsRecipe.get_browser(self)
if False and self.username and self.password:
# This site uses javascript in its login process
res = br.open('http://www.time.com/time/magazine')
br.select_form(nr=1)
br['username'] = self.username
# This site uses javascript in its login process
if self.username is not None and self.password is not None:
br.open('http://www.time.com/time/magazine')
br.select_form(predicate=lambda f: 'action' in f.attrs and f.attrs['action'] == 'https://auth.time.com/login.php')
br['username'] = self.username
br['password'] = self.password
res = br.submit()
raw = res.read()
if '>Log Out<' not in raw:
br['magcode'] = ['TD']
br.find_control('turl').readonly = False
br['turl'] = 'http://www.time.com/time/magazine'
br.find_control('rurl').readonly = False
br['rurl'] = 'http://www.time.com/time/magazine'
br['remember'] = False
raw = br.submit().read()
if False and '>Log Out<' not in raw:
# This check is disabled as it does not work (there is probably
# some cookie missing) however, the login is "sufficient" for
# the actual article downloads to work.
raise ValueError('Failed to login to time.com, check'
' your username and password')
' your username and password')
return br
def parse_index(self):
@ -70,6 +74,9 @@ class Time(BasicNewsRecipe):
except:
self.log.exception('Failed to fetch cover')
dates = ''.join(root.xpath('//time[@class="updated"]/text()'))
if dates:
self.timefmt = ' [%s]'%dates
feeds = []
parent = root.xpath('//div[@class="content-main-aside"]')[0]
@ -96,7 +103,8 @@ class Time(BasicNewsRecipe):
title = html.tostring(a[0], encoding=unicode,
method='text').strip()
if not title: continue
url = a[0].get('href')
url = a[0].get('href')
url = re.sub('/magazine/article/0,9171','/subscriber/printout/0,8816', url)
if url.startswith('/'):
url = 'http://www.time.com'+url
desc = ''
@ -111,9 +119,3 @@ class Time(BasicNewsRecipe):
'date' : '',
'description' : desc
}
def postprocess_html(self,soup,first):
for tag in soup.findAll(attrs ={'class':['artPag','pagination']}):
tag.extract()
return soup

18
recipes/times_online.recipe
View File

@ -1,6 +1,6 @@
__license__ = 'GPL v3'
__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
__copyright__ = '2009-2012, Darko Miletic <darko.miletic at gmail.com>'
'''
www.thetimes.co.uk
'''
@ -21,6 +21,7 @@ class TimesOnline(BasicNewsRecipe):
encoding = 'utf-8'
delay = 1
needs_subscription = True
auto_cleanup = False
publication_type = 'newspaper'
masthead_url = 'http://www.thetimes.co.uk/tto/public/img/the_times_460.gif'
INDEX = 'http://www.thetimes.co.uk'
@ -41,13 +42,14 @@ class TimesOnline(BasicNewsRecipe):
def get_browser(self):
br = BasicNewsRecipe.get_browser()
br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thetimes.co.uk/tto/news/?lightbox=false')
br.open('http://www.thetimes.co.uk/tto/news/')
if self.username is not None and self.password is not None:
data = urllib.urlencode({ 'userName':self.username
data = urllib.urlencode({
'gotoUrl' :self.INDEX
,'username':self.username
,'password':self.password
,'keepMeLoggedIn':'false'
})
br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
br.open('https://acs.thetimes.co.uk/user/login',data)
return br
remove_tags = [
@ -58,6 +60,7 @@ class TimesOnline(BasicNewsRecipe):
keep_only_tags = [
dict(attrs={'class':'heading' })
,dict(attrs={'class':'f-author'})
,dict(attrs={'class':['media','byline-timestamp']})
,dict(attrs={'id':'bodycopy'})
]
@ -79,11 +82,6 @@ class TimesOnline(BasicNewsRecipe):
,(u'Arts' , PREFIX + u'arts/?view=list' )
]
def preprocess_html(self, soup):
for item in soup.findAll(style=True):
del item['style']
return self.adeify_images(soup)
def parse_index(self):
totalfeeds = []
lfeeds = self.get_feeds()

8
recipes/timesnewroman.recipe
View File

@ -36,12 +36,14 @@ class TimesNewRoman(BasicNewsRecipe):
remove_tags = [
dict(name='p', attrs={'class':['articleinfo']})
, dict(name='div',attrs={'class':['vergefacebooklike']})
, dict(name='div', attrs={'class':'cleared'})
, dict(name='div', attrs={'class':['shareTools']})
, dict(name='div', attrs={'class':'fb_iframe_widget'})
, dict(name='div', attrs={'id':'jc'})
]
remove_tags_after = [
dict(name='div', attrs={'class':'cleared'})
dict(name='div', attrs={'class':'fb_iframe_widget'}),
dict(name='div', attrs={'id':'jc'})
]
feeds = [

36
recipes/variety.recipe
View File

@ -19,7 +19,13 @@ class Variety(BasicNewsRecipe):
category = 'Entertainment Industry News, Daily Variety, Movie Reviews, TV, Awards, Oscars, Cannes, Box Office, Hollywood'
language = 'en'
masthead_url = 'http://images1.variety.com/graphics/variety/Variety_logo_green_tm.gif'
extra_css = ' body{font-family: Georgia,"Times New Roman",Times,Courier,serif } img{margin-bottom: 1em} '
extra_css = """
body{font-family: Arial,Helvetica,sans-serif; font-size: 1.275em}
.date{font-size: small; border: 1px dotted rgb(204, 204, 204); font-style: italic; color: rgb(102, 102, 102); margin: 5px 0px; padding: 0.5em;}
.author{margin: 5px 0px 5px 20px; padding: 0.5em; background: none repeat scroll 0% 0% rgb(247, 247, 247);}
.art h2{color: rgb(153, 0, 0); font-size: 1.275em; font-weight: bold;}
img{margin-bottom: 1em}
"""
conversion_options = {
'comments' : description
@ -29,7 +35,7 @@ class Variety(BasicNewsRecipe):
}
remove_tags = [dict(name=['object','link','map'])]
remove_attributes=['lang','vspace','hspace','xmlns:ms','xmlns:dt']
keep_only_tags = [dict(name='div', attrs={'class':'art control'})]
feeds = [(u'News & Articles', u'http://feeds.feedburner.com/variety/headlines' )]
@ -37,3 +43,29 @@ class Variety(BasicNewsRecipe):
def print_version(self, url):
rpt = url.rpartition('.html')[0]
return rpt + '?printerfriendly=true'
def preprocess_raw_html(self, raw, url):
return '<html><head>'+raw[raw.find('</head>'):]
def get_article_url(self, article):
url = BasicNewsRecipe.get_article_url(self, article)
return url.rpartition('?')[0]
def preprocess_html(self, soup):
for item in soup.findAll('a'):
limg = item.find('img')
if item.string is not None:
str = item.string
item.replaceWith(str)
else:
if limg:
item.name = 'div'
item.attrs = []
else:
str = self.tag_to_string(item)
item.replaceWith(str)
for item in soup.findAll('img'):
if not item.has_key('alt'):
item['alt'] = 'image'
return soup

22
recipes/volksrant.recipe
View File

@ -73,14 +73,20 @@ class AdvancedUserRecipe1249039563(BasicNewsRecipe):
Change Log:
Date: 10/15/2010
Feeds updated by Martin Tarenskeen
Date: 09/09/2012
Feeds updated by Eric Lammerts
'''
feeds = [
(u'Laatste Nieuws', u'http://www.volkskrant.nl/rss/laatstenieuws.rss'),
(u'Binnenland', u'http://www.volkskrant.nl/rss/nederland.rss'),
(u'Buitenland', u'http://www.volkskrant.nl/rss/internationaal.rss'),
(u'Economie', u'http://www.volkskrant.nl/rss/economie.rss'),
(u'Sport', u'http://www.volkskrant.nl/rss/sport.rss'),
(u'Cultuur', u'http://www.volkskrant.nl/rss/kunst.rss'),
(u'Gezondheid & Wetenschap', u'http://www.volkskrant.nl/rss/wetenschap.rss'),
(u'Internet & Media', u'http://www.volkskrant.nl/rss/media.rss') ]
(u'Nieuws', u'http://www.volkskrant.nl/nieuws/rss.xml'),
(u'Binnenland', u'http://www.volkskrant.nl/nieuws/binnenland/rss.xml'),
(u'Buitenland', u'http://www.volkskrant.nl/buitenland/rss.xml'),
(u'Economie', u'http://www.volkskrant.nl/nieuws/economie/rss.xml'),
(u'Politiek', u'http://www.volkskrant.nl/politiek/rss.xml'),
(u'Sport', u'http://www.volkskrant.nl/sport/rss.xml'),
(u'Cultuur', u'http://www.volkskrant.nl/nieuws/cultuur/rss.xml'),
(u'Gezondheid & wetenschap', u'http://www.volkskrant.nl/nieuws/gezondheid--wetenschap/rss.xml'),
(u'Tech & Media', u'http://www.volkskrant.nl/tech-media/rss.xml'),
(u'Reizen', u'http://www.volkskrant.nl/nieuws/reizen/rss.xml'),
(u'Opinie', u'http://www.volkskrant.nl/opinie/rss.xml'),
(u'Opmerkelijk', u'http://www.volkskrant.nl/nieuws/opmerkelijk/rss.xml') ]

19
recipes/wsj.recipe
View File

@ -8,6 +8,12 @@ import copy
# http://online.wsj.com/page/us_in_todays_paper.html
def filter_classes(x):
if not x: return False
bad_classes = {'sTools', 'printSummary', 'mostPopular', 'relatedCollection'}
classes = frozenset(x.split())
return len(bad_classes.intersection(classes)) > 0
class WallStreetJournal(BasicNewsRecipe):
title = 'The Wall Street Journal'
@ -35,10 +41,17 @@ class WallStreetJournal(BasicNewsRecipe):
remove_tags_before = dict(name='h1')
remove_tags = [
dict(id=["articleTabs_tab_article", "articleTabs_tab_comments", "articleTabs_tab_interactive","articleTabs_tab_video","articleTabs_tab_map","articleTabs_tab_slideshow","articleTabs_tab_quotes","articleTabs_tab_document"]),
dict(id=["articleTabs_tab_article",
"articleTabs_tab_comments",
'articleTabs_panel_comments', 'footer',
"articleTabs_tab_interactive", "articleTabs_tab_video",
"articleTabs_tab_map", "articleTabs_tab_slideshow",
"articleTabs_tab_quotes", "articleTabs_tab_document",
"printModeAd", "aFbLikeAuth", "videoModule",
"mostRecommendations", "topDiscussions"]),
{'class':['footer_columns','network','insetCol3wide','interactive','video','slideshow','map','insettip','insetClose','more_in', "insetContent", 'articleTools_bottom', 'aTools', "tooltip", "adSummary", "nav-inline"]},
dict(rel='shortcut icon'),
{'class':lambda x: x and 'sTools' in x},
{'class':filter_classes},
]
remove_tags_after = [dict(id="article_story_body"), {'class':"article story"},]
@ -52,7 +65,7 @@ class WallStreetJournal(BasicNewsRecipe):
br['password'] = self.password
res = br.submit()
raw = res.read()
if 'Welcome,' not in raw and '>Logout<' not in raw:
if 'Welcome,' not in raw and '>Logout<' not in raw and '>Log Out<' not in raw:
raise ValueError('Failed to log in to wsj.com, check your '
'username and password')
return br

16
recipes/xkcd.recipe
View File

@ -2,6 +2,8 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
'''
Changelog:
2012-04-06
Fixed empty articles, added masthead img (NiLuJe)
2011-09-24
Changed cover (drMerry)
'''
@ -13,7 +15,8 @@ import time, re
from calibre.web.feeds.news import BasicNewsRecipe
class XkcdCom(BasicNewsRecipe):
cover_url = 'http://imgs.xkcd.com/s/9be30a7.png'
cover_url = 'http://imgs.xkcd.com/static/terrible_small_logo.png'
masthead_url = 'http://imgs.xkcd.com/static/terrible_small_logo.png'
title = 'xkcd'
description = 'A webcomic of romance and math humor.'
__author__ = 'Martin Pitt updated by DrMerry.'
@ -21,13 +24,14 @@ class XkcdCom(BasicNewsRecipe):
use_embedded_content = False
oldest_article = 60
keep_only_tags = [dict(id='middleContainer')]
remove_tags = [dict(name='ul'), dict(name='h3'), dict(name='br')]
#keep_only_tags = [dict(id='middleContainer')]
#remove_tags = [dict(name='ul'), dict(name='h3'), dict(name='br')]
keep_only_tags = [dict(id='comic')]
no_stylesheets = True
# turn image bubblehelp into a paragraph
# turn image bubblehelp into a paragraph, and put alt in a heading
preprocess_regexps = [
(re.compile(r'(<img.*title=")([^"]+)(".*>)'),
lambda m: '%s%s<p>%s</p>' % (m.group(1), m.group(3), m.group(2)))
(re.compile(r'(<img.*title=")([^"]+)(".alt=")([^"]+)(".*>)'),
lambda m: '<h1>%s</h1>%s%s%s<p>%s</p>' % (m.group(4), m.group(1), m.group(3), m.group(5), m.group(2)))
]
def parse_index(self):

8
recipes/zeitde_sub.recipe
View File

@ -118,13 +118,13 @@ class ZeitEPUBAbo(BasicNewsRecipe):
def build_index(self):
domain = "https://premium.zeit.de"
url = domain + "/abo/zeit_digital"
url = domain + "/abo/digitalpaket"
browser = self.get_browser()
# new login process
response = browser.open(url)
# Get rid of nested form
response.set_data(response.get_data().replace('<div><form action="/abo/zeit_digital?destination=node%2F94" accept-charset="UTF-8" method="post" id="user-login-form" class="zol_inlinelabel">', ''))
response.set_data(response.get_data().replace('<div><form action="/abo/digitalpaket?destination=node%2F94" accept-charset="UTF-8" method="post" id="user-login-form" class="zol_inlinelabel">', ''))
browser.set_response(response)
browser.select_form(nr=2)
browser.form['name']=self.username
@ -177,13 +177,13 @@ class ZeitEPUBAbo(BasicNewsRecipe):
try:
self.log.warning('Trying PDF-based cover')
domain = "https://premium.zeit.de"
url = domain + "/abo/zeit_digital"
url = domain + "/abo/digitalpaket"
browser = self.get_browser()
# new login process
response=browser.open(url)
# Get rid of nested form
response.set_data(response.get_data().replace('<div><form action="/abo/zeit_digital?destination=node%2F94" accept-charset="UTF-8" method="post" id="user-login-form" class="zol_inlinelabel">', ''))
response.set_data(response.get_data().replace('<div><form action="/abo/digitalpaket?destination=node%2F94" accept-charset="UTF-8" method="post" id="user-login-form" class="zol_inlinelabel">', ''))
browser.set_response(response)
browser.select_form(nr=2)

BIN
resources/compiled_coffeescript.zip
View File

Binary file not shown.

BIN
resources/images/mimetypes/azw2.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.1 KiB

BIN
resources/images/mimetypes/azw3.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.6 KiB

BIN
resources/images/mimetypes/mobi.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 7.3 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

BIN
resources/images/mimetypes/tpz.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 8.2 KiB

1207
resources/mime.types
View File

File diff suppressed because it is too large Load Diff

4
resources/templates/html.css
View File

@ -402,7 +402,3 @@ img, object, svg|svg {
height: auto;
}
/* These are needed because ADE renders anchors the same as links */
a { text-decoration: inherit; color: inherit; cursor: inherit }
a[href] { text-decoration: underline; color: blue; cursor: pointer }

2
resources/viewer/mathjax/jax/output/SVG/autoload/multiline.js
View File

@ -187,7 +187,7 @@ MathJax.Hub.Register.StartupHook("SVG Jax Ready",function () {
// fill it with the proper elements,
// and clean up the bbox
//
line = BBOX();
var line = BBOX();
state.first = broken; state.last = true;
this.SVGmoveLine(start,end,line,state,values);
line.Clean();

3
session.vim
View File

@ -3,6 +3,7 @@ let $PYFLAKES_BUILTINS = "_,dynamic_property,__,P,I,lopen,icu_lower,icu_upper,ic
" Include directories for C++ modules
let g:syntastic_cpp_include_dirs = [
\'/usr/include/python2.7',
\'/usr/include/podofo',
\'/usr/include/qt4/QtCore',
\'/usr/include/qt4/QtGui',
@ -12,6 +13,8 @@ let g:syntastic_cpp_include_dirs = [
\]
let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs
set wildignore+=resources/viewer/mathjax/**
fun! CalibreLog()
" Setup buffers to edit the calibre changelog and version info prior to
" making a release.

23
setup/extensions.py
View File

@ -137,11 +137,19 @@ extensions = [
['calibre/ebooks/compression/palmdoc.c']),
Extension('podofo',
['calibre/utils/podofo/podofo.cpp'],
[
'calibre/utils/podofo/utils.cpp',
'calibre/utils/podofo/output.cpp',
'calibre/utils/podofo/doc.cpp',
'calibre/utils/podofo/outline.cpp',
'calibre/utils/podofo/podofo.cpp',
],
headers=[
'calibre/utils/podofo/global.h',
],
libraries=['podofo'],
lib_dirs=[podofo_lib],
inc_dirs=[podofo_inc, os.path.dirname(podofo_inc)],
optional=True,
error=podofo_error),
Extension('pictureflow',
@ -179,7 +187,7 @@ if iswindows:
headers=[
'calibre/devices/mtp/windows/global.h',
],
libraries=['ole32', 'portabledeviceguids', 'user32'],
libraries=['ole32', 'oleaut32', 'portabledeviceguids', 'user32'],
# needs_ddk=True,
cflags=['/X']
),
@ -188,10 +196,15 @@ if iswindows:
if isosx:
extensions.append(Extension('usbobserver',
['calibre/devices/usbobserver/usbobserver.c'],
ldflags=['-framework', 'IOKit'])
ldflags=['-framework', 'CoreServices', '-framework', 'IOKit'])
)
if islinux:
if islinux or isosx:
extensions.append(Extension('libusb',
['calibre/devices/libusb/libusb.c'],
libraries=['usb-1.0']
))
extensions.append(Extension('libmtp',
[
'calibre/devices/mtp/unix/devices.c',

3
setup/installer/linux/freeze2.py
View File

@ -15,7 +15,8 @@ from setup import Command, modules, basenames, functions, __version__, \
SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt4', 'mechanize',
'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml',
'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', 'dbus_bindings.py',
'_dbus_glib_bindings.so']
'_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so',
'_psutil_linux.so', 'psutil']
QTDIR = '/usr/lib/qt4'
QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus')

9
setup/installer/osx/app/main.py
View File

@ -438,12 +438,15 @@ class Py2App(object):
@flush
def add_misc_libraries(self):
for x in ('usb', 'unrar', 'readline.6.1', 'wmflite-0.2.7', 'chm.0',
'sqlite3.0'):
for x in ('usb-1.0.0', 'mtp.9', 'unrar', 'readline.6.1',
'wmflite-0.2.7', 'chm.0', 'sqlite3.0'):
info('\nAdding', x)
x = 'lib%s.dylib'%x
shutil.copy2(join(SW, 'lib', x), self.frameworks_dir)
self.set_id(join(self.frameworks_dir, x), self.FID+'/'+x)
dest = join(self.frameworks_dir, x)
self.set_id(dest, self.FID+'/'+x)
if 'mtp' in x:
self.fix_dependencies_in_lib(dest)
@flush
def add_site_packages(self):

5
setup/installer/windows/main.c
View File

@ -11,7 +11,6 @@ wWinMain(HINSTANCE Inst, HINSTANCE PrevInst,
wchar_t *CmdLine, int CmdShow) {
wchar_t *stdout_redirect, *stderr_redirect, basename[50];
int ret = 0;
set_gui_app((char)1);
@ -20,7 +19,7 @@ wWinMain(HINSTANCE Inst, HINSTANCE PrevInst,
stdout_redirect = redirect_out_stream(basename, (char)1);
stderr_redirect = redirect_out_stream(basename, (char)0);
ret = execute_python_entrypoint(BASENAME, MODULE, FUNCTION,
execute_python_entrypoint(BASENAME, MODULE, FUNCTION,
stdout_redirect, stderr_redirect);
if (stdout != NULL) fclose(stdout);
@ -29,7 +28,7 @@ wWinMain(HINSTANCE Inst, HINSTANCE PrevInst,
DeleteFile(stdout_redirect);
DeleteFile(stderr_redirect);
return ret;
return 0; // This should really be returning the value set in the WM_QUIT message, but I cannot be bothered figuring out how to get that.
}
#else

22
setup/installer/windows/notes.rst
View File

@ -348,6 +348,28 @@ Remove the CORE_xlib, UTIL_Imdisplay and CORE_Magick++ projects.
F7 for build project, you will get one error due to the removal of xlib, ignore
it.
netifaces
------------
Download the source tarball from http://alastairs-place.net/projects/netifaces/
Rename netifaces.c to netifaces.cpp and make the same change in setup.py
Run
python setup.py build
cp build/lib.win32-2.7/netifaces.pyd /cygdrive/c/Python27/Lib/site-packages/
psutil
--------
Download the source tarball
Run
Python setup.py build
cp -r build/lib.win32-*/* /cygdrive/c/Python27/Lib/site-packages/
calibre
---------

1690
setup/iso_639/de.po
View File

File diff suppressed because it is too large Load Diff

2
setup/translations.py
View File

@ -152,7 +152,7 @@ class Translations(POT): # {{{
subprocess.check_call(['msgfmt', '-o', dest, iso639])
elif locale not in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc',
'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku',
'fr_CA', 'him'):
'fr_CA', 'him', 'jv', 'ka'):
self.warn('No ISO 639 translations for locale:', locale)
self.write_stats()

30
setup/upload.py
View File

@ -47,6 +47,21 @@ def installer_description(fname):
return 'Calibre Portable'
return 'Unknown file'
def upload_signatures():
tdir = mkdtemp()
for installer in installers():
if not os.path.exists(installer):
continue
with open(installer, 'rb') as f:
raw = f.read()
fingerprint = hashlib.sha512(raw).hexdigest()
fname = os.path.basename(installer+'.sha512')
with open(os.path.join(tdir, fname), 'wb') as f:
f.write(fingerprint)
check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS),
shell=True)
shutil.rmtree(tdir)
class ReUpload(Command): # {{{
description = 'Re-uplaod any installers present in dist/'
@ -57,6 +72,7 @@ class ReUpload(Command): # {{{
opts.replace = True
def run(self, opts):
upload_signatures()
for x in installers():
if os.path.exists(x):
os.remove(x)
@ -223,19 +239,7 @@ class UploadToServer(Command): # {{{
%(__version__, DOWNLOADS), shell=True)
check_call('ssh divok /etc/init.d/apache2 graceful',
shell=True)
tdir = mkdtemp()
for installer in installers():
if not os.path.exists(installer):
continue
with open(installer, 'rb') as f:
raw = f.read()
fingerprint = hashlib.sha512(raw).hexdigest()
fname = os.path.basename(installer+'.sha512')
with open(os.path.join(tdir, fname), 'wb') as f:
f.write(fingerprint)
check_call('scp %s/*.sha512 divok:%s/signatures/' % (tdir, DOWNLOADS),
shell=True)
shutil.rmtree(tdir)
upload_signatures()
# }}}
# Testing {{{

30
src/calibre/__init__.py
View File

@ -444,23 +444,6 @@ class CurrentDir(object):
pass
class StreamReadWrapper(object):
'''
Used primarily with pyPdf to ensure the stream is properly closed.
'''
def __init__(self, stream):
for x in ('read', 'seek', 'tell'):
setattr(self, x, getattr(stream, x))
def __exit__(self, *args):
for x in ('read', 'seek', 'tell'):
setattr(self, x, None)
def __enter__(self):
return self
def detect_ncpus():
"""Detects the number of effective CPUs in the system"""
import multiprocessing
@ -674,7 +657,7 @@ def get_download_filename(url, cookie_file=None):
return filename
def human_readable(size):
def human_readable(size, sep=' '):
""" Convert a size in bytes into a human readable form """
divisor, suffix = 1, "B"
for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
@ -686,7 +669,7 @@ def human_readable(size):
size = size[:size.find(".")+2]
if size.endswith('.0'):
size = size[:-2]
return size + " " + suffix
return size + sep + suffix
def remove_bracketed_text(src,
brackets={u'(':u')', u'[':u']', u'{':u'}'}):
@ -720,6 +703,15 @@ if isosx:
import traceback
traceback.print_exc()
def load_builtin_fonts():
import glob
from PyQt4.Qt import QFontDatabase
base = P('fonts/liberation/*.ttf')
for f in glob.glob(base):
QFontDatabase.addApplicationFont(f)
return 'Liberation Serif', 'Liberation Sans', 'Liberation Mono'
def ipython(user_ns=None):
from calibre.utils.ipython import ipython
ipython(user_ns=user_ns)

7
src/calibre/constants.py
View File

@ -4,7 +4,7 @@ __license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
__appname__ = u'calibre'
numeric_version = (0, 8, 65)
numeric_version = (0, 8, 69)
__version__ = u'.'.join(map(unicode, numeric_version))
__author__ = u"Kovid Goyal <kovid@kovidgoyal.net>"
@ -57,7 +57,7 @@ else:
# On linux, unicode arguments to os file functions are coerced to an ascii
# bytestring if sys.getfilesystemencoding() == 'ascii', which is
# just plain dumb. So issue a warning.
print ('WARNING: You do not have the LANG environment variable set. '
print ('WARNING: You do not have the LANG environment variable set correctly. '
'This will cause problems with non-ascii filenames. '
'Set it to something like en_US.UTF-8.\n')
except:
@ -94,7 +94,8 @@ class Plugins(collections.Mapping):
plugins.extend(['winutil', 'wpd'])
if isosx:
plugins.append('usbobserver')
if islinux:
if islinux or isosx:
plugins.append('libusb')
plugins.append('libmtp')
self.plugins = frozenset(plugins)

7
src/calibre/customize/builtins.py
View File

@ -675,7 +675,6 @@ from calibre.devices.bambook.driver import BAMBOOK
from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX
from calibre.devices.smart_device_app.driver import SMART_DEVICE_APP
# Order here matters. The first matched device is the one used.
plugins += [
HANLINV3,
@ -749,6 +748,12 @@ plugins += [
SMART_DEVICE_APP,
USER_DEFINED,
]
from calibre.utils.config_base import tweaks
if tweaks.get('test_mtp_driver', False):
from calibre.devices.mtp.driver import MTP_DEVICE
plugins.append(MTP_DEVICE)
# }}}
# New metadata download plugins {{{

103
src/calibre/devices/__init__.py
View File

@ -115,54 +115,65 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None):
out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in
devplugins])))
out(' ')
out('Looking for devices...')
found_dev = False
for dev in devplugins:
connected, det = s.is_device_connected(dev, debug=True)
if connected:
out('\t\tDetected possible device', dev.__class__.__name__)
connected_devices.append((dev, det))
out(' ')
errors = {}
success = False
out('Devices possibly connected:', end=' ')
for dev, det in connected_devices:
out(dev.name, end=', ')
if not connected_devices:
out('None', end='')
out(' ')
for dev, det in connected_devices:
out('Trying to open', dev.name, '...', end=' ')
try:
dev.reset(detected_device=det)
dev.open(det, None)
out('OK')
except:
import traceback
errors[dev] = traceback.format_exc()
out('failed')
continue
success = True
if hasattr(dev, '_main_prefix'):
out('Main memory:', repr(dev._main_prefix))
out('Total space:', dev.total_space())
break
if not success and errors:
out('Opening of the following devices failed')
for dev,msg in errors.items():
out(dev)
out(msg)
out(' ')
if ioreg is not None:
ioreg = 'IOREG Output\n'+ioreg
if not dev.MANAGES_DEVICE_PRESENCE: continue
out('Looking for devices of type:', dev.__class__.__name__)
if dev.debug_managed_device_detection(s.devices, buf):
found_dev = True
break
out(' ')
if ioreg_to_tmp:
open('/tmp/ioreg.txt', 'wb').write(ioreg)
out('Dont forget to send the contents of /tmp/ioreg.txt')
out('You can open it with the command: open /tmp/ioreg.txt')
else:
out(ioreg)
if not found_dev:
out('Looking for devices...')
for dev in devplugins:
if dev.MANAGES_DEVICE_PRESENCE: continue
connected, det = s.is_device_connected(dev, debug=True)
if connected:
out('\t\tDetected possible device', dev.__class__.__name__)
connected_devices.append((dev, det))
out(' ')
errors = {}
success = False
out('Devices possibly connected:', end=' ')
for dev, det in connected_devices:
out(dev.name, end=', ')
if not connected_devices:
out('None', end='')
out(' ')
for dev, det in connected_devices:
out('Trying to open', dev.name, '...', end=' ')
try:
dev.reset(detected_device=det)
dev.open(det, None)
out('OK')
except:
import traceback
errors[dev] = traceback.format_exc()
out('failed')
continue
success = True
if hasattr(dev, '_main_prefix'):
out('Main memory:', repr(dev._main_prefix))
out('Total space:', dev.total_space())
break
if not success and errors:
out('Opening of the following devices failed')
for dev,msg in errors.items():
out(dev)
out(msg)
out(' ')
if ioreg is not None:
ioreg = 'IOREG Output\n'+ioreg
out(' ')
if ioreg_to_tmp:
open('/tmp/ioreg.txt', 'wb').write(ioreg)
out('Dont forget to send the contents of /tmp/ioreg.txt')
out('You can open it with the command: open /tmp/ioreg.txt')
else:
out(ioreg)
if hasattr(buf, 'getvalue'):
return buf.getvalue().decode('utf-8')

66
src/calibre/devices/android/driver.py
View File

@ -45,6 +45,7 @@ class ANDROID(USBMS):
0xce5 : HTC_BCDS,
0xcec : HTC_BCDS,
0x2910 : HTC_BCDS,
0xe77 : HTC_BCDS,
0xff9 : HTC_BCDS,
},
@ -63,6 +64,7 @@ class ANDROID(USBMS):
0x42d7 : [0x216],
0x42f7 : [0x216],
0x4365 : [0x216],
0x4366 : [0x216],
},
# Freescale
0x15a2 : {
@ -123,6 +125,11 @@ class ANDROID(USBMS):
0xe115 : [0x0216], # PocketBook A10
},
# Another Viewsonic
0x0bb0 : {
0x2a2b : [0x0226, 0x0227],
},
# Acer
0x502 : { 0x3203 : [0x0100, 0x224]},
@ -186,10 +193,15 @@ class ANDROID(USBMS):
}
EBOOK_DIR_MAIN = ['eBooks/import', 'wordplayer/calibretransfer', 'Books',
'sdcard/ebooks']
EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to '
'send e-books to on the device. The first one that exists will '
EXTRA_CUSTOMIZATION_MESSAGE = [_('Comma separated list of directories to '
'send e-books to on the device\'s <b>main memory</b>. The first one that exists will '
'be used'),
_('Comma separated list of directories to '
'send e-books to on the device\'s <b>storage cards</b>. The first one that exists will '
'be used')
EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN)
]
EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(EBOOK_DIR_MAIN), '']
VENDOR_NAME = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
@ -197,7 +209,8 @@ class ANDROID(USBMS):
'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP',
'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD',
'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ']
'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0',
'COBY_MID', 'VS', 'AINOL']
WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
'__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID',
@ -212,11 +225,12 @@ class ANDROID(USBMS):
'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107',
'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK', 'MB855',
'XT910', 'BOOK_A10', 'USB_2.0_DRIVER', 'I9100T', 'P999DW',
'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER',
'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0',
'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX',
'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
'S5830I_CARD', 'MID7042', 'LINK-CREATE']
'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E',
'NOVO7']
WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@ -224,9 +238,10 @@ class ANDROID(USBMS):
'ANDROID_MID', 'P990_SD_CARD', '.K080', 'LTE_CARD', 'MB853',
'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910', 'BOOK_A10_CARD',
'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC',
'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0_DRIVER', 'XT875',
'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875',
'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
'USB_FLASH_DRIVER', 'ANDROID', 'MID7042']
'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E',
'NOVO7']
OSX_MAIN_MEM = 'Android Device Main Memory'
@ -236,23 +251,35 @@ class ANDROID(USBMS):
def post_open_callback(self):
opts = self.settings()
dirs = opts.extra_customization
if not dirs:
dirs = self.EBOOK_DIR_MAIN
else:
dirs = [x.strip() for x in dirs.split(',')]
self.EBOOK_DIR_MAIN = dirs
opts = opts.extra_customization
if not opts:
opts = [self.EBOOK_DIR_MAIN, '']
def strtolist(x):
if isinstance(x, basestring):
x = [y.strip() for y in x.split(',')]
return x or []
opts = [strtolist(x) for x in opts]
self._android_main_ebook_dir = opts[0]
self._android_card_ebook_dir = opts[1]
def get_main_ebook_dir(self, for_upload=False):
dirs = self.EBOOK_DIR_MAIN
dirs = self._android_main_ebook_dir
if not for_upload:
def aldiko_tweak(x):
return 'eBooks' if x == 'eBooks/import' else x
if isinstance(dirs, basestring):
dirs = [dirs]
dirs = list(map(aldiko_tweak, dirs))
return dirs
def get_carda_ebook_dir(self, for_upload=False):
if not for_upload:
return ''
return self._android_card_ebook_dir
def get_cardb_ebook_dir(self, for_upload=False):
return self.get_carda_ebook_dir()
def windows_sort_drives(self, drives):
try:
vid, pid, bcd = self.device_being_opened[:3]
@ -268,9 +295,10 @@ class ANDROID(USBMS):
@classmethod
def configure_for_kindle_app(cls):
proxy = cls._configProxy()
proxy['format_map'] = ['mobi', 'azw', 'azw1', 'azw4', 'pdf']
proxy['format_map'] = ['azw3', 'mobi', 'azw', 'azw1', 'azw4', 'pdf']
proxy['use_subdirs'] = False
proxy['extra_customization'] = ','.join(['kindle']+cls.EBOOK_DIR_MAIN)
proxy['extra_customization'] = [
','.join(['kindle']+cls.EBOOK_DIR_MAIN), '']
@classmethod
def configure_for_generic_epub_app(cls):

4
src/calibre/devices/apple/driver.py
View File

@ -13,7 +13,8 @@ from calibre.constants import isosx, iswindows
from calibre.devices.errors import OpenFeedback, UserFeedback
from calibre.devices.usbms.deviceconfig import DeviceConfig
from calibre.devices.interface import DevicePlugin
from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort
from calibre.ebooks.metadata import (author_to_author_sort, authors_to_string,
MetaInformation, title_sort)
from calibre.ebooks.metadata.book.base import Metadata
from calibre.utils.config import config_dir, dynamic, prefs
from calibre.utils.date import now, parse_date
@ -3478,6 +3479,7 @@ class Book(Metadata):
'''
def __init__(self,title,author):
Metadata.__init__(self, title, authors=author.split(' & '))
self.author_sort = author_to_author_sort(author)
@property
def title_sorter(self):

28
src/calibre/devices/prs500/cli/main.py → src/calibre/devices/cli.py
View File

@ -1,7 +1,7 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
"""
Provides a command-line and optional graphical interface to the SONY Reader PRS-500.
Provides a command-line interface to ebook devices.
For usage information run the script.
"""
@ -9,7 +9,7 @@ For usage information run the script.
import StringIO, sys, time, os
from optparse import OptionParser
from calibre import __version__, __appname__
from calibre import __version__, __appname__, human_readable
from calibre.devices.errors import PathError
from calibre.utils.terminfo import TerminalController
from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked
@ -18,16 +18,6 @@ from calibre.devices.scanner import DeviceScanner
MINIMUM_COL_WIDTH = 12 #: Minimum width of columns in ls output
def human_readable(size):
""" Convert a size in bytes into a human readle form """
if size < 1024: divisor, suffix = 1, ""
elif size < 1024*1024: divisor, suffix = 1024., "K"
elif size < 1024*1024*1024: divisor, suffix = 1024*1024, "M"
elif size < 1024*1024*1024*1024: divisor, suffix = 1024*1024, "G"
size = str(size/divisor)
if size.find(".") > -1: size = size[:size.find(".")+2]
return size + suffix
class FileFormatter(object):
def __init__(self, file, term):
self.term = term
@ -207,11 +197,19 @@ def main():
scanner = DeviceScanner()
scanner.scan()
connected_devices = []
for d in device_plugins():
try:
d.startup()
except:
print ('Startup failed for device plugin: %s'%d)
if d.MANAGES_DEVICE_PRESENCE:
cd = d.detect_managed_devices(scanner.devices)
if cd is not None:
connected_devices.append((cd, d))
dev = d
break
continue
ok, det = scanner.is_device_connected(d)
if ok:
dev = d
@ -275,7 +273,7 @@ def main():
elif command == "cp":
usage="usage: %prog cp [options] source destination\nCopy files to/from the device\n\n"+\
"One of source or destination must be a path on the device. \n\nDevice paths have the form\n"+\
"prs500:mountpoint/my/path\n"+\
"dev:mountpoint/my/path\n"+\
"where mountpoint is one of / or card:/\n\n"+\
"source must point to a file for which you have read permissions\n"+\
"destination must point to a file or directory for which you have write permissions"
@ -286,7 +284,7 @@ def main():
if len(args) != 2:
parser.print_help()
return 1
if args[0].startswith("prs500:"):
if args[0].startswith("dev:"):
outfile = args[1]
path = args[0][7:]
if path.endswith("/"): path = path[:-1]
@ -300,7 +298,7 @@ def main():
return 1
dev.get_file(path, outfile)
outfile.close()
elif args[1].startswith("prs500:"):
elif args[1].startswith("dev:"):
try:
infile = open(args[0], "rb")
except IOError as e:

6
src/calibre/devices/errors.py
View File

@ -110,3 +110,9 @@ class WrongDestinationError(PathError):
trying to send books to a non existant storage card.'''
pass
class BlacklistedDevice(OpenFailed):
''' Raise this error during open() when the device being opened has been
blacklisted by the user. Only used in drivers that manage device presence,
like the MTP driver. '''
pass

73
src/calibre/devices/interface.py
View File

@ -81,6 +81,23 @@ class DevicePlugin(Plugin):
#: by.
NUKE_COMMENTS = None
#: If True indicates that this driver completely manages device detection,
#: ejecting and so forth. If you set this to True, you *must* implement the
#: detect_managed_devices and debug_managed_device_detection methods.
#: A driver with this set to true is responsible for detection of devices,
#: managing a blacklist of devices, a list of ejected devices and so forth.
#: calibre will periodically call the detect_managed_devices() method and
#: is it returns a detected device, calibre will call open(). open() will
#: be called every time a device is returned even is previous calls to open()
#: failed, therefore the driver must maintain its own blacklist of failed
#: devices. Similarly, when ejecting, calibre will call eject() and then
#: assuming the next call to detect_managed_devices() returns None, it will
#: call post_yank_cleanup().
MANAGES_DEVICE_PRESENCE = False
#: If set the True, calibre will call the :meth:`get_driveinfo()` method
#: after the books lists have been loaded to get the driveinfo.
SLOW_DRIVEINFO = False
@classmethod
def get_gui_name(cls):
@ -196,6 +213,40 @@ class DevicePlugin(Plugin):
return True, dev
return False, None
def detect_managed_devices(self, devices_on_system, force_refresh=False):
'''
Called only if MANAGES_DEVICE_PRESENCE is True.
Scan for devices that this driver can handle. Should return a device
object if a device is found. This object will be passed to the open()
method as the connected_device. If no device is found, return None. The
returned object can be anything, calibre does not use it, it is only
passed to open().
This method is called periodically by the GUI, so make sure it is not
too resource intensive. Use a cache to avoid repeatedly scanning the
system.
:param devices_on_system: Set of USB devices found on the system.
:param force_refresh: If True and the driver uses a cache to prevent
repeated scanning, the cache must be flushed.
'''
raise NotImplementedError()
def debug_managed_device_detection(self, devices_on_system, output):
'''
Called only if MANAGES_DEVICE_PRESENCE is True.
Should write information about the devices detected on the system to
output, which is a file like object.
Should return True if a device was detected and successfully opened,
otherwise False.
'''
raise NotImplementedError()
# }}}
def reset(self, key='-1', log_packets=False, report_progress=None,
@ -270,6 +321,9 @@ class DevicePlugin(Plugin):
'''
Un-mount / eject the device from the OS. This does not check if there
are pending GUI jobs that need to communicate with the device.
NOTE: That this method may not be called on the same thread as the rest
of the device methods.
'''
raise NotImplementedError()
@ -302,6 +356,18 @@ class DevicePlugin(Plugin):
"""
raise NotImplementedError()
def get_driveinfo(self):
'''
Return the driveinfo dictionary. Usually called from
get_device_information(), but if loading the driveinfo is slow for this
driver, then it should set SLOW_DRIVEINFO. In this case, this method
will be called by calibre after the book lists have been loaded. Note
that it is not called on the device thread, so the driver should cache
the drive info in the books() method and this function should return
the cached data.
'''
return {}
def card_prefix(self, end_session=True):
'''
Return a 2 element list of the prefix to paths on the cards.
@ -487,7 +553,8 @@ class DevicePlugin(Plugin):
Set the device name in the driveinfo file to 'name'. This setting will
persist until the file is re-created or the name is changed again.
Non-disk devices will ignore this request.
Non-disk devices should implement this method based on the location
codes returned by the get_device_information() method.
'''
pass
@ -495,6 +562,10 @@ class DevicePlugin(Plugin):
'''
Given a list of paths, returns another list of paths. These paths
point to addable versions of the books.
If there is an error preparing a book, then instead of a path, the
position in the returned list for that book should be a three tuple:
(original_path, the exception instance, traceback)
'''
return paths

57
src/calibre/devices/kindle/apnx.py
View File

@ -12,7 +12,10 @@ import struct
from calibre.ebooks.mobi.reader.mobi6 import MobiReader
from calibre.ebooks.pdb.header import PdbHeaderReader
from calibre.ebooks.mobi.reader.headers import MetadataHeader
from calibre.utils.logging import default_log
from calibre import prints
from calibre.constants import DEBUG
class APNXBuilder(object):
'''
@ -25,11 +28,32 @@ class APNXBuilder(object):
pass in a value to page_count, otherwise a count will be estimated
using either the fast or accurate algorithm.
'''
# Check that this is really a MOBI file.
import uuid
apnx_meta = { 'guid': str(uuid.uuid4()).replace('-', '')[:8], 'asin':
'', 'cdetype': 'EBOK', 'format': 'MOBI_7', 'acr': '' }
with open(mobi_file_path, 'rb') as mf:
ident = PdbHeaderReader(mf).identity()
if ident != 'BOOKMOBI':
raise Exception(_('Not a valid MOBI file. Reports identity of %s') % ident)
if ident != 'BOOKMOBI':
# Check that this is really a MOBI file.
raise Exception(_('Not a valid MOBI file. Reports identity of %s') % ident)
apnx_meta['acr'] = str(PdbHeaderReader(mf).name())
# We'll need the PDB name, the MOBI version, and some metadata to make FW 3.4 happy with KF8 files...
with open(mobi_file_path, 'rb') as mf:
mh = MetadataHeader(mf, default_log)
if mh.mobi_version == 8:
apnx_meta['format'] = 'MOBI_8'
else:
apnx_meta['format'] = 'MOBI_7'
if mh.exth is None or not mh.exth.cdetype:
apnx_meta['cdetype'] = 'EBOK'
else:
apnx_meta['cdetype'] = str(mh.exth.cdetype)
if mh.exth is None or not mh.exth.uuid:
apnx_meta['asin'] = ''
else:
apnx_meta['asin'] = str(mh.exth.uuid)
# Get the pages depending on the chosen parser
pages = []
@ -51,23 +75,32 @@ class APNXBuilder(object):
raise Exception(_('Could not generate page mapping.'))
# Generate the APNX file from the page mapping.
apnx = self.generate_apnx(pages)
apnx = self.generate_apnx(pages, apnx_meta)
# Write the APNX.
with open(apnx_path, 'wb') as apnxf:
apnxf.write(apnx)
def generate_apnx(self, pages):
import uuid
def generate_apnx(self, pages, apnx_meta):
apnx = ''
content_vals = {
'guid': str(uuid.uuid4()).replace('-', '')[:8],
'isbn': '',
}
if DEBUG:
prints('APNX META: guid:', apnx_meta['guid'])
prints('APNX META: ASIN:', apnx_meta['asin'])
prints('APNX META: CDE:', apnx_meta['cdetype'])
prints('APNX META: format:', apnx_meta['format'])
prints('APNX META: Name:', apnx_meta['acr'])
content_header = '{"contentGuid":"%(guid)s","asin":"%(isbn)s","cdeType":"EBOK","fileRevisionId":"1"}' % content_vals
page_header = '{"asin":"%(isbn)s","pageMap":"(1,a,1)"}' % content_vals
# Updated header if we have a KF8 file...
if apnx_meta['format'] == 'MOBI_8':
content_header = '{"contentGuid":"%(guid)s","asin":"%(asin)s","cdeType":"%(cdetype)s","format":"%(format)s","fileRevisionId":"1","acr":"%(acr)s"}' % apnx_meta
else:
# My 5.1.x Touch & 3.4 K3 seem to handle the 'extended' header fine for legacy mobi files, too. But, since they still handle this one too, let's try not to break old devices, and keep using the simple header ;).
content_header = '{"contentGuid":"%(guid)s","asin":"%(asin)s","cdeType":"%(cdetype)s","fileRevisionId":"1"}' % apnx_meta
page_header = '{"asin":"%(asin)s","pageMap":"(1,a,1)"}' % apnx_meta
if DEBUG:
prints('APNX Content Header:', content_header)
apnx += struct.pack('>I', 65537)
apnx += struct.pack('>I', 12 + len(content_header))

8
src/calibre/devices/kindle/driver.py
View File

@ -288,8 +288,9 @@ class KINDLE2(KINDLE):
name = 'Kindle 2/3/4/Touch Device Interface'
description = _('Communicate with the Kindle 2/3/4/Touch eBook reader.')
FORMATS = KINDLE.FORMATS + ['pdf', 'azw4', 'pobi']
DELETE_EXTS = KINDLE.DELETE_EXTS + ['.mbp1', '.mbs', '.sdr']
FORMATS = ['azw3'] + KINDLE.FORMATS + ['pdf', 'azw4', 'pobi']
DELETE_EXTS = KINDLE.DELETE_EXTS + ['.mbp1', '.mbs', '.sdr', '.han']
# On the Touch, there's also .asc files, but not using the same basename (for X-Ray & End Actions), azw3f & azw3r files, but all of them are in the .sdr sidecar folder
PRODUCT_ID = [0x0002, 0x0004]
BCD = [0x0100]
@ -449,7 +450,7 @@ class KINDLE_DX(KINDLE2):
name = 'Kindle DX Device Interface'
description = _('Communicate with the Kindle DX eBook reader.')
FORMATS = KINDLE2.FORMATS[1:]
PRODUCT_ID = [0x0003]
BCD = [0x0100]
@ -462,7 +463,6 @@ class KINDLE_FIRE(KINDLE2):
description = _('Communicate with the Kindle Fire')
gui_name = 'Fire'
FORMATS = list(KINDLE2.FORMATS)
FORMATS.insert(0, 'azw3')
PRODUCT_ID = [0x0006]
BCD = [0x216, 0x100]

15
src/calibre/devices/kobo/books.py
View File

@ -6,7 +6,9 @@ __copyright__ = '2010, Timothy Legge <timlegge at gmail.com>'
import os
import time
from calibre.utils.date import parse_date
from calibre.devices.usbms.books import Book as Book_
from calibre.ebooks.metadata import author_to_author_sort
class Book(Book_):
@ -19,6 +21,7 @@ class Book(Book_):
self.authors = ['']
else:
self.authors = [authors]
self.author_sort = author_to_author_sort(self.authors[0])
if not title:
self.title = _('Unknown')
@ -28,7 +31,17 @@ class Book(Book_):
self.size = size # will be set later if None
if ContentType == '6' and date is not None:
self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
try:
self.datetime = time.strptime(date, "%Y-%m-%dT%H:%M:%S.%f")
except:
try:
self.datetime = parse_date(date,
assume_utc=True).timetuple()
except:
try:
self.datetime = time.gmtime(os.path.getctime(self.path))
except:
self.datetime = time.gmtime()
else:
try:
self.datetime = time.gmtime(os.path.getctime(self.path))

368
src/calibre/devices/libusb.py
View File

@ -1,368 +0,0 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
"""
This module provides a thin ctypes based wrapper around libusb.
"""
from ctypes import cdll, POINTER, byref, pointer, Structure as _Structure, \
c_ubyte, c_ushort, c_int, c_char, c_void_p, c_byte, c_uint
from errno import EBUSY, ENOMEM
from calibre import iswindows, isosx, isbsd, load_library
_libusb_name = 'libusb'
PATH_MAX = 511 if iswindows else 1024 if (isosx or isbsd) else 4096
if iswindows:
class Structure(_Structure):
_pack_ = 1
_libusb_name = 'libusb0'
else:
Structure = _Structure
try:
try:
_libusb = load_library(_libusb_name, cdll)
except OSError:
_libusb = cdll.LoadLibrary('libusb-0.1.so.4')
has_library = True
except:
_libusb = None
has_library = False
class DeviceDescriptor(Structure):
_fields_ = [\
('Length', c_ubyte), \
('DescriptorType', c_ubyte), \
('bcdUSB', c_ushort), \
('DeviceClass', c_ubyte), \
('DeviceSubClass', c_ubyte), \
('DeviceProtocol', c_ubyte), \
('MaxPacketSize0', c_ubyte), \
('idVendor', c_ushort), \
('idProduct', c_ushort), \
('bcdDevice', c_ushort), \
('Manufacturer', c_ubyte), \
('Product', c_ubyte), \
('SerialNumber', c_ubyte), \
('NumConfigurations', c_ubyte) \
]
class EndpointDescriptor(Structure):
_fields_ = [\
('Length', c_ubyte), \
('DescriptorType', c_ubyte), \
('EndpointAddress', c_ubyte), \
('Attributes', c_ubyte), \
('MaxPacketSize', c_ushort), \
('Interval', c_ubyte), \
('Refresh', c_ubyte), \
('SynchAddress', c_ubyte), \
('extra', POINTER(c_char)), \
('extralen', c_int)\
]
class InterfaceDescriptor(Structure):
_fields_ = [\
('Length', c_ubyte), \
('DescriptorType', c_ubyte), \
('InterfaceNumber', c_ubyte), \
('AlternateSetting', c_ubyte), \
('NumEndpoints', c_ubyte), \
('InterfaceClass', c_ubyte), \
('InterfaceSubClass', c_ubyte), \
('InterfaceProtocol', c_ubyte), \
('Interface', c_ubyte), \
('endpoint', POINTER(EndpointDescriptor)), \
('extra', POINTER(c_char)), \
('extralen', c_int)\
]
class Interface(Structure):
_fields_ = [\
('altsetting', POINTER(InterfaceDescriptor)), \
('num_altsetting', c_int)\
]
class ConfigDescriptor(Structure):
_fields_ = [\
('Length', c_ubyte), \
('DescriptorType', c_ubyte), \
('TotalLength', c_ushort), \
('NumInterfaces', c_ubyte), \
('Value', c_ubyte), \
('Configuration', c_ubyte), \
('Attributes', c_ubyte), \
('MaxPower', c_ubyte), \
('interface', POINTER(Interface)), \
('extra', POINTER(c_ubyte)), \
('extralen', c_int) \
]
def __str__(self):
ans = ""
for field in self._fields_:
ans += field[0] + ": " + str(eval('self.'+field[0])) + '\n'
return ans.strip()
class Error(Exception):
pass
class Device(Structure):
def open(self):
""" Open device for use. Return a DeviceHandle. """
handle = _libusb.usb_open(byref(self))
if not handle:
raise Error("Cannot open device")
return handle.contents
@dynamic_property
def configurations(self):
doc = """ List of device configurations. See L{ConfigDescriptor} """
def fget(self):
ans = []
for config in range(self.device_descriptor.NumConfigurations):
ans.append(self.config_descriptor[config])
return tuple(ans)
return property(doc=doc, fget=fget)
class Bus(Structure):
@dynamic_property
def device_list(self):
doc = \
"""
Flat list of devices on this bus.
Note: children are not explored
TODO: Check if exploring children is neccessary (e.g. with an external hub)
"""
def fget(self):
if _libusb is None:
return []
if _libusb.usb_find_devices() < 0:
raise Error('Unable to search for USB devices')
ndev = self.devices
ans = []
while ndev:
dev = ndev.contents
ans.append(dev)
ndev = dev.next
return ans
return property(doc=doc, fget=fget)
class DeviceHandle(Structure):
_fields_ = [\
('fd', c_int), \
('bus', POINTER(Bus)), \
('device', POINTER(Device)), \
('config', c_int), \
('interface', c_int), \
('altsetting', c_int), \
('impl_info', c_void_p)
]
def close(self):
""" Close this DeviceHandle """
_libusb.usb_close(byref(self))
def set_configuration(self, config):
"""
Set device configuration. This has to be called on windows before
trying to claim an interface.
@param config: A L{ConfigDescriptor} or a integer (the ConfigurationValue)
"""
try:
num = config.Value
except AttributeError:
num = config
ret = _libusb.usb_set_configuration(byref(self), num)
if ret < 0:
raise Error('Failed to set device configuration to: ' + str(num) + \
'. Error code: ' + str(ret))
def claim_interface(self, num):
"""
Claim interface C{num} on device.
Must be called before doing anything witht the device.
"""
ret = _libusb.usb_claim_interface(byref(self), num)
if -ret == ENOMEM:
raise Error("Insufficient memory to claim interface")
elif -ret == EBUSY:
raise Error('Device busy')
elif ret < 0:
raise Error('Unknown error occurred while trying to claim USB'\
' interface: ' + str(ret))
def control_msg(self, rtype, request, bytes, value=0, index=0, timeout=100):
"""
Perform a control request to the default control pipe on the device.
@param rtype: specifies the direction of data flow, the type
of request, and the recipient.
@param request: specifies the request.
@param bytes: if the transfer is a write transfer, buffer is a sequence
with the transfer data, otherwise, buffer is the number of
bytes to read.
@param value: specific information to pass to the device.
@param index: specific information to pass to the device.
"""
size = 0
try:
size = len(bytes)
except TypeError:
size = bytes
ArrayType = c_byte * size
_libusb.usb_control_msg.argtypes = [POINTER(DeviceHandle), c_int, \
c_int, c_int, c_int, \
POINTER(ArrayType), \
c_int, c_int]
arr = ArrayType()
rsize = _libusb.usb_control_msg(byref(self), rtype, request, \
value, index, byref(arr), \
size, timeout)
if rsize < size:
raise Error('Could not read ' + str(size) + ' bytes on the '\
'control bus. Read: ' + str(rsize) + ' bytes.')
return arr
else:
ArrayType = c_byte * size
_libusb.usb_control_msg.argtypes = [POINTER(DeviceHandle), c_int, \
c_int, c_int, c_int, \
POINTER(ArrayType), \
c_int, c_int]
arr = ArrayType(*bytes)
return _libusb.usb_control_msg(byref(self), rtype, request, \
value, index, byref(arr), \
size, timeout)
def bulk_read(self, endpoint, size, timeout=100):
"""
Read C{size} bytes via a bulk transfer from the device.
"""
ArrayType = c_byte * size
arr = ArrayType()
_libusb.usb_bulk_read.argtypes = [POINTER(DeviceHandle), c_int, \
POINTER(ArrayType), c_int, c_int
]
rsize = _libusb.usb_bulk_read(byref(self), endpoint, byref(arr), \
size, timeout)
if rsize < 0:
raise Error('Could not read ' + str(size) + ' bytes on the '\
'bulk bus. Error code: ' + str(rsize))
if rsize == 0:
raise Error('Device sent zero bytes')
if rsize < size:
arr = arr[:rsize]
return arr
def bulk_write(self, endpoint, bytes, timeout=100):
"""
Send C{bytes} to device via a bulk transfer.
"""
size = len(bytes)
ArrayType = c_byte * size
arr = ArrayType(*bytes)
_libusb.usb_bulk_write.argtypes = [POINTER(DeviceHandle), c_int, \
POINTER(ArrayType), c_int, c_int
]
_libusb.usb_bulk_write(byref(self), endpoint, byref(arr), size, timeout)
def release_interface(self, num):
ret = _libusb.usb_release_interface(pointer(self), num)
if ret < 0:
raise Error('Unknown error occurred while trying to release USB'\
' interface: ' + str(ret))
def reset(self):
ret = _libusb.usb_reset(pointer(self))
if ret < 0:
raise Error('Unknown error occurred while trying to reset '\
'USB device ' + str(ret))
Bus._fields_ = [ \
('next', POINTER(Bus)), \
('previous', POINTER(Bus)), \
('dirname', c_char * (PATH_MAX+1)), \
('devices', POINTER(Device)), \
('location', c_uint), \
('root_dev', POINTER(Device))\
]
Device._fields_ = [ \
('next', POINTER(Device)), \
('previous', POINTER(Device)), \
('filename', c_char * (PATH_MAX+1)), \
('bus', POINTER(Bus)), \
('device_descriptor', DeviceDescriptor), \
('config_descriptor', POINTER(ConfigDescriptor)), \
('dev', c_void_p), \
('devnum', c_ubyte), \
('num_children', c_ubyte), \
('children', POINTER(POINTER(Device)))
]
if _libusb is not None:
try:
_libusb.usb_get_busses.restype = POINTER(Bus)
_libusb.usb_open.restype = POINTER(DeviceHandle)
_libusb.usb_open.argtypes = [POINTER(Device)]
_libusb.usb_close.argtypes = [POINTER(DeviceHandle)]
_libusb.usb_claim_interface.argtypes = [POINTER(DeviceHandle), c_int]
_libusb.usb_claim_interface.restype = c_int
_libusb.usb_release_interface.argtypes = [POINTER(DeviceHandle), c_int]
_libusb.usb_release_interface.restype = c_int
_libusb.usb_reset.argtypes = [POINTER(DeviceHandle)]
_libusb.usb_reset.restype = c_int
_libusb.usb_control_msg.restype = c_int
_libusb.usb_bulk_read.restype = c_int
_libusb.usb_bulk_write.restype = c_int
_libusb.usb_set_configuration.argtypes = [POINTER(DeviceHandle), c_int]
_libusb.usb_set_configuration.restype = c_int
_libusb.usb_init()
except:
_libusb = None
def busses():
""" Get list of USB busses present on system """
if _libusb is None:
raise Error('Could not find libusb.')
if _libusb.usb_find_busses() < 0:
raise Error('Unable to search for USB busses')
if _libusb.usb_find_devices() < 0:
raise Error('Unable to search for USB devices')
ans = []
nbus = _libusb.usb_get_busses()
while nbus:
bus = nbus.contents
ans.append(bus)
nbus = bus.next
return ans
def get_device_by_id(idVendor, idProduct):
""" Return a L{Device} by vendor and prduct ids """
buslist = busses()
for bus in buslist:
devices = bus.device_list
for dev in devices:
if dev.device_descriptor.idVendor == idVendor and \
dev.device_descriptor.idProduct == idProduct:
return dev
def has_library():
return _libusb is not None
def get_devices():
buslist = busses()
ans = []
for bus in buslist:
devices = bus.device_list
for dev in devices:
device = (dev.device_descriptor.idVendor, dev.device_descriptor.idProduct, dev.device_descriptor.bcdDevice)
ans.append(device)
return ans

144
src/calibre/devices/libusb/libusb.c Normal file
View File

@ -0,0 +1,144 @@
/*
* libusb.c
* Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
*
* Distributed under terms of the GPL3 license.
*/
#define UNICODE
#include <Python.h>
#include <libusb-1.0/libusb.h>
static PyObject *Error = NULL;
static PyObject *cache = NULL;
static PyObject* format_err(int err) {
PyErr_SetString(Error, libusb_error_name(err));
return NULL;
}
static PyObject* read_string_property(libusb_device_handle *dev, uint8_t idx) {
unsigned char buf[301];
int err;
PyObject *ans = NULL;
Py_BEGIN_ALLOW_THREADS;
err = libusb_get_string_descriptor_ascii(dev, idx, buf, 300);
Py_END_ALLOW_THREADS;
if (err > 0) {
ans = PyUnicode_FromStringAndSize((char *)buf, err);
}
return ans;
}
static PyObject* read_string_data(libusb_device *dev, uint8_t manufacturer, uint8_t product, uint8_t serial) {
libusb_device_handle *handle;
int err;
PyObject *ans = NULL, *p;
ans = PyDict_New();
if (ans == NULL) return PyErr_NoMemory();
err = libusb_open(dev, &handle);
if (err == 0) {
p = read_string_property(handle, manufacturer);
if (p != NULL) { PyDict_SetItemString(ans, "manufacturer", p); Py_DECREF(p); }
p = read_string_property(handle, product);
if (p != NULL) { PyDict_SetItemString(ans, "product", p); Py_DECREF(p); };
p = read_string_property(handle, serial);
if (p != NULL) { PyDict_SetItemString(ans, "serial", p); Py_DECREF(p); };
libusb_close(handle);
}
return ans;
}
static PyObject* get_devices(PyObject *self, PyObject *args) {
PyObject *ans = NULL, *d = NULL, *t = NULL, *rec = NULL;
int err, i = 0;
libusb_device **devs = NULL, *dev = NULL;
ssize_t count;
ans = PyList_New(0);
if (ans == NULL) return PyErr_NoMemory();
Py_BEGIN_ALLOW_THREADS;
count = libusb_get_device_list(NULL, &devs);
Py_END_ALLOW_THREADS;
if (count < 0) { Py_DECREF(ans); return format_err((int)count); }
while ( (dev = devs[i++]) != NULL ) {
struct libusb_device_descriptor desc;
err = libusb_get_device_descriptor(dev, &desc);
if (err != 0) { format_err(err); break; }
if (desc.bDeviceClass == LIBUSB_CLASS_HUB) continue;
d = Py_BuildValue("(BBHHH)", (unsigned char)libusb_get_bus_number(dev),
(unsigned char)libusb_get_device_address(dev), (unsigned short)desc.idVendor, (unsigned short)desc.idProduct,
(unsigned short)desc.bcdDevice);
if (d == NULL) break;
t = PyDict_GetItem(cache, d);
if (t == NULL) {
t = read_string_data(dev, desc.iManufacturer, desc.iProduct, desc.iSerialNumber);
if (t == NULL) { Py_DECREF(d); break; }
PyDict_SetItem(cache, d, t);
Py_DECREF(t);
}
rec = Py_BuildValue("(NO)", d, t);
if (rec == NULL) { Py_DECREF(d); break; }
PyList_Append(ans, rec);
Py_DECREF(rec);
}
if (dev != NULL) {
// An error occurred
Py_DECREF(ans); ans = NULL;
}
if (devs != NULL) libusb_free_device_list(devs, 1);
return ans;
}
static PyMethodDef libusb_methods[] = {
{"get_devices", get_devices, METH_VARARGS,
"get_devices()\n\nGet the list of USB devices on the system."
},
{NULL, NULL, 0, NULL}
};
PyMODINIT_FUNC
initlibusb(void) {
PyObject *m;
// We deliberately use the default context. This is the context used by
// libmtp and we want to ensure that the busnum/devnum numbers are the same
// here and for libmtp.
if(libusb_init(NULL) != 0) return;
Error = PyErr_NewException("libusb.Error", NULL, NULL);
if (Error == NULL) return;
cache = PyDict_New();
if (cache == NULL) return;
m = Py_InitModule3("libusb", libusb_methods, "Interface to libusb.");
if (m == NULL) return;
PyModule_AddObject(m, "Error", Error);
PyModule_AddObject(m, "cache", cache);
}

37
src/calibre/devices/mtp/base.py
View File

@ -9,8 +9,14 @@ __docformat__ = 'restructuredtext en'
from functools import wraps
from calibre import prints
from calibre.constants import DEBUG
from calibre.devices.interface import DevicePlugin
def debug(*args, **kwargs):
if DEBUG:
prints('MTP:', *args, **kwargs)
def synchronous(func):
@wraps(func)
def synchronizer(self, *args, **kwargs):
@ -19,36 +25,41 @@ def synchronous(func):
return synchronizer
class MTPDeviceBase(DevicePlugin):
name = 'SmartDevice App Interface'
name = 'MTP Device Interface'
gui_name = _('MTP Device')
icon = I('devices/galaxy_s3.png')
description = _('Communicate with MTP devices')
author = 'Kovid Goyal'
version = (1, 0, 0)
# Invalid USB vendor information so the scanner will never match
VENDOR_ID = [0xffff]
PRODUCT_ID = [0xffff]
BCD = [0xffff]
THUMBNAIL_HEIGHT = 128
CAN_SET_METADATA = []
BACKLOADING_ERROR_MESSAGE = None
def __init__(self, *args, **kwargs):
DevicePlugin.__init__(self, *args, **kwargs)
self.progress_reporter = None
self.current_friendly_name = None
self.report_progress = lambda x, y: None
self.current_serial_num = None
def reset(self, key='-1', log_packets=False, report_progress=None,
detected_device=None):
pass
def set_progress_reporter(self, report_progress):
self.progress_reporter = report_progress
self.report_progress = report_progress
def get_gui_name(self):
return self.current_friendly_name or self.name
return getattr(self, 'current_friendly_name', self.gui_name)
def is_usb_connected(self, devices_on_system, debug=False,
only_presence=False):
# We manage device presence ourselves, so this method should always
# return False
return False
def build_template_regexp(self):
from calibre.devices.utils import build_template_regexp
return build_template_regexp(self.save_template)
def is_customizable(self):
return True

68
src/calibre/devices/mtp/books.py Normal file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os
from calibre.devices.interface import BookList as BL
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.book.json_codec import JsonCodec
from calibre.utils.date import utcnow
class BookList(BL):
def __init__(self, storage_id):
self.storage_id = storage_id
def supports_collections(self):
return False
def add_book(self, book, replace_metadata=True):
try:
b = self.index(book)
except (ValueError, IndexError):
b = None
if b is None:
self.append(book)
return book
if replace_metadata:
self[b].smart_update(book, replace_metadata=True)
return self[b]
return None
def remove_book(self, book):
self.remove(book)
class Book(Metadata):
def __init__(self, storage_id, lpath, other=None):
Metadata.__init__(self, _('Unknown'), other=other)
self.storage_id, self.lpath = storage_id, lpath
self.lpath = self.path = self.lpath.replace(os.sep, '/')
self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')])
self.datetime = utcnow().timetuple()
self.thumbail = None
def matches_file(self, mtp_file):
return (self.storage_id == mtp_file.storage_id and
self.mtp_relpath == mtp_file.mtp_relpath)
def __eq__(self, other):
return (isinstance(other, self.__class__) and (self.storage_id ==
other.storage_id and self.mtp_relpath == other.mtp_relpath))
def __ne__(self, other):
return not self.__eq__(other)
def __hash__(self):
return hash((self.storage_id, self.mtp_relpath))
class JSONCodec(JsonCodec):
pass

478
src/calibre/devices/mtp/driver.py Normal file
View File

@ -0,0 +1,478 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import json, traceback, posixpath, importlib, os
from io import BytesIO
from itertools import izip
from calibre import prints
from calibre.constants import iswindows, numeric_version
from calibre.devices.mtp.base import debug
from calibre.ptempfile import SpooledTemporaryFile, PersistentTemporaryDirectory
from calibre.utils.config import from_json, to_json, JSONConfig
from calibre.utils.date import now, isoformat, utcnow
BASE = importlib.import_module('calibre.devices.mtp.%s.driver'%(
'windows' if iswindows else 'unix')).MTP_DEVICE
class MTP_DEVICE(BASE):
METADATA_CACHE = 'metadata.calibre'
DRIVEINFO = 'driveinfo.calibre'
CAN_SET_METADATA = []
NEWS_IN_FOLDER = True
MAX_PATH_LEN = 230
THUMBNAIL_HEIGHT = 160
THUMBNAIL_WIDTH = 120
CAN_SET_METADATA = []
BACKLOADING_ERROR_MESSAGE = None
MANAGES_DEVICE_PRESENCE = True
FORMATS = ['epub', 'azw3', 'mobi', 'pdf']
DEVICE_PLUGBOARD_NAME = 'MTP_DEVICE'
SLOW_DRIVEINFO = True
def __init__(self, *args, **kwargs):
BASE.__init__(self, *args, **kwargs)
self.plugboards = self.plugboard_func = None
self._prefs = None
@property
def prefs(self):
if self._prefs is None:
from calibre.library.save_to_disk import config
self._prefs = p = JSONConfig('mtp_devices')
p.defaults['format_map'] = self.FORMATS
p.defaults['send_to'] = ['Calibre_Companion', 'Books',
'eBooks/import', 'eBooks', 'wordplayer/calibretransfer',
'sdcard/ebooks', 'kindle']
p.defaults['send_template'] = config().parse().send_template
p.defaults['blacklist'] = []
p.defaults['history'] = {}
p.defaults['rules'] = []
return self._prefs
def configure_for_kindle_app(self):
proxy = self.prefs
with proxy:
proxy['format_map'] = ['azw3', 'mobi', 'azw', 'azw1', 'azw4', 'pdf']
proxy['send_template'] = '{title} - {authors}'
orig = list(proxy['send_to'])
if 'kindle' in orig:
orig.remove('kindle')
orig.insert(0, 'kindle')
proxy['send_to'] = orig
def configure_for_generic_epub_app(self):
with self.prefs:
for x in ('format_map', 'send_template', 'send_to'):
del self.prefs[x]
def open(self, devices, library_uuid):
self.current_library_uuid = library_uuid
self.location_paths = None
self.driveinfo = {}
BASE.open(self, devices, library_uuid)
h = self.prefs['history']
if self.current_serial_num:
h[self.current_serial_num] = (self.current_friendly_name,
isoformat(utcnow()))
self.prefs['history'] = h
# Device information {{{
def _update_drive_info(self, storage, location_code, name=None):
import uuid
f = storage.find_path((self.DRIVEINFO,))
dinfo = {}
if f is not None:
stream = self.get_mtp_file(f)
try:
dinfo = json.load(stream, object_hook=from_json)
except:
dinfo = None
if dinfo.get('device_store_uuid', None) is None:
dinfo['device_store_uuid'] = unicode(uuid.uuid4())
if dinfo.get('device_name', None) is None:
dinfo['device_name'] = self.current_friendly_name
if name is not None:
dinfo['device_name'] = name
dinfo['location_code'] = location_code
dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version])
dinfo['date_last_connected'] = isoformat(now())
dinfo['mtp_prefix'] = storage.storage_prefix
raw = json.dumps(dinfo, default=to_json)
self.put_file(storage, self.DRIVEINFO, BytesIO(raw), len(raw))
self.driveinfo[location_code] = dinfo
def get_driveinfo(self):
if not self.driveinfo:
self.driveinfo = {}
for sid, location_code in ( (self._main_id, 'main'), (self._carda_id,
'A'), (self._cardb_id, 'B')):
if sid is None: continue
self._update_drive_info(self.filesystem_cache.storage(sid), location_code)
return self.driveinfo
def get_device_information(self, end_session=True):
self.report_progress(1.0, _('Get device information...'))
dinfo = self.get_basic_device_information()
return tuple( list(dinfo) + [self.driveinfo] )
def card_prefix(self, end_session=True):
return (self._carda_id, self._cardb_id)
def set_driveinfo_name(self, location_code, name):
sid = {'main':self._main_id, 'A':self._carda_id,
'B':self._cardb_id}.get(location_code, None)
if sid is None:
return
self._update_drive_info(self.filesystem_cache.storage(sid),
location_code, name=name)
# }}}
# Get list of books from device, with metadata {{{
def books(self, oncard=None, end_session=True):
from calibre.devices.mtp.books import JSONCodec
from calibre.devices.mtp.books import BookList, Book
self.get_driveinfo() # Ensure driveinfo is loaded
sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard,
self._main_id)
if sid is None:
return BookList(None)
bl = BookList(sid)
# If True then there is a mismatch between the ebooks on the device and
# the metadata cache
need_sync = False
all_books = list(self.filesystem_cache.iterebooks(sid))
steps = len(all_books) + 2
count = 0
self.report_progress(0, _('Reading metadata from device'))
# Read the cache if it exists
storage = self.filesystem_cache.storage(sid)
cache = storage.find_path((self.METADATA_CACHE,))
if cache is not None:
json_codec = JSONCodec()
try:
stream = self.get_mtp_file(cache)
json_codec.decode_from_file(stream, bl, Book, sid)
except:
need_sync = True
relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)}
for mtp_file in all_books:
count += 1
relpath = mtp_file.mtp_relpath
idx = relpath_cache.get(relpath, None)
if idx is not None:
cached_metadata = bl[idx]
del relpath_cache[relpath]
if cached_metadata.size == mtp_file.size:
cached_metadata.datetime = mtp_file.last_modified.timetuple()
cached_metadata.path = mtp_file.mtp_id_path
debug('Using cached metadata for',
'/'.join(mtp_file.full_path))
continue # No need to update metadata
book = cached_metadata
else:
book = Book(sid, '/'.join(relpath))
bl.append(book)
need_sync = True
self.report_progress(count/steps, _('Reading metadata from %s')%
('/'.join(relpath)))
try:
book.smart_update(self.read_file_metadata(mtp_file))
debug('Read metadata for', '/'.join(mtp_file.full_path))
except:
prints('Failed to read metadata from',
'/'.join(mtp_file.full_path))
traceback.print_exc()
book.size = mtp_file.size
book.datetime = mtp_file.last_modified.timetuple()
book.path = mtp_file.mtp_id_path
# Remove books in the cache that no longer exist
for idx in sorted(relpath_cache.itervalues(), reverse=True):
del bl[idx]
need_sync = True
if need_sync:
self.report_progress(count/steps, _('Updating metadata cache on device'))
self.write_metadata_cache(storage, bl)
self.report_progress(1, _('Finished reading metadata from device'))
return bl
def read_file_metadata(self, mtp_file):
from calibre.ebooks.metadata.meta import get_metadata
from calibre.customize.ui import quick_metadata
ext = mtp_file.name.rpartition('.')[-1].lower()
stream = self.get_mtp_file(mtp_file)
with quick_metadata:
return get_metadata(stream, stream_type=ext,
force_read_metadata=True,
pattern=self.build_template_regexp())
def write_metadata_cache(self, storage, bl):
from calibre.devices.mtp.books import JSONCodec
if bl.storage_id != storage.storage_id:
# Just a sanity check, should never happen
return
json_codec = JSONCodec()
stream = SpooledTemporaryFile(10*(1024**2))
json_codec.encode_to_file(stream, bl)
size = stream.tell()
stream.seek(0)
self.put_file(storage, self.METADATA_CACHE, stream, size)
def sync_booklists(self, booklists, end_session=True):
debug('sync_booklists() called')
for bl in booklists:
if getattr(bl, 'storage_id', None) is None:
continue
storage = self.filesystem_cache.storage(bl.storage_id)
if storage is None:
continue
self.write_metadata_cache(storage, bl)
debug('sync_booklists() ended')
# }}}
# Get files from the device {{{
def get_file(self, path, outfile, end_session=True):
f = self.filesystem_cache.resolve_mtp_id_path(path)
self.get_mtp_file(f, outfile)
def prepare_addable_books(self, paths):
tdir = PersistentTemporaryDirectory('_prepare_mtp')
ans = []
for path in paths:
try:
f = self.filesystem_cache.resolve_mtp_id_path(path)
except Exception as e:
ans.append((path, e, traceback.format_exc()))
continue
base = os.path.join(tdir, '%s'%f.object_id)
os.mkdir(base)
with open(os.path.join(base, f.name), 'wb') as out:
try:
self.get_mtp_file(f, out)
except Exception as e:
ans.append((path, e, traceback.format_exc()))
else:
ans.append(out.name)
return ans
# }}}
# Sending files to the device {{{
def set_plugboards(self, plugboards, pb_func):
self.plugboards = plugboards
self.plugboard_func = pb_func
def create_upload_path(self, path, mdata, fname, routing):
from calibre.devices.utils import create_upload_path
from calibre.utils.filenames import ascii_filename as sanitize
ext = fname.rpartition('.')[-1].lower()
path = routing.get(ext, path)
filepath = create_upload_path(mdata, fname, self.save_template, sanitize,
prefix_path=path,
path_type=posixpath,
maxlen=self.MAX_PATH_LEN,
use_subdirs=True,
news_in_folder=self.NEWS_IN_FOLDER,
)
return tuple(x for x in filepath.split('/'))
def prefix_for_location(self, on_card):
if self.location_paths is None:
self.location_paths = {}
for sid, loc in ( (self._main_id, None), (self._carda_id, 'carda'),
(self._cardb_id, 'cardb') ):
if sid is not None:
storage = self.filesystem_cache.storage(sid)
prefixes = self.get_pref('send_to')
p = None
for path in prefixes:
path = path.replace(os.sep, '/')
if storage.find_path(path.split('/')) is not None:
p = path
break
if p is None:
p = 'Books'
self.location_paths[loc] = p
return self.location_paths[on_card]
def ensure_parent(self, storage, path):
parent = storage
pos = list(path)[:-1]
while pos:
name = pos[0]
pos = pos[1:]
parent = self.create_folder(parent, name)
return parent
def upload_books(self, files, names, on_card=None, end_session=True,
metadata=None):
debug('upload_books() called')
from calibre.devices.utils import sanity_check
sanity_check(on_card, files, self.card_prefix(), self.free_space())
prefix = self.prefix_for_location(on_card)
sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(on_card,
self._main_id)
bl_idx = {'carda':1, 'cardb':2}.get(on_card, 0)
storage = self.filesystem_cache.storage(sid)
ans = []
self.report_progress(0, _('Transferring books to device...'))
i, total = 0, len(files)
routing = {fmt:dest for fmt,dest in self.get_pref('rules')}
for infile, fname, mi in izip(files, names, metadata):
path = self.create_upload_path(prefix, mi, fname, routing)
parent = self.ensure_parent(storage, path)
if hasattr(infile, 'read'):
pos = infile.tell()
infile.seek(0, 2)
sz = infile.tell()
infile.seek(pos)
stream = infile
close = False
else:
sz = os.path.getsize(infile)
stream = lopen(infile, 'rb')
close = True
try:
mtp_file = self.put_file(parent, path[-1], stream, sz)
finally:
if close:
stream.close()
ans.append((mtp_file, bl_idx))
i += 1
self.report_progress(i/total, _('Transferred %s to device')%mi.title)
self.report_progress(1, _('Transfer to device finished...'))
debug('upload_books() ended')
return ans
def add_books_to_metadata(self, mtp_files, metadata, booklists):
debug('add_books_to_metadata() called')
from calibre.devices.mtp.books import Book
i, total = 0, len(mtp_files)
self.report_progress(0, _('Adding books to device metadata listing...'))
for x, mi in izip(mtp_files, metadata):
mtp_file, bl_idx = x
bl = booklists[bl_idx]
book = Book(mtp_file.storage_id, '/'.join(mtp_file.mtp_relpath),
other=mi)
book = bl.add_book(book, replace_metadata=True)
if book is not None:
book.size = mtp_file.size
book.datetime = mtp_file.last_modified.timetuple()
book.path = mtp_file.mtp_id_path
i += 1
self.report_progress(i/total, _('Added %s')%mi.title)
self.report_progress(1, _('Adding complete'))
debug('add_books_to_metadata() ended')
# }}}
# Removing books from the device {{{
def recursive_delete(self, obj):
parent = self.delete_file_or_folder(obj)
if parent.empty and parent.can_delete and not parent.is_system:
try:
self.recursive_delete(parent)
except:
prints('Failed to delete parent: %s, ignoring'%(
'/'.join(parent.full_path)))
def delete_books(self, paths, end_session=True):
self.report_progress(0, _('Deleting books from device...'))
for i, path in enumerate(paths):
f = self.filesystem_cache.resolve_mtp_id_path(path)
self.recursive_delete(f)
self.report_progress((i+1) / float(len(paths)),
_('Deleted %s')%path)
self.report_progress(1, _('All books deleted'))
def remove_books_from_metadata(self, paths, booklists):
self.report_progress(0, _('Removing books from metadata'))
class NextPath(Exception): pass
for i, path in enumerate(paths):
try:
for bl in booklists:
for book in bl:
if book.path == path:
bl.remove_book(book)
raise NextPath('')
except NextPath:
pass
self.report_progress((i+1)/len(paths), _('Removed %s')%path)
self.report_progress(1, _('All books removed'))
# }}}
# Settings {{{
def get_pref(self, key):
return self.prefs.get('device-%s'%self.current_serial_num, {}).get(key,
self.prefs[key])
def config_widget(self):
from calibre.gui2.device_drivers.mtp_config import MTPConfig
return MTPConfig(self)
def save_settings(self, cw):
cw.commit()
def settings(self):
class Opts(object):
def __init__(s):
s.format_map = self.get_pref('format_map')
return Opts()
@property
def save_template(self):
return self.prefs['send_template']
# }}}
if __name__ == '__main__':
dev = MTP_DEVICE(None)
dev.startup()
try:
from calibre.devices.scanner import DeviceScanner
scanner = DeviceScanner()
scanner.scan()
devs = scanner.devices
cd = dev.detect_managed_devices(devs)
if cd is None:
raise ValueError('Failed to detect MTP device')
dev.set_progress_reporter(prints)
dev.open(cd, None)
dev.filesystem_cache.dump()
print ('Prefix for main mem:', dev.prefix_for_location(None))
finally:
dev.shutdown()

251
src/calibre/devices/mtp/filesystem_cache.py Normal file
View File

@ -0,0 +1,251 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import weakref, sys, json
from collections import deque
from operator import attrgetter
from future_builtins import map
from datetime import datetime
from calibre import human_readable, prints, force_unicode
from calibre.utils.date import local_tz, as_utc
from calibre.utils.icu import sort_key, lower
from calibre.ebooks import BOOK_EXTENSIONS
bexts = frozenset(BOOK_EXTENSIONS)
class FileOrFolder(object):
def __init__(self, entry, fs_cache):
self.all_storage_ids = fs_cache.all_storage_ids
self.object_id = entry['id']
self.is_folder = entry['is_folder']
self.storage_id = entry['storage_id']
# self.parent_id is None for storage objects
self.parent_id = entry.get('parent_id', None)
n = entry.get('name', None)
if not n: n = '___'
self.name = force_unicode(n, 'utf-8')
self.persistent_id = entry.get('persistent_id', self.object_id)
self.size = entry.get('size', 0)
md = entry.get('modified', 0)
try:
if isinstance(md, tuple):
self.last_modified = datetime(*(list(md)+[local_tz]))
else:
self.last_modified = datetime.fromtimestamp(md, local_tz)
except:
self.last_modified = datetime.fromtimestamp(0, local_tz)
self.last_mod_string = self.last_modified.strftime('%Y/%m/%d %H:%M')
self.last_modified = as_utc(self.last_modified)
if self.storage_id not in self.all_storage_ids:
raise ValueError('Storage id %s not valid for %s, valid values: %s'%(self.storage_id,
entry, self.all_storage_ids))
if self.parent_id == 0:
self.parent_id = self.storage_id
self.is_hidden = entry.get('is_hidden', False)
self.is_system = entry.get('is_system', False)
self.can_delete = entry.get('can_delete', True)
self.files = []
self.folders = []
fs_cache.id_map[self.object_id] = self
self.fs_cache = weakref.ref(fs_cache)
self.deleted = False
if self.storage_id == self.object_id:
self.storage_prefix = 'mtp:::%s:::'%self.persistent_id
self.is_ebook = (not self.is_folder and
self.name.rpartition('.')[-1].lower() in bexts)
def __repr__(self):
name = 'Folder' if self.is_folder else 'File'
try:
path = unicode(self.full_path)
except:
path = ''
datum = 'size=%s'%(self.size)
if self.is_folder:
datum = 'children=%s'%(len(self.files) + len(self.folders))
return '%s(id=%s, storage_id=%s, %s, path=%s, modified=%s)'%(name, self.object_id,
self.storage_id, datum, path, self.last_mod_string)
__str__ = __repr__
__unicode__ = __repr__
@property
def empty(self):
return not self.files and not self.folders
@property
def id_map(self):
return self.fs_cache().id_map
@property
def parent(self):
return None if self.parent_id is None else self.id_map[self.parent_id]
@property
def full_path(self):
parts = deque()
parts.append(self.name)
p = self.parent
while p is not None:
parts.appendleft(p.name)
p = p.parent
return tuple(parts)
def __iter__(self):
for e in self.folders:
yield e
for e in self.files:
yield e
def add_child(self, entry):
ans = FileOrFolder(entry, self.fs_cache())
t = self.folders if ans.is_folder else self.files
t.append(ans)
return ans
def remove_child(self, entry):
for x in (self.files, self.folders):
try:
x.remove(entry)
except ValueError:
pass
self.id_map.pop(entry.object_id, None)
entry.deleted = True
def dump(self, prefix='', out=sys.stdout):
c = '+' if self.is_folder else '-'
data = ('%s children'%(sum(map(len, (self.files, self.folders))))
if self.is_folder else human_readable(self.size))
data += ' modified=%s'%self.last_mod_string
line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data)
prints(line, file=out)
for c in (self.folders, self.files):
for e in sorted(c, key=lambda x:sort_key(x.name)):
e.dump(prefix=prefix+' ', out=out)
def folder_named(self, name):
name = lower(name)
for e in self.folders:
if e.name and lower(e.name) == name:
return e
return None
def file_named(self, name):
name = lower(name)
for e in self.files:
if e.name and lower(e.name) == name:
return e
return None
def find_path(self, path):
'''
Find a path in this folder, where path is a
tuple of folder and file names like ('eBooks', 'newest',
'calibre.epub'). Finding is case-insensitive.
'''
parent = self
components = list(path)
while components:
child = components[0]
components = components[1:]
c = parent.folder_named(child)
if c is None:
c = parent.file_named(child)
if c is None:
return None
parent = c
return parent
@property
def mtp_relpath(self):
return tuple(x.lower() for x in self.full_path[1:])
@property
def mtp_id_path(self):
return 'mtp:::' + json.dumps(self.object_id) + ':::' + '/'.join(self.full_path)
class FilesystemCache(object):
def __init__(self, all_storage, entries):
self.entries = []
self.id_map = {}
self.all_storage_ids = tuple(x['id'] for x in all_storage)
for storage in all_storage:
storage['storage_id'] = storage['id']
e = FileOrFolder(storage, self)
self.entries.append(e)
self.entries.sort(key=attrgetter('object_id'))
all_storage_ids = [x.storage_id for x in self.entries]
self.all_storage_ids = tuple(all_storage_ids)
for entry in entries:
FileOrFolder(entry, self)
for item in self.id_map.itervalues():
try:
p = item.parent
except KeyError:
# Parent does not exist, set the parent to be the storage
# object
sid = item.storage_id
if sid not in all_storage_ids:
sid = all_storage_ids[0]
item.parent_id = sid
p = item.parent
if p is not None:
t = p.folders if item.is_folder else p.files
t.append(item)
def dump(self, out=sys.stdout):
for e in self.entries:
e.dump(out=out)
def storage(self, storage_id):
for e in self.entries:
if e.storage_id == storage_id:
return e
def iterebooks(self, storage_id):
for x in self.id_map.itervalues():
if x.storage_id == storage_id and x.is_ebook:
if x.parent_id == storage_id and x.name.lower().endswith('.txt'):
continue # Ignore .txt files in the root
yield x
def __len__(self):
return len(self.id_map)
def resolve_mtp_id_path(self, path):
if not path.startswith('mtp:::'):
raise ValueError('%s is not a valid MTP path'%path)
parts = path.split(':::')
if len(parts) < 3:
raise ValueError('%s is not a valid MTP path'%path)
try:
object_id = json.loads(parts[1])
except:
raise ValueError('%s is not a valid MTP path'%path)
try:
return self.id_map[object_id]
except KeyError:
raise ValueError('No object found with MTP path: %s'%path)

261
src/calibre/devices/mtp/test.py Normal file
View File

@ -0,0 +1,261 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import unittest, gc, io
from calibre.constants import iswindows, islinux
from calibre.utils.icu import lower
from calibre.devices.mtp.driver import MTP_DEVICE
from calibre.devices.scanner import DeviceScanner
class ProgressCallback(object):
def __init__(self):
self.count = 0
self.end_called = False
def __call__(self, pos, total):
if pos == total:
self.end_called = True
self.count += 1
class TestDeviceInteraction(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.dev = cls.storage = None
cls.dev = MTP_DEVICE(None)
cls.dev.startup()
cls.scanner = DeviceScanner()
cls.scanner.scan()
cd = cls.dev.detect_managed_devices(cls.scanner.devices)
if cd is None:
cls.dev.shutdown()
cls.dev = None
return
cls.dev.open(cd, 'test_library')
if cls.dev.free_space()[0] < 10*(1024**2):
return
cls.dev.filesystem_cache
cls.storage = cls.dev.filesystem_cache.entries[0]
@classmethod
def tearDownClass(cls):
if cls.dev is not None:
cls.dev.shutdown()
cls.dev = None
def setUp(self):
self.cleanup = []
def tearDown(self):
for obj in reversed(self.cleanup):
self.dev.delete_file_or_folder(obj)
def check_setup(self):
if self.dev is None:
self.skipTest('No MTP device detected')
if self.storage is None:
self.skipTest('The connected device does not have enough free space')
def test_folder_operations(self):
''' Test the creation of folders, duplicate folders and sub folders '''
self.check_setup()
# Create a folder
name = 'zzz-test-folder'
folder = self.dev.create_folder(self.storage, name)
self.cleanup.append(folder)
self.assertTrue(folder.is_folder)
self.assertEqual(folder.parent_id, self.storage.object_id)
self.assertEqual(folder.storage_id, self.storage.object_id)
self.assertEqual(lower(name), lower(folder.name))
# Create a sub-folder
name = 'sub-folder'
subfolder = self.dev.create_folder(folder, name)
self.assertTrue(subfolder.is_folder)
self.assertEqual(subfolder.parent_id, folder.object_id)
self.assertEqual(subfolder.storage_id, self.storage.object_id)
self.assertEqual(lower(name), lower(subfolder.name))
self.cleanup.append(subfolder)
# Check that creating an existing folder returns that folder (case
# insensitively)
self.assertIs(subfolder, self.dev.create_folder(folder,
'SUB-FOLDER'),
msg='Creating an existing folder did not return the existing folder')
# Check that creating folders as children of files is not allowed
root_file = [f for f in self.dev.filesystem_cache.entries[0].files if
not f.is_folder]
if root_file:
with self.assertRaises(ValueError):
self.dev.create_folder(root_file[0], 'sub-folder')
def test_file_transfer(self):
''' Test transferring files to and from the device '''
self.check_setup()
# Create a folder
name = 'zzz-test-folder'
folder = self.dev.create_folder(self.storage, name)
self.cleanup.append(folder)
self.assertTrue(folder.is_folder)
self.assertEqual(folder.parent_id, self.storage.object_id)
# Check simple file put/get
size = 1024**2
raw = io.BytesIO(b'a'*size)
raw.seek(0)
name = 'test-file.txt'
pc = ProgressCallback()
f = self.dev.put_file(folder, name, raw, size, callback=pc)
self.cleanup.append(f)
self.assertEqual(f.name, name)
self.assertEqual(f.size, size)
self.assertEqual(f.parent_id, folder.object_id)
self.assertEqual(f.storage_id, folder.storage_id)
self.assertTrue(pc.end_called,
msg='Progress callback not called with equal values (put_file)')
self.assertTrue(pc.count > 1,
msg='Progress callback only called once (put_file)')
raw2 = io.BytesIO()
pc = ProgressCallback()
self.dev.get_mtp_file(f, raw2, callback=pc)
self.assertEqual(raw.getvalue(), raw2.getvalue())
self.assertTrue(pc.end_called,
msg='Progress callback not called with equal values (get_file)')
self.assertTrue(pc.count > 1,
msg='Progress callback only called once (get_file)')
# Check file replacement
raw = io.BytesIO(b'abcd')
raw.seek(0)
size = 4
f = self.dev.put_file(folder, name, raw, size)
self.cleanup.append(f)
self.assertEqual(f.name, name)
self.assertEqual(f.size, size)
self.assertEqual(f.parent_id, folder.object_id)
self.assertEqual(f.storage_id, folder.storage_id)
# Check that we get an error with replace=False
raw.seek(0)
with self.assertRaises(ValueError):
self.dev.put_file(folder, name, raw, size, replace=False)
# Check that we can put a file into the root
raw.seek(0)
name = 'zzz-test-file.txt'
f = self.dev.put_file(self.storage, name, raw, size)
self.cleanup.append(f)
self.assertEqual(f.name, name)
self.assertEqual(f.size, size)
self.assertEqual(f.parent_id, self.storage.object_id)
self.assertEqual(f.storage_id, self.storage.storage_id)
raw2 = io.BytesIO()
self.dev.get_mtp_file(f, raw2)
self.assertEqual(raw.getvalue(), raw2.getvalue())
def measure_memory_usage(self, repetitions, func, *args, **kwargs):
from calibre.utils.mem import memory
gc.disable()
try:
start_mem = memory()
for i in xrange(repetitions):
func(*args, **kwargs)
for i in xrange(3): gc.collect()
end_mem = memory()
finally:
gc.enable()
return end_mem - start_mem
def check_memory(self, once, many, msg, factor=2):
msg += ' for once: %g for many: %g'%(once, many)
if once > 0:
self.assertTrue(many <= once*factor, msg=msg)
else:
self.assertTrue(many <= 0.01, msg=msg)
@unittest.skipUnless(iswindows or islinux, 'Can only test for leaks on windows and linux')
def test_memory_leaks(self):
''' Test for memory leaks in the C module '''
self.check_setup()
# Test device scanning
used_by_one = self.measure_memory_usage(1,
self.dev.detect_managed_devices, self.scanner.devices,
force_refresh=True)
used_by_many = self.measure_memory_usage(100,
self.dev.detect_managed_devices, self.scanner.devices,
force_refresh=True)
self.check_memory(used_by_one, used_by_many,
'Memory consumption during device scan')
# Test file transfer
size = 1024*100
raw = io.BytesIO(b'a'*size)
raw.seek(0)
name = 'zzz-test-file.txt'
def send_file(storage, name, raw, size):
raw.seek(0)
pc = ProgressCallback()
f = self.dev.put_file(storage, name, raw, size, callback=pc)
self.cleanup.append(f)
del pc
used_once = self.measure_memory_usage(1, send_file, self.storage, name,
raw, size)
used_many = self.measure_memory_usage(20, send_file, self.storage, name,
raw, size)
self.check_memory(used_once, used_many,
'Memory consumption during put_file:')
def get_file(f):
raw = io.BytesIO()
pc = ProgressCallback()
self.dev.get_mtp_file(f, raw, callback=pc)
raw.truncate(0)
del raw
del pc
f = self.storage.file_named(name)
used_once = self.measure_memory_usage(1, get_file, f)
used_many = self.measure_memory_usage(20, get_file, f)
self.check_memory(used_once, used_many,
'Memory consumption during get_file:')
# Test get_filesystem
used_by_one = self.measure_memory_usage(1,
self.dev.dev.get_filesystem, self.storage.object_id)
used_by_many = self.measure_memory_usage(5,
self.dev.dev.get_filesystem, self.storage.object_id)
self.check_memory(used_by_one, used_by_many,
'Memory consumption during get_filesystem')
def tests():
tl = unittest.TestLoader()
# return tl.loadTestsFromName('test.TestDeviceInteraction.test_memory_leaks')
return tl.loadTestsFromTestCase(TestDeviceInteraction)
def run():
unittest.TextTestRunner(verbosity=2).run(tests())
if __name__ == '__main__':
run()

71
src/calibre/devices/mtp/unix/detect.py
View File

@ -1,71 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from calibre.constants import plugins
class MTPDetect(object):
def __init__(self):
p = plugins['libmtp']
self.libmtp = p[0]
if self.libmtp is None:
print ('Failed to load libmtp, MTP device detection disabled')
print (p[1])
self.cache = {}
def __call__(self, devices):
'''
Given a list of devices as returned by LinuxScanner, return the set of
devices that are likely to be MTP devices. This class maintains a cache
to minimize USB polling. Note that detection is partially based on a
list of known vendor and product ids. This is because polling some
older devices causes problems. Therefore, if this method identifies a
device as MTP, it is not actually guaranteed that it will be a working
MTP device.
'''
# First drop devices that have been disconnected from the cache
connected_devices = {(d.busnum, d.devnum, d.vendor_id, d.product_id,
d.bcd, d.serial) for d in devices}
for d in tuple(self.cache.iterkeys()):
if d not in connected_devices:
del self.cache[d]
# Since is_mtp_device() can cause USB traffic by probing the device, we
# cache its result
mtp_devices = set()
if self.libmtp is None:
return mtp_devices
for d in devices:
ans = self.cache.get((d.busnum, d.devnum, d.vendor_id, d.product_id,
d.bcd, d.serial), None)
if ans is None:
ans = self.libmtp.is_mtp_device(d.busnum, d.devnum,
d.vendor_id, d.product_id)
self.cache[(d.busnum, d.devnum, d.vendor_id, d.product_id,
d.bcd, d.serial)] = ans
if ans:
mtp_devices.add(d)
return mtp_devices
def create_device(self, connected_device):
d = connected_device
return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
d.product_id, d.manufacturer, d.product, d.serial)
if __name__ == '__main__':
from calibre.devices.scanner import linux_scanner
mtp_detect = MTPDetect()
devs = mtp_detect(linux_scanner())
print ('Found %d MTP devices:'%len(devs))
for dev in devs:
print (dev, 'at busnum=%d and devnum=%d'%(dev.busnum, dev.devnum))
print()

444
src/calibre/devices/mtp/unix/driver.py
View File

@ -7,140 +7,153 @@ __license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import time, operator
import operator, traceback, pprint, sys, time
from threading import RLock
from itertools import chain
from collections import deque, OrderedDict
from io import BytesIO
from collections import namedtuple
from functools import partial
from calibre import prints
from calibre.devices.errors import OpenFailed, DeviceError
from calibre.devices.mtp.base import MTPDeviceBase, synchronous
from calibre.devices.mtp.unix.detect import MTPDetect
from calibre import prints, as_unicode
from calibre.constants import plugins
from calibre.ptempfile import SpooledTemporaryFile
from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice
from calibre.devices.mtp.base import MTPDeviceBase, synchronous, debug
class FilesystemCache(object):
MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
'bcd serial manufacturer product')
def __init__(self, files, folders):
self.files = files
self.folders = folders
self.file_id_map = {f['id']:f for f in files}
self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)}
# Set the parents of each file
self.files_in_root = OrderedDict()
for f in files:
parents = deque()
pid = f['parent_id']
while pid is not None and pid > 0:
try:
parent = self.folder_id_map[pid]
except KeyError:
break
parents.appendleft(pid)
pid = parent['parent_id']
f['parents'] = parents
if not parents:
self.files_in_root[f['id']] = f
# Set the files in each folder
for f in self.iterfolders():
f['files'] = [i for i in files if i['parent_id'] ==
f['id']]
# Decode the file and folder names
for f in chain(files, folders):
try:
name = f['name'].decode('utf-8')
except UnicodeDecodeError:
name = 'undecodable_%d'%f['id']
f['name'] = name
def iterfolders(self, folders=None, set_level=None):
clevel = None if set_level is None else set_level + 1
if folders is None:
folders = self.folders
for f in folders:
if set_level is not None:
f['level'] = set_level
yield f
for c in f['children']:
for child in self.iterfolders([c], set_level=clevel):
yield child
def dump_filesystem(self):
indent = 2
for f in self.iterfolders():
prefix = ' '*(indent*f['level'])
prints(prefix, '+', f['name'], 'id=%s'%f['id'])
for leaf in f['files']:
prints(prefix, ' '*indent, '-', leaf['name'],
'id=%d'%leaf['id'], 'size=%d'%leaf['size'],
'modtime=%d'%leaf['modtime'])
for leaf in self.files_in_root.itervalues():
prints('-', leaf['name'], 'id=%d'%leaf['id'],
'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime'])
def fingerprint(d):
return MTPDevice(d.busnum, d.devnum, d.vendor_id, d.product_id, d.bcd,
d.serial, d.manufacturer, d.product)
class MTP_DEVICE(MTPDeviceBase):
# libusb(x) does not work on OS X. So no MTP support for OS X
supported_platforms = ['linux']
def __init__(self, *args, **kwargs):
MTPDeviceBase.__init__(self, *args, **kwargs)
self.libmtp = None
self.known_devices = None
self.detect_cache = {}
self.dev = None
self.filesystem_cache = None
self._filesystem_cache = None
self.lock = RLock()
self.blacklisted_devices = set()
self.ejected_devices = set()
self.currently_connected_dev = None
def set_debug_level(self, lvl):
self.detect.libmtp.set_debug_level(lvl)
def report_progress(self, sent, total):
try:
p = int(sent/total * 100)
except ZeroDivisionError:
p = 100
if self.progress_reporter is not None:
self.progress_reporter(p)
self.libmtp.set_debug_level(lvl)
@synchronous
def is_usb_connected(self, devices_on_system, debug=False,
only_presence=False):
def detect_managed_devices(self, devices_on_system, force_refresh=False):
if self.libmtp is None: return None
# First remove blacklisted devices.
devs = []
devs = set()
for d in devices_on_system:
if (d.busnum, d.devnum, d.vendor_id,
d.product_id, d.bcd, d.serial) not in self.blacklisted_devices:
devs.append(d)
fp = fingerprint(d)
if fp not in self.blacklisted_devices:
devs.add(fp)
devs = self.detect(devs)
if self.dev is not None:
# Check if the currently opened device is still connected
ids = self.dev.ids
found = False
for d in devs:
if ( (d.busnum, d.devnum, d.vendor_id, d.product_id, d.serial)
== ids ):
found = True
break
return found
# Check if any MTP capable device is present
return len(devs) > 0
# Clean up ejected devices
self.ejected_devices = devs.intersection(self.ejected_devices)
# Check if the currently connected device is still present
if self.currently_connected_dev is not None:
return (self.currently_connected_dev if
self.currently_connected_dev in devs else None)
# Remove ejected devices
devs = devs - self.ejected_devices
# Now check for MTP devices
if force_refresh:
self.detect_cache = {}
cache = self.detect_cache
for d in devs:
ans = cache.get(d, None)
if ans is None:
ans = (d.vendor_id, d.product_id) in self.known_devices
cache[d] = ans
if ans:
return d
return None
@synchronous
def debug_managed_device_detection(self, devices_on_system, output):
if self.currently_connected_dev is not None:
return True
p = partial(prints, file=output)
if self.libmtp is None:
err = plugins['libmtp'][1]
if not err:
err = 'startup() not called on this device driver'
p(err)
return False
devs = [d for d in devices_on_system if (d.vendor_id, d.product_id)
in self.known_devices]
if not devs:
p('No known MTP devices connected to system')
return False
p('Known MTP devices connected:')
for d in devs: p(d)
for d in devs:
p('\nTrying to open:', d)
try:
self.open(d, 'debug')
except BlacklistedDevice:
p('This device has been blacklisted by the user')
continue
except:
p('Opening device failed:')
p(traceback.format_exc())
return False
else:
p('Opened', self.current_friendly_name, 'successfully')
p('Storage info:')
p(pprint.pformat(self.dev.storage_info))
self.post_yank_cleanup()
return True
return False
@synchronous
def create_device(self, connected_device):
d = connected_device
return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
d.product_id, d.manufacturer, d.product, d.serial)
@synchronous
def eject(self):
if self.currently_connected_dev is None: return
self.ejected_devices.add(self.currently_connected_dev)
self.post_yank_cleanup()
@synchronous
def post_yank_cleanup(self):
self.dev = self.filesystem_cache = self.current_friendly_name = None
self.dev = self._filesystem_cache = self.current_friendly_name = None
self.currently_connected_dev = None
self.current_serial_num = None
@synchronous
def startup(self):
self.detect = MTPDetect()
for x in vars(self.detect.libmtp):
p = plugins['libmtp']
self.libmtp = p[0]
if self.libmtp is None:
print ('Failed to load libmtp, MTP device detection disabled')
print (p[1])
else:
self.known_devices = frozenset(self.libmtp.known_devices())
for x in vars(self.libmtp):
if x.startswith('LIBMTP'):
setattr(self, x, getattr(self.detect.libmtp, x))
setattr(self, x, getattr(self.libmtp, x))
@synchronous
def shutdown(self):
self.dev = self.filesystem_cache = None
self.dev = self._filesystem_cache = None
def format_errorstack(self, errs):
return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for
@ -148,69 +161,76 @@ class MTP_DEVICE(MTPDeviceBase):
@synchronous
def open(self, connected_device, library_uuid):
self.dev = self.filesystem_cache = None
def blacklist_device():
d = connected_device
self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
d.product_id, d.bcd, d.serial))
self.dev = self._filesystem_cache = None
try:
self.dev = self.detect.create_device(connected_device)
except ValueError:
# Give the device some time to settle
time.sleep(2)
try:
self.dev = self.detect.create_device(connected_device)
except ValueError:
# Black list this device so that it is ignored for the
# remainder of this session.
blacklist_device()
raise OpenFailed('%s is not a MTP device'%(connected_device,))
except TypeError:
blacklist_device()
raise OpenFailed('')
self.dev = self.create_device(connected_device)
except Exception as e:
raise OpenFailed('Failed to open %s: Error: %s'%(
connected_device, as_unicode(e)))
storage = sorted(self.dev.storage_info, key=operator.itemgetter('id'))
storage = [x for x in storage if x.get('rw', False)]
if not storage:
blacklist_device()
self.blacklisted_devices.add(connected_device)
raise OpenFailed('No storage found for device %s'%(connected_device,))
snum = self.dev.serial_number
if snum in self.prefs.get('blacklist', []):
self.blacklisted_devices.add(connected_device)
self.dev = None
raise BlacklistedDevice(
'The %s device has been blacklisted by the user'%(connected_device,))
self._main_id = storage[0]['id']
self._carda_id = self._cardb_id = None
if len(storage) > 1:
self._carda_id = storage[1]['id']
if len(storage) > 2:
self._cardb_id = storage[2]['id']
self.current_friendly_name = self.dev.name
self.current_friendly_name = self.dev.friendly_name
if not self.current_friendly_name:
self.current_friendly_name = self.dev.model_name or _('Unknown MTP device')
self.current_serial_num = snum
@property
def filesystem_cache(self):
if self._filesystem_cache is None:
st = time.time()
debug('Loading filesystem metadata...')
from calibre.devices.mtp.filesystem_cache import FilesystemCache
with self.lock:
storage, all_items, all_errs = [], [], []
for sid, capacity in zip([self._main_id, self._carda_id,
self._cardb_id], self.total_space()):
if sid is None: continue
name = _('Unknown')
for x in self.dev.storage_info:
if x['id'] == sid:
name = x['name']
break
storage.append({'id':sid, 'size':capacity,
'is_folder':True, 'name':name, 'can_delete':False,
'is_system':True})
items, errs = self.dev.get_filesystem(sid)
all_items.extend(items), all_errs.extend(errs)
if not all_items and all_errs:
raise DeviceError(
'Failed to read filesystem from %s with errors: %s'
%(self.current_friendly_name,
self.format_errorstack(all_errs)))
if all_errs:
prints('There were some errors while getting the '
' filesystem from %s: %s'%(
self.current_friendly_name,
self.format_errorstack(all_errs)))
self._filesystem_cache = FilesystemCache(storage, all_items)
debug('Filesystem metadata loaded in %g seconds (%d objects)'%(
time.time()-st, len(self._filesystem_cache)))
return self._filesystem_cache
@synchronous
def read_filesystem_cache(self):
try:
files, errs = self.dev.get_filelist(self)
if errs and not files:
raise DeviceError('Failed to read files from device. Underlying errors:\n'
+self.format_errorstack(errs))
folders, errs = self.dev.get_folderlist()
if errs and not folders:
raise DeviceError('Failed to read folders from device. Underlying errors:\n'
+self.format_errorstack(errs))
self.filesystem_cache = FilesystemCache(files, folders)
except:
self.dev = self._main_id = self._carda_id = self._cardb_id = None
raise
@synchronous
def get_device_information(self, end_session=True):
def get_basic_device_information(self):
d = self.dev
return (self.current_friendly_name, d.device_version, d.device_version, '')
@synchronous
def card_prefix(self, end_session=True):
ans = [None, None]
if self._carda_id is not None:
ans[0] = 'mtp:::%d:::'%self._carda_id
if self._cardb_id is not None:
ans[1] = 'mtp:::%d:::'%self._cardb_id
return tuple(ans)
@synchronous
def total_space(self, end_session=True):
ans = [0, 0, 0]
@ -232,39 +252,105 @@ class MTP_DEVICE(MTPDeviceBase):
ans[i] = s['freespace_bytes']
return tuple(ans)
@synchronous
def create_folder(self, parent, name):
if not parent.is_folder:
raise ValueError('%s is not a folder'%(parent.full_path,))
e = parent.folder_named(name)
if e is not None:
return e
ename = name.encode('utf-8') if isinstance(name, unicode) else name
sid, pid = parent.storage_id, parent.object_id
if pid == sid:
pid = 0
ans, errs = self.dev.create_folder(sid, pid, ename)
if ans is None:
raise DeviceError(
'Failed to create folder named %s in %s with error: %s'%
(name, parent.full_path, self.format_errorstack(errs)))
return parent.add_child(ans)
if __name__ == '__main__':
BytesIO
class PR:
def report_progress(self, sent, total):
print (sent, total, end=', ')
@synchronous
def put_file(self, parent, name, stream, size, callback=None, replace=True):
e = parent.folder_named(name)
if e is not None:
raise ValueError('Cannot upload file, %s already has a folder named: %s'%(
parent.full_path, e.name))
e = parent.file_named(name)
if e is not None:
if not replace:
raise ValueError('Cannot upload file %s, it already exists'%(
e.full_path,))
self.delete_file_or_folder(e)
ename = name.encode('utf-8') if isinstance(name, unicode) else name
sid, pid = parent.storage_id, parent.object_id
if pid == sid:
pid = 0
from pprint import pprint
ans, errs = self.dev.put_file(sid, pid, ename, stream, size, callback)
if ans is None:
raise DeviceError('Failed to upload file named: %s to %s: %s'
%(name, parent.full_path, self.format_errorstack(errs)))
return parent.add_child(ans)
@synchronous
def get_mtp_file(self, f, stream=None, callback=None):
if f.is_folder:
raise ValueError('%s if a folder'%(f.full_path,))
if stream is None:
stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat')
stream.name = f.name
ok, errs = self.dev.get_file(f.object_id, stream, callback)
if not ok:
raise DeviceError('Failed to get file: %s with errors: %s'%(
f.full_path, self.format_errorstack(errs)))
stream.seek(0)
return stream
@synchronous
def delete_file_or_folder(self, obj):
if obj.deleted:
return
if not obj.can_delete:
raise ValueError('Cannot delete %s as deletion not allowed'%
(obj.full_path,))
if obj.is_system:
raise ValueError('Cannot delete %s as it is a system object'%
(obj.full_path,))
if obj.files or obj.folders:
raise ValueError('Cannot delete %s as it is not empty'%
(obj.full_path,))
parent = obj.parent
ok, errs = self.dev.delete_object(obj.object_id)
if not ok:
raise DeviceError('Failed to delete %s with error: %s'%
(obj.full_path, self.format_errorstack(errs)))
parent.remove_child(obj)
return parent
def develop():
from calibre.devices.scanner import DeviceScanner
scanner = DeviceScanner()
scanner.scan()
dev = MTP_DEVICE(None)
dev.startup()
from calibre.devices.scanner import linux_scanner
devs = linux_scanner()
mtp_devs = dev.detect(devs)
dev.open(list(mtp_devs)[0], 'xxx')
dev.read_filesystem_cache()
d = dev.dev
print ("Opened device:", dev.get_gui_name())
print ("Storage info:")
pprint(d.storage_info)
print("Free space:", dev.free_space())
# print (d.create_folder(dev._main_id, 0, 'testf'))
# raw = b'test'
# fname = b'moose.txt'
# src = BytesIO(raw)
# print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR()))
dev.filesystem_cache.dump_filesystem()
# with open('/tmp/flint.epub', 'wb') as f:
# print(d.get_file(786, f, PR()))
# print()
# with open('/tmp/bleak.epub', 'wb') as f:
# print(d.get_file(601, f, PR()))
# print()
try:
cd = dev.detect_managed_devices(scanner.devices)
if cd is None: raise RuntimeError('No MTP device found')
dev.open(cd, 'develop')
pprint.pprint(dev.dev.storage_info)
dev.filesystem_cache
finally:
dev.shutdown()
if __name__ == '__main__':
dev = MTP_DEVICE(None)
dev.startup()
from calibre.devices.scanner import DeviceScanner
scanner = DeviceScanner()
scanner.scan()
devs = scanner.devices
dev.debug_managed_device_detection(devs, sys.stdout)
dev.set_debug_level(dev.LIBMTP_DEBUG_ALL)
del d
dev.shutdown()

463
src/calibre/devices/mtp/unix/libmtp.c
View File

@ -14,7 +14,7 @@
#include "devices.h"
// Macros and utilities
// Macros and utilities {{{
static PyObject *MTPError = NULL;
#define ENSURE_DEV(rval) \
@ -55,7 +55,7 @@ static int report_progress(uint64_t const sent, uint64_t const total, void const
cb = (ProgressCallback *)data;
if (cb->obj != NULL) {
PyEval_RestoreThread(cb->state);
res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total);
res = PyObject_CallFunction(cb->obj, "KK", (unsigned long long)sent, (unsigned long long)total);
Py_XDECREF(res);
cb->state = PyEval_SaveThread();
}
@ -67,7 +67,7 @@ static void dump_errorstack(LIBMTP_mtpdevice_t *dev, PyObject *list) {
PyObject *err;
for(stack = LIBMTP_Get_Errorstack(dev); stack != NULL; stack=stack->next) {
err = Py_BuildValue("Is", stack->errornumber, stack->error_text);
err = Py_BuildValue("is", stack->errornumber, stack->error_text);
if (err == NULL) break;
PyList_Append(list, err);
Py_DECREF(err);
@ -84,7 +84,7 @@ static uint16_t data_to_python(void *params, void *priv, uint32_t sendlen, unsig
cb = (ProgressCallback *)priv;
*putlen = sendlen;
PyEval_RestoreThread(cb->state);
res = PyObject_CallMethod(cb->extra, "write", "s#", data, sendlen);
res = PyObject_CallMethod(cb->extra, "write", "s#", data, (Py_ssize_t)sendlen);
if (res == NULL) {
ret = LIBMTP_HANDLER_RETURN_ERROR;
*putlen = 0;
@ -106,7 +106,7 @@ static uint16_t data_from_python(void *params, void *priv, uint32_t wantlen, uns
cb = (ProgressCallback *)priv;
PyEval_RestoreThread(cb->state);
res = PyObject_CallMethod(cb->extra, "read", "k", wantlen);
res = PyObject_CallMethod(cb->extra, "read", "k", (unsigned long)wantlen);
if (res != NULL && PyBytes_AsStringAndSize(res, &buf, &len) != -1 && len <= wantlen) {
memcpy(data, buf, len);
*gotlen = len;
@ -118,6 +118,36 @@ static uint16_t data_from_python(void *params, void *priv, uint32_t wantlen, uns
return ret;
}
static PyObject* build_file_metadata(LIBMTP_file_t *nf, uint32_t storage_id) {
PyObject *ans = NULL;
ans = Py_BuildValue("{s:s, s:k, s:k, s:k, s:K, s:L, s:O}",
"name", (unsigned long)nf->filename,
"id", (unsigned long)nf->item_id,
"parent_id", (unsigned long)nf->parent_id,
"storage_id", (unsigned long)storage_id,
"size", nf->filesize,
"modified", (PY_LONG_LONG)nf->modificationdate,
"is_folder", (nf->filetype == LIBMTP_FILETYPE_FOLDER) ? Py_True : Py_False
);
return ans;
}
static PyObject* file_metadata(LIBMTP_mtpdevice_t *device, PyObject *errs, uint32_t item_id, uint32_t storage_id) {
LIBMTP_file_t *nf;
PyObject *ans = NULL;
Py_BEGIN_ALLOW_THREADS;
nf = LIBMTP_Get_Filemetadata(device, item_id);
Py_END_ALLOW_THREADS;
if (nf == NULL) dump_errorstack(device, errs);
else {
ans = build_file_metadata(nf, storage_id);
LIBMTP_destroy_file_t(nf);
}
return ans;
}
// }}}
// Device object definition {{{
@ -132,11 +162,11 @@ typedef struct {
PyObject *serial_number;
PyObject *device_version;
} libmtp_Device;
} Device;
// Device.__init__() {{{
static void
libmtp_Device_dealloc(libmtp_Device* self)
Device_dealloc(Device* self)
{
if (self->device != NULL) {
Py_BEGIN_ALLOW_THREADS;
@ -156,49 +186,48 @@ libmtp_Device_dealloc(libmtp_Device* self)
}
static int
libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
Device_init(Device *self, PyObject *args, PyObject *kwds)
{
int busnum, devnum, vendor_id, product_id;
unsigned long busnum;
unsigned char devnum;
unsigned short vendor_id, product_id;
PyObject *usb_serialnum;
char *vendor, *product, *friendly_name, *manufacturer_name, *model_name, *serial_number, *device_version;
LIBMTP_raw_device_t rawdev;
LIBMTP_mtpdevice_t *dev;
size_t i;
LIBMTP_raw_device_t *rawdevs = NULL, rdev;
int numdevs, c;
LIBMTP_mtpdevice_t *dev = NULL;
LIBMTP_error_number_t err;
if (!PyArg_ParseTuple(args, "iiiissO", &busnum, &devnum, &vendor_id, &product_id, &vendor, &product, &usb_serialnum)) return -1;
if (devnum < 0 || devnum > 255 || busnum < 0) { PyErr_SetString(PyExc_TypeError, "Invalid busnum/devnum"); return -1; }
self->ids = Py_BuildValue("iiiiO", busnum, devnum, vendor_id, product_id, usb_serialnum);
if (self->ids == NULL) return -1;
rawdev.bus_location = (uint32_t)busnum;
rawdev.devnum = (uint8_t)devnum;
rawdev.device_entry.vendor = vendor;
rawdev.device_entry.product = product;
rawdev.device_entry.vendor_id = vendor_id;
rawdev.device_entry.product_id = product_id;
rawdev.device_entry.device_flags = 0x00000000U;
if (!PyArg_ParseTuple(args, "kBHHssO", &busnum, &devnum, &vendor_id, &product_id, &vendor, &product, &usb_serialnum)) return -1;
// We have to build and search the rawdevice list instead of creating a
// rawdevice directly as otherwise, dynamic bug flag assignment in libmtp
// does not work
Py_BEGIN_ALLOW_THREADS;
for (i = 0; ; i++) {
if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == product_id) {
rawdev.device_entry.device_flags = calibre_mtp_device_table[i].device_flags;
err = LIBMTP_Detect_Raw_Devices(&rawdevs, &numdevs);
Py_END_ALLOW_THREADS;
if (err == LIBMTP_ERROR_NO_DEVICE_ATTACHED) { PyErr_SetString(MTPError, "No raw devices found"); return -1; }
if (err == LIBMTP_ERROR_CONNECTING) { PyErr_SetString(MTPError, "There has been an error connecting"); return -1; }
if (err == LIBMTP_ERROR_MEMORY_ALLOCATION) { PyErr_NoMemory(); return -1; }
if (err != LIBMTP_ERROR_NONE) { PyErr_SetString(MTPError, "Failed to detect raw MTP devices"); return -1; }
for (c = 0; c < numdevs; c++) {
rdev = rawdevs[c];
if (rdev.bus_location == (uint32_t)busnum && rdev.devnum == (uint8_t)devnum) {
Py_BEGIN_ALLOW_THREADS;
dev = LIBMTP_Open_Raw_Device_Uncached(&rdev);
Py_END_ALLOW_THREADS;
if (dev == NULL) { free(rawdevs); PyErr_SetString(MTPError, "Unable to open raw device."); return -1; }
break;
}
}
// Note that contrary to what the libmtp docs imply, we cannot use
// LIBMTP_Open_Raw_Device_Uncached as using it causes file listing to fail
dev = LIBMTP_Open_Raw_Device(&rawdev);
Py_END_ALLOW_THREADS;
if (dev == NULL) {
PyErr_SetString(MTPError, "Unable to open raw device.");
return -1;
}
if (rawdevs != NULL) free(rawdevs);
if (dev == NULL) { PyErr_Format(MTPError, "No device with busnum=%lu and devnum=%u found", busnum, devnum); return -1; }
self->device = dev;
self->ids = Py_BuildValue("kBHHO", busnum, devnum, vendor_id, product_id, usb_serialnum);
if (self->ids == NULL) return -1;
Py_BEGIN_ALLOW_THREADS;
friendly_name = LIBMTP_Get_Friendlyname(self->device);
@ -244,46 +273,46 @@ libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
// Device.friendly_name {{{
static PyObject *
libmtp_Device_friendly_name(libmtp_Device *self, void *closure) {
Device_friendly_name(Device *self, void *closure) {
Py_INCREF(self->friendly_name); return self->friendly_name;
} // }}}
// Device.manufacturer_name {{{
static PyObject *
libmtp_Device_manufacturer_name(libmtp_Device *self, void *closure) {
Device_manufacturer_name(Device *self, void *closure) {
Py_INCREF(self->manufacturer_name); return self->manufacturer_name;
} // }}}
// Device.model_name {{{
static PyObject *
libmtp_Device_model_name(libmtp_Device *self, void *closure) {
Device_model_name(Device *self, void *closure) {
Py_INCREF(self->model_name); return self->model_name;
} // }}}
// Device.serial_number {{{
static PyObject *
libmtp_Device_serial_number(libmtp_Device *self, void *closure) {
Device_serial_number(Device *self, void *closure) {
Py_INCREF(self->serial_number); return self->serial_number;
} // }}}
// Device.device_version {{{
static PyObject *
libmtp_Device_device_version(libmtp_Device *self, void *closure) {
Device_device_version(Device *self, void *closure) {
Py_INCREF(self->device_version); return self->device_version;
} // }}}
// Device.ids {{{
static PyObject *
libmtp_Device_ids(libmtp_Device *self, void *closure) {
Device_ids(Device *self, void *closure) {
Py_INCREF(self->ids); return self->ids;
} // }}}
// Device.update_storage_info() {{{
static PyObject*
libmtp_Device_update_storage_info(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
Device_update_storage_info(Device *self, PyObject *args) {
ENSURE_DEV(NULL);
if (LIBMTP_Get_Storage(self->device, LIBMTP_STORAGE_SORTBY_NOTSORTED) < 0) {
PyErr_SetString(MTPError, "Failed to get storage infor for device.");
PyErr_SetString(MTPError, "Failed to get storage info for device.");
return NULL;
}
Py_RETURN_NONE;
@ -292,31 +321,29 @@ libmtp_Device_update_storage_info(libmtp_Device *self, PyObject *args, PyObject
// Device.storage_info {{{
static PyObject *
libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
Device_storage_info(Device *self, void *closure) {
PyObject *ans, *loc;
LIBMTP_devicestorage_t *storage;
int ro = 0;
ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
ans = PyList_New(0);
if (ans == NULL) { PyErr_NoMemory(); return NULL; }
for (storage = self->device->storage; storage != NULL; storage = storage->next) {
// Ignore read only storage
if (storage->StorageType == ST_FixedROM || storage->StorageType == ST_RemovableROM) continue;
// Storage IDs with the lower 16 bits 0x0000 are not supposed to be
// writeable.
if ((storage->id & 0x0000FFFFU) == 0x00000000U) continue;
// Also check the access capability to avoid e.g. deletable only storages
if (storage->AccessCapability == AC_ReadOnly || storage->AccessCapability == AC_ReadOnly_with_Object_Deletion) continue;
ro = 0;
// Check if read only storage
if (storage->StorageType == ST_FixedROM || storage->StorageType == ST_RemovableROM || (storage->id & 0x0000FFFFU) == 0x00000000U || storage->AccessCapability == AC_ReadOnly || storage->AccessCapability == AC_ReadOnly_with_Object_Deletion) ro = 1;
loc = Py_BuildValue("{s:k,s:O,s:K,s:K,s:K,s:s,s:s}",
"id", storage->id,
loc = Py_BuildValue("{s:k,s:O,s:K,s:K,s:K,s:s,s:s,s:O}",
"id", (unsigned long)storage->id,
"removable", ((storage->StorageType == ST_RemovableRAM) ? Py_True : Py_False),
"capacity", storage->MaxCapacity,
"freespace_bytes", storage->FreeSpaceInBytes,
"freespace_objects", storage->FreeSpaceInObjects,
"storage_desc", storage->StorageDescription,
"volume_id", storage->VolumeIdentifier
"capacity", (unsigned long long)storage->MaxCapacity,
"freespace_bytes", (unsigned long long)storage->FreeSpaceInBytes,
"freespace_objects", (unsigned long long)storage->FreeSpaceInObjects,
"name", storage->StorageDescription,
"volume_id", storage->VolumeIdentifier,
"rw", (ro) ? Py_False : Py_True
);
if (loc == NULL) return NULL;
@ -328,124 +355,75 @@ libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
return ans;
} // }}}
// Device.get_filelist {{{
static PyObject *
libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
PyObject *ans, *fo, *callback = NULL, *errs;
ProgressCallback cb;
LIBMTP_file_t *f, *tf;
// Device.get_filesystem {{{
ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
static int recursive_get_files(LIBMTP_mtpdevice_t *dev, uint32_t storage_id, uint32_t parent_id, PyObject *ans, PyObject *errs) {
LIBMTP_file_t *f, *files;
PyObject *entry;
int ok = 1;
Py_BEGIN_ALLOW_THREADS;
files = LIBMTP_Get_Files_And_Folders(dev, storage_id, parent_id);
Py_END_ALLOW_THREADS;
if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL;
cb.obj = callback;
if (files == NULL) return ok;
ans = PyList_New(0);
errs = PyList_New(0);
if (ans == NULL || errs == NULL) { PyErr_NoMemory(); return NULL; }
for (f = files; ok && f != NULL; f = f->next) {
entry = build_file_metadata(f, storage_id);
if (entry == NULL) { ok = 0; }
else {
if (PyList_Append(ans, entry) != 0) { ok = 0; }
Py_DECREF(entry);
}
Py_XINCREF(callback);
cb.state = PyEval_SaveThread();
tf = LIBMTP_Get_Filelisting_With_Callback(self->device, report_progress, &cb);
PyEval_RestoreThread(cb.state);
Py_XDECREF(callback);
if (tf == NULL) {
dump_errorstack(self->device, errs);
return Py_BuildValue("NN", ans, errs);
}
for (f=tf; f != NULL; f=f->next) {
fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
"id", f->item_id,
"parent_id", f->parent_id,
"storage_id", f->storage_id,
"name", f->filename,
"size", f->filesize,
"modtime", f->modificationdate
);
if (fo == NULL || PyList_Append(ans, fo) != 0) break;
Py_DECREF(fo);
if (ok && f->filetype == LIBMTP_FILETYPE_FOLDER) {
if (!recursive_get_files(dev, storage_id, f->item_id, ans, errs)) {
ok = 0;
}
}
}
// Release memory
f = tf;
f = files;
while (f != NULL) {
tf = f; f = f->next; LIBMTP_destroy_file_t(tf);
files = f; f = f->next; LIBMTP_destroy_file_t(files);
}
if (callback != NULL) {
// Bug in libmtp where it does not call callback with 100%
fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans));
Py_XDECREF(fo);
}
return Py_BuildValue("NN", ans, errs);
} // }}}
// Device.get_folderlist {{{
int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
PyObject *folder, *children;
children = PyList_New(0);
if (children == NULL) { PyErr_NoMemory(); return 1;}
folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
"id", f->folder_id,
"parent_id", f->parent_id,
"storage_id", f->storage_id,
"name", f->name,
"children", children);
if (folder == NULL) return 1;
PyList_Append(parent, folder);
Py_DECREF(folder);
if (f->sibling != NULL) {
if (folderiter(f->sibling, parent)) return 1;
}
if (f->child != NULL) {
if (folderiter(f->child, children)) return 1;
}
return 0;
return ok;
}
static PyObject *
libmtp_Device_get_folderlist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
Device_get_filesystem(Device *self, PyObject *args) {
PyObject *ans, *errs;
LIBMTP_folder_t *f;
unsigned long storage_id;
int ok = 0;
ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
if (!PyArg_ParseTuple(args, "k", &storage_id)) return NULL;
ans = PyList_New(0);
errs = PyList_New(0);
if (errs == NULL || ans == NULL) { PyErr_NoMemory(); return NULL; }
Py_BEGIN_ALLOW_THREADS;
f = LIBMTP_Get_Folder_List(self->device);
Py_END_ALLOW_THREADS;
if (f == NULL) {
dump_errorstack(self->device, errs);
return Py_BuildValue("NN", ans, errs);
LIBMTP_Clear_Errorstack(self->device);
ok = recursive_get_files(self->device, (uint32_t)storage_id, 0, ans, errs);
dump_errorstack(self->device, errs);
if (!ok) {
Py_DECREF(ans);
Py_DECREF(errs);
return NULL;
}
if (folderiter(f, ans)) return NULL;
LIBMTP_destroy_folder_t(f);
return Py_BuildValue("NN", ans, errs);
} // }}}
// Device.get_file {{{
static PyObject *
libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
Device_get_file(Device *self, PyObject *args) {
PyObject *stream, *callback = NULL, *errs;
ProgressCallback cb;
uint32_t fileid;
unsigned long fileid;
int ret;
ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
@ -454,11 +432,12 @@ libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
if (!PyArg_ParseTuple(args, "kO|O", &fileid, &stream, &callback)) return NULL;
errs = PyList_New(0);
if (errs == NULL) { PyErr_NoMemory(); return NULL; }
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
cb.obj = callback; cb.extra = stream;
Py_XINCREF(callback); Py_INCREF(stream);
cb.state = PyEval_SaveThread();
ret = LIBMTP_Get_File_To_Handler(self->device, fileid, data_to_python, &cb, report_progress, &cb);
ret = LIBMTP_Get_File_To_Handler(self->device, (uint32_t)fileid, data_to_python, &cb, report_progress, &cb);
PyEval_RestoreThread(cb.state);
Py_XDECREF(callback); Py_DECREF(stream);
@ -472,49 +451,33 @@ libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
// Device.put_file {{{
static PyObject *
libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
PyObject *stream, *callback = NULL, *errs, *fo;
Device_put_file(Device *self, PyObject *args) {
PyObject *stream, *callback = NULL, *errs, *fo = NULL;
ProgressCallback cb;
uint32_t parent_id, storage_id;
uint64_t filesize;
unsigned long parent_id, storage_id;
unsigned long long filesize;
int ret;
char *name;
LIBMTP_file_t f, *nf;
LIBMTP_file_t f;
ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
if (!PyArg_ParseTuple(args, "kksOK|O", &storage_id, &parent_id, &name, &stream, &filesize, &callback)) return NULL;
errs = PyList_New(0);
if (errs == NULL) { PyErr_NoMemory(); return NULL; }
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
cb.obj = callback; cb.extra = stream;
f.parent_id = parent_id; f.storage_id = storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = filesize;
f.parent_id = (uint32_t)parent_id; f.storage_id = (uint32_t)storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = (uint64_t)filesize;
Py_XINCREF(callback); Py_INCREF(stream);
cb.state = PyEval_SaveThread();
ret = LIBMTP_Send_File_From_Handler(self->device, data_from_python, &cb, &f, report_progress, &cb);
PyEval_RestoreThread(cb.state);
Py_XDECREF(callback); Py_DECREF(stream);
fo = Py_None; Py_INCREF(fo);
if (ret != 0) dump_errorstack(self->device, errs);
else {
Py_BEGIN_ALLOW_THREADS;
nf = LIBMTP_Get_Filemetadata(self->device, f.item_id);
Py_END_ALLOW_THREADS;
if (nf == NULL) dump_errorstack(self->device, errs);
else {
Py_DECREF(fo);
fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
"id", nf->item_id,
"parent_id", nf->parent_id,
"storage_id", nf->storage_id,
"name", nf->filename,
"size", nf->filesize,
"modtime", nf->modificationdate
);
LIBMTP_destroy_file_t(nf);
}
}
else fo = file_metadata(self->device, errs, f.item_id, storage_id);
if (fo == NULL) { fo = Py_None; Py_INCREF(fo); }
return Py_BuildValue("NN", fo, errs);
@ -522,9 +485,9 @@ libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
// Device.delete_object {{{
static PyObject *
libmtp_Device_delete_object(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
Device_delete_object(Device *self, PyObject *args) {
PyObject *errs;
uint32_t id;
unsigned long id;
int res;
ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
@ -534,7 +497,7 @@ libmtp_Device_delete_object(libmtp_Device *self, PyObject *args, PyObject *kwarg
if (errs == NULL) { PyErr_NoMemory(); return NULL; }
Py_BEGIN_ALLOW_THREADS;
res = LIBMTP_Delete_Object(self->device, id);
res = LIBMTP_Delete_Object(self->device, (uint32_t)id);
Py_END_ALLOW_THREADS;
if (res != 0) dump_errorstack(self->device, errs);
@ -543,12 +506,11 @@ libmtp_Device_delete_object(libmtp_Device *self, PyObject *args, PyObject *kwarg
// Device.create_folder {{{
static PyObject *
libmtp_Device_create_folder(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
PyObject *errs, *fo, *children, *temp;
uint32_t parent_id, storage_id;
char *name;
Device_create_folder(Device *self, PyObject *args) {
PyObject *errs, *fo = NULL;
unsigned long storage_id, parent_id;
uint32_t folder_id;
LIBMTP_folder_t *f = NULL, *cf;
char *name;
ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
@ -556,69 +518,39 @@ libmtp_Device_create_folder(libmtp_Device *self, PyObject *args, PyObject *kwarg
errs = PyList_New(0);
if (errs == NULL) { PyErr_NoMemory(); return NULL; }
fo = Py_None; Py_INCREF(fo);
Py_BEGIN_ALLOW_THREADS;
folder_id = LIBMTP_Create_Folder(self->device, name, parent_id, storage_id);
folder_id = LIBMTP_Create_Folder(self->device, name, (uint32_t)parent_id, (uint32_t)storage_id);
Py_END_ALLOW_THREADS;
if (folder_id == 0) dump_errorstack(self->device, errs);
else {
Py_BEGIN_ALLOW_THREADS;
// Cannot use Get_Folder_List_For_Storage as it fails
f = LIBMTP_Get_Folder_List(self->device);
Py_END_ALLOW_THREADS;
if (f == NULL) dump_errorstack(self->device, errs);
else {
cf = LIBMTP_Find_Folder(f, folder_id);
if (cf == NULL) {
temp = Py_BuildValue("is", 1, "Newly created folder not present on device!");
if (temp == NULL) { PyErr_NoMemory(); return NULL;}
PyList_Append(errs, temp);
Py_DECREF(temp);
} else {
Py_DECREF(fo);
children = PyList_New(0);
if (children == NULL) { PyErr_NoMemory(); return NULL; }
fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
"id", cf->folder_id,
"parent_id", cf->parent_id,
"storage_id", cf->storage_id,
"name", cf->name,
"children", children);
}
LIBMTP_destroy_folder_t(f);
}
}
else fo = file_metadata(self->device, errs, folder_id, storage_id);
if (fo == NULL) { fo = Py_None; Py_INCREF(fo); }
return Py_BuildValue("NN", fo, errs);
} // }}}
static PyMethodDef libmtp_Device_methods[] = {
{"update_storage_info", (PyCFunction)libmtp_Device_update_storage_info, METH_VARARGS,
static PyMethodDef Device_methods[] = {
{"update_storage_info", (PyCFunction)Device_update_storage_info, METH_VARARGS,
"update_storage_info() -> Reread the storage info from the device (total, space, free space, storage locations, etc.)"
},
{"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS,
"get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors."
{"get_filesystem", (PyCFunction)Device_get_filesystem, METH_VARARGS,
"get_filesystem(storage_id) -> Get the list of files and folders on the device in storage_id. Returns files, errors."
},
{"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
"get_folderlist() -> Get the list of folders on the device. Returns files, errors."
},
{"get_file", (PyCFunction)libmtp_Device_get_file, METH_VARARGS,
{"get_file", (PyCFunction)Device_get_file, METH_VARARGS,
"get_file(fileid, stream, callback=None) -> Get the file specified by fileid from the device. stream must be a file-like object. The file will be written to it. callback works the same as in get_filelist(). Returns ok, errs, where errs is a list of errors (if any)."
},
{"put_file", (PyCFunction)libmtp_Device_put_file, METH_VARARGS,
{"put_file", (PyCFunction)Device_put_file, METH_VARARGS,
"put_file(storage_id, parent_id, filename, stream, size, callback=None) -> Put a file on the device. The file is read from stream. It is put inside the folder identified by parent_id on the storage identified by storage_id. Use parent_id=0 to put it in the root. stream must be a file-like object. size is the size in bytes of the data in stream. callback works the same as in get_filelist(). Returns fileinfo, errs, where errs is a list of errors (if any), and fileinfo is a file information dictionary, as returned by get_filelist(). fileinfo will be None if case or errors."
},
{"create_folder", (PyCFunction)libmtp_Device_create_folder, METH_VARARGS,
{"create_folder", (PyCFunction)Device_create_folder, METH_VARARGS,
"create_folder(storage_id, parent_id, name) -> Create a folder named name under parent parent_id (use 0 for root) in the storage identified by storage_id. Returns folderinfo, errors, where folderinfo is the same dict as returned by get_folderlist(), it will be None if there are errors."
},
{"delete_object", (PyCFunction)libmtp_Device_delete_object, METH_VARARGS,
{"delete_object", (PyCFunction)Device_delete_object, METH_VARARGS,
"delete_object(id) -> Delete the object identified by id from the device. Can be used to delete files, folders, etc. Returns ok, errs."
},
@ -626,52 +558,52 @@ static PyMethodDef libmtp_Device_methods[] = {
{NULL} /* Sentinel */
};
static PyGetSetDef libmtp_Device_getsetters[] = {
static PyGetSetDef Device_getsetters[] = {
{(char *)"friendly_name",
(getter)libmtp_Device_friendly_name, NULL,
(getter)Device_friendly_name, NULL,
(char *)"The friendly name of this device, can be None.",
NULL},
{(char *)"manufacturer_name",
(getter)libmtp_Device_manufacturer_name, NULL,
(getter)Device_manufacturer_name, NULL,
(char *)"The manufacturer name of this device, can be None.",
NULL},
{(char *)"model_name",
(getter)libmtp_Device_model_name, NULL,
(getter)Device_model_name, NULL,
(char *)"The model name of this device, can be None.",
NULL},
{(char *)"serial_number",
(getter)libmtp_Device_serial_number, NULL,
(getter)Device_serial_number, NULL,
(char *)"The serial number of this device, can be None.",
NULL},
{(char *)"device_version",
(getter)libmtp_Device_device_version, NULL,
(getter)Device_device_version, NULL,
(char *)"The device version of this device, can be None.",
NULL},
{(char *)"ids",
(getter)libmtp_Device_ids, NULL,
(getter)Device_ids, NULL,
(char *)"The ids of the device (busnum, devnum, vendor_id, product_id, usb_serialnum)",
NULL},
{(char *)"storage_info",
(getter)libmtp_Device_storage_info, NULL,
(getter)Device_storage_info, NULL,
(char *)"Information about the storage locations on the device. Returns a list of dictionaries where each dictionary corresponds to the LIBMTP_devicestorage_struct.",
NULL},
{NULL} /* Sentinel */
};
static PyTypeObject libmtp_DeviceType = { // {{{
static PyTypeObject DeviceType = { // {{{
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
"libmtp.Device", /*tp_name*/
sizeof(libmtp_Device), /*tp_basicsize*/
sizeof(Device), /*tp_basicsize*/
0, /*tp_itemsize*/
(destructor)libmtp_Device_dealloc, /*tp_dealloc*/
(destructor)Device_dealloc, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
@ -694,15 +626,15 @@ static PyTypeObject libmtp_DeviceType = { // {{{
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
libmtp_Device_methods, /* tp_methods */
Device_methods, /* tp_methods */
0, /* tp_members */
libmtp_Device_getsetters, /* tp_getset */
Device_getsetters, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)libmtp_Device_init, /* tp_init */
(initproc)Device_init, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
}; // }}}
@ -710,7 +642,7 @@ static PyTypeObject libmtp_DeviceType = { // {{{
// }}} End Device object definition
static PyObject *
libmtp_set_debug_level(PyObject *self, PyObject *args) {
set_debug_level(PyObject *self, PyObject *args) {
int level;
if (!PyArg_ParseTuple(args, "i", &level)) return NULL;
LIBMTP_Set_Debug(level);
@ -719,18 +651,10 @@ libmtp_set_debug_level(PyObject *self, PyObject *args) {
static PyObject *
libmtp_is_mtp_device(PyObject *self, PyObject *args) {
int busnum, devnum, vendor_id, prod_id, ans = 0;
size_t i;
is_mtp_device(PyObject *self, PyObject *args) {
int busnum, devnum, ans = 0;
if (!PyArg_ParseTuple(args, "iiii", &busnum, &devnum, &vendor_id, &prod_id)) return NULL;
for (i = 0; ; i++) {
if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == prod_id) {
Py_RETURN_TRUE;
}
}
if (!PyArg_ParseTuple(args, "ii", &busnum, &devnum)) return NULL;
/*
* LIBMTP_Check_Specific_Device does not seem to work at least on my linux
@ -749,13 +673,36 @@ libmtp_is_mtp_device(PyObject *self, PyObject *args) {
}
static PyObject*
known_devices(PyObject *self, PyObject *args) {
PyObject *ans, *d;
size_t i;
ans = PyList_New(0);
if (ans == NULL) return PyErr_NoMemory();
for (i = 0; ; i++) {
if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
d = Py_BuildValue("(HH)", (unsigned short)calibre_mtp_device_table[i].vendor_id, (unsigned short)calibre_mtp_device_table[i].product_id);
if (d == NULL) { Py_DECREF(ans); ans = NULL; break; }
if (PyList_Append(ans, d) != 0) { Py_DECREF(d); Py_DECREF(ans); ans = NULL; PyErr_NoMemory(); break; }
Py_DECREF(d);
}
return ans;
}
static PyMethodDef libmtp_methods[] = {
{"set_debug_level", libmtp_set_debug_level, METH_VARARGS,
{"set_debug_level", set_debug_level, METH_VARARGS,
"set_debug_level(level)\n\nSet the debug level bit mask, see LIBMTP_DEBUG_* constants."
},
{"is_mtp_device", libmtp_is_mtp_device, METH_VARARGS,
"is_mtp_device(busnum, devnum, vendor_id, prod_id)\n\nReturn True if the device is recognized as an MTP device by its vendor/product ids. If it is not recognized a probe is done and True returned if the probe succeeds. Note that probing can cause some devices to malfunction, and it is not very reliable, which is why we prefer to use the device database."
{"is_mtp_device", is_mtp_device, METH_VARARGS,
"is_mtp_device(busnum, devnum)\n\nA probe is done and True returned if the probe succeeds. Note that probing can cause some devices to malfunction, and it is not very reliable, which is why we prefer to use the device database."
},
{"known_devices", known_devices, METH_VARARGS,
"known_devices() -> Return the list of known (vendor_id, product_id) combinations."
},
{NULL, NULL, 0, NULL}
@ -766,20 +713,22 @@ PyMODINIT_FUNC
initlibmtp(void) {
PyObject *m;
libmtp_DeviceType.tp_new = PyType_GenericNew;
if (PyType_Ready(&libmtp_DeviceType) < 0)
DeviceType.tp_new = PyType_GenericNew;
if (PyType_Ready(&DeviceType) < 0)
return;
m = Py_InitModule3("libmtp", libmtp_methods, "Interface to libmtp.");
if (m == NULL) return;
MTPError = PyErr_NewException("libmtp.MTPError", NULL, NULL);
if (MTPError == NULL) return;
PyModule_AddObject(m, "MTPError", MTPError);
LIBMTP_Init();
LIBMTP_Set_Debug(LIBMTP_DEBUG_NONE);
Py_INCREF(&libmtp_DeviceType);
PyModule_AddObject(m, "Device", (PyObject *)&libmtp_DeviceType);
Py_INCREF(&DeviceType);
PyModule_AddObject(m, "Device", (PyObject *)&DeviceType);
PyModule_AddStringMacro(m, LIBMTP_VERSION_STRING);
PyModule_AddIntMacro(m, LIBMTP_DEBUG_NONE);

365
src/calibre/devices/mtp/unix/upstream/music-players.h
View File

@ -292,6 +292,7 @@
DEVICE_FLAG_PLAYLIST_SPL_V1 |
DEVICE_FLAG_UNIQUE_FILENAMES |
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST },
// The "YP-R2" (0x04e8/0x512d) is NOT MTP, it is UMS only.
// From Manuel Carro
// Copied from Q2
{ "Samsung", 0x04e8, "YP-Q3", 0x5130,
@ -350,24 +351,30 @@
{ "Samsung", 0x04e8, "GT-S8500", 0x6819,
DEVICE_FLAG_UNLOAD_DRIVER |
DEVICE_FLAG_PLAYLIST_SPL_V1 },
// From Harrison Metzger <harrisonmetz@gmail.com>
{ "Samsung", 0x04e8,
"Galaxy Nexus/Galaxy S i9000/i9250, Android 4.0 updates", 0x685c,
DEVICE_FLAGS_ANDROID_BUGS |
DEVICE_FLAG_PLAYLIST_SPL_V2 },
// Reported by David Goodenough <dfgdga@users.sourceforge.net>
// Guessing on flags.
{ "Samsung", 0x04e8, "Galaxy Y", 0x685e,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL |
DEVICE_FLAG_UNLOAD_DRIVER |
DEVICE_FLAG_LONG_TIMEOUT |
DEVICE_FLAG_PROPLIST_OVERRIDES_OI },
/*
* This entry (device 0x6860) seems to be used on a *lot* of Samsung
* Android (gingerbread, 2.3) phones. It is *not* the Android MTP stack
* but an internal Samsung stack.
* These entries seems to be used on a *lot* of Samsung
* Android phones. It is *not* the Android MTP stack but an internal
* Samsung stack. The devices present a few different product IDs
* depending on mode:
*
* Popular devices: Galaxy S2 and S3.
* 0x685b - UMS
* 0x685c - MTP + ADB
* 0x685e - UMS + CDC
* 0x6860 - MTP mode (default)
* 0x6863 - USB CDC RNDIS (not MTP)
* 0x6865 - PTP mode (not MTP)
* 0x6877 - Kies mode? Does it have MTP?
*
* Used on these samsung devices:
* GT P7310/P7510/N7000/I9100/I9250/I9300
* Galaxy Nexus
* Galaxy Tab 7.7/10.1
* Galaxy S GT-I9000
* Galaxy S Advance GT-I9070
* Galaxy S2
* Galaxy S3
* Galaxy Note
* Galaxy Y
*
* - It seems that some PTP commands are broken.
* - Devices seem to have a connection timeout, the session must be
@ -377,13 +384,30 @@
* US markets for some weird reason.
*
* From: Ignacio Martínez <ignacio.martinezrivera@yahoo.es> and others
* From Harrison Metzger <harrisonmetz@gmail.com>
*/
{ "Samsung", 0x04e8,
"GT P7310/P7510/N7000/I9070/I9100/I9300 Galaxy Tab 7.7/10.1/S2/S3/Nexus/Note/Y", 0x6860,
"Galaxy models (MTP+ADB)", 0x685c,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL |
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
DEVICE_FLAG_UNLOAD_DRIVER |
DEVICE_FLAG_LONG_TIMEOUT |
DEVICE_FLAG_PROPLIST_OVERRIDES_OI },
// Reported by David Goodenough <dfgdga@users.sourceforge.net>
// Guessing on flags.
{ "Samsung", 0x04e8, "Galaxy Y", 0x685e,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL |
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
DEVICE_FLAG_UNLOAD_DRIVER |
DEVICE_FLAG_LONG_TIMEOUT |
DEVICE_FLAG_PROPLIST_OVERRIDES_OI },
{ "Samsung", 0x04e8,
"Galaxy models (MTP)", 0x6860,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL |
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
DEVICE_FLAG_UNLOAD_DRIVER |
DEVICE_FLAG_LONG_TIMEOUT |
DEVICE_FLAG_PROPLIST_OVERRIDES_OI },
// Note: ID 0x6865 is some PTP mode! Don't add it.
// From: Erik Berglund <erikjber@users.sourceforge.net>
// Logs indicate this needs DEVICE_FLAG_NO_ZERO_READS
// No Samsung platlists on this device.
@ -391,7 +415,7 @@
// i5800 duplicate reported by igel <igel-kun@users.sourceforge.net>
// Guessing this has the same problematic MTP stack as the device
// above.
{ "Samsung", 0x04e8, "Galaxy S GT-I9000/Galaxy 3 i5800/Kies mode", 0x6877,
{ "Samsung", 0x04e8, "Galaxy models Kies mode", 0x6877,
DEVICE_FLAG_UNLOAD_DRIVER |
DEVICE_FLAG_LONG_TIMEOUT |
DEVICE_FLAG_PROPLIST_OVERRIDES_OI },
@ -499,17 +523,26 @@
* Acer
*/
// Reported by anonymous sourceforge user
{ "Acer", 0x0502, "Iconia TAB A500 v1", 0x3325, DEVICE_FLAGS_ANDROID_BUGS },
{ "Acer", 0x0502, "Iconia TAB A500 (ID1)", 0x3325, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by: Franck VDL <franckv@users.sourceforge.net>
{ "Acer", 0x0502, "Iconia TAB A500 v2", 0x3341, DEVICE_FLAGS_ANDROID_BUGS },
{ "Acer", 0x0502, "Iconia TAB A500 (ID2)", 0x3341, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by: Matthias Arndt <simonsunnyboy@users.sourceforge.net>
{ "Acer", 0x0502, "Iconia TAB A501", 0x3344, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by: anonymous sourceforge user
{ "Acer", 0x0502, "Iconia TAB A100", 0x3348, DEVICE_FLAGS_ANDROID_BUGS },
{ "Acer", 0x0502, "Iconia TAB A100 (ID1)", 0x3348, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by: Arvin Schnell <arvins@users.sourceforge.net>
{ "Acer", 0x0502, "Iconia TAB A100 ID2", 0x3349, DEVICE_FLAGS_ANDROID_BUGS },
{ "Acer", 0x0502, "Iconia TAB A100 (ID2)", 0x3349, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Philippe Marzouk <philm@users.sourceforge.net>
{ "Acer", 0x0502, "Iconia TAB A700", 0x3378, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by anonymous sourceforge user
{ "Acer", 0x0502, "Iconia TAB A200", 0x337c, DEVICE_FLAGS_ANDROID_BUGS },
{ "Acer", 0x0502, "Iconia TAB A200 (ID1)", 0x337c, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by anonymous sourceforge user
{ "Acer", 0x0502, "Iconia TAB A200 (ID2)", 0x337d, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by nE0sIghT <ne0sight@users.sourceforge.net>
{ "Acer", 0x0502, "Iconia TAB A510", 0x338a, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Maxime de Roucy <maxime1986@users.sourceforge.net>
{ "Acer", 0x0502, "E350 Liquid Gallant Duo", 0x33c3,
DEVICE_FLAGS_ANDROID_BUGS },
/*
* SanDisk
@ -843,6 +876,7 @@
{ "Archos", 0x0e79, "SPOD (MTP mode)", 0x1341, DEVICE_FLAG_UNLOAD_DRIVER },
{ "Archos", 0x0e79, "5S IT (MTP mode)", 0x1351, DEVICE_FLAG_UNLOAD_DRIVER },
{ "Archos", 0x0e79, "5H IT (MTP mode)", 0x1357, DEVICE_FLAG_UNLOAD_DRIVER },
{ "Archos", 0x0e79, "Arnova Childpad", 0x1458, DEVICE_FLAG_UNLOAD_DRIVER },
// Reported by anonymous Sourceforge user
{ "Archos", 0x0e79, "8o G9 (MTP mode)", 0x1508, DEVICE_FLAG_UNLOAD_DRIVER },
// Reported by Clément <clemvangelis@users.sourceforge.net>
@ -1248,6 +1282,9 @@
{ "LG Electronics Inc.", 0x1004, "V909 G-Slate", 0x61f9,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST |
DEVICE_FLAG_UNLOAD_DRIVER },
// Reported by Brian J. Murrell
{ "LG Electronics Inc.", 0x1004, "LG-E617G/P700", 0x631c,
DEVICE_FLAGS_ANDROID_BUGS },
/*
* Sony
@ -1330,6 +1367,9 @@
// Reported by Jan Rheinlaender <jrheinlaender@users.sourceforge.net>
{ "Sony", 0x054c, "NWZ-S765", 0x05a8,
DEVICE_FLAGS_SONY_NWZ_BUGS },
// Olivier Keshavjee <olivierkes@users.sourceforge.net>
{ "Sony", 0x054c, "Sony Tablet S", 0x05b3,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by ghalambaz <ghalambaz@users.sourceforge.net>
{ "Sony", 0x054c, "Sony Tablet S1", 0x05b4,
DEVICE_FLAGS_ANDROID_BUGS },
@ -1396,102 +1436,9 @@
// Reported by Serge Chirik <schirik@users.sourceforge.net>
{ "SonyEricsson", 0x0fce, "j108i (Cedar)", 0x014e,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST },
/*
* SonyEricsson/SONY Android devices usually have three personalities due to
* using composite descriptors and the fact that Windows cannot distinguish
* the device unless each composite descriptor is unique.
*
* 0x0nnn = MTP
* 0x4nnn = MTP + mass storage (for CD-ROM)
* 0x5nnn = MTP + ADB (Android debug bridge)
*
*/
// Reported by Jonas Salling <>
// Erroneous MTP implementation seems to be from Aricent, returns
// broken transaction ID.
{ "SonyEricsson", 0x0fce, "LT15i (Xperia arc S)", 0x014f,
DEVICE_FLAGS_ARICENT_BUGS },
// Reported by Eamonn Webster <eweb@users.sourceforge.net>
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "MT11i Xperia Neo", 0x0156,
DEVICE_FLAG_NONE },
// Reported by Alejandro DC <Alejandro_DC@users.sourceforge.ne>
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "MK16i Xperia", 0x015a,
DEVICE_FLAG_NONE },
// Reported by <wealas@users.sourceforge.net>
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "ST18a Xperia Ray", 0x0161,
DEVICE_FLAG_NONE },
/*
* Reported by StehpanKa <stehp@users.sourceforge.net>
* Android with homebrew MTP stack in one firmware, possibly Aricent
* Android with Android stack in another one, so let the run-time
* detector look up the device bug flags, set to NONE initially.
*/
{ "SonyEricsson", 0x0fce, "SK17i Xperia mini pro", 0x0166,
DEVICE_FLAG_NONE },
// Reported by hdhoang <hdhoang@users.sourceforge.net>
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "ST15i Xperia Mini", 0x0167,
DEVICE_FLAG_NONE },
// Reported by Paul Taylor
{ "SONY", 0x0fce, "Xperia S", 0x0169,
DEVICE_FLAG_NO_ZERO_READS | DEVICE_FLAGS_ANDROID_BUGS },
// Reported by equaeghe <equaeghe@users.sourceforge.net>
{ "SONY", 0x0fce, "ST15i Xperia U", 0x0171,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Ondra Lengal
{ "SONY", 0x0fce, "Xperia P", 0x0172,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Jonas Nyrén <spectralmks@users.sourceforge.net>
{ "SonyEricsson", 0x0fce, "W302", 0x10c8,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST },
/*
* MTP+MSC personalities of MTP devices (see above)
*/
// Reported by equaeghe <equaeghe@users.sourceforge.net>
{ "SONY", 0x0fce, "ST25i Xperia U (MTP+MSC mode)", 0x4171,
DEVICE_FLAGS_ANDROID_BUGS },
// Guessing on this one
{ "SONY", 0x0fce, "Xperia P (MTP+MSC mode)", 0x4172,
DEVICE_FLAGS_ANDROID_BUGS },
/*
* MTP+ADB personalities of MTP devices (see above)
*/
// Reported by anonymous sourceforge user
// Suspect Aricent stack, guessing on these bug flags
{ "SonyEricsson", 0x0fce, "LT15i Xperia Arc (MTP+ADB mode)", 0x514f,
DEVICE_FLAGS_ARICENT_BUGS },
// Reported by Michael K. <kmike@users.sourceforge.net>
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "MT11i Xperia Neo (MTP+ADB mode)", 0x5156,
DEVICE_FLAG_NONE },
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "MK16i Xperia (MTP+ADB mode)", 0x515a,
DEVICE_FLAG_NONE },
// Reported by Eduard Bloch <blade@debian.org>
// Xperia Ray (2012), SE Android 2.3.4, flags from ST18a
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "ST18i Xperia Ray (MTP+ADB mode)", 0x5161,
DEVICE_FLAG_NONE },
// Reported by StehpanKa <stehp@users.sourceforge.net>
// Android with homebrew MTP stack, possibly Aricent
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "SK17i Xperia mini pro (MTP+ADB mode)", 0x5166,
DEVICE_FLAG_NONE },
// Android with homebrew MTP stack, possibly Aricent
// Runtime detect the Aricent or Android stack
{ "SonyEricsson", 0x0fce, "ST15i Xperia Mini (MTP+ADB mode)", 0x5167,
DEVICE_FLAG_NONE },
// Reported by equaeghe <equaeghe@users.sourceforge.net>
{ "SONY", 0x0fce, "ST25i Xperia U (MTP+ADB mode)", 0x5171,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Ondra Lengál
{ "SONY", 0x0fce, "Xperia P (MTP+ADB mode)", 0x5172,
DEVICE_FLAGS_ANDROID_BUGS },
{ "SONY", 0x0fce, "MT27i Xperia Sola (MTP+MSC+? mode)", 0xa173,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Anonymous Sourceforge user
{ "SonyEricsson", 0x0fce, "j10i (Elm)", 0xd144,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST },
@ -1499,6 +1446,133 @@
{ "SonyEricsson", 0x0fce, "K550i", 0xe000,
DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST },
/*
* SonyEricsson/SONY Android devices usually have three personalities due to
* using composite descriptors and the fact that Windows cannot distinguish
* the device unless each composite descriptor is unique.
*
* Legend:
* MTP = Media Transfer Protocol
* UMS = USB Mass Storage Protocol
* ADB = Android Debug Bridge Protocol
* CDC = Communications Device Class, Internet Sharing
*
* 0x0nnn = MTP
* 0x4nnn = MTP + UMS (for CD-ROM)
* 0x5nnn = MTP + ADB
* 0x6nnn = UMS + ADB
* 0x7nnn = MTP + CDC
* 0x8nnn = MTP + CDC + ADB
* 0xannn = MTP + UMS + ?
* 0xennn = UMS only
*
* The SonyEricsson and SONY devices have (at least)two deployed MTP
* stacks: Aricent and Android. These have different bug flags, and
* sometimes the same device has firmware upgrades moving it from
* the Aricent to Android MTP stack without changing the device
* VID+PID (first observed on the SK17i Xperia Mini Pro), so the
* detection has to be more elaborate. The code in libmtp.c will do
* this and assign the proper bug flags (hopefully).
* That is why DEVICE_FLAG_NONE is used for these devices.
*
* Devices reported by:
* Jonas Salling
* Eamonn Webster <eweb@users.sourceforge.net>
* Alejandro DC <Alejandro_DC@users.sourceforge.ne>
* StehpanKa <stehp@users.sourceforge.net>
* hdhoang <hdhoang@users.sourceforge.net>
* Paul Taylor
* Bruno Basilio <bbasilio@users.sourceforge.net>
* Christoffer Holmstedt <christofferh@users.sourceforge.net>
* equaeghe <equaeghe@users.sourceforge.net>
* Ondra Lengal
* Michael K. <kmike@users.sourceforge.net>
* Jean-François B. <changi67@users.sourceforge.net>
* Eduard Bloch <blade@debian.org>
* Ah Hong <hongster@users.sourceforge.net>
*/
{ "SonyEricsson", 0x0fce, "LT15i (Xperia arc S)", 0x014f,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "MT11i Xperia Neo", 0x0156,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "MK16i Xperia", 0x015a,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "ST18a Xperia Ray", 0x0161,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "SK17i Xperia Mini Pro", 0x0166,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "ST15i Xperia Mini", 0x0167,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "ST17i Xperia Active", 0x0168,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT26i Xperia S", 0x0169,
DEVICE_FLAG_NO_ZERO_READS },
{ "SONY", 0x0fce, "WT19i Live Walkman", 0x016d,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "ST21i Xperia Tipo", 0x0170,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "ST15i Xperia U", 0x0171,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT22i Xperia P", 0x0172,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT26w Xperia Acro S", 0x0176,
DEVICE_FLAG_NONE },
/*
* MTP+UMS personalities of MTP devices (see above)
*/
{ "SonyEricsson", 0x0fce, "ST17i Xperia Active (MTP+UMS mode)", 0x4168,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT26i Xperia S (MTP+UMS mode)", 0x4169,
DEVICE_FLAG_NO_ZERO_READS },
{ "SONY", 0x0fce, "ST21i Xperia Tipo (MTP+UMS mode)", 0x4170,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "ST25i Xperia U (MTP+UMS mode)", 0x4171,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT22i Xperia P (MTP+UMS mode)", 0x4172,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT26w Xperia Acro S (MTP+UMS mode)", 0x4176,
DEVICE_FLAG_NONE },
/*
* MTP+ADB personalities of MTP devices (see above)
*/
{ "SonyEricsson", 0x0fce, "LT15i Xperia Arc (MTP+ADB mode)", 0x514f,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "MT11i Xperia Neo (MTP+ADB mode)", 0x5156,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "ST17i Xperia Active (MTP+ADB mode)", 0x5168,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT26i Xperia S (MTP+ADB mode)", 0x5169,
DEVICE_FLAG_NO_ZERO_READS },
{ "SonyEricsson", 0x0fce, "MK16i Xperia (MTP+ADB mode)", 0x515a,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "ST18i Xperia Ray (MTP+ADB mode)", 0x5161,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "SK17i Xperia Mini Pro (MTP+ADB mode)", 0x5166,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "ST15i Xperia Mini (MTP+ADB mode)", 0x5167,
DEVICE_FLAG_NONE },
{ "SonyEricsson", 0x0fce, "SK17i Xperia Mini Pro (MTP+ADB mode)", 0x516d,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "ST21i Xperia Tipo (MTP+ADB mode)", 0x5170,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "ST25i Xperia U (MTP+ADB mode)", 0x5171,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT22i Xperia P (MTP+ADB mode)", 0x5172,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "LT26w Xperia Acro S (MTP+ADB mode)", 0x5176,
DEVICE_FLAG_NONE },
/*
* MTP+UMS+? modes
* No reports on other personalities on these devices.
*/
{ "SONY", 0x0fce, "MT27i Xperia Sola (MTP+UMS+? mode)", 0xa173,
DEVICE_FLAG_NONE },
{ "SONY", 0x0fce, "ST27i Xperia Go (MTP+UMS+? mode)", 0xa17e,
DEVICE_FLAG_NONE },
/*
* Motorola
* Assume DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST on all of these.
@ -1521,7 +1595,11 @@
DEVICE_FLAGS_ANDROID_BUGS },
{ "Motorola", 0x22b8, "Xoom 2 Media Edition", 0x4311,
DEVICE_FLAGS_ANDROID_BUGS },
{ "Motorola", 0x22b8, "XT912", 0x4362,
// Reported by B,H,Kissinger <mrkissinger@users.sourceforge.net>
{ "Motorola", 0x22b8, "XT912/XT928", 0x4362,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Lundgren <alundgren@users.sourceforge.net>
{ "Motorola", 0x22b8, "DROID4", 0x437f,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Marcus Meissner to libptp2
{ "Motorola", 0x22b8, "IdeaPad K1", 0x4811,
@ -1535,10 +1613,32 @@
// Reported by anonymous user
{ "Motorola", 0x22b8, "RAZR2 V8/U9/Z6", 0x6415,
DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST },
// Reported by Google Inc's Yavor Goulishev <yavor@google.com>
// Android 3.0 MTP stack seems to announce that it supports the
// list operations, but they do not work?
{ "Motorola", 0x22b8, "Xoom (ID 1)", 0x70a8, DEVICE_FLAGS_ANDROID_BUGS },
/*
* Motorola Xoom (Wingray) variants
*
* These devices seem to use these product IDs simulatenously
* https://code.google.com/p/android-source-browsing/source/browse/init.stingray.usb.rc?repo=device--moto--wingray
*
* 0x70a3 - Factory test - reported as early MTP ID
* 0x70a8 - MTP
* 0x70a9 - MTP+ADB
* 0x70ae - RNDIS
* 0x70af - RNDIS+ADB
* 0x70b0 - ACM
* 0x70b1 - ACM+ADB
* 0x70b2 - ACM+RNDIS
* 0x70b3 - ACM+RNDIS+ADB
* 0x70b4 - PTP
* 0x70b5 - PTP+ADB
*
* Reported by Google Inc's Yavor Goulishev <yavor@google.com>
*/
{ "Motorola", 0x22b8, "Xoom (Factory test)", 0x70a3,
DEVICE_FLAGS_ANDROID_BUGS },
{ "Motorola", 0x22b8, "Xoom (MTP)", 0x70a8,
DEVICE_FLAGS_ANDROID_BUGS },
{ "Motorola", 0x22b8, "Xoom (MTP+ADB)", 0x70a9,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by anonymous Sourceforge user
// "carried by C Spire and other CDMA US carriers"
{ "Motorola", 0x22b8, "Milestone X2", 0x70ca, DEVICE_FLAGS_ANDROID_BUGS },
@ -1561,17 +1661,15 @@
{ "Google Inc (for Samsung)", 0x18d1, "Nexus S", 0x4e21,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Chris Smith <tcgsmythe@users.sourceforge.net>
{ "Google Inc (for Asus)", 0x18d1, "Nexus 7 (mode 1)", 0x4e41,
{ "Google Inc (for Asus)", 0x18d1, "Nexus 7 (MTP)", 0x4e41,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Michael Hess <mhess126@gmail.com>
{ "Google Inc (for Asus)", 0x18d1, "Nexus 7 (mode 2)", 0x4e42,
{ "Google Inc (for Asus)", 0x18d1, "Nexus 7 (MTP+ADB)", 0x4e42,
DEVICE_FLAGS_ANDROID_BUGS },
// WiFi-only version of Xoom
// See: http://bugzilla.gnome.org/show_bug.cgi?id=647506
{ "Google Inc (for Motorola)", 0x18d1, "Xoom (MZ604)", 0x70a8,
DEVICE_FLAGS_ANDROID_BUGS },
{ "Google Inc (for Motorola)", 0x22b8, "Xoom (ID 2)", 0x70a9,
DEVICE_FLAGS_ANDROID_BUGS },
{ "Google Inc (for Toshiba)", 0x18d1, "Thrive 7/AT105", 0x7102,
DEVICE_FLAGS_ANDROID_BUGS },
{ "Google Inc (for Lenovo)", 0x18d1, "Ideapad K1", 0x740a,
@ -1701,6 +1799,9 @@
// Reported by jaile <jaile@users.sourceforge.net>
{ "Asus", 0x0b05, "TF300 Transformer (USB debug mode)", 0x4c81,
DEVICE_FLAGS_ANDROID_BUGS },
// Repored by Florian Apolloner <f-apolloner@users.sourceforge.net>
{ "Asus", 0x0b05, "TF700 Transformer", 0x4c90,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by anonymous Sourceforge user
{ "Asus", 0x0b05, "TF201 Transformer Prime (keyboard dock)", 0x4d00,
DEVICE_FLAGS_ANDROID_BUGS },
@ -1727,6 +1828,13 @@
// Adding Android default bug flags since it appears to be an Android
{ "Lenovo", 0x17ef, "ThinkPad Tablet", 0x741c,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by: XChesser <XChesser@users.sourceforge.net>
{ "Lenovo", 0x17ef, "P700", 0x7497,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by: anonymous sourceforge user
{ "Lenovo", 0x17ef, "Lifetab S9512", 0x74cc,
DEVICE_FLAGS_ANDROID_BUGS },
/*
* Huawei
@ -1748,7 +1856,7 @@
/*
* HTC (High Tech Computer Corp)
*/
{ "HTC", 0x0bb4, "Zopo ZP100", 0x0c02,
{ "HTC", 0x0bb4, "Zopo ZP100 (ID1)", 0x0c02,
DEVICE_FLAGS_ANDROID_BUGS },
// Reported by Steven Eastland <grassmonk@users.sourceforge.net>
{ "HTC", 0x0bb4, "EVO 4G LTE", 0x0c93,
@ -1762,6 +1870,9 @@
DEVICE_FLAGS_ANDROID_BUGS },
{ "Hewlett-Packard", 0x0bb4, "HP Touchpad (debug mode)",
0x6860, DEVICE_FLAGS_ANDROID_BUGS },
// Reported by anonymous SourceForge user
{ "HTC", 0x0bb4, "Zopo ZP100 (ID2)", 0x2008,
DEVICE_FLAGS_ANDROID_BUGS },
/*
* NEC

441
src/calibre/devices/mtp/windows/content_enumeration.cpp
View File

@ -14,7 +14,7 @@
namespace wpd {
static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{
IPortableDeviceKeyCollection *properties;
IPortableDeviceKeyCollection *properties = NULL;
HRESULT hr;
Py_BEGIN_ALLOW_THREADS;
@ -28,11 +28,13 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() {
ADDPROP(WPD_OBJECT_PARENT_ID);
ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID);
ADDPROP(WPD_OBJECT_NAME);
ADDPROP(WPD_OBJECT_SYNC_ID);
ADDPROP(WPD_OBJECT_ORIGINAL_FILE_NAME);
// ADDPROP(WPD_OBJECT_SYNC_ID);
ADDPROP(WPD_OBJECT_ISSYSTEM);
ADDPROP(WPD_OBJECT_ISHIDDEN);
ADDPROP(WPD_OBJECT_CAN_DELETE);
ADDPROP(WPD_OBJECT_SIZE);
ADDPROP(WPD_OBJECT_DATE_MODIFIED);
return properties;
@ -72,7 +74,7 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py
hr = properties->GetUnsignedLargeIntegerValue(key, &val);
if (SUCCEEDED(hr)) {
pval = PyInt_FromSsize_t((Py_ssize_t)val);
pval = PyLong_FromUnsignedLongLong(val);
if (pval != NULL) {
PyDict_SetItemString(dict, pykey, pval);
Py_DECREF(pval);
@ -80,6 +82,24 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py
}
}
static void set_date_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) {
FLOAT val = 0;
SYSTEMTIME st;
unsigned int microseconds;
PyObject *t;
if (SUCCEEDED(properties->GetFloatValue(key, &val))) {
if (VariantTimeToSystemTime(val, &st)) {
microseconds = 1000 * st.wMilliseconds;
t = Py_BuildValue("H H H H H H I", (unsigned short)st.wYear,
(unsigned short)st.wMonth, (unsigned short)st.wDay,
(unsigned short)st.wHour, (unsigned short)st.wMinute,
(unsigned short)st.wSecond, microseconds);
if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); }
}
}
}
static void set_content_type_property(PyObject *dict, IPortableDeviceValues *properties) {
GUID guid = GUID_NULL;
BOOL is_folder = 0;
@ -87,8 +107,28 @@ static void set_content_type_property(PyObject *dict, IPortableDeviceValues *pro
if (SUCCEEDED(properties->GetGuidValue(WPD_OBJECT_CONTENT_TYPE, &guid)) && IsEqualGUID(guid, WPD_CONTENT_TYPE_FOLDER)) is_folder = 1;
PyDict_SetItemString(dict, "is_folder", (is_folder) ? Py_True : Py_False);
}
static void set_properties(PyObject *obj, IPortableDeviceValues *values) {
set_content_type_property(obj, values);
set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values);
set_string_property(obj, WPD_OBJECT_NAME, "nominal_name", values);
// set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values);
set_string_property(obj, WPD_OBJECT_ORIGINAL_FILE_NAME, "name", values);
set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values);
set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values);
set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", values);
set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values);
set_size_property(obj, WPD_OBJECT_SIZE, "size", values);
set_date_property(obj, WPD_OBJECT_DATE_MODIFIED, "modified", values);
}
// }}}
// Bulk get filesystem {{{
class GetBulkCallback : public IPortableDevicePropertiesBulkCallback {
public:
@ -154,19 +194,8 @@ public:
}
Py_DECREF(temp);
set_content_type_property(obj, properties);
set_properties(obj, properties);
set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", properties);
set_string_property(obj, WPD_OBJECT_NAME, "name", properties);
set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", properties);
set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", properties);
set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", properties);
set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", properties);
set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", properties);
set_size_property(obj, WPD_OBJECT_SIZE, "size", properties);
properties->Release(); properties = NULL;
}
} // end for loop
@ -240,6 +269,9 @@ end:
return folders;
}
// }}}
// find_all_objects_in() {{{
static BOOL find_all_objects_in(IPortableDeviceContent *content, IPortableDevicePropVariantCollection *object_ids, const wchar_t *parent_id) {
/*
* Find all children of the object identified by parent_id, recursively.
@ -286,9 +318,118 @@ end:
if (children != NULL) children->Release();
PropVariantClear(&pv);
return ok;
} // }}}
// Single get filesystem {{{
static PyObject* get_object_properties(IPortableDeviceProperties *devprops, IPortableDeviceKeyCollection *properties, const wchar_t *object_id) {
IPortableDeviceValues *values = NULL;
HRESULT hr;
PyObject *ans = NULL, *temp = NULL;
Py_BEGIN_ALLOW_THREADS;
hr = devprops->GetValues(object_id, properties, &values);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to get properties for object", hr); goto end; }
temp = wchar_to_unicode(object_id);
if (temp == NULL) goto end;
ans = PyDict_New();
if (ans == NULL) { PyErr_NoMemory(); goto end; }
if (PyDict_SetItemString(ans, "id", temp) != 0) { Py_DECREF(ans); ans = NULL; PyErr_NoMemory(); goto end; }
set_properties(ans, values);
end:
Py_XDECREF(temp);
if (values != NULL) values->Release();
return ans;
}
PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) {
static PyObject* single_get_filesystem(IPortableDeviceContent *content, const wchar_t *storage_id, IPortableDevicePropVariantCollection *object_ids) {
DWORD num, i;
PROPVARIANT pv;
HRESULT hr;
BOOL ok = 1;
PyObject *ans = NULL, *item = NULL;
IPortableDeviceProperties *devprops = NULL;
IPortableDeviceKeyCollection *properties = NULL;
hr = content->Properties(&devprops);
if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
properties = create_filesystem_properties_collection();
if (properties == NULL) goto end;
hr = object_ids->GetCount(&num);
if (FAILED(hr)) { hresult_set_exc("Failed to get object id count", hr); goto end; }
ans = PyDict_New();
if (ans == NULL) goto end;
for (i = 0; i < num; i++) {
ok = 0;
PropVariantInit(&pv);
hr = object_ids->GetAt(i, &pv);
if (SUCCEEDED(hr) && pv.pwszVal != NULL) {
item = get_object_properties(devprops, properties, pv.pwszVal);
if (item != NULL) {
PyDict_SetItem(ans, PyDict_GetItemString(item, "id"), item);
Py_DECREF(item); item = NULL;
ok = 1;
}
} else hresult_set_exc("Failed to get item from IPortableDevicePropVariantCollection", hr);
PropVariantClear(&pv);
if (!ok) { Py_DECREF(ans); ans = NULL; break; }
}
end:
if (devprops != NULL) devprops->Release();
if (properties != NULL) properties->Release();
return ans;
}
// }}}
static IPortableDeviceValues* create_object_properties(const wchar_t *parent_id, const wchar_t *name, const GUID content_type, unsigned PY_LONG_LONG size) { // {{{
IPortableDeviceValues *values = NULL;
HRESULT hr;
BOOL ok = FALSE;
hr = CoCreateInstance(CLSID_PortableDeviceValues, NULL,
CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&values));
if (FAILED(hr)) { hresult_set_exc("Failed to create values interface", hr); goto end; }
hr = values->SetStringValue(WPD_OBJECT_PARENT_ID, parent_id);
if (FAILED(hr)) { hresult_set_exc("Failed to set parent_id value", hr); goto end; }
hr = values->SetStringValue(WPD_OBJECT_NAME, name);
if (FAILED(hr)) { hresult_set_exc("Failed to set name value", hr); goto end; }
hr = values->SetStringValue(WPD_OBJECT_ORIGINAL_FILE_NAME, name);
if (FAILED(hr)) { hresult_set_exc("Failed to set original_file_name value", hr); goto end; }
hr = values->SetGuidValue(WPD_OBJECT_FORMAT, WPD_OBJECT_FORMAT_UNSPECIFIED);
if (FAILED(hr)) { hresult_set_exc("Failed to set object_format value", hr); goto end; }
hr = values->SetGuidValue(WPD_OBJECT_CONTENT_TYPE, content_type);
if (FAILED(hr)) { hresult_set_exc("Failed to set content_type value", hr); goto end; }
if (!IsEqualGUID(WPD_CONTENT_TYPE_FOLDER, content_type)) {
hr = values->SetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, size);
if (FAILED(hr)) { hresult_set_exc("Failed to set size value", hr); goto end; }
}
ok = TRUE;
end:
if (!ok && values != NULL) { values->Release(); values = NULL; }
return values;
} // }}}
PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{
PyObject *folders = NULL;
IPortableDevicePropVariantCollection *object_ids = NULL;
IPortableDeviceContent *content = NULL;
@ -310,12 +451,278 @@ PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id
if (!ok) goto end;
if (bulk_properties != NULL) folders = bulk_get_filesystem(device, bulk_properties, storage_id, object_ids);
else folders = single_get_filesystem(content, storage_id, object_ids);
end:
if (content != NULL) content->Release();
if (object_ids != NULL) object_ids->Release();
return folders;
}
} // }}}
PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback) { // {{{
IPortableDeviceContent *content = NULL;
IPortableDeviceResources *resources = NULL;
IPortableDeviceProperties *devprops = NULL;
IPortableDeviceValues *values = NULL;
IPortableDeviceKeyCollection *properties = NULL;
IStream *stream = NULL;
HRESULT hr;
DWORD bufsize = 4096;
char *buf = NULL;
ULONG bytes_read = 0, total_read = 0;
BOOL ok = FALSE;
PyObject *res = NULL;
ULONGLONG filesize = 0;
Py_BEGIN_ALLOW_THREADS;
hr = device->Content(&content);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
Py_BEGIN_ALLOW_THREADS;
hr = content->Properties(&devprops);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
Py_BEGIN_ALLOW_THREADS;
hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL,
CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties));
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); goto end; }
hr = properties->Add(WPD_OBJECT_SIZE);
if (FAILED(hr)) { hresult_set_exc("Failed to add filesize property to properties collection", hr); goto end; }
Py_BEGIN_ALLOW_THREADS;
hr = devprops->GetValues(object_id, properties, &values);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to get filesize for object", hr); goto end; }
hr = values->GetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, &filesize);
if (FAILED(hr)) { hresult_set_exc("Failed to get filesize from values collection", hr); goto end; }
Py_BEGIN_ALLOW_THREADS;
hr = content->Transfer(&resources);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to create resources interface", hr); goto end; }
Py_BEGIN_ALLOW_THREADS;
hr = resources->GetStream(object_id, WPD_RESOURCE_DEFAULT, STGM_READ, &bufsize, &stream);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) {
if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) {
PyErr_SetString(WPDFileBusy, "Object is in use");
} else hresult_set_exc("Failed to create stream interface to read from object", hr);
goto end;
}
buf = (char *)calloc(bufsize+10, 1);
if (buf == NULL) { PyErr_NoMemory(); goto end; }
while (TRUE) {
bytes_read = 0;
Py_BEGIN_ALLOW_THREADS;
hr = stream->Read(buf, bufsize, &bytes_read);
Py_END_ALLOW_THREADS;
total_read = total_read + bytes_read;
if (hr == STG_E_ACCESSDENIED) {
PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break;
} else if (SUCCEEDED(hr)) {
if (bytes_read > 0) {
res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read);
if (res == NULL) break;
Py_DECREF(res); res = NULL;
if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_read, filesize));
}
} else { hresult_set_exc("Failed to read file from device", hr); break; }
if (bytes_read == 0) {
ok = TRUE;
Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL));
break;
}
}
if (ok && total_read != filesize) {
ok = FALSE;
PyErr_SetString(WPDError, "Failed to read all data from file");
}
end:
if (content != NULL) content->Release();
if (devprops != NULL) devprops->Release();
if (resources != NULL) resources->Release();
if (stream != NULL) stream->Release();
if (values != NULL) values->Release();
if (properties != NULL) properties->Release();
if (buf != NULL) free(buf);
if (!ok) return NULL;
Py_RETURN_NONE;
} // }}}
PyObject* wpd::create_folder(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name) { // {{{
IPortableDeviceContent *content = NULL;
IPortableDeviceValues *values = NULL;
IPortableDeviceProperties *devprops = NULL;
IPortableDeviceKeyCollection *properties = NULL;
wchar_t *newid = NULL;
PyObject *ans = NULL;
HRESULT hr;
values = create_object_properties(parent_id, name, WPD_CONTENT_TYPE_FOLDER, 0);
if (values == NULL) goto end;
Py_BEGIN_ALLOW_THREADS;
hr = device->Content(&content);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
hr = content->Properties(&devprops);
if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
properties = create_filesystem_properties_collection();
if (properties == NULL) goto end;
Py_BEGIN_ALLOW_THREADS;
hr = content->CreateObjectWithPropertiesOnly(values, &newid);
Py_END_ALLOW_THREADS;
if (FAILED(hr) || newid == NULL) { hresult_set_exc("Failed to create folder", hr); goto end; }
ans = get_object_properties(devprops, properties, newid);
end:
if (content != NULL) content->Release();
if (values != NULL) values->Release();
if (devprops != NULL) devprops->Release();
if (properties != NULL) properties->Release();
if (newid != NULL) CoTaskMemFree(newid);
return ans;
} // }}}
PyObject* wpd::delete_object(IPortableDevice *device, const wchar_t *object_id) { // {{{
IPortableDeviceContent *content = NULL;
HRESULT hr;
BOOL ok = FALSE;
PROPVARIANT pv;
IPortableDevicePropVariantCollection *object_ids = NULL;
PropVariantInit(&pv);
pv.vt = VT_LPWSTR;
Py_BEGIN_ALLOW_THREADS;
hr = CoCreateInstance(CLSID_PortableDevicePropVariantCollection, NULL,
CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&object_ids));
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to create propvariantcollection", hr); goto end; }
pv.pwszVal = (wchar_t*)object_id;
hr = object_ids->Add(&pv);
pv.pwszVal = NULL;
if (FAILED(hr)) { hresult_set_exc("Failed to add device id to propvariantcollection", hr); goto end; }
Py_BEGIN_ALLOW_THREADS;
hr = device->Content(&content);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
hr = content->Delete(PORTABLE_DEVICE_DELETE_NO_RECURSION, object_ids, NULL);
if (hr == E_ACCESSDENIED) PyErr_SetString(WPDError, "Do not have permission to delete this object");
else if (hr == HRESULT_FROM_WIN32(ERROR_DIR_NOT_EMPTY) || hr == HRESULT_FROM_WIN32(ERROR_INVALID_OPERATION)) PyErr_SetString(WPDError, "Cannot delete object as it has children");
else if (hr == HRESULT_FROM_WIN32(ERROR_NOT_FOUND) || SUCCEEDED(hr)) ok = TRUE;
else hresult_set_exc("Cannot delete object", hr);
end:
PropVariantClear(&pv);
if (content != NULL) content->Release();
if (object_ids != NULL) object_ids->Release();
if (!ok) return NULL;
Py_RETURN_NONE;
} // }}}
PyObject* wpd::put_file(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name, PyObject *src, unsigned PY_LONG_LONG size, PyObject *callback) { // {{{
IPortableDeviceContent *content = NULL;
IPortableDeviceValues *values = NULL;
IPortableDeviceProperties *devprops = NULL;
IPortableDeviceKeyCollection *properties = NULL;
IStream *temp = NULL;
IPortableDeviceDataStream *dest = NULL;
char *buf = NULL;
wchar_t *newid = NULL;
PyObject *ans = NULL, *raw;
HRESULT hr;
DWORD bufsize = 0;
BOOL ok = FALSE;
Py_ssize_t bytes_read = 0;
ULONG bytes_written = 0, total_written = 0;
values = create_object_properties(parent_id, name, WPD_CONTENT_TYPE_GENERIC_FILE, size);
if (values == NULL) goto end;
Py_BEGIN_ALLOW_THREADS;
hr = device->Content(&content);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
hr = content->Properties(&devprops);
if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
properties = create_filesystem_properties_collection();
if (properties == NULL) goto end;
Py_BEGIN_ALLOW_THREADS;
hr = content->CreateObjectWithPropertiesAndData(values, &temp, &bufsize, NULL);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) {
if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) {
PyErr_SetString(WPDFileBusy, "Object is in use");
} else hresult_set_exc("Failed to create stream interface to write to object", hr);
goto end;
}
hr = temp->QueryInterface(IID_PPV_ARGS(&dest));
if (FAILED(hr)) { hresult_set_exc("Failed to create IPortableDeviceStream", hr); goto end; }
while(TRUE) {
raw = PyObject_CallMethod(src, "read", "k", bufsize);
if (raw == NULL) break;
PyBytes_AsStringAndSize(raw, &buf, &bytes_read);
if (bytes_read > 0) {
Py_BEGIN_ALLOW_THREADS;
hr = dest->Write(buf, bytes_read, &bytes_written);
Py_END_ALLOW_THREADS;
Py_DECREF(raw);
if (hr == STG_E_MEDIUMFULL) { PyErr_SetString(WPDError, "Cannot write to device as it is full"); break; }
if (hr == STG_E_ACCESSDENIED) { PyErr_SetString(WPDError, "Cannot write to file as access is denied"); break; }
if (hr == STG_E_WRITEFAULT) { PyErr_SetString(WPDError, "Cannot write to file as there was a disk I/O error"); break; }
if (FAILED(hr)) { hresult_set_exc("Cannot write to file", hr); break; }
if (bytes_written != bytes_read) { PyErr_SetString(WPDError, "Writing to file failed, not all bytes were written"); break; }
total_written += bytes_written;
if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_written, size));
} else Py_DECREF(raw);
if (bytes_read == 0) { ok = TRUE; break; }
}
if (!ok) {dest->Revert(); goto end;}
Py_BEGIN_ALLOW_THREADS;
hr = dest->Commit(STGC_DEFAULT);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to write data to file, commit failed", hr); goto end; }
if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_written, size));
Py_BEGIN_ALLOW_THREADS;
hr = dest->GetObjectID(&newid);
Py_END_ALLOW_THREADS;
if (FAILED(hr)) { hresult_set_exc("Failed to get id of newly created file", hr); goto end; }
ans = get_object_properties(devprops, properties, newid);
end:
if (content != NULL) content->Release();
if (values != NULL) values->Release();
if (devprops != NULL) devprops->Release();
if (properties != NULL) properties->Release();
if (temp != NULL) temp->Release();
if (dest != NULL) dest->Release();
if (newid != NULL) CoTaskMemFree(newid);
return ans;
} // }}}
} // namespace wpd

93
src/calibre/devices/mtp/windows/device.cpp
View File

@ -67,7 +67,7 @@ init(Device *self, PyObject *args, PyObject *kwds)
// update_device_data() {{{
static PyObject*
update_data(Device *self, PyObject *args, PyObject *kwargs) {
update_data(Device *self, PyObject *args) {
PyObject *di = NULL;
di = get_device_information(self->device, NULL);
if (di == NULL) return NULL;
@ -77,15 +77,84 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) {
// get_filesystem() {{{
static PyObject*
py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
PyObject *storage_id, *ans = NULL;
py_get_filesystem(Device *self, PyObject *args) {
PyObject *storage_id, *ret;
wchar_t *storage;
if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL;
storage = unicode_to_wchar(storage_id);
if (storage == NULL) return NULL;
return wpd::get_filesystem(self->device, storage, self->bulk_properties);
ret = wpd::get_filesystem(self->device, storage, self->bulk_properties);
free(storage);
return ret;
} // }}}
// get_file() {{{
static PyObject*
py_get_file(Device *self, PyObject *args) {
PyObject *object_id, *stream, *callback = NULL, *ret;
wchar_t *object;
if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL;
object = unicode_to_wchar(object_id);
if (object == NULL) return NULL;
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
ret = wpd::get_file(self->device, object, stream, callback);
free(object);
return ret;
} // }}}
// create_folder() {{{
static PyObject*
py_create_folder(Device *self, PyObject *args) {
PyObject *pparent_id, *pname, *ret;
wchar_t *parent_id, *name;
if (!PyArg_ParseTuple(args, "OO", &pparent_id, &pname)) return NULL;
parent_id = unicode_to_wchar(pparent_id);
name = unicode_to_wchar(pname);
if (parent_id == NULL || name == NULL) return NULL;
ret = wpd::create_folder(self->device, parent_id, name);
free(parent_id); free(name);
return ret;
} // }}}
// delete_object() {{{
static PyObject*
py_delete_object(Device *self, PyObject *args) {
PyObject *pobject_id, *ret;
wchar_t *object_id;
if (!PyArg_ParseTuple(args, "O", &pobject_id)) return NULL;
object_id = unicode_to_wchar(pobject_id);
if (object_id == NULL) return NULL;
ret = wpd::delete_object(self->device, object_id);
free(object_id);
return ret;
} // }}}
// get_file() {{{
static PyObject*
py_put_file(Device *self, PyObject *args) {
PyObject *pparent_id, *pname, *stream, *callback = NULL, *ret;
wchar_t *parent_id, *name;
unsigned long long size;
if (!PyArg_ParseTuple(args, "OOOK|O", &pparent_id, &pname, &stream, &size, &callback)) return NULL;
parent_id = unicode_to_wchar(pparent_id);
name = unicode_to_wchar(pname);
if (parent_id == NULL || name == NULL) return NULL;
if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
ret = wpd::put_file(self->device, parent_id, name, stream, size, callback);
free(parent_id); free(name);
return ret;
} // }}}
static PyMethodDef Device_methods[] = {
@ -97,6 +166,22 @@ static PyMethodDef Device_methods[] = {
"get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible."
},
{"get_file", (PyCFunction)py_get_file, METH_VARARGS,
"get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values."
},
{"create_folder", (PyCFunction)py_create_folder, METH_VARARGS,
"create_folder(parent_id, name) -> Create a folder. Returns the folder metadata."
},
{"delete_object", (PyCFunction)py_delete_object, METH_VARARGS,
"delete_object(object_id) -> Delete the object identified by object_id. Note that trying to delete a non-empty folder will raise an error."
},
{"put_file", (PyCFunction)py_put_file, METH_VARARGS,
"put_file(parent_id, name, stream, size_in_bytes, callback=None) -> Copy a file from the stream object, creating a new file on the device with parent identified by parent_id. Returns the file metadata of the newly created file. callback should be a callable that accepts two argument: (bytes_written, total_size). It will be called after each chunk is written to the device. Note that it can be called multiple times with the same arguments."
},
{NULL}
};

2
src/calibre/devices/mtp/windows/device_enumeration.cpp
View File

@ -149,7 +149,7 @@ PyObject* get_storage_info(IPortableDevice *device) { // {{{
if (SUCCEEDED(values->GetUnsignedIntegerValue(WPD_STORAGE_ACCESS_CAPABILITY, &access)) && access == WPD_STORAGE_ACCESS_CAPABILITY_READWRITE) desc = Py_True;
soid = PyUnicode_FromWideChar(object_ids[i], wcslen(object_ids[i]));
if (soid == NULL) { PyErr_NoMemory(); goto end; }
so = Py_BuildValue("{s:K,s:K,s:K,s:K,s:O,s:N}",
so = Py_BuildValue("{s:K, s:K, s:K, s:K, s:O, s:N}",
"capacity", capacity, "capacity_objects", capacity_objects, "free_space", free_space, "free_objects", free_objects, "rw", desc, "id", soid);
if (so == NULL) { PyErr_NoMemory(); goto end; }
if (SUCCEEDED(values->GetStringValue(WPD_STORAGE_DESCRIPTION, &storage_desc))) {

250
src/calibre/devices/mtp/windows/driver.py
View File

@ -7,13 +7,32 @@ __license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import time
from threading import RLock
import time, threading, traceback
from functools import wraps, partial
from future_builtins import zip
from itertools import chain
from calibre import as_unicode, prints
from calibre.constants import plugins, __appname__, numeric_version
from calibre.devices.errors import OpenFailed
from calibre.devices.mtp.base import MTPDeviceBase, synchronous
from calibre.ptempfile import SpooledTemporaryFile
from calibre.devices.errors import OpenFailed, DeviceError, BlacklistedDevice
from calibre.devices.mtp.base import MTPDeviceBase, debug
class ThreadingViolation(Exception):
def __init__(self):
Exception.__init__(self,
'You cannot use the MTP driver from a thread other than the '
' thread in which startup() was called')
def same_thread(func):
@wraps(func)
def check_thread(self, *args, **kwargs):
if self.start_thread is not threading.current_thread():
raise ThreadingViolation()
return func(self, *args, **kwargs)
return check_thread
class MTP_DEVICE(MTPDeviceBase):
@ -22,7 +41,6 @@ class MTP_DEVICE(MTPDeviceBase):
def __init__(self, *args, **kwargs):
MTPDeviceBase.__init__(self, *args, **kwargs)
self.dev = None
self.lock = RLock()
self.blacklisted_devices = set()
self.ejected_devices = set()
self.currently_connected_pnp_id = None
@ -31,9 +49,12 @@ class MTP_DEVICE(MTPDeviceBase):
self.last_refresh_devices_time = time.time()
self.wpd = self.wpd_error = None
self._main_id = self._carda_id = self._cardb_id = None
self.start_thread = None
self._filesystem_cache = None
self.eject_dev_on_next_scan = False
@synchronous
def startup(self):
self.start_thread = threading.current_thread()
self.wpd, self.wpd_error = plugins['wpd']
if self.wpd is not None:
try:
@ -46,19 +67,24 @@ class MTP_DEVICE(MTPDeviceBase):
except Exception as e:
self.wpd_error = as_unicode(e)
@synchronous
@same_thread
def shutdown(self):
self.dev = self.filesystem_cache = None
self.dev = self._filesystem_cache = self.start_thread = None
if self.wpd is not None:
self.wpd.uninit()
@synchronous
def detect_managed_devices(self, devices_on_system):
@same_thread
def detect_managed_devices(self, devices_on_system, force_refresh=False):
if self.wpd is None: return None
if self.eject_dev_on_next_scan:
self.eject_dev_on_next_scan = False
if self.currently_connected_pnp_id is not None:
self.do_eject()
devices_on_system = frozenset(devices_on_system)
if (devices_on_system != self.previous_devices_on_system or time.time()
- self.last_refresh_devices_time > 10):
if (force_refresh or
devices_on_system != self.previous_devices_on_system or
time.time() - self.last_refresh_devices_time > 10):
self.previous_devices_on_system = devices_on_system
self.last_refresh_devices_time = time.time()
try:
@ -103,6 +129,57 @@ class MTP_DEVICE(MTPDeviceBase):
return None
@same_thread
def debug_managed_device_detection(self, devices_on_system, output):
import pprint
p = partial(prints, file=output)
if self.currently_connected_pnp_id is not None:
return True
if self.wpd_error:
p('Cannot detect MTP devices')
p(self.wpd_error)
return False
try:
pnp_ids = frozenset(self.wpd.enumerate_devices())
except:
p("Failed to get list of PNP ids on system")
p(traceback.format_exc())
return False
for pnp_id in pnp_ids:
try:
data = self.wpd.device_info(pnp_id)
except:
p('Failed to get data for device:', pnp_id)
p(traceback.format_exc())
continue
protocol = data.get('protocol', '').lower()
if not protocol.startswith('mtp:'): continue
p('MTP device:', pnp_id)
p(pprint.pformat(data))
if not self.is_suitable_wpd_device(data):
p('Not a suitable MTP device, ignoring\n')
continue
p('\nTrying to open:', pnp_id)
try:
self.open(pnp_id, 'debug-detection')
except BlacklistedDevice:
p('This device has been blacklisted by the user')
continue
except:
p('Open failed:')
p(traceback.format_exc())
continue
break
if self.currently_connected_pnp_id:
p('Opened', self.current_friendly_name, 'successfully')
p('Device info:')
p(pprint.pformat(self.dev.data))
self.post_yank_cleanup()
return True
p('No suitable MTP devices found')
return False
def is_suitable_wpd_device(self, devdata):
# Check that protocol is MTP
protocol = devdata.get('protocol', '').lower()
@ -119,23 +196,57 @@ class MTP_DEVICE(MTPDeviceBase):
return True
@synchronous
def post_yank_cleanup(self):
self.currently_connected_pnp_id = self.current_friendly_name = None
self._main_id = self._carda_id = self._cardb_id = None
self.dev = self.filesystem_cache = None
@property
def filesystem_cache(self):
if self._filesystem_cache is None:
debug('Loading filesystem metadata...')
st = time.time()
from calibre.devices.mtp.filesystem_cache import FilesystemCache
ts = self.total_space()
all_storage = []
items = []
for storage_id, capacity in zip([self._main_id, self._carda_id,
self._cardb_id], ts):
if storage_id is None: continue
name = _('Unknown')
for s in self.dev.data['storage']:
if s['id'] == storage_id:
name = s['name']
break
storage = {'id':storage_id, 'size':capacity, 'name':name,
'is_folder':True, 'can_delete':False, 'is_system':True}
id_map = self.dev.get_filesystem(storage_id)
for x in id_map.itervalues(): x['storage_id'] = storage_id
all_storage.append(storage)
items.append(id_map.itervalues())
self._filesystem_cache = FilesystemCache(all_storage, chain(*items))
debug('Filesystem metadata loaded in %g seconds (%d objects)'%(
time.time()-st, len(self._filesystem_cache)))
return self._filesystem_cache
@synchronous
def eject(self):
@same_thread
def do_eject(self):
if self.currently_connected_pnp_id is None: return
self.ejected_devices.add(self.currently_connected_pnp_id)
self.currently_connected_pnp_id = self.current_friendly_name = None
self._main_id = self._carda_id = self._cardb_id = None
self.dev = self.filesystem_cache = None
self.dev = self._filesystem_cache = None
@synchronous
@same_thread
def post_yank_cleanup(self):
self.currently_connected_pnp_id = self.current_friendly_name = None
self._main_id = self._carda_id = self._cardb_id = None
self.dev = self._filesystem_cache = None
self.current_serial_num = None
def eject(self):
if self.currently_connected_pnp_id is None: return
self.eject_dev_on_next_scan = True
self.current_serial_num = None
@same_thread
def open(self, connected_device, library_uuid):
self.dev = self.filesystem_cache = None
self.dev = self._filesystem_cache = None
try:
self.dev = self.wpd.Device(connected_device)
except self.wpd.WPDError:
@ -151,29 +262,32 @@ class MTP_DEVICE(MTPDeviceBase):
if not storage:
self.blacklisted_devices.add(connected_device)
raise OpenFailed('No storage found for device %s'%(connected_device,))
snum = devdata.get('serial_number', None)
if snum in self.prefs.get('blacklist', []):
self.blacklisted_devices.add(connected_device)
self.dev = None
raise BlacklistedDevice(
'The %s device has been blacklisted by the user'%(connected_device,))
self._main_id = storage[0]['id']
if len(storage) > 1:
self._carda_id = storage[1]['id']
if len(storage) > 2:
self._cardb_id = storage[2]['id']
self.current_friendly_name = devdata.get('friendly_name', None)
self.current_friendly_name = devdata.get('friendly_name', '')
if not self.current_friendly_name:
self.current_friendly_name = devdata.get('model_name',
_('Unknown MTP device'))
self.currently_connected_pnp_id = connected_device
self.current_serial_num = snum
@synchronous
def get_device_information(self, end_session=True):
@same_thread
def get_basic_device_information(self):
d = self.dev.data
dv = d.get('device_version', '')
return (self.current_friendly_name, dv, dv, '')
@synchronous
def card_prefix(self, end_session=True):
ans = [None, None]
if self._carda_id is not None:
ans[0] = 'mtp:::%s:::'%self._carda_id
if self._cardb_id is not None:
ans[1] = 'mtp:::%s:::'%self._cardb_id
return tuple(ans)
@synchronous
@same_thread
def total_space(self, end_session=True):
ans = [0, 0, 0]
dd = self.dev.data
@ -184,7 +298,7 @@ class MTP_DEVICE(MTPDeviceBase):
ans[i] = s['capacity']
return tuple(ans)
@synchronous
@same_thread
def free_space(self, end_session=True):
self.dev.update_data()
ans = [0, 0, 0]
@ -196,5 +310,69 @@ class MTP_DEVICE(MTPDeviceBase):
ans[i] = s['free_space']
return tuple(ans)
@same_thread
def get_mtp_file(self, f, stream=None, callback=None):
if f.is_folder:
raise ValueError('%s if a folder'%(f.full_path,))
if stream is None:
stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat')
stream.name = f.name
try:
try:
self.dev.get_file(f.object_id, stream, callback)
except self.wpd.WPDFileBusy:
time.sleep(2)
self.dev.get_file(f.object_id, stream, callback)
except Exception as e:
raise DeviceError('Failed to fetch the file %s with error: %s'%
f.full_path, as_unicode(e))
stream.seek(0)
return stream
@same_thread
def create_folder(self, parent, name):
if not parent.is_folder:
raise ValueError('%s is not a folder'%(parent.full_path,))
e = parent.folder_named(name)
if e is not None:
return e
ans = self.dev.create_folder(parent.object_id, name)
ans['storage_id'] = parent.storage_id
return parent.add_child(ans)
@same_thread
def delete_file_or_folder(self, obj):
if obj.deleted:
return
if not obj.can_delete:
raise ValueError('Cannot delete %s as deletion not allowed'%
(obj.full_path,))
if obj.is_system:
raise ValueError('Cannot delete %s as it is a system object'%
(obj.full_path,))
if obj.files or obj.folders:
raise ValueError('Cannot delete %s as it is not empty'%
(obj.full_path,))
parent = obj.parent
self.dev.delete_object(obj.object_id)
parent.remove_child(obj)
return parent
@same_thread
def put_file(self, parent, name, stream, size, callback=None, replace=True):
e = parent.folder_named(name)
if e is not None:
raise ValueError('Cannot upload file, %s already has a folder named: %s'%(
parent.full_path, e.name))
e = parent.file_named(name)
if e is not None:
if not replace:
raise ValueError('Cannot upload file %s, it already exists'%(
e.full_path,))
self.delete_file_or_folder(e)
sid, pid = parent.storage_id, parent.object_id
ans = self.dev.put_file(pid, name, stream, size, callback)
ans['storage_id'] = sid
return parent.add_child(ans)

8
src/calibre/devices/mtp/windows/global.h
View File

@ -20,7 +20,7 @@
namespace wpd {
// Module exception types
extern PyObject *WPDError, *NoWPD;
extern PyObject *WPDError, *NoWPD, *WPDFileBusy;
// The global device manager
extern IPortableDeviceManager *portable_device_manager;
@ -50,13 +50,17 @@ extern PyTypeObject DeviceType;
// Utility functions
PyObject *hresult_set_exc(const char *msg, HRESULT hr);
wchar_t *unicode_to_wchar(PyObject *o);
PyObject *wchar_to_unicode(wchar_t *o);
PyObject *wchar_to_unicode(const wchar_t *o);
int pump_waiting_messages();
extern IPortableDeviceValues* get_client_information();
extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information);
extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties);
extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties);
extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback);
extern PyObject* create_folder(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name);
extern PyObject* delete_object(IPortableDevice *device, const wchar_t *object_id);
extern PyObject* put_file(IPortableDevice *device, const wchar_t *parent_id, const wchar_t *name, PyObject *src, unsigned PY_LONG_LONG size, PyObject *callback);
}

25
src/calibre/devices/mtp/windows/remote.py
View File

@ -7,8 +7,8 @@ __license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import subprocess, sys, os, pprint, signal, time, glob
pprint
import subprocess, sys, os, pprint, signal, time, glob, io
pprint, io
def build(mod='wpd'):
master = subprocess.Popen('ssh -MN getafix'.split())
@ -54,6 +54,10 @@ def main():
plugins._plugins['wpd'] = (wpd, '')
sys.path.pop(0)
# from calibre.devices.mtp.test import run
# run()
# return
from calibre.devices.scanner import win_scanner
from calibre.devices.mtp.windows.driver import MTP_DEVICE
dev = MTP_DEVICE(None)
@ -63,6 +67,8 @@ def main():
try:
devices = win_scanner()
pnp_id = dev.detect_managed_devices(devices)
if not pnp_id:
raise ValueError('Failed to detect device')
# pprint.pprint(dev.detected_devices)
print ('Trying to connect to:', pnp_id)
dev.open(pnp_id, '')
@ -70,10 +76,23 @@ def main():
print ('Connected to:', dev.get_gui_name())
print ('Total space', dev.total_space())
print ('Free space', dev.free_space())
pprint.pprint(dev.dev.get_filesystem(dev._main_id))
# pprint.pprint(dev.dev.create_folder(dev.filesystem_cache.entries[0].object_id,
# 'zzz'))
# print ('Fetching file: oFF (198214 bytes)')
# stream = dev.get_file('oFF')
# print ("Fetched size: ", stream.tell())
# size = 4
# stream = io.BytesIO(b'a'*size)
# name = 'zzz-test-file.txt'
# stream.seek(0)
# f = dev.put_file(dev.filesystem_cache.entries[0], name, stream, size)
# print ('Put file:', f)
dev.filesystem_cache.dump()
finally:
dev.shutdown()
print ('Device connection shutdown')
if __name__ == '__main__':
main()

Some files were not shown because too many files have changed in this diff Show More