Merge pull request #2391 from mikiher/binary-manager

Add a binary manager that finds ffmpeg and ffprobe and installs them if not found

.gitignore

@@ -13,6 +13,8 @@
 /deploy/
 /coverage/
 /.nyc_output/
+/ffmpeg*
+/ffprobe*
 
 sw.*
 .DS_STORE

server/Server.js

@@ -33,6 +33,7 @@ const AudioMetadataMangaer = require('./managers/AudioMetadataManager')
 const RssFeedManager = require('./managers/RssFeedManager')
 const CronManager = require('./managers/CronManager')
 const ApiCacheManager = require('./managers/ApiCacheManager')
+const BinaryManager = require('./managers/BinaryManager')
 const LibraryScanner = require('./scanner/LibraryScanner')
 
 //Import the main Passport and Express-Session library
@@ -74,6 +75,7 @@ class Server {
     this.rssFeedManager = new RssFeedManager()
     this.cronManager = new CronManager(this.podcastManager)
     this.apiCacheManager = new ApiCacheManager()
+    this.binaryManager = new BinaryManager()
 
     // Routers
     this.apiRouter = new ApiRouter(this)
@@ -120,6 +122,11 @@ class Server {
     await this.cronManager.init(libraries)
     this.apiCacheManager.init()
 
+    // Download ffmpeg & ffprobe if not found (Currently only in use for Windows installs)
+    if (global.isWin || Logger.isDev) {
+      await this.binaryManager.init()
+    }
+
     if (Database.serverSettings.scannerDisableWatcher) {
       Logger.info(`[Server] Watcher is disabled`)
       this.watcher.disabled = true

server/libs/ffbinaries/index.js

@@ -0,0 +1,315 @@
+const os = require('os')
+const path = require('path')
+const axios = require('axios')
+const fse = require('../fsExtra')
+const async = require('../async')
+const StreamZip = require('../nodeStreamZip')
+const { finished } = require('stream/promises')
+
+var API_URL = 'https://ffbinaries.com/api/v1'
+
+var RUNTIME_CACHE = {}
+var errorMsgs = {
+  connectionIssues: 'Couldn\'t connect to ffbinaries.com API. Check your Internet connection.',
+  parsingVersionData: 'Couldn\'t parse retrieved version data.',
+  parsingVersionList: 'Couldn\'t parse the list of available versions.',
+  notFound: 'Requested data not found.',
+  incorrectVersionParam: '"version" parameter must be a string.'
+}
+
+function ensureDirSync(dir) {
+  try {
+    fse.accessSync(dir)
+  } catch (e) {
+    fse.mkdirSync(dir)
+  }
+}
+
+/**
+ * Resolves the platform key based on input string
+ */
+function resolvePlatform(input) {
+  var rtn = null
+
+  switch (input) {
+    case 'mac':
+    case 'osx':
+    case 'mac-64':
+    case 'osx-64':
+      rtn = 'osx-64'
+      break
+
+    case 'linux':
+    case 'linux-32':
+      rtn = 'linux-32'
+      break
+
+    case 'linux-64':
+      rtn = 'linux-64'
+      break
+
+    case 'linux-arm':
+    case 'linux-armel':
+      rtn = 'linux-armel'
+      break
+
+    case 'linux-armhf':
+      rtn = 'linux-armhf'
+      break
+
+    case 'win':
+    case 'win-32':
+    case 'windows':
+    case 'windows-32':
+      rtn = 'windows-32'
+      break
+
+    case 'win-64':
+    case 'windows-64':
+      rtn = 'windows-64'
+      break
+
+    default:
+      rtn = null
+  }
+
+  return rtn
+}
+/**
+ * Detects the platform of the machine the script is executed on.
+ * Object can be provided to detect platform from info derived elsewhere.
+ *
+ * @param {object} osinfo Contains "type" and "arch" properties
+ */
+function detectPlatform(osinfo) {
+  var inputIsValid = typeof osinfo === 'object' && typeof osinfo.type === 'string' && typeof osinfo.arch === 'string'
+  var type = (inputIsValid ? osinfo.type : os.type()).toLowerCase()
+  var arch = (inputIsValid ? osinfo.arch : os.arch()).toLowerCase()
+
+  if (type === 'darwin') {
+    return 'osx-64'
+  }
+
+  if (type === 'windows_nt') {
+    return arch === 'x64' ? 'windows-64' : 'windows-32'
+  }
+
+  if (type === 'linux') {
+    if (arch === 'arm' || arch === 'arm64') {
+      return 'linux-armel'
+    }
+    return arch === 'x64' ? 'linux-64' : 'linux-32'
+  }
+
+  return null
+}
+/**
+ * Gets the binary filename (appends exe in Windows)
+ *
+ * @param {string} component "ffmpeg", "ffplay", "ffprobe" or "ffserver"
+ * @param {platform} platform "ffmpeg", "ffplay", "ffprobe" or "ffserver"
+ */
+function getBinaryFilename(component, platform) {
+  var platformCode = resolvePlatform(platform)
+  if (platformCode === 'windows-32' || platformCode === 'windows-64') {
+    return component + '.exe'
+  }
+  return component
+}
+
+function listPlatforms() {
+  return ['osx-64', 'linux-32', 'linux-64', 'linux-armel', 'linux-armhf', 'windows-32', 'windows-64']
+}
+
+/**
+ * 
+ * @returns {Promise<string[]>} array of version strings
+ */
+function listVersions() {
+  if (RUNTIME_CACHE.versionsAll) {
+    return RUNTIME_CACHE.versionsAll
+  }
+  return axios.get(API_URL).then((res) => {
+    if (!res.data?.versions || !Object.keys(res.data.versions)?.length) {
+      throw new Error(errorMsgs.parsingVersionList)
+    }
+    const versionKeys = Object.keys(res.data.versions)
+    RUNTIME_CACHE.versionsAll = versionKeys
+    return versionKeys
+  })
+}
+/**
+ * Gets full data set from ffbinaries.com
+ */
+function getVersionData(version) {
+  if (RUNTIME_CACHE[version]) {
+    return RUNTIME_CACHE[version]
+  }
+
+  if (version && typeof version !== 'string') {
+    throw new Error(errorMsgs.incorrectVersionParam)
+  }
+
+  var url = version ? '/version/' + version : '/latest'
+
+  return axios.get(`${API_URL}${url}`).then((res) => {
+    RUNTIME_CACHE[version] = res.data
+    return res.data
+  }).catch((error) => {
+    if (error.response?.status == 404) {
+      throw new Error(errorMsgs.notFound)
+    } else {
+      throw new Error(errorMsgs.connectionIssues)
+    }
+  })
+}
+
+/**
+ * Download file(s) and save them in the specified directory
+ */
+async function downloadUrls(components, urls, opts) {
+  const destinationDir = opts.destination
+  const results = []
+  const remappedUrls = []
+
+  if (components && !Array.isArray(components)) {
+    components = [components]
+  } else if (!components || !Array.isArray(components)) {
+    components = []
+  }
+
+  // returns an array of objects like this: {component: 'ffmpeg', url: 'https://...'}
+  if (typeof urls === 'object') {
+    for (const key in urls) {
+      if (components.includes(key) && urls[key]) {
+        remappedUrls.push({
+          component: key,
+          url: urls[key]
+        })
+      }
+    }
+  }
+
+
+  async function extractZipToDestination(zipFilename) {
+    const oldpath = path.join(destinationDir, zipFilename)
+    const zip = new StreamZip.async({ file: oldpath })
+    const count = await zip.extract(null, destinationDir)
+    await zip.close()
+  }
+
+
+  await async.each(remappedUrls, async function (urlObject) {
+    try {
+      const url = urlObject.url
+
+      const zipFilename = url.split('/').pop()
+      const binFilenameBase = urlObject.component
+      const binFilename = getBinaryFilename(binFilenameBase, opts.platform || detectPlatform())
+      
+      let runningTotal = 0
+      let totalFilesize
+      let interval
+
+      
+      if (typeof opts.tickerFn === 'function') {
+        opts.tickerInterval = parseInt(opts.tickerInterval, 10)
+        const tickerInterval = (!Number.isNaN(opts.tickerInterval)) ? opts.tickerInterval : 1000
+        const tickData = { filename: zipFilename, progress: 0 }
+
+        // Schedule next ticks
+        interval = setInterval(function () {
+          if (totalFilesize && runningTotal == totalFilesize) {
+            return clearInterval(interval)
+          }
+          tickData.progress = totalFilesize > -1 ? runningTotal / totalFilesize : 0
+
+          opts.tickerFn(tickData)
+        }, tickerInterval)
+      }
+      
+
+      // Check if file already exists in target directory
+      const binPath = path.join(destinationDir, binFilename)
+      if (!opts.force && await fse.pathExists(binPath)) {
+        // if the accessSync method doesn't throw we know the binary already exists
+        results.push({
+          filename: binFilename,
+          path: destinationDir,
+          status: 'File exists',
+          code: 'FILE_EXISTS'
+        })
+        clearInterval(interval)
+        return
+      }
+
+      if (opts.quiet) clearInterval(interval)
+
+      const zipPath = path.join(destinationDir, zipFilename)
+      const zipFileTempName = zipPath + '.part'
+      const zipFileFinalName = zipPath
+
+      const response = await axios({
+        url,
+        method: 'GET',
+        responseType: 'stream'
+      })
+      totalFilesize = response.headers?.['content-length'] || []
+
+      const writer = fse.createWriteStream(zipFileTempName)
+      response.data.on('data', (chunk) => {        
+        runningTotal += chunk.length
+      }) 
+      response.data.pipe(writer)
+      await finished(writer)
+      await fse.rename(zipFileTempName, zipFileFinalName)
+      await extractZipToDestination(zipFilename)
+      await fse.remove(zipFileFinalName)
+
+      results.push({
+        filename: binFilename,
+        path: destinationDir,
+        size: Math.floor(totalFilesize / 1024 / 1024 * 1000) / 1000 + 'MB',
+        status: 'File extracted to destination (downloaded from "' + url + '")',
+        code: 'DONE_CLEAN'
+      })
+    } catch (err) {
+      console.error(`Failed to download or extract file for component: ${urlObject.component}`, err)
+    }
+  })
+
+  return results
+}
+
+/**
+ * Gets binaries for the platform
+ * It will get the data from ffbinaries, pick the correct files
+ * and save it to the specified directory
+ *
+ * @param {Array} components
+ * @param {Object} [opts]
+ */
+async function downloadBinaries(components, opts = {}) {
+  var platform = resolvePlatform(opts.platform) || detectPlatform()
+
+  opts.destination = path.resolve(opts.destination || '.')
+  ensureDirSync(opts.destination)
+
+  const versionData = await getVersionData(opts.version)
+  const urls = versionData?.bin?.[platform]
+  if (!urls) {
+    throw new Error('No URLs!')
+  }
+
+  return await downloadUrls(components, urls, opts)
+}
+
+module.exports = {
+  downloadBinaries: downloadBinaries,
+  getVersionData: getVersionData,
+  listVersions: listVersions,
+  listPlatforms: listPlatforms,
+  detectPlatform: detectPlatform,
+  resolvePlatform: resolvePlatform,
+  getBinaryFilename: getBinaryFilename
+}

server/managers/BinaryManager.js

@@ -0,0 +1,74 @@
+const path = require('path')
+const which = require('../libs/which')
+const fs = require('../libs/fsExtra')
+const ffbinaries = require('../libs/ffbinaries')
+const Logger = require('../Logger')
+const fileUtils = require('../utils/fileUtils')
+
+class BinaryManager {
+
+  defaultRequiredBinaries = [
+    { name: 'ffmpeg', envVariable: 'FFMPEG_PATH' },
+    { name: 'ffprobe', envVariable: 'FFPROBE_PATH' }
+  ]
+
+  constructor(requiredBinaries = this.defaultRequiredBinaries) {
+    this.requiredBinaries = requiredBinaries
+    this.mainInstallPath = process.pkg ? path.dirname(process.execPath) : global.appRoot
+    this.altInstallPath = global.ConfigPath
+  }
+
+  async init() {
+    if (this.initialized) return
+    const missingBinaries = await this.findRequiredBinaries()
+    if (missingBinaries.length == 0) return
+    await this.install(missingBinaries)
+    const missingBinariesAfterInstall = await this.findRequiredBinaries()
+    if (missingBinariesAfterInstall.length != 0) {
+      Logger.error(`[BinaryManager] Failed to find or install required binaries: ${missingBinariesAfterInstall.join(', ')}`)
+      process.exit(1)
+    }
+    this.initialized = true
+  }
+
+  async findRequiredBinaries() {
+    const missingBinaries = []
+    for (const binary of this.requiredBinaries) {
+      const binaryPath = await this.findBinary(binary.name, binary.envVariable)
+      if (binaryPath) {
+        Logger.info(`[BinaryManager] Found ${binary.name} at ${binaryPath}`)
+        if (process.env[binary.envVariable] !== binaryPath) {
+          Logger.info(`[BinaryManager] Updating process.env.${binary.envVariable}`)
+          process.env[binary.envVariable] = binaryPath
+        }
+      } else {
+        Logger.info(`[BinaryManager] ${binary.name} not found`)
+        missingBinaries.push(binary.name)
+      }
+    }
+    return missingBinaries
+  }
+
+  async findBinary(name, envVariable) {
+    const executable = name + (process.platform == 'win32' ? '.exe' : '')
+    const defaultPath = process.env[envVariable]
+    if (defaultPath && await fs.pathExists(defaultPath)) return defaultPath
+    const whichPath = which.sync(executable, { nothrow: true })
+    if (whichPath) return whichPath
+    const mainInstallPath = path.join(this.mainInstallPath, executable)
+    if (await fs.pathExists(mainInstallPath)) return mainInstallPath
+    const altInstallPath = path.join(this.altInstallPath, executable)
+    if (await fs.pathExists(altInstallPath)) return altInstallPath
+    return null
+  }
+
+  async install(binaries) {
+    if (binaries.length == 0) return
+    Logger.info(`[BinaryManager] Installing binaries: ${binaries.join(', ')}`)
+    let destination = await fileUtils.isWritable(this.mainInstallPath) ? this.mainInstallPath : this.altInstallPath
+    await ffbinaries.downloadBinaries(binaries, { destination })
+    Logger.info(`[BinaryManager] Binaries installed to ${destination}`)
+  }
+}
+
+module.exports = BinaryManager

server/utils/fileUtils.js

@@ -359,3 +359,22 @@ module.exports.encodeUriPath = (path) => {
   const uri = new URL(path, "file://")
   return uri.pathname
 }
+
+/**
+ * Check if directory is writable.
+ * This method is necessary because fs.access(directory, fs.constants.W_OK) does not work on Windows
+ * 
+ * @param {string} directory 
+ * @returns {boolean}
+ */
+module.exports.isWritable = async (directory) => {
+  try {
+    const accessTestFile = path.join(directory, 'accessTest')
+    await fs.writeFile(accessTestFile, '')
+    await fs.remove(accessTestFile)
+    return true
+  } catch (err) {
+    return false
+  }
+}
+

test/server/managers/BinaryManager.test.js

@@ -0,0 +1,264 @@
+const chai = require('chai')
+const sinon = require('sinon')
+const fs = require('../../../server/libs/fsExtra')
+const fileUtils = require('../../../server/utils/fileUtils')
+const which = require('../../../server/libs/which')
+const ffbinaries = require('../../../server/libs/ffbinaries')
+const path = require('path')
+const BinaryManager = require('../../../server/managers/BinaryManager')
+
+const expect = chai.expect
+
+describe('BinaryManager', () => {
+  let binaryManager
+
+  describe('init', () => {
+    let findStub
+    let installStub
+    let errorStub
+    let exitStub
+
+    beforeEach(() => {
+      binaryManager = new BinaryManager()
+      findStub = sinon.stub(binaryManager, 'findRequiredBinaries')
+      installStub = sinon.stub(binaryManager, 'install')
+      errorStub = sinon.stub(console, 'error')
+      exitStub = sinon.stub(process, 'exit')
+    })
+
+    afterEach(() => {
+      findStub.restore()
+      installStub.restore()
+      errorStub.restore()
+      exitStub.restore()
+    })
+
+    it('should not install binaries if they are already found', async () => {
+      findStub.resolves([])
+      
+      await binaryManager.init()
+
+      expect(installStub.called).to.be.false
+      expect(findStub.calledOnce).to.be.true
+      expect(errorStub.called).to.be.false
+      expect(exitStub.called).to.be.false
+    })
+
+    it('should install missing binaries', async () => {
+      const missingBinaries = ['ffmpeg', 'ffprobe']
+      const missingBinariesAfterInstall = []
+      findStub.onFirstCall().resolves(missingBinaries)
+      findStub.onSecondCall().resolves(missingBinariesAfterInstall)
+
+      await binaryManager.init()
+
+      expect(findStub.calledTwice).to.be.true
+      expect(installStub.calledOnce).to.be.true
+      expect(errorStub.called).to.be.false
+      expect(exitStub.called).to.be.false
+    })
+
+    it('exit if binaries are not found after installation', async () => {
+      const missingBinaries = ['ffmpeg', 'ffprobe']
+      const missingBinariesAfterInstall = ['ffmpeg', 'ffprobe']
+      findStub.onFirstCall().resolves(missingBinaries)
+      findStub.onSecondCall().resolves(missingBinariesAfterInstall)
+
+      await binaryManager.init()
+
+      expect(findStub.calledTwice).to.be.true
+      expect(installStub.calledOnce).to.be.true
+      expect(errorStub.calledOnce).to.be.true
+      expect(exitStub.calledOnce).to.be.true
+      expect(exitStub.calledWith(1)).to.be.true
+    })
+  })
+
+  
+  describe('findRequiredBinaries', () => {
+    let findBinaryStub
+
+    beforeEach(() => {
+      const requiredBinaries = [{ name: 'ffmpeg', envVariable: 'FFMPEG_PATH' }]
+      binaryManager = new BinaryManager(requiredBinaries)
+      findBinaryStub = sinon.stub(binaryManager, 'findBinary')
+    })
+
+    afterEach(() => {
+      findBinaryStub.restore()
+    })
+
+    it('should put found paths in the correct environment variables', async () => {
+      const pathToFFmpeg = '/path/to/ffmpeg'
+      const missingBinaries = []
+      delete process.env.FFMPEG_PATH
+      findBinaryStub.resolves(pathToFFmpeg)
+
+      const result = await binaryManager.findRequiredBinaries()
+
+      expect(result).to.deep.equal(missingBinaries)
+      expect(findBinaryStub.calledOnce).to.be.true
+      expect(process.env.FFMPEG_PATH).to.equal(pathToFFmpeg)
+    })
+
+    it('should add missing binaries to result', async () => {
+      const missingBinaries = ['ffmpeg']
+      delete process.env.FFMPEG_PATH    
+      findBinaryStub.resolves(null)
+
+      const result = await binaryManager.findRequiredBinaries()
+
+      expect(result).to.deep.equal(missingBinaries)
+      expect(findBinaryStub.calledOnce).to.be.true
+      expect(process.env.FFMPEG_PATH).to.be.undefined
+    })
+  })
+  
+  describe('install', () => {
+    let isWritableStub
+    let downloadBinariesStub
+
+    beforeEach(() => {
+      binaryManager = new BinaryManager()
+      isWritableStub = sinon.stub(fileUtils, 'isWritable')
+      downloadBinariesStub = sinon.stub(ffbinaries, 'downloadBinaries')
+      binaryManager.mainInstallPath = '/path/to/main/install'
+      binaryManager.altInstallPath = '/path/to/alt/install'
+    })
+
+    afterEach(() => {
+      isWritableStub.restore()
+      downloadBinariesStub.restore()
+    })
+
+    it('should not install binaries if no binaries are passed', async () => {
+      const binaries = []
+
+      await binaryManager.install(binaries)
+
+      expect(isWritableStub.called).to.be.false
+      expect(downloadBinariesStub.called).to.be.false
+    })
+
+    it('should install binaries in main install path if has access', async () => {
+      const binaries = ['ffmpeg']
+      const destination = binaryManager.mainInstallPath
+      isWritableStub.withArgs(destination).resolves(true)
+      downloadBinariesStub.resolves()
+      
+      await binaryManager.install(binaries)
+
+      expect(isWritableStub.calledOnce).to.be.true
+      expect(downloadBinariesStub.calledOnce).to.be.true
+      expect(downloadBinariesStub.calledWith(binaries, sinon.match({ destination: destination }))).to.be.true  
+    })
+
+    it('should install binaries in alt install path if has no access to main', async () => {
+      const binaries = ['ffmpeg']
+      const mainDestination = binaryManager.mainInstallPath
+      const destination = binaryManager.altInstallPath
+      isWritableStub.withArgs(mainDestination).resolves(false)
+      downloadBinariesStub.resolves()
+      
+      await binaryManager.install(binaries)
+
+      expect(isWritableStub.calledOnce).to.be.true
+      expect(downloadBinariesStub.calledOnce).to.be.true
+      expect(downloadBinariesStub.calledWith(binaries, sinon.match({ destination: destination }))).to.be.true  
+    })
+  })
+})
+
+describe('findBinary', () => {
+  let binaryManager
+  let fsPathExistsStub
+  let whichSyncStub
+  let mainInstallPath
+  let altInstallPath
+
+  const name = 'ffmpeg'
+  const envVariable = 'FFMPEG_PATH'
+  const defaultPath = '/path/to/ffmpeg'
+  const executable = name + (process.platform == 'win32' ? '.exe' : '')
+  const whichPath = '/usr/bin/ffmpeg'
+
+
+  beforeEach(() => {
+    binaryManager = new BinaryManager()
+    fsPathExistsStub = sinon.stub(fs, 'pathExists')
+    whichSyncStub = sinon.stub(which, 'sync')
+    binaryManager.mainInstallPath = '/path/to/main/install'
+    mainInstallPath = path.join(binaryManager.mainInstallPath, executable)
+    binaryManager.altInstallPath = '/path/to/alt/install'
+    altInstallPath = path.join(binaryManager.altInstallPath, executable)
+  })
+
+  afterEach(() => {
+    fsPathExistsStub.restore()
+    whichSyncStub.restore()
+  })
+
+  it('should return defaultPath if it exists', async () => {
+    process.env[envVariable] = defaultPath
+    fsPathExistsStub.withArgs(defaultPath).resolves(true)
+
+    const result = await binaryManager.findBinary(name, envVariable)
+
+    expect(result).to.equal(defaultPath)
+    expect(fsPathExistsStub.calledOnceWith(defaultPath)).to.be.true
+    expect(whichSyncStub.notCalled).to.be.true
+  })
+
+  it('should return whichPath if it exists', async () => {
+    delete process.env[envVariable]
+    whichSyncStub.returns(whichPath)
+
+    const result = await binaryManager.findBinary(name, envVariable)
+
+    expect(result).to.equal(whichPath)
+    expect(fsPathExistsStub.notCalled).to.be.true
+    expect(whichSyncStub.calledOnce).to.be.true
+  })
+
+  it('should return mainInstallPath if it exists', async () => {
+    delete process.env[envVariable]
+    whichSyncStub.returns(null)
+    fsPathExistsStub.withArgs(mainInstallPath).resolves(true)
+
+    const result = await binaryManager.findBinary(name, envVariable)
+
+    expect(result).to.equal(mainInstallPath)
+    expect(whichSyncStub.calledOnce).to.be.true
+    expect(fsPathExistsStub.calledOnceWith(mainInstallPath)).to.be.true
+  })
+
+  it('should return altInstallPath if it exists', async () => {
+    delete process.env[envVariable]
+    whichSyncStub.returns(null)
+    fsPathExistsStub.withArgs(mainInstallPath).resolves(false)
+    fsPathExistsStub.withArgs(altInstallPath).resolves(true)
+
+    const result = await binaryManager.findBinary(name, envVariable)
+
+    expect(result).to.equal(altInstallPath)
+    expect(whichSyncStub.calledOnce).to.be.true
+    expect(fsPathExistsStub.calledTwice).to.be.true
+    expect(fsPathExistsStub.calledWith(mainInstallPath)).to.be.true
+    expect(fsPathExistsStub.calledWith(altInstallPath)).to.be.true
+  })
+
+  it('should return null if binary is not found', async () => {
+    delete process.env[envVariable]
+    whichSyncStub.returns(null)
+    fsPathExistsStub.withArgs(mainInstallPath).resolves(false)
+    fsPathExistsStub.withArgs(altInstallPath).resolves(false)
+
+    const result = await binaryManager.findBinary(name, envVariable)
+
+    expect(result).to.be.null
+    expect(whichSyncStub.calledOnce).to.be.true
+    expect(fsPathExistsStub.calledTwice).to.be.true
+    expect(fsPathExistsStub.calledWith(mainInstallPath)).to.be.true
+    expect(fsPathExistsStub.calledWith(altInstallPath)).to.be.true
+  })
+})