Pyinstaller .exe cannot find _tiffile module - Loading of some compressed images will be very slow

Question

When I run my code from Pyinstaller the tiff reader works fine. After freezing using Pyinstaller I get the following warning:

UserWarning: ImportError: No module named '_tifffile'. Loading of some compressed images will be very slow. Tifffile.c can be obtained at http://www.lfd.uci.edu/~gohlke

And sure enough, a tiff file that used to take seconds to load into a numpy array may now take minutes.

Here is a simplified form of my code to focus on the problem. If you load an example tiff like this one it should load fast without problems.

If you use C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies tiffile_problems.py you should get a functional .exe with the above error message when you run it. When you try to load the same tiff it now takes much longer.

tiffile_problems.py

#!/usr/bin/env python3

import os
import sys
import traceback
import numpy as np
import matplotlib.pyplot as plt

from PyQt4.QtGui import *
from PyQt4.QtCore import *

sys.path.append('..')

from MBE_for_SO.util import fileloader, fileconverter

class NotConvertedError(Exception):
  pass

class FileAlreadyInProjectError(Exception):
  def __init__(self, filename):
    self.filename = filename

class Widget(QWidget):
  def __init__(self, project, parent=None):
    super(Widget, self).__init__(parent)

    if not project:
        self.setup_ui()
        return

  def setup_ui(self):
    vbox = QVBoxLayout()

    ## Related to importing Raws
    self.setWindowTitle('Import Raw File')

    #vbox.addWidget(QLabel('Set the size all data are to be rescaled to'))

    grid = QGridLayout()

    vbox.addLayout(grid)
    vbox.addStretch()

    self.setLayout(vbox)
    self.resize(400, 220)

    self.listview = QListView()
    self.listview.setStyleSheet('QListView::item { height: 26px; }')
    self.listview.setSelectionMode(QAbstractItemView.NoSelection)
    vbox.addWidget(self.listview)

    hbox = QVBoxLayout()
    pb = QPushButton('New Video')
    pb.clicked.connect(self.new_video)
    hbox.addWidget(pb)

    vbox.addLayout(hbox)
    vbox.addStretch()
    self.setLayout(vbox)


  def convert_tif(self, filename):
    path = os.path.splitext(os.path.basename(filename))[0] + '.npy'
    #path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting tif to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.tif2npy(filename, path, callback)
      print('Tifffile saved to wherever this script is')
    except:
      # qtutil.critical('Converting tiff to npy failed.')
      progress.close()
    return path

  def to_npy(self, filename):
    if filename.endswith('.raw'):
      print('No raws allowed')
      #filename = self.convert_raw(filename)
    elif filename.endswith('.tif'):
      filename = self.convert_tif(filename)
    else:
      raise fileloader.UnknownFileFormatError()
    return filename

  def import_file(self, filename):
    if not filename.endswith('.npy'):
      new_filename = self.to_npy(filename)
      if not new_filename:
        raise NotConvertedError()
      else:
        filename = new_filename

    return filename

  def import_files(self, filenames):
    for filename in filenames:
      try:
        filename = self.import_file(filename)
      except NotConvertedError:
        # qtutil.warning('Skipping file \'{}\' since not converted.'.format(filename))
        print('Skipping file \'{}\' since not converted.'.format(filename))
      except FileAlreadyInProjectError as e:
        # qtutil.warning('Skipping file \'{}\' since already in project.'.format(e.filename))
        print('Skipping file \'{}\' since already in project.'.format(e.filename))
      except:
        # qtutil.critical('Import of \'{}\' failed:\n'.format(filename) +\
        #   traceback.format_exc())
        print('Import of \'{}\' failed:\n'.format(filename) + traceback.format_exc())
      # else:
      #   self.listview.model().appendRow(QStandardItem(filename))

  def new_video(self):
    filenames = QFileDialog.getOpenFileNames(
      self, 'Load images', QSettings().value('last_load_data_path'),
      'Video files (*.npy *.tif *.raw)')
    if not filenames:
      return
    QSettings().setValue('last_load_data_path', os.path.dirname(filenames[0]))
    self.import_files(filenames)

class MyPlugin:
  def __init__(self, project):
    self.name = 'Import video files'
    self.widget = Widget(project)

  def run(self):
    pass

if __name__ == '__main__':
  app = QApplication(sys.argv)
  app.aboutToQuit.connect(app.deleteLater)
  w = QMainWindow()
  w.setCentralWidget(Widget(None))
  w.show()
  app.exec_()
  sys.exit()

fileconverter.py

#!/usr/bin/env python3

import os
import numpy as np

import tifffile as tiff

class ConvertError(Exception):
  pass

def tif2npy(filename_from, filename_to, progress_callback):
  with tiff.TiffFile(filename_from) as tif:
    w, h = tif[0].shape
    shape = len(tif), w, h
    np.save(filename_to, np.empty(shape, tif[0].dtype))
    fp = np.load(filename_to, mmap_mode='r+')
    for i, page in enumerate(tif):
      progress_callback(i / float(shape[0]-1))
      fp[i] = page.asarray()

def raw2npy(filename_from, filename_to, dtype, width, height,
  num_channels, channel, progress_callback):
    fp = np.memmap(filename_from, dtype, 'r')
    frame_size = width * height * num_channels
    if len(fp) % frame_size:
      raise ConvertError()
    num_frames = len(fp) / frame_size
    fp = np.memmap(filename_from, dtype, 'r',
      shape=(num_frames, width, height, num_channels))
    np.save(filename_to, np.empty((num_frames, width, height), dtype))
    fp_to = np.load(filename_to, mmap_mode='r+')
    for i, frame in enumerate(fp):
      progress_callback(i / float(len(fp)-1))
      fp_to[i] = frame[:,:,channel-1]

fileloader.py

#!/usr/bin/env python3

import numpy as np

class UnknownFileFormatError(Exception):
  pass

def load_npy(filename):
  frames = np.load(filename)
  # frames[np.isnan(frames)] = 0
  return frames

def load_file(filename):
  if filename.endswith('.npy'):
    frames = load_npy(filename)
  else:
    raise UnknownFileFormatError()
  return frames

def load_reference_frame_npy(filename, offset):
  frames_mmap = np.load(filename, mmap_mode='c')
  if frames_mmap is None:
    return None
  frame = np.array(frames_mmap[offset])
  frame[np.isnan(frame)] = 0
  frame = frame.swapaxes(0, 1)
  if frame.ndim == 2:
    frame = frame[:, ::-1]
  elif frame.ndim == 3:
    frame = frame[:, ::-1, :]
  return frame

def load_reference_frame(filename, offset=0):
  if filename.endswith('.npy'):
    frame = load_reference_frame_npy(filename, offset)
  else:
    raise UnknownFileFormatError()
  return frame

Why? And how do I fix this? I've located tifffile.py, tifffile.cpython-35.pyc, tifffile.c and placed them all in the same directory as the .exe. No effect. _tifffile.cp35-win_amd64.pyd is created by pyinstaller and placed in the same dir as the .exe. I don't know what other options are available to me.

tifffile_problems.spec

# -*- mode: python -*-

block_cipher = None


a = Analysis(['tiffile_problems.py'],
             pathex=['C:\\Users\\Cornelis\\PycharmProjects\\tester\\MBE_for_SO'],
             binaries=None,
             datas=None,
             hiddenimports=[],
             hookspath=[],
             runtime_hooks=[],
             excludes=[],
             win_no_prefer_redirects=False,
             win_private_assemblies=True,
             cipher=block_cipher)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)
exe = EXE(pyz,
          a.scripts,
          exclude_binaries=True,
          name='tiffile_problems',
          debug=False,
          strip=False,
          upx=True,
          console=True )
coll = COLLECT(exe,
               a.binaries,
               a.zipfiles,
               a.datas,
               strip=False,
               upx=True,
               name='tiffile_problems')

tiffile.spec when using C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies --onefile tiffile_problems.py

# -*- mode: python -*-

block_cipher = None


a = Analysis(['tiffile_problems.py'],
             pathex=['C:\\Users\\Cornelis\\PycharmProjects\\tester\\MBE_for_SO'],
             binaries=None,
             datas=None,
             hiddenimports=[],
             hookspath=['.'],
             runtime_hooks=[],
             excludes=[],
             win_no_prefer_redirects=False,
             win_private_assemblies=True,
             cipher=block_cipher)
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)
exe = EXE(pyz,
          a.scripts,
          a.binaries,
          a.zipfiles,
          a.datas,
          name='tiffile_problems',
          debug=False,
          strip=False,
          upx=True,
          console=True )

Answer 1

I think muggy is right about he weirdness with __package__ causing the issue here. I haven't tracked down the exact reason for the fix, but this seems to be resolved using the latest update to pyinstaller. Check your version with:

→ pyinstaller --version 3.2.1

and upgrade with

→ pip3 install --upgrade pyinstaller

The update was only made on January 15, 2017 so this wouldn't have helped when you originally asked, but it sure helps now.

Answer 2

I actually seen this via upwork whilst I was just browsing around the net and decided to have a play around.

kazemakase was on the right track at the very start, when you run normally, the __package__ is None, and when its packaged __package__ is set to tifffile and the first condition is executed, and it becomes relative to the module tifffile.

if __package__:
    from . import _tifffile
else:
    import _tifffile

I just converted tifffile to a module manually, by creating in site-packages a tifffile folder, creating an empty __init__.py file in the new folder, placing tifffile.py, _tifffile.pyd from site-packages into the tifffile folder and changing the import statement, admittedly in a simple skeleton.

import tifffile.tifffile as tiff

If this helps across your whole project I don't know. And should note I used the wheel from http://www.lfd.uci.edu/~gohlke/pythonlibs/ to install originally to save the compilation step so your mileage may vary. I did the above initially on 2.7 but it also appears to work fine on 3.5 from some testing I was able to do. And didn't need to put anything in .spec files when I tested from the one originally generated.