Hi I am reviving this thread as I am having issues with headless running for my data.
I have big 2 color movies (~ 700Mb to 1.1Gb each) with 200 timepoints.
We have cellprofiler 3.1.9 installed from source on a Ubuntu 16.04 machine with 125Gb RAM
I managed to have the first part running in headless (see below) using several posts from this forum and batch instructions and it is running and saving segmentations in the 2 channels and some classes but not the measurements (CSV and database).
Any ideas?
Cheers
Debbi
here is the output from terminal after my command line
dolivierhub@ficoides:~$ cellprofiler -c -r -p /home/dolivierhub/data/output/Batch_data.h5 -f 0 -l 211 --plugins-directory /home/dolivierhub/CellProfiler-plugins -o /home/dolivierhub/data/output/batch1
Could not load runimagej
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/runimagej.py", line 11, in <module>
import imagej
File "/home/dolivierhub/.local/lib/python2.7/site-packages/imagej/__init__.py", line 1, in <module>
from .imagej import *
File "/home/dolivierhub/.local/lib/python2.7/site-packages/imagej/imagej.py", line 10, in <module>
import jnius_config
ImportError: No module named jnius_config
Could not load DoGNet
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/DoGNet.py", line 6, in <module>
import dognet
ImportError: No module named dognet
Could not load transformfilters
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 326, in add_module
cp_module = find_cpmodule(m)
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 313, in find_cpmodule
raise ValueError("Could not find cellprofiler.module.Module class in %s" % m.__file__)
ValueError: Could not find cellprofiler.module.Module class in /home/dolivierhub/CellProfiler-plugins/transformfilters.pyc
Could not load save_16bit_pngs
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/save_16bit_pngs.py", line 11, in <module>
import imageio
ImportError: No module named imageio
Could not load enhancedmeasuretexture
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/enhancedmeasuretexture.py", line 124, in <module>
import calculatemoments as cpmoments
File "/home/dolivierhub/CellProfiler-plugins/calculatemoments.py", line 38, in <module>
import cellprofiler.cpimage as cpi
ImportError: No module named cpimage
Could not load calculatemoments
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/calculatemoments.py", line 38, in <module>
import cellprofiler.cpimage as cpi
ImportError: No module named cpimage
Could not load rescale_mode_percentile
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/rescale_mode_percentile.py", line 11, in <module>
import statistics
ImportError: No module named statistics
Could not load identifylinearobjects
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/identifylinearobjects.py", line 14, in <module>
import cellprofiler.cpmodule as cpm
ImportError: No module named cpmodule
Could not load transform
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/transform.py", line 31, in <module>
import cellprofiler.cpimage as cpi
ImportError: No module named cpimage
/home/dolivierhub/.local/lib/python2.7/site-packages/requests/__init__.py:83: RequestsDependencyWarning: Old version of cryptography ([1, 2, 3]) may cause slowdown.
warnings.warn(warning, RequestsDependencyWarning)
Using TensorFlow backend.
Could not load calculatehistogram
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/calculatehistogram.py", line 18, in <module>
import cellprofiler.cpimage as cpi
ImportError: No module named cpimage
/home/dolivierhub/CellProfiler-plugins/cellstar/utils/debug_util.py:38: UserWarning:
This call to matplotlib.use() has no effect because the backend has already
been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
or matplotlib.backends is imported for the first time.
The backend was *originally* set to 'TkAgg' by the following code:
File "/home/dolivierhub/.local/bin/cellprofiler", line 11, in <module>
load_entry_point('CellProfiler', 'console_scripts', 'cellprofiler')()
File "/home/dolivierhub/CellProfiler/cellprofiler/__main__.py", line 171, in main
exit_code = run_pipeline_headless(options, args)
File "/home/dolivierhub/CellProfiler/cellprofiler/__main__.py", line 692, in run_pipeline_headless
pipeline.load(six.moves.StringIO(pipeline_text))
File "/home/dolivierhub/CellProfiler/cellprofiler/pipeline.py", line 796, in load
self.loadtxt(fd)
File "/home/dolivierhub/CellProfiler/cellprofiler/pipeline.py", line 1024, in loadtxt
module = self.instantiate_module(module_name)
File "/home/dolivierhub/CellProfiler/cellprofiler/pipeline.py", line 671, in instantiate_module
import cellprofiler.modules
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 385, in <module>
fill_modules()
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 371, in fill_modules
add_module(mod, False)
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/__init__.py", line 325, in add_module
m = __import__(mod, globals(), locals(), ['__all__'], 0)
File "/home/dolivierhub/CellProfiler-plugins/exporttocellh5.py", line 26, in <module>
import cellh5
File "/home/dolivierhub/.local/lib/python2.7/site-packages/cellh5/__init__.py", line 12, in <module>
from cellh5 import *
File "/home/dolivierhub/.local/lib/python2.7/site-packages/cellh5/cellh5.py", line 18, in <module>
import matplotlib.pyplot as plt
File "/home/dolivierhub/.local/lib/python2.7/site-packages/matplotlib/pyplot.py", line 71, in <module>
from matplotlib.backends import pylab_setup
File "/home/dolivierhub/.local/lib/python2.7/site-packages/matplotlib/backends/__init__.py", line 17, in <module>
line for line in traceback.format_stack()
matplotlib.use('Agg')
could not load these modules: runimagej,DoGNet,transformfilters,save_16bit_pngs,enhancedmeasuretexture,calculatemoments,rescale_mode_percentile,identifylinearobjects,transform,calculatehistogram
/home/dolivierhub/CellProfiler/cellprofiler/utilities/hdf5_dict.py:539: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.
np.issubdtype(hdf5_type, int) or
/home/dolivierhub/CellProfiler/cellprofiler/utilities/hdf5_dict.py:541: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
hdf5_type_is_float = np.issubdtype(hdf5_type, float)
Times reported are CPU and Wall-clock times for each module
Thu Apr 9 16:03:43 2020: Image # 1, module Images # 1: CPU_time = 0.00 secs, Wall_time = 0.00 secs
Thu Apr 9 16:03:43 2020: Image # 1, module Metadata # 2: CPU_time = 0.00 secs, Wall_time = 0.00 secs
Thu Apr 9 16:03:43 2020: Image # 1, module NamesAndTypes # 3: CPU_time = 4.40 secs, Wall_time = 2.88 secs
Thu Apr 9 16:03:46 2020: Image # 1, module Groups # 4: CPU_time = 0.00 secs, Wall_time = 0.00 secs
Thu Apr 9 16:03:46 2020: Image # 1, module Smooth # 5: CPU_time = 0.16 secs, Wall_time = 0.11 secs
Thu Apr 9 16:03:47 2020: Image # 1, module Smooth # 6: CPU_time = 0.09 secs, Wall_time = 0.10 secs
Thu Apr 9 16:03:47 2020: Image # 1, module Smooth # 7: CPU_time = 0.10 secs, Wall_time = 0.10 secs
/home/dolivierhub/.local/lib/python2.7/site-packages/skimage/transform/_warps.py:110: UserWarning: Anti-aliasing will be enabled by default in skimage 0.15 to avoid aliasing artifacts when down-sampling images.
warn("Anti-aliasing will be enabled by default in skimage 0.15 to "
/home/dolivierhub/.local/lib/python2.7/site-packages/skimage/util/dtype.py:137: UserWarning: Possible sign loss when converting negative image of type float64 to positive image of type bool.
.format(dtypeobj_in, dtypeobj_out))
/home/dolivierhub/.local/lib/python2.7/site-packages/skimage/util/dtype.py:141: UserWarning: Possible precision loss when converting from float64 to bool
.format(dtypeobj_in, dtypeobj_out))
Thu Apr 9 16:03:47 2020: Image # 1, module Resize # 8: CPU_time = 0.07 secs, Wall_time = 0.01 secs
/home/dolivierhub/.local/lib/python2.7/site-packages/skimage/transform/_warps.py:105: UserWarning: The default mode, 'constant', will be changed to 'reflect' in skimage 0.15.
warn("The default mode, 'constant', will be changed to 'reflect' in "
Thu Apr 9 16:03:47 2020: Image # 1, module ClassifyPixels-Unet # 9: CPU_time = 16.85 secs, Wall_time = 3.13 secs
Thu Apr 9 16:03:50 2020: Image # 1, module Resize # 10: CPU_time = 0.42 secs, Wall_time = 0.14 secs
Thu Apr 9 16:03:50 2020: Image # 1, module ColorToGray # 11: CPU_time = 0.00 secs, Wall_time = 0.00 secs
Thu Apr 9 16:03:50 2020: Image # 1, module IdentifyPrimaryObjects # 12: CPU_time = 0.74 secs, Wall_time = 0.63 secs
Thu Apr 9 16:03:51 2020: Image # 1, module IdentifySecondaryObjects # 13: CPU_time = 0.72 secs, Wall_time = 0.72 secs
Thu Apr 9 16:03:51 2020: Image # 1, module MeasureObjectIntensity # 14: CPU_time = 0.20 secs, Wall_time = 0.19 secs
Thu Apr 9 16:03:52 2020: Image # 1, module FilterObjects # 15: CPU_time = 0.16 secs, Wall_time = 0.16 secs
Thu Apr 9 16:03:52 2020: Image # 1, module TrackObjects # 17: CPU_time = 0.84 secs, Wall_time = 0.83 secs
Thu Apr 9 16:03:53 2020: Image # 1, module EnhanceOrSuppressFeatures # 18: CPU_time = 0.40 secs, Wall_time = 0.40 secs
Thu Apr 9 16:03:53 2020: Image # 1, module MaskImage # 19: CPU_time = 0.01 secs, Wall_time = 0.01 secs
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:4209: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
big_labels[[slice(fe,-fe) for fe in footprint_extent]] = labels
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:416: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
index_i, index_j, image = prepare_for_index_lookup(image, False)
Thu Apr 9 16:03:53 2020: Image # 1, module IdentifyPrimaryObjects # 20: CPU_time = 0.77 secs, Wall_time = 0.77 secs
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2535: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
I00[slice_00] = labels
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2536: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
I01[slice_01] = labels
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2537: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
I10[slice_10] = labels
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2538: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
I11[slice_11] = labels
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2560: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
Q1_condition[slice_00] += (NE01_00 & NE01_10 & NE01_11)[slice_01]
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2563: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
Q1_condition[slice_00] += (NE10_00 & NE10_01 & NE10_11)[slice_10]
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2566: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
Q1_condition[slice_00] += (NE11_00 & NE11_01 & NE11_10)[slice_11]
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2574: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
Q3_condition[slice_00] += (NE11_00 & EQ11_10 & EQ11_01)[slice_11]
/home/dolivierhub/.local/lib/python2.7/site-packages/centrosome/cpmorphology.py:2587: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
QD_condition[slice_00] += (NE01_00 & NE01_11 & EQ01_10)[slice_01]
Thu Apr 9 16:03:54 2020: Image # 1, module MeasureObjectSizeShape # 21: CPU_time = 2.30 secs, Wall_time = 2.30 secs
Thu Apr 9 16:03:56 2020: Image # 1, module MeasureGranularity # 22: CPU_time = 2.30 secs, Wall_time = 2.29 secs
Thu Apr 9 16:03:58 2020: Image # 1, module MeasureObjectIntensityDistribution # 23: CPU_time = 1.16 secs, Wall_time = 1.15 secs
Thu Apr 9 16:04:00 2020: Image # 1, module MeasureObjectIntensity # 24: CPU_time = 0.65 secs, Wall_time = 0.65 secs
Thu Apr 9 16:04:00 2020: Image # 1, module FilterObjects # 25: CPU_time = 0.21 secs, Wall_time = 0.21 secs
Thu Apr 9 16:04:00 2020: Image # 1, module FilterObjects # 26: CPU_time = 0.19 secs, Wall_time = 0.19 secs
Thu Apr 9 16:04:01 2020: Image # 1, module FilterObjects # 27: CPU_time = 0.19 secs, Wall_time = 0.19 secs
Thu Apr 9 16:04:01 2020: Image # 1, module FilterObjects # 28: CPU_time = 0.19 secs, Wall_time = 0.19 secs
Thu Apr 9 16:04:01 2020: Image # 1, module FilterObjects # 29: CPU_time = 0.19 secs, Wall_time = 0.19 secs
Thu Apr 9 16:04:01 2020: Image # 1, module FilterObjects # 30: CPU_time = 0.18 secs, Wall_time = 0.18 secs
Thu Apr 9 16:04:01 2020: Image # 1, module RescaleIntensity # 31: CPU_time = 0.20 secs, Wall_time = 0.20 secs
Thu Apr 9 16:04:02 2020: Image # 1, module RescaleIntensity # 32: CPU_time = 0.22 secs, Wall_time = 0.22 secs
/home/dolivierhub/CellProfiler/cellprofiler/modules/relateobjects.py:599: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.
perim_idx = pperim[perim_loc.transpose().tolist()]
Thu Apr 9 16:04:02 2020: Image # 1, module RelateObjects # 33: CPU_time = 0.45 secs, Wall_time = 0.45 secs
Thu Apr 9 16:04:02 2020: Image # 1, module OverlayOutlines # 34: CPU_time = 0.11 secs, Wall_time = 0.11 secs
/home/dolivierhub/.local/lib/python2.7/site-packages/skimage/util/dtype.py:141: UserWarning: Possible precision loss when converting from float32 to uint8
.format(dtypeobj_in, dtypeobj_out))
Thu Apr 9 16:04:02 2020: Image # 1, module SaveImages # 35: CPU_time = 0.30 secs, Wall_time = 0.30 secs
Thu Apr 9 16:04:03 2020: Image # 1, module OverlayOutlines # 36: CPU_time = 0.21 secs, Wall_time = 0.21 secs
/home/dolivierhub/.local/lib/python2.7/site-packages/skimage/io/_io.py:140: UserWarning: /home/dolivierhub/data/output/batch1/RESULTS/siCTRL/20191010/Pos01/NucleiClasses/20191010_Pos01_cl7-class-0.png is a low contrast image
warn('%s is a low contrast image' % fname)
Thu Apr 9 16:04:03 2020: Image # 1, module SaveImages # 37: CPU_time = 0.30 secs, Wall_time = 0.30 secs
Thu Apr 9 16:04:03 2020: Image # 1, module SaveImages # 38: CPU_time = 0.14 secs, Wall_time = 0.14 secs
Thu Apr 9 16:04:03 2020: Image # 1, module OverlayOutlines # 43: CPU_time = 0.09 secs, Wall_time = 0.09 secs
/home/dolivierhub/.local/lib/python2.7/site-packages/skimage/io/_io.py:140: UserWarning: /home/dolivierhub/data/output/batch1/RESULTS/siCTRL/20191010/Pos01/outline/20191010_Pos01_cl7-0.png is a low contrast image
warn('%s is a low contrast image' % fname)
Thu Apr 9 16:04:03 2020: Image # 1, module SaveImages # 44: CPU_time = 0.15 secs, Wall_time = 0.14 secs
Thu Apr 9 16:04:04 2020: Image # 1, module ExportToSpreadsheet # 45: CPU_time = 0.00 secs, Wall_time = 0.00 secs
Failed to write measurements to database
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 3545, in write_data_to_db
object_name2)
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 2927, in get_relationship_type_id
self.handle_interaction_get_relationship_types()
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 2212, in handle_interaction_get_relationship_types
return self.get_relationship_types(cursor).items()
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 2238, in get_relationship_types
execute(cursor, statement)])
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 281, in execute
cursor.execute(query)
OperationalError: no such table: Batch_tablePer_RelationshipTypes
Error detected during run of module ExportToDatabase
Traceback (most recent call last):
File "/home/dolivierhub/CellProfiler/cellprofiler/pipeline.py", line 1782, in run_with_yield
self.run_module(module, workspace)
File "/home/dolivierhub/CellProfiler/cellprofiler/pipeline.py", line 2034, in run_module
module.run(workspace)
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 2159, in run
self.write_data_to_db(workspace)
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 3545, in write_data_to_db
object_name2)
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 2927, in get_relationship_type_id
self.handle_interaction_get_relationship_types()
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 2212, in handle_interaction_get_relationship_types
return self.get_relationship_types(cursor).items()
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 2238, in get_relationship_types
execute(cursor, statement)])
File "/home/dolivierhub/CellProfiler/cellprofiler/modules/exporttodatabase.py", line 281, in execute
cursor.execute(query)
OperationalError: no such table: Batch_tablePer_RelationshipTypes
Thu Apr 9 16:04:04 2020: Image # 1, module ExportToDatabase # 46: CPU_time = 0.38 secs, Wall_time = 0.38 secs