More issues with numpy arrays

[nicolocin]

Still having errors with comparing arrays, now in copyfile_workflow. It's somewhat confusing that value can be the actual output or the path to output file.

Error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-2-418998bcfc4f> in <module>
     27 
     28 with pydra.Submitter(plugin="cf") as sub:
---> 29     sub(wf2)
     30 
     31 wf2.result(return_inputs=True)

~/Desktop/nlo/pydra/pydra/engine/submitter.py in __call__(self, runnable, cache_locations, rerun)
     57             self.loop.run_until_complete(self.submit_workflow(runnable, rerun=rerun))
     58         else:
---> 59             self.loop.run_until_complete(self.submit(runnable, wait=True, rerun=rerun))
     60         if is_workflow(runnable):
     61             # resetting all connections with LazyFields

~/anaconda3/envs/pydra/lib/python3.7/asyncio/base_events.py in run_until_complete(self, future)
    581             raise RuntimeError('Event loop stopped before Future completed.')
    582 
--> 583         return future.result()
    584 
    585     def stop(self):

~/Desktop/nlo/pydra/pydra/engine/submitter.py in submit(self, runnable, wait, rerun)
    122             # run coroutines concurrently and wait for execution
    123             # wait until all states complete or error
--> 124             await asyncio.gather(*futures)
    125             return
    126         # pass along futures to be awaited independently

~/Desktop/nlo/pydra/pydra/engine/submitter.py in submit_workflow(self, workflow, rerun)
     69             await self.worker.run_el(workflow, rerun=rerun)
     70         else:
---> 71             await workflow._run(self, rerun=rerun)
     72 
     73     async def submit(self, runnable, wait=False, rerun=False):

~/Desktop/nlo/pydra/pydra/engine/core.py in _run(self, submitter, rerun, **kwargs)
    907                 self.hooks.post_run_task(self, result)
    908                 self.audit.finalize_audit(result=result)
--> 909                 save(odir, result=result, task=self)
    910                 os.chdir(cwd)
    911         self.hooks.post_run(self, result)

~/Desktop/nlo/pydra/pydra/engine/helpers.py in save(task_path, result, task)
    115         if Path(task_path).name.startswith("Workflow"):
    116             # copy files to the workflow directory
--> 117             result = copyfile_workflow(wf_path=task_path, result=result)
    118         with (task_path / "_result.pklz").open("wb") as fp:
    119             cp.dump(result, fp)

~/Desktop/nlo/pydra/pydra/engine/helpers.py in copyfile_workflow(wf_path, result)
    128         value = getattr(result.output, field.name)
    129         new_value = _copyfile_single_value(wf_path=wf_path, value=value)
--> 130         if new_value != value:
    131             setattr(result.output, field.name, new_value)
    132     return result

ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()

To replicate:

import pydra
import typing as ty

@pydra.mark.task
@pydra.mark.annotate({"return": {'scores': ty.Any}})
def calc_metric(score):
    from sklearn import datasets
    from sklearn.model_selection import train_test_split
    from sklearn.svm import SVC
    from sklearn.model_selection import cross_val_score
    X, y = datasets.load_digits(n_class=10, return_X_y=True)
    X_tr, X_tt, y_tr, y_tt = train_test_split(X, y)
    
    clf = SVC()
    clf.fit(X_tr, y_tr)    
    cv_scores = cross_val_score(clf, X_tt, y_tt, scoring=f'{score}_macro')
    return(cv_scores)

# workflow
wf2 = pydra.Workflow(name="svm", 
                     input_spec=['score'], score=['precision', 'recall'])

wf2.split('score')
wf2.add(calc_metric(name='calc_metric', score=wf2.lzin.score))  
wf2.set_output([("scores", wf2.calc_metric.lzout.scores)])

with pydra.Submitter(plugin="cf") as sub:
    sub(wf2)
    
wf2.result(return_inputs=True)

[djarecka]

could you please provide a minimal example, that doesn't involve sklearn

[nicolocin]

Oh wait nevermind I got my conda environments messed up, my bad

[nicolocin]

My bad again, it's a bug in the current f042738 commit. I'll find a simpler example

[nicolocin]

Minimal example. Same error as above

wf = pydra.Workflow("wf", input_spec=["val"], val=[0, 1, 2])
wf.split("val")
wf.add(arrayout(name="mo", val=wf.lzin.val))

wf.set_output([("array", wf.mo.lzout.b)])

with pydra.Submitter(plugin=plugin, n_procs=2) as sub:
    sub(runnable=wf)

results = wf.result(return_inputs=True)

[djarecka]

@nicolocin - i believe this is fixed, if not please reopen

[daniel-ge]

Hi guys,

It seems that the issue with numpy arrays is still there. The following code fails:

import numpy as np

@pydra.mark.task
def repeat_to_array(val):
    return np.repeat(val,20)

wf = pydra.Workflow("wf", input_spec=["val"], val=2)
wf.add(repeat_to_array(name="repeat_to_array", val=wf.lzin.val))
wf.set_output([("array", wf.repeat_to_array.lzout.out)])

with pydra.Submitter(plugin="cf", n_procs=2) as sub:
    sub(runnable=wf)

# raises exception:
~/anaconda3/envs/resting_state_longi/lib/python3.8/site-packages/pydra/engine/helpers.py in copyfile_workflow(wf_path, result)
    157         value = getattr(result.output, field.name)
    158         new_value = _copyfile_single_value(wf_path=wf_path, value=value)
--> 159         if new_value != value:
    160             setattr(result.output, field.name, new_value)
    161     return result

ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()

An ugly solution would be to change lines 159 and 160 in helpers.py:

if isinstance(value, np.ndarray): 
     if not np.array_equal(value, new_value, equal_nan=True):
            setattr(result.output, field.name, new_value)
elif new_value != value:
    setattr(result.output, field.name, new_value)    

This introduces new dependencies (numpy in this case), contains code duplication, and will fail in future for other objects (like xarray).