使用 python 导入代码的 julia 中的分布式微积分:UndefVarError: __anon__ not defined

问题描述

我有这个代码文件名是 test.jl),它是一个更复杂代码的简化版本:

using distributed,SharedArrays
using Linearalgebra
using PyCall
@everywhere @pyimport scipy.optimize as so

function fun()                              # coupling constants

    
    Ntraj = 2
    Ntime = 10
    
    result = @distributed (+) for ktraj = 1 : Ntraj
    println("Step_1")

        
    # One cycle one trajectory
    for jt=1:Ntime
            println(jt)
            fidelity = x -> x[1]*x[2]*x[3]-x[1]-x[2]-x[3]
            x0 = [0 0 0]
            println(fidelity(x0))
            println(so.minimize(fidelity,x0))                                

    end
    end

    return 0
end

我通过以下方式从笔记本中调用

using distributed
using PyCall
@pyimport scipy.optimize as so

addprocs(2)

@everywhere include("test.jl")
@time fun()

除了警告

┌ Warning: `@pyimport foo` is deprecated in favor of `foo = pyimport("foo")`.
│   caller = _pywrap_pyimport(::PyObject) at PyCall.jl:410
└ @ PyCall ~/.julia/packages/PyCall/tqyST/src/PyCall.jl:410
┌ Warning: `@pyimport foo` is deprecated in favor of `foo = pyimport("foo")`.
│   caller = _pywrap_pyimport(::PyObject) at PyCall.jl:410
└ @ PyCall ~/.julia/packages/PyCall/tqyST/src/PyCall.jl:410
┌ Warning: `@pyimport foo` is deprecated in favor of `foo = pyimport("foo")`.
│   caller = _pywrap_pyimport(::PyObject) at PyCall.jl:410
└ @ PyCall ~/.julia/packages/PyCall/tqyST/src/PyCall.jl:410
┌ Warning: `@pyimport foo` is deprecated in favor of `foo = pyimport("foo")`.
│   caller = _pywrap_pyimport(::PyObject) at PyCall.jl:410
└ @ PyCall ~/.julia/packages/PyCall/tqyST/src/PyCall.jl:410
┌ Warning: `@pyimport foo` is deprecated in favor of `foo = pyimport("foo")`.
│   caller = _pywrap_pyimport(::PyObject) at PyCall.jl:410
└ @ PyCall ~/.julia/packages/PyCall/tqyST/src/PyCall.jl:410


┌ Warning: `@pyimport foo` is deprecated in favor of `foo = pyimport("foo")`.
│   caller = _pywrap_pyimport(::PyObject) at PyCall.jl:410
└ @ PyCall ~/.julia/packages/PyCall/tqyST/src/PyCall.jl:410

我收到此错误

TaskFailedException:
On worker 2:
undefvarerror: __anon__ not defined
deserialize_module at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:915
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:812
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:735
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:820
deserialize_fillarray! at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:1112
deserialize_array at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:1104
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:786
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:735 [inlined]
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:1010
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:878
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:735 [inlined]
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:947
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:878
deserialize_fillarray! at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:1112
deserialize_array at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:1104
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:786
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:735 [inlined]
deserialize_typename at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:1177
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/clusterserialize.jl:68
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:878
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:735
deserialize_datatype at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:0
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:790
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:735
handle_deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:795
deserialize at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/Serialization/src/Serialization.jl:735 [inlined]
deserialize_msg at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/messages.jl:99
#invokelatest#1 at ./essentials.jl:712 [inlined]
invokelatest at ./essentials.jl:711 [inlined]
message_handler_loop at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/process_messages.jl:185
process_tcp_streams at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/process_messages.jl:142
#97 at ./task.jl:358
Stacktrace:
 [1] remotecall_fetch(::Function,::distributed.Worker,::Function,::Vararg{Any,N} where N; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/remotecall.jl:390
 [2] remotecall_fetch(::Function,N} where N) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/remotecall.jl:382
 [3] remotecall_fetch(::Function,::Int64,Tuple{}}}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/remotecall.jl:417
 [4] remotecall_fetch at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/remotecall.jl:417 [inlined]
 [5] (::distributed.var"#155#156"{typeof(+),var"#9#12"{Int64},UnitRange{Int64},Array{UnitRange{Int64},1},Int64,Int64})() at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/macros.jl:270

Stacktrace:
 [1] wait at ./task.jl:267 [inlined]
 [2] fetch at ./task.jl:282 [inlined]
 [3] iterate at ./generator.jl:47 [inlined]
 [4] collect(::Base.Generator{Array{Task,typeof(fetch)}) at ./array.jl:665
 [5] preduce(::Function,::UnitRange{Int64}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.4/distributed/src/macros.jl:274
 [6] fun at /home/candeloro/Feedback_qw/Feedback_Adjacency_2Measurements_N5_5HF/test.jl:12 [inlined]
 [7] macro expansion at ./util.jl:175 [inlined]
 [8] top-level scope at ./In[6]:8

我真的不知道如何修复它

解决方法

这是清理后有效的代码。 基本上,主要问题是 @distributed 宏如何尝试在集群周围移动 Python 模块(它似乎不知道它是一个库)。所以我把它打包成一个函数,这个函数总是在每个给定的工作进程本地调用(没有复制的风险)。


using Distributed,LinearAlgebra,PyCall
addprocs(2)
@everywhere using Distributed,PyCall

@everywhere const myso = pyimport("scipy.optimize")
@everywhere getso() = myso

function fun()
    Ntraj = 2
    Ntime = 10
    result = @distributed (+) for ktraj = 1 : Ntraj
        for jt=1:Ntime
            println(jt)
            fidelity = x -> x[1]*x[2]*x[3]-x[1]-x[2]-x[3]
            x0 = [0 0 0]
            println(fidelity(x0))
            println(getso().minimize(fidelity,x0))                                
        end
        1 # you need to add something or use @sync @distributed for ktraj = 1 : Ntraj
    end
end

fun()