如何从cython访问python共享内存?

我有两个过程,一个创建共享内存,第二个访问共享内存。我正在使用Python 3.8。

第二个进程可以选择仅使用python函数或通过cython访问共享内存。 cython选项失败-我得到了SIGSEGV。

是否有一些特殊的方法可以在cython中获取共享内存? docs似乎没有显示如何实际获取python共享内存。

进程1(setup_shm.py):

from multiprocessing import shared_memory
import numpy as np
import argparse
from package.get_shm import main as get_shm
from multiprocessing import Process


def main(cython=None):
    arr_share = np.array([[1.3424,23.24324],[1.4234,.08682]],dtype='f')
    shape,dtype = arr_share.shape,arr_share.dtype
    # Create a shared memory of size arr_share.nbytes
    shm = shared_memory.SharedMemory(create=True,size=arr_share.nbytes)
    # Create array using the buffer of shm
    shm_np_array = np.ndarray(shape=shape,dtype=dtype,buffer=shm.buf)
    # copy the data into the shared memory
    np.copyto(shm_np_array,arr_share)
    print('shared name {}'.format(shm.name))
    print('shared size {}'.format(arr_share.nbytes))
    print(arr_share)
    p = Process(target=get_shm,args=(shm.name,),kwargs={'cython': cython})
    p.start()
    p.join()
    shm.close()
    shm.unlink()


def setup():
    parser = argparse.ArgumentParser(description='Shared mem getter')
    parser.add_argument('--cython',action='store_true',help='Use cython to get shared mem')
    args = parser.parse_args()
    main(cython=args.cython)


if __name__ == "__main__":
    setup()

进程2(get_shm.py):

from multiprocessing.shared_memory import SharedMemory
import numpy as np
import argparse
import sys
from c_get_shm import Vectors

def main(name,cython=None):
    val = None
    if cython:
        print('getting vector using cython - {}'.format(name))
        vectors = Vectors(name)
        try:
            val = vectors.data.base[0]
        except Exception as e:
            raise (e)
    else:
        shm = SharedMemory(name=name)
        np_array = np.ndarray(shape=(2,2),dtype='f',buffer=shm.buf)
        print('getting vector using python - {}'.format(name))
        val = np_array[0]
        shm.close()
    print('val is {}'.format(val))
    sys.stdout.flush()
    return


def setup():
    parser = argparse.ArgumentParser(description='Shared mem getter')
    parser.add_argument('name',type=str,help='Shared memory name')
    parser.add_argument('--cython',help='Use cython to get shared mem')
    args = parser.parse_args()
    main(name=args.name,cython=args.cython)


if __name__ == "__main__":
    setup()

Cython类(c_get_shm.pyx)

#!python
#cython: language_level=3,boundscheck=False
from multiprocessing import shared_memory
cimport numpy as np
np.import_array()

cdef class Vectors:

    cdef public object shm_name
    cdef public float[:,:] data

    def __init__(self,shm_name):
        print('Vectors Class for getting shared memory - {}'.format(shm_name))
        self.__get_shared_array(shm_name)
        print('data is {},{}'.format(self.data,self.data.shape))

    def __get_shared_array(self,shm_name: str):
        print('get_shared_array {}'.format(shm_name))
        shm = shared_memory.SharedMemory(name=shm_name)
        print('shm buf is {} size {}'.format(shm.buf,shm.buf.nbytes))
        self.data = np.ndarray(shape=(2,buffer=shm.buf)

Setup.py

from setuptools import setup,Extension
from Cython.Build import cythonize

extensions = [
    Extension("c_get_shm",["./c_get_shm.pyx"],libraries=['rt'])
]
setup(
    ext_modules=cythonize(extensions,gdb_debug=True)
)

我正在使用任一脚本运行

  • python ./setup_shm.py --cython
  • python ./setup_shm.py

我设法看到SIGSEGV的唯一方法是使用strace:

Vectors Class for getting shared memory - psm_23cd04df
get_shared_array psm_23cd04df
shm buf is <memory at 0x7f8b4d66df40> size 16
data is <memoryview of 'ndarray' object>,[2,2,0]
[{WIFSIGNALED(s) && WTERMSIG(s) == SIGSEGV && WCOREDUMP(s)}],NULL) = 868
--- SIGCHLD {si_signo=SIGCHLD,si_code=CLD_DUMPED,si_pid=868,si_uid=1000,si_status=SIGSEGV,si_utime=0,si_stime=0} ---
close(5)                                = 0
munmap(0x7f8b65698000,16)              = 0
close(4)                                = 0
unlink("/dev/shm/psm_23cd04df")         = 0

我尝试了gdb python并从那里运行脚本,但是没有堆栈跟踪。

相关文章

功能概要:(目前已实现功能)公共展示部分:1.网站首页展示...
大体上把Python中的数据类型分为如下几类: Number(数字) ...
开发之前第一步,就是构造整个的项目结构。这就好比作一幅画...
源码编译方式安装Apache首先下载Apache源码压缩包,地址为ht...
前面说完了此项目的创建及数据模型设计的过程。如果未看过,...
python中常用的写爬虫的库有urllib2、requests,对于大多数比...