问题描述
我已将名为 ofunctions
的个人通用函数上传到 github,以便在我的项目之间共享它们,并进行单独的 CI 和覆盖测试。链接到 github 项目 here。
到目前为止一切顺利,我有一个名为 ofunctions
的包,它有几个子包,如 ofunctions.network
。
我希望能够安装子包而不必安装整个包,即pip install ofunctions.network
。
所以我创建了一个 setup.py
文件,用于创建必要的 dist 文件以上传到 PyPI。
我的问题:
每当我使用 python setup.py sdist bdist_wheel
时,它都会为每个子包生成完整的 ofunctions
包和一个包,但是:
- 像
ofunctions.network-0.5.0.tar.gz
这样的源包只包含子包(预期行为) - 像
ofunctions.network-0.5.0-py3-non-any.whl
这样的包含整个包的轮包(意外行为)
wheel 包包含整个 ofunctions
库,包括所有子包,这些子包显然应该只包含与源 dist 文件相同的子包。
谁能看看我的 setup.py
文件并告诉我为什么 sdist 和 wheel 文件不只包含严格相同的子包?
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of ofunctions package
"""
Namespace packaging here
# Make sure we declare an __init__.py file as namespace holder in the package root containing the following
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__,__name__)
"""
import codecs
import os
import pkg_resources
import setuptools
def get_Metadata(package_file):
"""
Read Metadata from pacakge file
"""
def _read(_package_file):
here = os.path.abspath(os.path.dirname(__file__))
with codecs.open(os.path.join(here,_package_file),'r') as fp:
return fp.read()
_Metadata = {}
for line in _read(package_file).splitlines():
if line.startswith('__version__'):
delim = '"' if '"' in line else "'"
_Metadata['version'] = line.split(delim)[1]
if line.startswith('__description__'):
delim = '"' if '"' in line else "'"
_Metadata['description'] = line.split(delim)[1]
return _Metadata
def parse_requirements(filename):
"""
There is a parse_requirements function in pip but it keeps changing import path
Let's build a simple one
"""
try:
with open(filename,'r') as requirements_txt:
install_requires = [
str(requirement)
for requirement
in pkg_resources.parse_requirements(requirements_txt)
]
return install_requires
except OSError:
print('WARNING: No requirements.txt file found as "{}". Please check path or create an empty one'
.format(filename))
def get_long_description(filename):
with open(filename,'r',encoding='utf-8') as readme_file:
_long_description = readme_file.read()
return _long_description
# ######### ACTUAL SCRIPT ENTRY POINT
NAMESPACE_PACKAGE_NAME = 'ofunctions'
namespace_package_path = os.path.abspath(NAMESPACE_PACKAGE_NAME)
namespace_package_file = os.path.join(namespace_package_path,'__init__.py')
Metadata = get_Metadata(namespace_package_file)
requirements = parse_requirements(os.path.join(namespace_package_path,'requirements.txt'))
# Generic namespace package
setuptools.setup(
name=NAMESPACE_PACKAGE_NAME,namespace_packages=[NAMESPACE_PACKAGE_NAME],packages=setuptools.find_namespace_packages(include=['ofunctions.*']),version=Metadata['version'],install_requires=requirements,classifiers=[
"Development Status :: 5 - Production/Stable","Intended Audience :: Developers","Topic :: Software Development","Topic :: System","Topic :: System :: Operating System","Topic :: System :: Shells","Programming Language :: Python","Programming Language :: Python :: 3","Programming Language :: Python :: Implementation :: cpython","Programming Language :: Python :: Implementation :: PyPy","Operating System :: POSIX :: Linux","Operating System :: POSIX :: BSD :: FreeBSD","Operating System :: POSIX :: BSD :: NetBSD","Operating System :: POSIX :: BSD :: OpenBSD","Operating System :: Microsoft","Operating System :: Microsoft :: Windows","License :: OSI Approved :: BSD License",],description=Metadata['description'],author='NetInvent - Orsiris de Jong',author_email='contact@netinvent.fr',url='https://github.com/netinvent/ofunctions',keywords=['network','bisection','logging'],long_description=get_long_description('README.md'),long_description_content_type="text/markdown",python_requires='>=3.5',# namespace packages don't work well with zipped eggs
# ref https://packaging.python.org/guides/packaging-namespace-packages/
zip_safe=False
)
for package in setuptools.find_namespace_packages(include=['ofunctions.*']):
package_path = os.path.abspath(package.replace('.',os.sep))
package_file = os.path.join(package_path,'__init__.py')
Metadata = get_Metadata(package_file)
requirements = parse_requirements(os.path.join(package_path,'requirements.txt'))
print(package_path)
print(package_file)
print(Metadata)
print(requirements)
setuptools.setup(
name=package,packages=[package],package_data={package: ['__init__.py']},classifiers=[
"Development Status :: 5 - Production/Stable",# namespace packages don't work well with zipped eggs
# ref https://packaging.python.org/guides/packaging-namespace-packages/
zip_safe=False
)
谢谢 8-|
解决方法
好的,我想我发现了问题。
在 setuptools 运行之间不会清除 build
目录。
更糟糕的是,除非您手动将其删除,否则构建目录从不被清除,因此旧的构建文件可能最终会出现在较新的轮包构建中,即使在我认为的单个包构建中也是如此。
我在运行 clear_package_build_path()
run 之前添加了一个函数 setuptools.setup()
,它只是清理 build/lib/package
目录。
现在我的轮文件只用必要的文件构建,不再膨胀。
例如这里是完整的工作代码:
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of ofunctions package
"""
Namespace packaging here
# Make sure we declare an __init__.py file as namespace holder in the package root containing the following
try:
__import__('pkg_resources').declare_namespace(__name__)
except ImportError:
from pkgutil import extend_path
__path__ = extend_path(__path__,__name__)
"""
import codecs
import os
import shutil
import pkg_resources
import setuptools
def get_metadata(package_file):
"""
Read metadata from package file
"""
def _read(_package_file):
here = os.path.abspath(os.path.dirname(__file__))
with codecs.open(os.path.join(here,_package_file),'r') as fp:
return fp.read()
_metadata = {}
for line in _read(package_file).splitlines():
if line.startswith('__version__'):
delim = '"' if '"' in line else "'"
_metadata['version'] = line.split(delim)[1]
if line.startswith('__description__'):
delim = '"' if '"' in line else "'"
_metadata['description'] = line.split(delim)[1]
return _metadata
def parse_requirements(filename):
"""
There is a parse_requirements function in pip but it keeps changing import path
Let's build a simple one
"""
try:
with open(filename,'r') as requirements_txt:
install_requires = [
str(requirement)
for requirement
in pkg_resources.parse_requirements(requirements_txt)
]
return install_requires
except OSError:
print('WARNING: No requirements.txt file found as "{}". Please check path or create an empty one'
.format(filename))
def get_long_description(filename):
with open(filename,'r',encoding='utf-8') as readme_file:
_long_description = readme_file.read()
return _long_description
def clear_package_build_path(package_rel_path):
"""
We need to clean build path,but setuptools will wait for build/lib/package_name so we need to create that
"""
build_path = os.path.abspath(os.path.join('build','lib',package_rel_path))
try:
# We need to use shutil.rmtree() instead of os.remove() since the latter implementation
# produces "WindowsError: [Error 5] Access is denied"
shutil.rmtree('build')
except FileNotFoundError:
print('build path: {} does not exist'.format(build_path))
# Now we need to create the 'build/lib/package/subpackage' path so setuptools won't fail
os.makedirs(build_path)
# ######### ACTUAL SCRIPT ENTRY POINT
NAMESPACE_PACKAGE_NAME = 'ofunctions'
namespace_package_path = os.path.abspath(NAMESPACE_PACKAGE_NAME)
namespace_package_file = os.path.join(namespace_package_path,'__init__.py')
metadata = get_metadata(namespace_package_file)
requirements = parse_requirements(os.path.join(namespace_package_path,'requirements.txt'))
# First lets make sure build path is clean (avoiding namespace package pollution in subpackages)
# Clean build dir before every run so we don't make cumulative wheel files
clear_package_build_path(NAMESPACE_PACKAGE_NAME)
# Generic namespace package
setuptools.setup(
name=NAMESPACE_PACKAGE_NAME,namespace_packages=[NAMESPACE_PACKAGE_NAME],packages=setuptools.find_namespace_packages(include=['ofunctions.*']),version=metadata['version'],install_requires=requirements,classifiers=[
"Development Status :: 5 - Production/Stable","Intended Audience :: Developers","Topic :: Software Development","Topic :: System","Topic :: System :: Operating System","Topic :: System :: Shells","Programming Language :: Python","Programming Language :: Python :: 3","Programming Language :: Python :: Implementation :: CPython","Programming Language :: Python :: Implementation :: PyPy","Operating System :: POSIX :: Linux","Operating System :: POSIX :: BSD :: FreeBSD","Operating System :: POSIX :: BSD :: NetBSD","Operating System :: POSIX :: BSD :: OpenBSD","Operating System :: Microsoft","Operating System :: Microsoft :: Windows","License :: OSI Approved :: BSD License",],description=metadata['description'],author='NetInvent - Orsiris de Jong',author_email='contact@netinvent.fr',url='https://github.com/netinvent/ofunctions',keywords=['network','bisection','logging'],long_description=get_long_description('README.md'),long_description_content_type="text/markdown",python_requires='>=3.5',# namespace packages don't work well with zipped eggs
# ref https://packaging.python.org/guides/packaging-namespace-packages/
zip_safe=False
)
for package in setuptools.find_namespace_packages(include=['ofunctions.*']):
rel_package_path = package.replace('.',os.sep)
package_path = os.path.abspath(rel_package_path)
package_file = os.path.join(package_path,'__init__.py')
metadata = get_metadata(package_file)
requirements = parse_requirements(os.path.join(package_path,'requirements.txt'))
print(package_path)
print(package_file)
print(metadata)
print(requirements)
# Again,we need to clean build paths between runs
clear_package_build_path(rel_package_path)
setuptools.setup(
name=package,packages=[package],package_data={package: ['__init__.py']},classifiers=[
"Development Status :: 5 - Production/Stable",# namespace packages don't work well with zipped eggs
# ref https://packaging.python.org/guides/packaging-namespace-packages/
zip_safe=False
)
作为一个侧节点,我注意到 os.remove()
会不时以 WindowsError: [Error 5] Access is denied
失败,因为 os.remove()
等待所有句柄关闭,这可能需要时间,因为垃圾收集器(据我所知)。在任何情况下都可以使用 shutil.rmtree()
。