如何在 Linux 上为 netCDF4 安装 I/O 并行？

问题描述

我正在尝试为 netCDF4 安装 I/O 并行。我正在运行的脚本如下：

#!/bin/bash

module purge
module load gcc/5.3.1
module load mpich/3.2

# zlib
cd /storage/home/htn5098/
mkdir -p local_lib/R40
cd local_lib/R40  
wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4/zlib-1.2.8.tar.gz
tar -xf zlib-1.2.8.tar.gz && cd zlib-1.2.8
./configure --prefix=../zlib
make install
cd ../zlib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib
cd ..
rm zlib-1.2.8.tar.gz
rm zlib-1.2.8 -rf

# HDF5
wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4/hdf5-1.8.13.tar.gz
tar -xf hdf5-1.8.13.tar.gz && cd hdf5-1.8.13
prefix="/storage/work/htn5098/local_lib/R40/hdf5"
export HDF5_DIR=$prefix
./configure --enable-shared --enable-hl --enable-parallel --prefix=$HDF5_DIR CFLAGS=-fPIC # important for parallel processing 
#./configure --enable-shared --enable-hl --prefix=$HDF5_DIR
make -j4
make install
cd $HDF5_DIR
export CPPFLAGS=-I$HDF5_DIR/include 
export LDFLAGS=-L$HDF5_DIR/lib 
cd ..
rm hdf5-1.8.13.tar.gz
rm hdf5-1.8.13 -rf

# NETCDF4
wget http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-c-4.7.3.tar.gz
tar -xf netcdf-c-4.7.3.tar.gz && cd netcdf-c-4.7.3
prefix="/storage/work/htn5098/local_lib/R40/netcdf"
export NETCDF4_DIR=$prefix 
./configure --enable-netcdf-4 -with-gnu-ld --enable-logging --enable-shared --enable-dap --enable-parallel-tests --prefix=$NETCDF4_DIR 
make -j4
make install
cd $prefix
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib
export PATH=$PATH:$PWD/bin
export PKG_CONfig_PATH=$PKG_CONfig_PATH:/storage/work/htn5098/local_lib/R40/netcdf/lib/pkgconfig
cd ..
rm netcdf-c-4.7.3.tar.gz
rm netcdf-c-4.7.3 -rf

# R ncdf4 package
module load r
cd ~/work/local_lib/R40
wget http://cirrus.ucsd.edu/~pierce/ncdf/ncdf4_1.13.tar.gz
R CMD INSTALL ncdf4_1.13.tar.gz --library=.
rm ncdf4_1.13.tar.gz

但是，我不断收到错误：

checking hdf5.h presence... no
checking for hdf5.h... no
configure: error: Compiling a test with HDF5 Failed.  Either hdf5.h cannot be found,or config.log should be checked for other reason.

我认为可能是 HDF5 库路径未与 netCDF4 共享的情况。你能告诉我如何编译 HDF5 来为 netCDF4 启用并行吗？

非常感谢。

解决方法

这是我通常在我的 Ubuntu 服务器上使用的。它最近也可以在 OpenSuse 机器上运行，只需少量修改。

请注意，安装目录 /opt/software 有多次手动插入，作为 LDFLAGS 等中的参数。您还需要根据您的具体情况调整这些内容。

# export installation directorie
export INSDIR=/opt/software

1) SZIP v2.1.1 - https://support.hdfgroup.org/doc_resource/SZIP

./configure --prefix=${INSDIR} \
    CC=gcc \
    CFLAGS="-march=native -g -O2 -ansi -pedantic -Wall -W -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wcast-align -Wwrite-strings -Wconversion -Wsign-compare -Waggregate-return -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wnested-externs -Winline -O -fomit-frame-pointer -finline-functions"

make && sudo make install && make all check

-----------------------------------------------------------------------------------
2) HDF5 v1.12.0 - https://www.hdfgroup.org/downloads/hdf5/source-code

# parallel version
./configure --prefix=${INSDIR} --disable-dependency-tracking \
    --disable-maintainer-mode --disable-silent-rules \
    --build=x86_64-linux-gnu --host=x86_64-linux-gnu \
    --with-pthread --enable-hl --enable-shared \
    --enable-build-mode=production --with-zlib \
    --enable-parallel --with-szlib=${INSDIR} \
    --disable-threadsafe --disable-fortran --disable-cxx \
    CC=/usr/bin/mpicc CXX=/usr/bin/mpicxx FC=/usr/bin/mpifort \
    "CFLAGS=-march=native -g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
    "FCFLAGS=-march=native -g -O2 -fstack-protector-strong" \
    "CXXFLAGS=-march=native -g -O2 -fstack-protector-strong -Wformat -Werror=format-security"

make && sudo make install && sudo make check-install

-----------------------------------------------------------------------------------
3) NetCDF C v4.7.4 - https://www.unidata.ucar.edu/downloads/netcdf/index.jsp

# parallel version
./configure -prefix=${INSDIR} --enable-shared --enable-parallel-tests \
    CC=/usr/bin/mpicc CXX=/usr/bin/mpicxx \
    "CFLAGS=-march=native -g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64" \
    "CPPFLAGS=-I/opt/software/include" \
    "LDFLAGS=-L/opt/software/lib" \
    "LIBS=-lz -ldl -lm -lcurl -lpthread -L/opt/software/lib -lsz -lhdf5_hl -lhdf5"

make && sudo make install && sudo make check

-----------------------------------------------------------------------------------
4) NetCDF Fortran v4.5.3 - https://www.unidata.ucar.edu/downloads/netcdf/index.jsp

# parallel version
./configure --prefix=${INSDIR} --enable-parallel-tests \
    CC=/usr/bin/mpicc FC=/usr/bin/mpifort F77=/usr/bin/mpifort \
    "CFLAGS=-march=native -g -O2 -Wdate-time -D_FORTIFY_SOURCE=2" \
    "FCFLAGS=-march=native -g -O2 " \
    "CPPFLAGS=-I/opt/software/include" \
    "LDFLAGS=-L/opt/software/lib" \
    "LIBS=-L/opt/software/lib -lnetcdf -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,now"

make && sudo make install && sudo make check

另外，请注意，在这种情况下启用并行功能的关键步骤不是使用 make -j4，而是手动通知您要使用 MPI 编译器来构建库。这意味着您必须将 CC、CXX、FC 等标志明确定义为 MPI 版本（mpicc、mpicxx、mpifort 等）。

这应该可行，但请注意 make check 的输出以了解构建的库实际发生了什么。