Skip to content

Commit 8ab1eb6

Browse files
matthew-brettmatthew-brett
committed
Refactor to multibuild; add Linux wheel build
1 parent bf4d0b3 commit 8ab1eb6

8 files changed

Lines changed: 188 additions & 159 deletions

File tree

‎.gitmodules‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
[submodule "netcdf4-python"]
22
path = netcdf4-python
33
url = https://github.com/Unidata/netcdf4-python
4-
[submodule "terryfy"]
5-
path = terryfy
6-
url = git://github.com/MacPython/terryfy.git
4+
[submodule "multibuild"]
5+
path = multibuild
6+
url = https://github.com/matthew-brett/multibuild
7+
[submodule "h5py-wheels"]
8+
path = h5py-wheels
9+
url = https://github.com/MacPython/h5py-wheels

‎.travis.yml‎

Lines changed: 95 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1,102 +1,98 @@
1-
language:
2-
- objective-c
31
env:
4-
global:
5-
- BUILD_COMMIT='latest-tag' # comment out to build version in submodule
6-
- REPO_DIR=netcdf4-python
7-
- SZIP_VERSION=2.1
8-
- HDF5_VERSION=1.8.14
9-
- NETCDF_VERSION=4.3.3.1
10-
- NETCDF_CXX_VERSION=4.2.1
11-
- LIBCURL_VERSION=7.43.0
12-
matrix:
13-
- VERSION=2.7.10 NUMPY_VERSION=1.7.1
14-
- VERSION=3.3.5 NUMPY_VERSION=1.7.1
15-
- VERSION=3.4.3 NUMPY_VERSION=1.7.1
16-
- VERSION=3.5.0 NUMPY_VERSION=1.7.1
2+
global:
3+
- REPO_DIR=netcdf4-python
4+
- BUILD_COMMIT=v1.2.4rel
5+
- PLAT=x86_64
6+
- UNICODE_WIDTH=32
7+
- NP_BUILD_DEP=1.7.1
8+
- NP_TEST_DEP=1.7.1
9+
- MANYLINUX_URL=https://nipy.bic.berkeley.edu/manylinux
10+
- WHEELHOUSE_UPLOADER_USERNAME=travis-worker
11+
# Following generated with
12+
# travis encrypt -r MacPython/netcdf4-python-wheels WHEELHOUSE_UPLOADER_SECRET=<the api key>
13+
- secure:
14+
"qgMf5wjB1OU7OJT/5AN1+0yfHWlRGCpLn1BT1+RbLTMB3LLHz0qrnzDSuJ3GjlJlYWDyDkgSKCBUFWwy4BnazdRWVj8MTqgpMYlIFKT4XWYYpL82vZ+QCQHe91ExvqF+QAtwxtVhJK1g0bRe908y8IPDkbKqlj4maLm+oUak/DU="
15+
16+
language: python
17+
# Default Python version is usually 2.7
18+
python: 3.5
19+
sudo: required
20+
dist: trusty
21+
services: docker
22+
23+
matrix:
24+
exclude:
25+
# Exclude the default Python 3.5 build
26+
- python: 3.5
27+
include:
28+
- os: linux
29+
env:
30+
- MB_PYTHON_VERSION=2.7
31+
- os: linux
32+
env:
33+
- MB_PYTHON_VERSION=2.7
34+
- UNICODE_WIDTH=16
35+
- os: linux
36+
env:
37+
- MB_PYTHON_VERSION=2.7
38+
- PLAT=i686
39+
- os: linux
40+
env:
41+
- MB_PYTHON_VERSION=2.7
42+
- PLAT=i686
43+
- UNICODE_WIDTH=16
44+
- os: linux
45+
env:
46+
- MB_PYTHON_VERSION=3.4
47+
- os: linux
48+
env:
49+
- MB_PYTHON_VERSION=3.4
50+
- PLAT=i686
51+
- os: linux
52+
env:
53+
- MB_PYTHON_VERSION=3.5
54+
- NP_BUILD_DEP="1.9.3"
55+
- NP_TEST_DEP="1.9.3"
56+
- os: linux
57+
env:
58+
- MB_PYTHON_VERSION=3.5
59+
- PLAT=i686
60+
- NP_BUILD_DEP="1.9.3"
61+
- NP_TEST_DEP="1.9.3"
62+
- os: osx
63+
language: objective-c
64+
env:
65+
- MB_PYTHON_VERSION=2.7
66+
- os: osx
67+
language: objective-c
68+
env:
69+
- MB_PYTHON_VERSION=3.4
70+
- os: osx
71+
language: objective-c
72+
env:
73+
- MB_PYTHON_VERSION=3.5
74+
- NP_BUILD_DEP="1.9.3"
75+
- NP_TEST_DEP="1.9.3"
76+
77+
before_install:
78+
- BUILD_DEPENDS="numpy==$NP_BUILD_DEP Cython"
79+
- TEST_DEPENDS="numpy==$NP_TEST_DEP nose"
80+
- source multibuild/common_utils.sh
81+
- source multibuild/travis_steps.sh
82+
- before_install
83+
1784
install:
18-
- source terryfy/travis_tools.sh
19-
- get_python_environment macpython $VERSION venv
20-
# Use clang to avoid warnings on Python 3.5
21-
- export CC=clang
22-
- export CXX=clang++
23-
# Update to latest wheel version for Python 3.5
24-
- pip install -U wheel
25-
- pip install delocate numpy==$NUMPY_VERSION
26-
- pip install cython
27-
- source terryfy/library_installers.sh
28-
- clean_builds
29-
# We have to build separately for 32 and 64 bit because libcurl headers
30-
# set the pointer size, and so libcurl won't do dual-arch compile, and
31-
# netcdf can't compile and link against these headers in dual arch. So,
32-
# build the whole stack in 32 and 64 bit, fuse them after.
33-
# Build for 32-bit
34-
- set_32_prefix
35-
- standard_install szip $SZIP_VERSION .tar.gz szip- --enable-encoding=no
36-
- standard_install hdf5 $HDF5_VERSION .tar.bz2 hdf5- --with-szlib=$BUILD_PREFIX
37-
- standard_install curl $LIBCURL_VERSION .tar.gz curl- --with-darwinssl
38-
- standard_install netcdf-c $NETCDF_VERSION .tar.gz netcdf-
39-
# Clean sources and rebuild for 64-bit
40-
- set_64_prefix
41-
- rm -rf $SRC_PREFIX
42-
- mkdir $SRC_PREFIX
43-
- standard_install szip $SZIP_VERSION .tar.gz szip- --enable-encoding=no
44-
- standard_install hdf5 $HDF5_VERSION .tar.bz2 hdf5- --with-szlib=$BUILD_PREFIX
45-
- standard_install curl $LIBCURL_VERSION .tar.gz curl- --with-darwinssl
46-
- standard_install netcdf-c $NETCDF_VERSION .tar.gz netcdf-
47-
- set_dual_prefix
48-
# Reset library linking to point to local directory
49-
- python $TERRYFY_DIR/repath_lib_names.py
50-
$BUILD_PREFIX_32/lib
51-
$BUILD_PREFIX/lib
52-
$BUILD_PREFIX_32/lib/*
53-
- python $TERRYFY_DIR/repath_lib_names.py
54-
$BUILD_PREFIX_64/lib
55-
$BUILD_PREFIX/lib
56-
$BUILD_PREFIX_64/lib/*
57-
# Fuse 32 and 64-bit libraries
58-
- cp -a $BUILD_PREFIX_64/* $BUILD_PREFIX
59-
- python $TERRYFY_DIR/fuse_suff_real_libs.py
60-
$BUILD_PREFIX/lib
61-
$BUILD_PREFIX_32/lib
62-
$BUILD_PREFIX_64/lib
63-
# Point pkgconfig and nc-config at fused library location
64-
- perl -pi -e "s#$BUILD_PREFIX_64#$BUILD_PREFIX#g" $BUILD_PREFIX/lib/pkgconfig/*
65-
- perl -pi -e "s#$BUILD_PREFIX_64#$BUILD_PREFIX#g" $BUILD_PREFIX/bin/*
66-
# Delete single-arch builds for safety
67-
- rm -rf $BUILD_CONFIG_32 $BUILD_CONFIG_64
68-
# Checkout named commit and build wheel
69-
- if [ -n "$BUILD_COMMIT" ]; then
70-
checkout_commit $REPO_DIR $BUILD_COMMIT;
71-
fi
72-
- cd $REPO_DIR
73-
- python setup.py bdist_wheel
74-
- delocate-listdeps dist/*.whl # lists library dependencies
75-
- delocate-wheel dist/*.whl # copies library dependencies into wheel
76-
- delocate-addplat --rm-orig -x 10_9 -x 10_10 dist/*.whl
77-
- pip install dist/*.whl
78-
# Remove built libraries to make sure wheel is using own libraries
79-
- clean_builds
85+
# Maybe get and clean and patch source
86+
- clean_code $REPO_DIR $BUILD_COMMIT
87+
- build_wheel $REPO_DIR $PLAT
88+
8089
script:
81-
- cd test
82-
# Python 3.5 needs more recent numpy
83-
- if [ -n "$(pyver_ge $VERSION 3.5.0)" ]; then
84-
pip install --upgrade numpy;
85-
fi
86-
# Run the tests in default 64-bit mode
87-
- python run_all.py
88-
# Run tests for 32-bit Python.org Python
89-
- arch -i386 python run_all.py
90-
- cd ..
91-
before_deploy:
92-
- cd dist
93-
# Fix for https://github.com/travis-ci/travis-ci/issues/4635
94-
- rvm 1.9.3 do gem install net-ssh -v 2.9.2
95-
deploy:
96-
provider: cloudfiles
97-
username: travis-worker
98-
api_key:
99-
secure: "LCjZLORUfF9gZx5Bw0RYZacLV6wgdN+vpNggJmt36FyFwXNdYfc3qT98XT83D29rNTFW7wh3N8WsfWm4zWn8X/N7Ndl545Kq3zIJwnlaHdnd8a7LT6TKXEdrSTPnZQL5Uqxffj2DYYuMvXkTfNm47f3caxQdXVeLTC5YGtTWiBY="
100-
region: ORD
101-
container: wheels
102-
skip_cleanup: true
90+
- install_run $PLAT
91+
92+
after_success:
93+
# Upload wheels to Rackspace container
94+
- pip install wheelhouse-uploader
95+
- python -m wheelhouse_uploader upload --local-folder
96+
${TRAVIS_BUILD_DIR}/wheelhouse/
97+
--no-update-index
98+
wheels

‎README.rst‎

Lines changed: 67 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,35 +2,32 @@
22
Building and uploading netcdf4-python wheels
33
############################################
44

5-
*******
6-
For OSX
7-
*******
5+
We automate wheel building using this custom github repository that builds on
6+
the travis-ci OSX machines and the travis-ci Linux machines.
87

9-
We automate OSX wheel building using this custom github repository that builds
10-
on the travis-ci OSX machines.
11-
12-
The travis-ci interface for the builds is :
8+
The travis-ci interface for the builds is
139
https://travis-ci.org/MacPython/netcdf4-python-wheels
1410

15-
The driving github repository is :
11+
The driving github repository is
1612
https://github.com/MacPython/netcdf4-python-wheels
1713

1814
How it works
1915
============
2016

2117
The wheel-building repository:
2218

23-
* does a fresh build of the required C / C++ libraries;
19+
* does a fresh build of any required C / C++ libraries;
2420
* builds a netcdf4-python wheel, linking against these fresh builds;
25-
* processes the wheel using `delocate
26-
<https://pypi.python.org/pypi/delocate>`_. `delocate` copies the required
27-
dynamic libraries into the wheel and relinks the extension modules against
28-
the copied libraries;
29-
* uploads the built wheel to http://wheels.scipy.org (a Rackspace container
21+
* processes the wheel using delocate_ (OSX) or auditwheel_ ``repair``
22+
(Manylinux1_). ``delocate`` and ``auditwheel`` copy the required dynamic
23+
libraries into the wheel and relinks the extension modules against the
24+
copied libraries;
25+
* uploads the built wheels to http://wheels.scipy.org (a Rackspace container
3026
kindly donated by Rackspace to scikit-learn).
3127

32-
The resulting wheel is therefore self-contained and does not need any external
33-
dynamic libraries apart from those provided as standard by OSX.
28+
The resulting wheels are therefore self-contained and do not need any external
29+
dynamic libraries apart from those provided as standard by OSX / Linux as
30+
defined by the manylinux1 standard.
3431

3532
The ``.travis.yml`` file in this repository has a line containing the API key
3633
for the Rackspace container encrypted with an RSA key that is unique to the
@@ -41,13 +38,16 @@ directory pointed to by http://wheels.scipy.org.
4138
Triggering a build
4239
==================
4340

41+
You will likely want to edit the ``.travis.yml`` file to specify the
42+
``BUILD_COMMIT`` before triggering a build - see below.
43+
4444
You will need write permission to the github repository to trigger new builds
4545
on the travis-ci interface. Contact us on the mailing list if you need this.
4646

4747
You can trigger a build by:
4848

49-
* making a commit to the `netcdf4-python-wheels` repository (e.g. with `git
50-
commit --allow-empty`); or
49+
* making a commit to the ``netcdf4-python-wheels`` repository (e.g. with ``git
50+
commit --allow-empty``); or
5151
* clicking on the circular arrow icon towards the top right of the travis-ci
5252
page, to rerun the previous build.
5353

@@ -57,23 +57,11 @@ Keeping the old build logs helps us keep track of previous problems and
5757
successful builds.
5858

5959
Which netcdf4-python commit does the repository build?
60-
======================================================
61-
62-
By default, the `netcdf4-python-wheels` repository is usually set up to build
63-
the latest git tag. By "latest" we mean the tag on the branch most recently
64-
branched from master - see http://stackoverflow.com/a/24557377/1939576. To
65-
check whether you are building the latest tag have a look around line 5 of
66-
`.travis.yml` in the `netcdf4-python-wheels` repository. You should see something
67-
like::
68-
69-
- BUILD_COMMIT='latest-tag'
60+
============================================
7061

71-
If this is commented out, then the repository is set up to build the current
72-
commit in the `netcdf4-python` submodule of the repository. If it is set to
73-
another value then it will be specifying a commit to build.
74-
75-
You can therefore build any arbitrary commit by specifying the commit hash or
76-
branch name or tag name in this line of the `.travis.yml` file.
62+
The ``netcdf4-python-wheels`` repository will build the commit specified in the
63+
``BUILD_COMMIT`` at the top of the ``.travis.yml`` file. This can be any
64+
naming of a commit, including branch name, tag name or commit hash.
7765

7866
Uploading the built wheels to pypi
7967
==================================
@@ -82,42 +70,66 @@ Be careful, http://wheels.scipy.org points to a container on a distributed
8270
content delivery network. It can take up to 15 minutes for the new wheel file
8371
to get updated into the container at http://wheels.scipy.org.
8472

85-
When the wheels are updated, you can of course just download them to your
86-
machine manually, and then upload them manually to pypi, or by using
87-
twine_. You can also use a script for doing this, housed at :
73+
The same contents appear at
74+
https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com;
75+
you might prefer this address because it is https.
76+
77+
When the wheels are updated, you can download them to your machine manually,
78+
and then upload them manually to pypi, or by using twine_. You can also use a
79+
script for doing this, housed at :
8880
https://github.com/MacPython/terryfy/blob/master/wheel-uploader
8981

90-
You'll need twine and `beautiful soup 4 <bs4>`_.
82+
For the ``wheel-uploader`` script, you'll need twine and `beautiful soup 4
83+
<bs4>`_.
9184

9285
You will typically have a directory on your machine where you store wheels,
9386
called a `wheelhouse`. The typical call for `wheel-uploader` would then
9487
be something like::
9588

96-
wheel-uploader -v -s -w ~/wheelhouse netcdf4-python 1.0.3
89+
VERSION=1.2.4
90+
CDN_URL=https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com
91+
wheel-uploader -r warehouse -u $CDN_URL -s -v -w ~/wheelhouse -t macosx netCDF4 $VERSION
92+
wheel-uploader -r warehouse -u $CDN_URL -s -v -w ~/wheelhouse -t manylinux1 netCDF4 $VERSION
9793

9894
where:
9995

100-
* `-v` means give verbose messages;
101-
* `-s` asks to use your GPG key to sign the wheels before upload;
102-
* `-w ~/wheelhouse` means download the wheels from https://wheels.scipy.org to
103-
the directory `~/wheelhouse`;
104-
* `netcdf4-python` is the root name of the wheel(s) to download / upload;
105-
* `1.0.3` is the version to download / upload.
96+
* ``-r warehouse`` uses the upcoming Warehouse PyPI server (it is more
97+
reliable than the current PyPI service for uploads);
98+
* ``-u`` gives the URL from which to fetch the wheels, here the https address,
99+
for some extra security;
100+
* ``-s`` causes twine to sign the wheels with your GPG key;
101+
* ``-v`` means give verbose messages;
102+
* ``-w ~/wheelhouse`` means download the wheels from to the local directory
103+
``~/wheelhouse``.
104+
105+
``netCDF4`` is the root name of the wheel(s) to download / upload, and ``1.2.4``
106+
is the version to download / upload.
107+
108+
In order to use the Warehouse PyPI server, you will need something like this
109+
in your ``~/.pypirc`` file::
106110

107-
So, in this case, `wheel-uploader` will download all wheels starting with
108-
`netcdf4-python-1.0.3-` from http://wheels.scipy.org to `~/wheelhouse`, then upload
109-
them to pypi.
111+
[distutils]
112+
index-servers =
113+
pypi
114+
warehouse
110115

111-
Of course, you will need permissions to upload to pypi, for this to work.
116+
[pypi]
117+
username:your_user_name
118+
password:your_password
112119

113-
Maintaining the build repo
114-
==========================
120+
[warehouse]
121+
repository: https://upload.pypi.io/legacy/
122+
username: your_user_name
123+
password: your_password
115124

116-
* Check minimum numpy versions to build against in ``.travis.yml`` file. You
117-
need to build against the earliest numpy that netcdf4-python is compatible with; see
118-
`forward, backward numpy compatibility
119-
<http://stackoverflow.com/questions/17709641/valueerror-numpy-dtype-has-the-wrong-size-try-recompiling/18369312#18369312>`_
125+
So, in this case, ``wheel-uploader`` will download all wheels starting with
126+
``netCDF4-1.2.4-`` from http://wheels.scipy.org to ``~/wheelhouse``, then
127+
upload them to PyPI.
120128

129+
Of course, you will need permissions to upload to PyPI, for this to work.
121130

131+
.. _manylinux1: https://www.python.org/dev/peps/pep-0513
122132
.. _twine: https://pypi.python.org/pypi/twine
123133
.. _bs4: https://pypi.python.org/pypi/beautifulsoup4
134+
.. _delocate: https://pypi.python.org/pypi/delocate
135+
.. _auditwheel: https://pypi.python.org/pypi/auditwheel

0 commit comments

Comments
 (0)