diff --git a/.travis.yml b/.travis.yml index 32ae33e..dc4a492 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,38 +34,36 @@ matrix: - stage: test os: osx env: SCRIPT=osx - - if: branch = develop + - if: branch IN (master, develop) stage: deploy os: linux + env: SCRIPT=manylinux2010 BINARY_PACKAGE=yes language: python python: 3.5 - script: - - export PRIMITIV_PYTHON_BUILD_NUMBER="dev${TRAVIS_BUILD_NUMBER}"; - - pip install cython numpy scikit-build twine - - $TRAVIS_BUILD_DIR/setup.py sdist --bundle-core-library - deploy: - skip_cleanup: true - provider: script - script: twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD $TRAVIS_BUILD_DIR/dist/*.tar.gz - on: - tags: false - branch: develop - - if: branch = master + - if: branch IN (master, develop) stage: deploy os: linux + env: SCRIPT=manylinux2010 BINARY_PACKAGE=yes language: python - python: 3.5 - script: - - export PRIMITIV_PYTHON_BUILD_NUMBER="${TRAVIS_BUILD_NUMBER}"; - - pip install cython numpy scikit-build twine - - $TRAVIS_BUILD_DIR/setup.py sdist --bundle-core-library - deploy: - skip_cleanup: true - provider: script - script: twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD $TRAVIS_BUILD_DIR/dist/*.tar.gz - on: - tags: false - branch: master + python: 3.6 + - if: branch IN (master, develop) + stage: deploy + os: linux + env: SCRIPT=sdist + language: python + python: 3.6 script: - $TRAVIS_BUILD_DIR/.travis/${SCRIPT}.sh + +deploy: + skip_cleanup: true + provider: script + script: + - $TRAVIS_BUILD_DIR/.travis/deploy.sh + on: + tags: false + condition: $TRAVIS_BUILD_STAGE_NAME = Deploy + branch: + - develop + - master diff --git a/.travis/debian.sh b/.travis/debian.sh index b9a2cad..4e2a292 100755 --- a/.travis/debian.sh +++ b/.travis/debian.sh @@ -7,11 +7,31 @@ docker run --name travis-ci -v $TRAVIS_BUILD_DIR:/primitiv-python -td debian:sta # install docker exec travis-ci bash -c "apt update" -docker exec travis-ci bash -c "apt install -y git build-essential cmake python3-dev python3-pip python3-numpy" -docker exec travis-ci bash -c "pip3 install cython scikit-build" +docker exec travis-ci bash -c "apt install -y build-essential cmake python3-dev python3-pip" +docker exec travis-ci bash -c "pip3 install -U pip setuptools" +docker exec travis-ci bash -c "pip3 install cython scikit-build numpy" + +# TODO(vbkaisetsu): +# Debian stretch contains Eigen 3.3.2. It has a bug around EIGEN_MPL2_ONLY +# mode and SparseCholesky module. It is fixed in newer version. +# +# For more details, see: http://eigen.tuxfamily.org/bz/show_bug.cgi?id=1392 + +# install Eigen +docker exec travis-ci bash -c "apt install -y wget" +docker exec travis-ci bash -c "wget http://bitbucket.org/eigen/eigen/get/3.3.4.tar.bz2 -O ./eigen.tar.bz2" +docker exec travis-ci bash -c "mkdir ./eigen" +docker exec travis-ci bash -c "tar xf ./eigen.tar.bz2 -C ./eigen --strip-components 1" +docker exec travis-ci bash -c "mkdir ./eigen/build" +docker exec travis-ci bash -c "cd ./eigen/build && cmake .." +docker exec travis-ci bash -c "cd ./eigen/build && make && make install" # install OpenCL environment -docker exec travis-ci bash -c "apt install -y opencl-headers libclblas-dev pkg-config libhwloc-dev libltdl-dev ocl-icd-dev ocl-icd-opencl-dev clang-3.8 llvm-3.8-dev libclang-3.8-dev libz-dev" +docker exec travis-ci bash -c "apt install -y opencl-headers git pkg-config libhwloc-dev libltdl-dev ocl-icd-dev ocl-icd-opencl-dev clang-4.0 llvm-4.0-dev libclang-4.0-dev libz-dev" +docker exec travis-ci bash -c "wget https://github.com/CNugteren/CLBlast/archive/1.2.0.tar.gz -O ./clblast.tar.gz" +docker exec travis-ci bash -c "mkdir ./clblast" +docker exec travis-ci bash -c "tar xf ./clblast.tar.gz -C ./clblast --strip-components 1" +docker exec travis-ci bash -c "cd ./clblast && cmake . && make && make install" # pocl 0.13 does not contain mem_fence() function that is used by primitiv. # We build the latest pocl instead of using distribution's package. # See: https://github.com/pocl/pocl/issues/294 @@ -21,38 +41,40 @@ docker exec travis-ci bash -c "cd ./pocl && make && make install" if [ "${WITH_CORE_LIBRARY}" = "yes" ]; then # script - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-opencl" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-opencl" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen --enable-opencl -- -DCMAKE_VERBOSE_MAKEFILE=ON" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build_ext -i --enable-eigen --enable-opencl -- -DCMAKE_VERBOSE_MAKEFILE=ON" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-eigen --enable-opencl" # test installing by "pip install" docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py sdist --bundle-core-library" - docker exec travis-ci bash -c "pip3 install /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "pip3 install /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen --global-option --enable-opencl" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" - docker exec travis-ci bash -c "pip3 install --user /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "pip3 install --user /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen --global-option --enable-opencl" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" # test installing by "./setup.py install" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen --enable-opencl" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" else # install core library - docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && cmake . -DPRIMITIV_USE_OPENCL=ON" + docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && cmake . -DPRIMITIV_USE_EIGEN=ON -DPRIMITIV_USE_OPENCL=ON -DCMAKE_VERBOSE_MAKEFILE=ON" docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && make" docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && make install" docker exec travis-ci bash -c "ldconfig" # script - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-opencl --no-build-core-library" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build_ext -i --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-eigen --enable-opencl --no-build-core-library" # test installing by "./setup.py install" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-opencl --no-build-core-library" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" fi diff --git a/.travis/deploy.sh b/.travis/deploy.sh new file mode 100755 index 0000000..ccb064b --- /dev/null +++ b/.travis/deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -xe + +pip install twine +if [ "${BINARY_PACKAGE}" = "yes" ]; then + twine upload -u "${PYPI_USERNAME}" -p "${PYPI_PASSWORD}" $TRAVIS_BUILD_DIR/wheelhouse/primitiv-*.whl; +else + twine upload -u "${PYPI_USERNAME}" -p "${PYPI_PASSWORD}" $TRAVIS_BUILD_DIR/dist/primitiv-*.tar.gz; +fi diff --git a/.travis/fedora.sh b/.travis/fedora.sh index e4fa48b..ecdbca8 100755 --- a/.travis/fedora.sh +++ b/.travis/fedora.sh @@ -7,50 +7,68 @@ docker run --name travis-ci -v $TRAVIS_BUILD_DIR:/primitiv-python -td fedora:lat # install docker exec travis-ci bash -c "dnf update -y" -docker exec travis-ci bash -c "dnf install -y git rpm-build gcc-c++ cmake python3-devel python3-numpy" +docker exec travis-ci bash -c "dnf install -y rpm-build gcc-c++ cmake make python3-devel python3-numpy eigen3-devel" docker exec travis-ci bash -c "pip3 install cython scikit-build" -# install OpenCL environment -docker exec travis-ci bash -c "dnf install -y opencl-headers hwloc-devel libtool-ltdl-devel ocl-icd-devel ocl-icd clang llvm-devel clang-devel zlib-devel blas-devel boost-devel patch --setopt=install_weak_deps=False" -docker exec travis-ci bash -c "git clone https://github.com/clMathLibraries/clBLAS.git" -docker exec travis-ci bash -c "cd ./clBLAS/src && cmake . -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TEST=OFF -DBUILD_KTEST=OFF" -docker exec travis-ci bash -c "cd ./clBLAS/src && make && make install" -# pocl 0.13 does not contain mem_fence() function that is used by primitiv. -# We build the latest pocl instead of using distribution's package. -# See: https://github.com/pocl/pocl/issues/294 -docker exec travis-ci bash -c "git clone https://github.com/pocl/pocl.git" -docker exec travis-ci bash -c "cd ./pocl && cmake . -DCMAKE_INSTALL_PREFIX=/usr" -docker exec travis-ci bash -c "cd ./pocl && make && make install" +# NOTE(vbkaisetsu): +# OpenCL test is disabled because irreproducible memory error is occured +# in Python+OpenCL+Fedora(+Travis?) combination. +# +# For developers: +# If you have Fedora machine and the bug is reproducible, please fix the bug +# in your environment. + +# # install OpenCL environment +# docker exec travis-ci bash -c "dnf install -y opencl-headers hwloc-devel libtool-ltdl-devel ocl-icd-devel ocl-icd clang llvm-devel clang-devel zlib-devel blas-devel boost-devel patch --setopt=install_weak_deps=False" +# docker exec travis-ci bash -c "git clone https://github.com/clMathLibraries/clBLAS.git" +# docker exec travis-ci bash -c "cd ./clBLAS/src && cmake . -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TEST=OFF -DBUILD_KTEST=OFF" +# docker exec travis-ci bash -c "cd ./clBLAS/src && make && make install" +# # pocl 0.13 does not contain mem_fence() function that is used by primitiv. +# # We build the latest pocl instead of using distribution's package. +# # See: https://github.com/pocl/pocl/issues/294 +# docker exec travis-ci bash -c "git clone https://github.com/pocl/pocl.git" +# docker exec travis-ci bash -c "cd ./pocl && cmake . -DCMAKE_INSTALL_PREFIX=/usr" +# docker exec travis-ci bash -c "cd ./pocl && make && make install" if [ "${WITH_CORE_LIBRARY}" = "yes" ]; then # script - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-opencl" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-opencl" +# docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen --enable-opencl" +# docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-eigen --enable-opencl" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen -- -DCMAKE_VERBOSE_MAKEFILE=ON" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build_ext -i --enable-eigen -- -DCMAKE_VERBOSE_MAKEFILE=ON" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-eigen" # test installing by "pip install" docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py sdist --bundle-core-library" - docker exec travis-ci bash -c "pip3 install --user /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" +# docker exec travis-ci bash -c "pip3 install --user /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen --global-option --enable-opencl" + docker exec travis-ci bash -c "pip3 install --user /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" # test installing by "./setup.py install" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" +# docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen --enable-opencl" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" else # install core library - docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && cmake . -DPRIMITIV_USE_OPENCL=ON" +# docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && cmake . -DPRIMITIV_USE_EIGEN=ON -DPRIMITIV_USE_OPENCL=ON" + docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && cmake . -DPRIMITIV_USE_EIGEN=ON -DCMAKE_VERBOSE_MAKEFILE=ON" docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && make" docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && make install" # script - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-opencl --no-build-core-library" - docker exec travis-ci bash -c "export LD_LIBRARY_PATH=/usr/local/lib && cd /primitiv-python && ./setup.py test --enable-opencl --no-build-core-library" +# docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen --enable-opencl --no-build-core-library" +# docker exec travis-ci bash -c "export LD_LIBRARY_PATH=/usr/local/lib && cd /primitiv-python && ./setup.py test --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build_ext -i --enable-eigen --no-build-core-library" + docker exec travis-ci bash -c "export LD_LIBRARY_PATH=/usr/local/lib && cd /primitiv-python && ./setup.py test --enable-eigen --no-build-core-library" # test installing by "./setup.py install" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-opencl --no-build-core-library" - docker exec travis-ci bash -c "export LD_LIBRARY_PATH=/usr/local/lib && python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" +# docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen --no-build-core-library" + docker exec travis-ci bash -c "export LD_LIBRARY_PATH=/usr/local/lib && python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" fi diff --git a/.travis/manylinux2010.sh b/.travis/manylinux2010.sh new file mode 100755 index 0000000..b62e5ce --- /dev/null +++ b/.travis/manylinux2010.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -xe + +if [ "${TRAVIS_BRANCH}" = "develop" ]; then + PRIMITIV_PYTHON_BUILD_NUMBER="dev${TRAVIS_BUILD_NUMBER}" +else + PRIMITIV_PYTHON_BUILD_NUMBER="${TRAVIS_BUILD_NUMBER}" +fi + +# before_install +docker pull vbkaisetsu/manylinux2010-py3-cmake +docker run --name travis-ci -v ${TRAVIS_BUILD_DIR}:/primitiv-python --env PRIMITIV_PYTHON_BUILD_NUMBER=${PRIMITIV_PYTHON_BUILD_NUMBER} -td vbkaisetsu/manylinux2010-py3-cmake:${TRAVIS_PYTHON_VERSION} /bin/bash + +# install +docker exec travis-ci bash -c "pip${TRAVIS_PYTHON_VERSION} install numpy==1.16.1 cython scikit-build auditwheel wheel==0.31.1" + +docker exec travis-ci bash -c "cd /primitiv-python && wget -q http://bitbucket.org/eigen/eigen/get/3.3.4.tar.bz2 -O eigen-downloaded.tar.gz" +docker exec travis-ci bash -c "cd /primitiv-python && mkdir ./eigen-downloaded" +docker exec travis-ci bash -c "cd /primitiv-python && tar xf ./eigen-downloaded.tar.gz --strip-components=1 -C ./eigen-downloaded" + +# source package +docker exec travis-ci bash -c "cd /primitiv-python && python${TRAVIS_PYTHON_VERSION} ./setup.py sdist --bundle-core-library --bundle-eigen-headers ./eigen-downloaded" + +# script +docker exec travis-ci bash -c "cd /primitiv-python && python${TRAVIS_PYTHON_VERSION} ./setup.py build --enable-eigen -- -DCMAKE_VERBOSE_MAKEFILE=ON" +docker exec travis-ci bash -c "cd /primitiv-python && python${TRAVIS_PYTHON_VERSION} ./setup.py build_ext -i --enable-eigen -- -DCMAKE_VERBOSE_MAKEFILE=ON" +docker exec travis-ci bash -c "cd /primitiv-python && python${TRAVIS_PYTHON_VERSION} ./setup.py test" + +# binary package +docker exec travis-ci bash -c "cd /primitiv-python && python${TRAVIS_PYTHON_VERSION} ./setup.py bdist_wheel" +docker exec travis-ci bash -c "cd /primitiv-python && /usr/bin/auditwheel repair --plat manylinux2010_x86_64 dist/primitiv-*.whl" + +# after_script +docker stop travis-ci + +pip install -U pip +pip install ${TRAVIS_BUILD_DIR}/wheelhouse/primitiv-*.whl +mkdir ./work +pushd ./work +python -c "import primitiv; dev = primitiv.devices.Eigen()" +popd diff --git a/.travis/osx.sh b/.travis/osx.sh index 90351f4..c7ffe2a 100755 --- a/.travis/osx.sh +++ b/.travis/osx.sh @@ -3,7 +3,8 @@ set -xe # install brew update -brew install python3 +brew upgrade python +brew install eigen pip3 install cython numpy scikit-build pushd $TRAVIS_BUILD_DIR @@ -13,43 +14,45 @@ mkdir work if [ "${WITH_CORE_LIBRARY}" = "yes" ]; then # script git submodule update --init - ./setup.py build - ./setup.py test + ./setup.py build --enable-eigen -- -DCMAKE_VERBOSE_MAKEFILE=ON + ./setup.py build_ext -i --enable-eigen -- -DCMAKE_VERBOSE_MAKEFILE=ON + ./setup.py test --enable-eigen # test installing by "pip install" ./setup.py sdist --bundle-core-library - pip3 install dist/primitiv-*.tar.gz --verbose + pip3 install dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen pushd work - python3 -c 'import primitiv; dev = primitiv.devices.Naive()' + python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()' popd pip3 uninstall -y primitiv - pip3 install --user dist/primitiv-*.tar.gz --verbose + pip3 install --user dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen pushd work - python3 -c 'import primitiv; dev = primitiv.devices.Naive()' + python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()' popd pip3 uninstall -y primitiv # test installing by "./setup.py install" - ./setup.py install + ./setup.py install --enable-eigen pushd work - python3 -c 'import primitiv; dev = primitiv.devices.Naive()' + python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()' popd pip3 uninstall -y primitiv else pushd primitiv-core - cmake . + cmake . -DPRIMITIV_USE_EIGEN=ON -DCMAKE_VERBOSE_MAKEFILE=ON make make install popd - ./setup.py build --no-build-core-library - ./setup.py test --no-build-core-library + ./setup.py build --enable-eigen --no-build-core-library + ./setup.py build_ext -i --enable-eigen --no-build-core-library + ./setup.py test --enable-eigen --no-build-core-library # test installing by "./setup.py install" - ./setup.py install --no-build-core-library + ./setup.py install --enable-eigen --no-build-core-library pushd work - python3 -c 'import primitiv; dev = primitiv.devices.Naive()' + python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()' popd pip3 uninstall -y primitiv fi diff --git a/.travis/sdist.sh b/.travis/sdist.sh new file mode 100755 index 0000000..e8028ee --- /dev/null +++ b/.travis/sdist.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -xe + +if [ "${TRAVIS_BRANCH}" = "develop" ]; then + PRIMITIV_PYTHON_BUILD_NUMBER="dev${TRAVIS_BUILD_NUMBER}" +else + PRIMITIV_PYTHON_BUILD_NUMBER="${TRAVIS_BUILD_NUMBER}" +fi + +export PRIMITIV_PYTHON_BUILD_NUMBER + +pip install cython numpy scikit-build +wget -q "http://bitbucket.org/eigen/eigen/get/3.3.4.tar.bz2" -O eigen-downloaded.tar.gz +mkdir ./eigen-downloaded +tar xf ./eigen-downloaded.tar.gz --strip-components=1 -C ./eigen-downloaded +${TRAVIS_BUILD_DIR}/setup.py sdist --bundle-core-library --bundle-eigen-headers ./eigen-downloaded +pip install ${TRAVIS_BUILD_DIR}/dist/primitiv-*.tar.gz +mkdir ./work +pushd ./work +python -c "import primitiv; dev = primitiv.devices.Eigen()" +popd diff --git a/.travis/ubuntu.sh b/.travis/ubuntu.sh index df662fe..98352a0 100755 --- a/.travis/ubuntu.sh +++ b/.travis/ubuntu.sh @@ -7,11 +7,15 @@ docker run --name travis-ci -v $TRAVIS_BUILD_DIR:/primitiv-python -td ubuntu:rol # install docker exec travis-ci bash -c "apt update" -docker exec travis-ci bash -c "apt install -y git build-essential cmake python3-dev python3-pip python3-numpy" +docker exec travis-ci bash -c "apt install -y build-essential cmake python3-dev python3-pip python3-numpy libeigen3-dev" docker exec travis-ci bash -c "pip3 install cython scikit-build" # install OpenCL environment -docker exec travis-ci bash -c "apt install -y opencl-headers libclblas-dev pkg-config libhwloc-dev libltdl-dev ocl-icd-dev ocl-icd-opencl-dev clang-3.8 llvm-3.8-dev libclang-3.8-dev libz-dev" +docker exec travis-ci bash -c "apt install -y opencl-headers git wget pkg-config libhwloc-dev libltdl-dev ocl-icd-dev ocl-icd-opencl-dev clang llvm-dev libclang-dev libz-dev" +docker exec travis-ci bash -c "wget https://github.com/CNugteren/CLBlast/archive/1.2.0.tar.gz -O ./clblast.tar.gz" +docker exec travis-ci bash -c "mkdir ./clblast" +docker exec travis-ci bash -c "tar xf ./clblast.tar.gz -C ./clblast --strip-components 1" +docker exec travis-ci bash -c "cd ./clblast && cmake . && make && make install" # pocl 0.13 does not contain mem_fence() function that is used by primitiv. # We build the latest pocl instead of using distribution's package. # See: https://github.com/pocl/pocl/issues/294 @@ -21,38 +25,40 @@ docker exec travis-ci bash -c "cd ./pocl && make && make install" if [ "${WITH_CORE_LIBRARY}" = "yes" ]; then # script - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-opencl" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-opencl" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen --enable-opencl -- -DCMAKE_VERBOSE_MAKEFILE=ON" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build_ext -i --enable-eigen --enable-opencl -- -DCMAKE_VERBOSE_MAKEFILE=ON" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-eigen --enable-opencl" # test installing by "pip install" docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py sdist --bundle-core-library" - docker exec travis-ci bash -c "pip3 install /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "pip3 install /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen --global-option --enable-opencl" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" - docker exec travis-ci bash -c "pip3 install --user /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "pip3 install --user /primitiv-python/dist/primitiv-*.tar.gz --verbose --global-option --enable-eigen --global-option --enable-opencl" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" # test installing by "./setup.py install" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-opencl" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen --enable-opencl" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" else # install core library - docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && cmake . -DPRIMITIV_USE_OPENCL=ON" + docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && cmake . -DPRIMITIV_USE_EIGEN=ON -DPRIMITIV_USE_OPENCL=ON -DCMAKE_VERBOSE_MAKEFILE=ON" docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && make" docker exec travis-ci bash -c "cd /primitiv-python/primitiv-core && make install" docker exec travis-ci bash -c "ldconfig" # script - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-opencl --no-build-core-library" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py build_ext -i --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py test --enable-eigen --enable-opencl --no-build-core-library" # test installing by "./setup.py install" - docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-opencl --no-build-core-library" - docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive()'" + docker exec travis-ci bash -c "cd /primitiv-python && ./setup.py install --enable-eigen --enable-opencl --no-build-core-library" + docker exec travis-ci bash -c "python3 -c 'import primitiv; dev = primitiv.devices.Naive(); dev = primitiv.devices.Eigen()'" docker exec travis-ci bash -c "pip3 uninstall -y primitiv" fi diff --git a/README.md b/README.md index fc78c07..c5cd45f 100644 --- a/README.md +++ b/README.md @@ -6,65 +6,70 @@ [![Build Status (develop)](https://img.shields.io/travis/primitiv/primitiv-python/develop.svg?label=build+%28develop%29)](https://travis-ci.org/primitiv/primitiv-python) [![PyPI version](https://badge.fury.io/py/primitiv.svg)](https://pypi.python.org/pypi/primitiv) + Python Frontend of primitiv =========================== +This frontend is compatible with **primitiv core library 0.4.x**. + + Dependencies ------------ -* Python 3 (3.5 or later) +* Python3 (3.5 or later) * NumPy (1.11.0 or later) * Cython (0.27 or later) -* scikit-build (0.6.1 or later, only for building) +* CMake (3.1.0 or later) +* scikit-build (0.6.1 or later) * (optional) CUDA (7.5 or later) +* (optional) Eigen (Eigen 3.3.0 or later) * (optional) OpenCL (1.2 or later) and OpenCL C++ binding v2 + Getting Started --------------- -### Automatic Install using `pip` - -To install primitiv without CUDA and OpenCL, run the following commands: +### Using `pip` ``` -$ pip3 install numpy cython scikit-build [--user] +$ pip3 install numpy cython cmake scikit-build [--user] $ pip3 install primitiv [--user] + [--global-option --enable-cuda] + [--global-option --enable-opencl] ``` To enable CUDA and/or OpenCL support, specify `--enable-cuda` or -`--enable-opencl` with `--global-option` flag of `pip` like the following -example: +`--enable-opencl` with `--global-option` flag. -``` -$ pip3 install primitiv --global-option --enable-cuda \ - --global-option --enable-opncl -``` +`--enable-eigen` flag that enables `Eigen` backend is added by default in the +package contained in PyPI. To disable the Eigen backend, use `--disable-eigen` +flag. Note that Eigen is bundled with the package contained in PyPI. -### Compiling Step by Step +We are providing only a source pacakge for now, and `pip` command +downloads the source package and builds it before installing. +This is mainly because of keeping compatibility with the `manylinux1` standard +described in [PEP 513](https://www.python.org/dev/peps/pep-0513/) +while maintaining supports of non-standard backends such as CUDA/OpenCL. -1. Install NumPy, Cython and scikit-build with Python 3 - -``` -$ sudo pip3 install numpy cython scikit-build -``` - -2. Run the following commands in `primitiv-python` directory: +### Installing from source ``` +$ git clone https://github.com/primitiv/primitiv-python +$ cd primitiv-python $ git submodule update --init -$ python3 ./setup.py build [--enable-cuda] [--enable-opencl] -$ python3 ./setup.py test [--enable-cuda] [--enable-opencl] # (optional) -$ python3 ./setup.py install [--user] [--enable-cuda] [--enable-opencl] +$ pip3 install -r requirements.txt +$ python3 ./setup.py build [--enable-cuda] [--enable-eigen] [--enable-opencl] +$ python3 ./setup.py test [--enable-cuda] [--enable-eigen] [--enable-opencl] # (optional) +$ python3 ./setup.py install [--user] [--enable-cuda] [--enable-eigen] [--enable-opencl] ``` -To enable CUDA and/or OpenCL support, run setup script with `--enable-DEVICE` option. - -primitiv-python repository contains the core library as a git submodule. +*primitiv-python* repository contains the core library as a git submodule. Note that you have to update the working tree of the core library manually by `git submodule update` after you run `git pull` or `git checkout` commands. + Resources --------- * [C++ core library of primitiv](https://github.com/primitiv/primitiv) -* [Examples](https://github.com/primitiv/primitiv-python/tree/develop/examples) +* [Examples/tutorials](https://github.com/primitiv/primitiv-python/tree/develop/examples) diff --git a/examples/encdec/encdec.py b/examples/encdec/encdec.py index 8b5648c..d032a00 100755 --- a/examples/encdec/encdec.py +++ b/examples/encdec/encdec.py @@ -52,7 +52,6 @@ def __init__(self): self.pby = Parameter() self.src_lstm = LSTM() self.trg_lstm = LSTM() - self.scan_attributes() def init(self, src_vocab_size, trg_vocab_size, embed_size, hidden_size): """Creates a new EncoderDecoder object.""" diff --git a/examples/encdec/encdec_attention.py b/examples/encdec/encdec_attention.py index 5ba2d13..732ff59 100755 --- a/examples/encdec/encdec_attention.py +++ b/examples/encdec/encdec_attention.py @@ -54,7 +54,6 @@ def __init__(self): self.src_fw_lstm = LSTM() self.src_bw_lstm = LSTM() self.trg_lstm = LSTM() - self.scan_attributes() def init(self, src_vocab_size, trg_vocab_size, embed_size, hidden_size): """Creates a new AttentionalEncoderDecoder object.""" diff --git a/examples/encdec/lstm.py b/examples/encdec/lstm.py index ed596e6..5aa63bc 100644 --- a/examples/encdec/lstm.py +++ b/examples/encdec/lstm.py @@ -21,7 +21,6 @@ def __init__(self): self.pwxh = Parameter() self.pwhh = Parameter() self.pbh = Parameter() - self.scan_attributes() def init(self, in_size, out_size): """Creates a new LSTM.""" diff --git a/examples/mnist/mnist_cnn.py b/examples/mnist/mnist_cnn.py new file mode 100755 index 0000000..378b16c --- /dev/null +++ b/examples/mnist/mnist_cnn.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +# Python example of Convolutional Neural Network. +# Please refer primitiv repository for more details. +# +# Usage: +# $ ./download_data.sh +# $ python3 ./mnist_cnn.py + +import random + +import numpy as np + +from primitiv import functions as F +from primitiv import initializers as I +from primitiv import optimizers as O +from primitiv import devices as D +from primitiv import Device, Graph, Parameter, Shape + +NUM_TRAIN_SAMPLES = 60000 +NUM_TEST_SAMPLES = 10000 +BATCH_SIZE = 200 +NUM_TRAIN_BATCHES = NUM_TRAIN_SAMPLES // BATCH_SIZE +NUM_TEST_BATCHES = NUM_TEST_SAMPLES // BATCH_SIZE +MAX_EPOCH = 100 + +IMAGE_HEIGHT = 28 +IMAGE_WIDTH = 28 + +KERNEL_SIZE1 = 5 # should be an odd number +KERNEL_SIZE2 = 5 # ditto +NUM_CHANNELS1 = 8 +NUM_CHANNELS2 = 16 +PADDING1 = KERNEL_SIZE1 // 2 +PADDING2 = KERNEL_SIZE2 // 2 + +NUM_INPUT_UNITS = (IMAGE_HEIGHT // 4) * (IMAGE_WIDTH // 4) * NUM_CHANNELS2 +NUM_HIDDEN_UNITS = 256 +NUM_OUTPUT_UNITS = 10 + + +def load_images(filename, n): + with open(filename, "rb") as ifs: + ifs.seek(16) # header + return (np.fromfile(ifs, dtype=np.uint8, count=n*NUM_INPUT_UNITS) / 255) \ + .astype(np.float32) \ + .reshape((n, IMAGE_HEIGHT, IMAGE_WIDTH)) + + +def load_labels(filename, n): + with open(filename, "rb") as ifs: + ifs.seek(8) # header + return np.fromfile(ifs, dtype=np.uint8, count=n) \ + .astype(np.uint32) + +def main(): + # Loads data + train_inputs = load_images("data/train-images-idx3-ubyte", NUM_TRAIN_SAMPLES) + train_labels = load_labels("data/train-labels-idx1-ubyte", NUM_TRAIN_SAMPLES) + test_inputs = load_images("data/t10k-images-idx3-ubyte", NUM_TEST_SAMPLES) + test_labels = load_labels("data/t10k-labels-idx1-ubyte", NUM_TEST_SAMPLES) + + dev = D.CUDA(0); + Device.set_default(dev) + g = Graph() + Graph.set_default(g) + + # Parameters of CNNs + # Shape: {kernel_height, kernel_width, in_channels, out_channels} + pw_cnn1 = Parameter( + Shape([KERNEL_SIZE1, KERNEL_SIZE1, 1, NUM_CHANNELS1]), + I.XavierUniformConv2D()) + pw_cnn2 = Parameter( + Shape([KERNEL_SIZE2, KERNEL_SIZE2, NUM_CHANNELS1, NUM_CHANNELS2]), + I.XavierUniformConv2D()) + + # Parameters of FC layers + pw_fc1 = Parameter(Shape([NUM_HIDDEN_UNITS, NUM_INPUT_UNITS]), I.XavierUniform()) + pw_fc2 = Parameter(Shape([NUM_OUTPUT_UNITS, NUM_HIDDEN_UNITS]), I.XavierUniform()) + pb_fc1 = Parameter(Shape([NUM_HIDDEN_UNITS]), I.Constant(0)) + pb_fc2 = Parameter(Shape([NUM_OUTPUT_UNITS]), I.Constant(0)) + + # Optimizer + optimizer = O.SGD(.1) + optimizer.add(pw_cnn1, pw_cnn2, pw_fc1, pw_fc2, pb_fc1, pb_fc2) + + # Helper lambda to construct the predictor network. + def make_graph(inputs, train): + # Input and parameters. + #x = F.input(Shape([IMAGE_HEIGHT, IMAGE_WIDTH], BATCH_SIZE), inputs) + x = F.input(inputs) + w_cnn1 = F.parameter(pw_cnn1) + w_cnn2 = F.parameter(pw_cnn2) + w_fc1 = F.parameter(pw_fc1) + w_fc2 = F.parameter(pw_fc2) + b_fc1 = F.parameter(pb_fc1) + b_fc2 = F.parameter(pb_fc2) + # CNNs + h_cnn1 = F.relu(F.conv2d(x, w_cnn1, PADDING1, PADDING1, 1, 1, 1, 1)) + h_pool1 = F.max_pool2d(h_cnn1, 2, 2, 0, 0, 2, 2) + h_cnn2 = F.relu(F.conv2d(h_pool1, w_cnn2, PADDING2, PADDING2, 1, 1, 1, 1)) + h_pool2 = F.max_pool2d(h_cnn2, 2, 2, 0, 0, 2, 2) + # FC layers + x_fc = F.dropout(F.flatten(h_pool2), .5, train) + h_fc = F.dropout( + F.relu(F.matmul(w_fc1, x_fc) + b_fc1), .5, train) + return F.matmul(w_fc2, h_fc) + b_fc2 + + # Batch randomizer + ids = list(range(NUM_TRAIN_SAMPLES)) + + for epoch in range(MAX_EPOCH): + # Shuffles sample IDs. + random.shuffle(ids) + + # Training loop + for batch in range(NUM_TRAIN_BATCHES): + print("\rTraining... %d / %d" % (batch + 1, NUM_TRAIN_BATCHES), end="") + # Makes a minibatch for training. + inputs = [train_inputs[ids[batch * BATCH_SIZE + i]] for i in range(BATCH_SIZE)] + labels = [train_labels[ids[batch * BATCH_SIZE + i]] for i in range(BATCH_SIZE)] + + # Constructs the graph. + g.clear(); + y = make_graph(inputs, True); + loss = F.softmax_cross_entropy(y, labels, 0) + avg_loss = F.batch.mean(loss) + + # Dump computation graph at the first time. + # if epoch == 0 and batch == 0: + # print(g.dump("dot")) + + # Implicit forward, backward, and updates parameters. + optimizer.reset_gradients() + avg_loss.backward() + optimizer.update() + + print() + + match = 0 + + # Test loop + for batch in range(NUM_TEST_BATCHES): + print("\rTesting... %d / %d" % (batch + 1, NUM_TEST_BATCHES), end="") + # Makes a test minibatch. + inputs = [test_inputs[batch * BATCH_SIZE + i] for i in range(BATCH_SIZE)] + + # Constructs the graph. + g.clear() + y = make_graph(inputs, False) + + # Gets outputs, argmax, and compares them with the label. + y_val = y.to_list() + for i in range(BATCH_SIZE): + maxval = -1e10 + argmax = -1 + for j in range(NUM_OUTPUT_UNITS): + v = y_val[j + i * NUM_OUTPUT_UNITS] + if v > maxval: + maxval = v + argmax = j + + if argmax == test_labels[i + batch * BATCH_SIZE]: + match += 1 + + accuracy = 100.0 * match / NUM_TEST_SAMPLES; + print("epoch %d: accuracy: %.2f%%" % (epoch, accuracy)) + + return 0 + + +if __name__ == "__main__": + main() diff --git a/examples/ptb/ptb_rnnlm.py b/examples/ptb/ptb_rnnlm.py index 960d9f8..9f62e71 100755 --- a/examples/ptb/ptb_rnnlm.py +++ b/examples/ptb/ptb_rnnlm.py @@ -24,7 +24,6 @@ def __init__(self, vocab_size, eos_id): self.pwlookup = Parameter([NUM_HIDDEN_UNITS, vocab_size], I.XavierUniform()) self.pwxs = Parameter([NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS], I.XavierUniform()) self.pwsy = Parameter([vocab_size, NUM_HIDDEN_UNITS], I.XavierUniform()) - self.scan_attributes() # Forward function of RNNLM. Input data should be arranged below: # inputs = { diff --git a/examples/ptb/ptb_rnnlm_lstm.py b/examples/ptb/ptb_rnnlm_lstm.py index 962e6c8..4f2b14b 100755 --- a/examples/ptb/ptb_rnnlm_lstm.py +++ b/examples/ptb/ptb_rnnlm_lstm.py @@ -25,7 +25,6 @@ class Affine(Model): def __init__(self, in_size, out_size): self.pw = Parameter([out_size, in_size], I.Uniform(-0.1, 0.1)) self.pb = Parameter([out_size], I.Constant(0)) - self.scan_attributes() # Initializes internal values. def reset(self): @@ -52,7 +51,6 @@ def __init__(self, in_size, out_size): self.pwxh = Parameter([4 * out_size, in_size], I.Uniform(-0.1, 0.1)) self.pwhh = Parameter([4 * out_size, out_size], I.Uniform(-0.1, 0.1)) self.pbh = Parameter([4 * out_size], I.Constant(0)) - self.scan_attributes() # Initializes internal values. def restart(self): @@ -82,7 +80,6 @@ def __init__(self, vocab_size, eos_id): self.rnn1 = LSTM(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS) self.rnn2 = LSTM(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS) self.hy = Affine(NUM_HIDDEN_UNITS, vocab_size) - self.scan_attributes() # Forward function of RNNLM. Input data should be arranged below: diff --git a/examples/ptb/ptb_rnnlm_sru.py b/examples/ptb/ptb_rnnlm_sru.py index b6b627a..4cdfa99 100755 --- a/examples/ptb/ptb_rnnlm_sru.py +++ b/examples/ptb/ptb_rnnlm_sru.py @@ -25,7 +25,6 @@ class Affine(Model): def __init__(self, in_size, out_size): self.pw = Parameter([out_size, in_size], I.Uniform(-0.1, 0.1)) self.pb = Parameter([out_size], I.Constant(0)) - self.scan_attributes() # Initializes internal values. def reset(self): @@ -51,7 +50,6 @@ def __init__(self, in_size, out_size): self.pw = Parameter([3 * out_size, in_size], I.Uniform(-0.1, 0.1)) self.pbf = Parameter([out_size], I.Constant(0)) self.pbr = Parameter([out_size], I.Constant(0)) - self.scan_attributes() # Initializes internal values. def restart(self): @@ -91,7 +89,6 @@ def __init__(self, vocab_size, eos_id): self.rnn1 = SRU(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS) self.rnn2 = SRU(NUM_HIDDEN_UNITS, NUM_HIDDEN_UNITS) self.hy = Affine(NUM_HIDDEN_UNITS, vocab_size) - self.scan_attributes() # Forward function of RNNLM. Input data should be arranged below: # inputs = { diff --git a/examples/tutorial1_xor.ipynb b/examples/tutorial1_xor.ipynb index f56413b..6b993d1 100644 --- a/examples/tutorial1_xor.ipynb +++ b/examples/tutorial1_xor.ipynb @@ -26,7 +26,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 2. Sets the default device and the default computation graph. ##" + "## 2. Defines a default device and a default computation graph. ##" ] }, { @@ -45,7 +45,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 3. Initializes parameters. ##" + "## 3. Defines parameters to be optimized. ##" ] }, { @@ -64,7 +64,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 4. Initializes the trainer. ##" + "## 4. Defines an optimizer. ##" ] }, { @@ -73,8 +73,8 @@ "metadata": {}, "outputs": [], "source": [ - "trainer = optimizers.SGD()\n", - "trainer.add(w1, b1, w2, b2)" + "opt = optimizers.SGD()\n", + "opt.add(w1, b1, w2, b2)" ] }, { @@ -106,7 +106,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 6. Defines the graph structure (as the function). ##" + "## 6. Defines an actual graph structure. ##" ] }, { @@ -361,26 +361,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "y = [-0.61339909, -0.23394600, +0.23394600, +0.61339909] , loss = 4.86197805\n", - "y = [-0.22889039, -0.06932333, -0.53593642, -0.21425165] , loss = 4.06609297\n", - "y = [+0.27250409, -0.25272337, +0.37463596, +0.32917529] , loss = 3.42730260\n", - "y = [+0.19238855, -0.07796110, -0.43207717, +0.05877838] , loss = 2.71082640\n", - "y = [+0.29339507, -0.39842471, +0.03626741, +0.41060734] , loss = 2.28241730\n", - "y = [+0.39863276, -0.11305720, -0.69823319, +0.16493824] , loss = 1.93670154\n", - "y = [+0.37555519, -0.63617033, -0.08534528, +0.59322888] , loss = 1.52435946\n", - "y = [+0.59360415, -0.23211496, -0.82383120, +0.32651529] , loss = 1.23942208\n", - "y = [+0.47169027, -0.73102200, -0.48567823, +0.65091115] , loss = 0.73785019\n", - "y = [+0.72777152, -0.55034208, -0.71488351, +0.61249226] , loss = 0.50775427\n", - "y = [+0.68653172, -0.70507240, -0.68368751, +0.67842233] , loss = 0.38871044\n", - "y = [+0.73128694, -0.69765669, -0.72570342, +0.71620476] , loss = 0.31939653\n", - "y = [+0.75017446, -0.72703147, -0.74771553, +0.73718238] , loss = 0.26964518\n", - "y = [+0.76873147, -0.74654895, -0.76606297, +0.75702387] , loss = 0.23148648\n", - "y = [+0.78418005, -0.76339120, -0.78164792, +0.77347887] , loss = 0.20155144\n", - "y = [+0.79737693, -0.77779418, -0.79496622, +0.78751719] , loss = 0.17761934\n", - "y = [+0.80877757, -0.79024386, -0.80647415, +0.79963452] , loss = 0.15816224\n", - "y = [+0.81872499, -0.80111289, -0.81651759, +0.81020015] , loss = 0.14210647\n", - "y = [+0.82748288, -0.81068623, -0.82536149, +0.81949651] , loss = 0.12868197\n", - "y = [+0.83525544, -0.81918591, -0.83321190, +0.82774210] , loss = 0.11732556\n" + "y = [-0.61339909, -0.23394600, +0.23394600, +0.61339909], loss = 4.86197805\n", + "y = [-0.22889039, -0.06932333, -0.53593642, -0.21425165], loss = 4.06609297\n", + "y = [+0.27250409, -0.25272337, +0.37463596, +0.32917529], loss = 3.42730260\n", + "y = [+0.19238855, -0.07796110, -0.43207717, +0.05877838], loss = 2.71082640\n", + "y = [+0.29339507, -0.39842471, +0.03626741, +0.41060734], loss = 2.28241730\n", + "y = [+0.39863276, -0.11305720, -0.69823319, +0.16493824], loss = 1.93670154\n", + "y = [+0.37555519, -0.63617033, -0.08534528, +0.59322888], loss = 1.52435946\n", + "y = [+0.59360415, -0.23211496, -0.82383120, +0.32651529], loss = 1.23942208\n", + "y = [+0.47169027, -0.73102200, -0.48567823, +0.65091115], loss = 0.73785019\n", + "y = [+0.72777152, -0.55034208, -0.71488351, +0.61249226], loss = 0.50775427\n", + "y = [+0.68653172, -0.70507240, -0.68368751, +0.67842233], loss = 0.38871044\n", + "y = [+0.73128694, -0.69765669, -0.72570342, +0.71620476], loss = 0.31939653\n", + "y = [+0.75017446, -0.72703147, -0.74771553, +0.73718238], loss = 0.26964518\n", + "y = [+0.76873147, -0.74654895, -0.76606297, +0.75702387], loss = 0.23148648\n", + "y = [+0.78418005, -0.76339120, -0.78164792, +0.77347887], loss = 0.20155144\n", + "y = [+0.79737693, -0.77779418, -0.79496622, +0.78751719], loss = 0.17761934\n", + "y = [+0.80877757, -0.79024386, -0.80647415, +0.79963452], loss = 0.15816224\n", + "y = [+0.81872499, -0.80111289, -0.81651759, +0.81020015], loss = 0.14210647\n", + "y = [+0.82748288, -0.81068623, -0.82536149, +0.81949651], loss = 0.12868197\n", + "y = [+0.83525544, -0.81918591, -0.83321190, +0.82774210], loss = 0.11732556\n" ] } ], @@ -393,12 +393,12 @@ " # Calculates values\n", " y_vals = y.to_list()\n", " loss_val = loss.to_float()\n", - " print(\"y =\", \"[\" + \", \".join(\"%+.8f\" % x for x in y_vals) + \"]\", \", loss =\", \"%.8f\" % loss_val)\n", + " print(\"y =\", \"[\" + \", \".join(\"%+.8f\" % x for x in y_vals) + \"], loss =\", \"%.8f\" % loss_val)\n", " \n", " # Train\n", - " trainer.reset_gradients()\n", + " opt.reset_gradients()\n", " loss.backward()\n", - " trainer.update()" + " opt.update()" ] } ], diff --git a/package_description.rst b/package_description.rst index f6f2a9c..f327146 100644 --- a/package_description.rst +++ b/package_description.rst @@ -1,6 +1,7 @@ primitiv: A Neural Network Toolkit. (Python frontend) ===================================================== + Features -------- @@ -10,6 +11,7 @@ Features - Mostly device-independent - Simple usage + Install ------- @@ -18,19 +20,20 @@ Prerequisites: - Python 3 (3.5 or later) - NumPy (1.11.0 or later) - Cython (0.27 or later) -- scikit-build (0.6.1 or later, only for building) +- CMake (3.1.0 or later) +- scikit-build (0.6.1 or later) - (optional) CUDA (7.5 or later) - (optional) OpenCL (1.2 or later) and OpenCL C++ binding v2 -Install dependencies:: +Install required packages:: - pip3 install numpy cython scikit-build + pip3 install numpy cython cmake scikit-build -Install primitiv without CUDA and OpenCL:: +Build and install primitiv without CUDA and OpenCL:: pip3 install primitiv -Install primitiv with CUDA and/or OpenCL support:: +Build and install primitiv with CUDA and/or OpenCL support:: # Enable only CUDA pip3 install primitiv --global-option --enable-cuda @@ -38,8 +41,24 @@ Install primitiv with CUDA and/or OpenCL support:: # Enable both CUDA and OpenCL pip3 install primitiv --global-option --enable-cuda --global-option --enable-opencl +``--enable-eigen`` flag that enables ``Eigen`` backend is added by default in +the package contained in PyPI. To disable the Eigen backend, use +``--disable-eigen`` flag. Note that Eigen is bundled with the package contained +in PyPI. + + +Notes +----- + +We are providing only a source pacakge for now, and ``pip`` command +downloads the source package and builds it before installing. +This is mainly because of keeping compatibility with the ``manylinux1`` standard +described in `PEP 513 `_ +while maintaining supports of non-standard backends such as CUDA/OpenCL. + + Resources --------- -* `Homepage `_ +* `Official repository `_ * `Examples `_ diff --git a/primitiv-core b/primitiv-core index a8717eb..c57b022 160000 --- a/primitiv-core +++ b/primitiv-core @@ -1 +1 @@ -Subproject commit a8717eb74e0aa7fcdda9345b57ca51ce0ecf0b3a +Subproject commit c57b02248afac42b3b001901d33f5dfe0bf4ad2a diff --git a/primitiv/_device.pxd b/primitiv/_device.pxd index 17c6797..fa3740f 100644 --- a/primitiv/_device.pxd +++ b/primitiv/_device.pxd @@ -1,7 +1,5 @@ -cdef extern from "primitiv/device.h": +cdef extern from "primitiv/core/device.h": cdef cppclass CppDevice "primitiv::Device": - @staticmethod - void set_default(CppDevice &dev) except + void dump_description() except + diff --git a/primitiv/_device.pyx b/primitiv/_device.pyx index 74e875a..7187e91 100644 --- a/primitiv/_device.pyx +++ b/primitiv/_device.pyx @@ -10,6 +10,8 @@ from weakref import WeakValueDictionary # It means that users can not compare instances by using "is" operator. cdef object py_primitiv_device_weak_dict = WeakValueDictionary() +cdef object py_primitiv_default_device = None + cdef class Device: """Interface of the Tensor provider. @@ -17,14 +19,27 @@ cdef class Device: """ @staticmethod - def set_default(Device dev): + def set_default(Device device): """Specifies a new default device. - :param dev: Reference of the new default device. - :type dev: primitiv.Device + :param device: Reference of the new default device. + :type device: primitiv.Device + + """ + global py_primitiv_default_device + py_primitiv_default_device = device + + @staticmethod + def get_default(): + """Retrieves the current default device. + + :return: The current default device + :rtype: primitiv.Device """ - CppDevice.set_default(dev.wrapped[0]) + if py_primitiv_default_device is None: + raise RuntimeError("Default object is null.") + return py_primitiv_default_device def dump_description(self): """Prints device description to stderr. diff --git a/primitiv/_function.pxd b/primitiv/_function.pxd index a60767a..ba4e390 100644 --- a/primitiv/_function.pxd +++ b/primitiv/_function.pxd @@ -9,7 +9,7 @@ from primitiv._shape cimport CppShape from primitiv._parameter cimport CppParameter -cdef extern from "primitiv/functions.h": +cdef extern from "primitiv/core/functions.h": CppTensor func_input_tensor "primitiv::functions::input_tensor" (const CppShape &shape, const vector[float] &data, CppDevice *dev) except + CppNode func_input_node "primitiv::functions::input_node" (const CppShape &shape, const vector[float] &data, CppDevice *dev, CppGraph *g) except + CppTensor func_parameter_tensor "primitiv::functions::parameter_tensor" (CppParameter ¶m) except + @@ -25,7 +25,7 @@ cdef extern from "primitiv/functions.h": Var func_sqrt "primitiv::functions::sqrt" [Var](const Var &x) except + Var func_exp "primitiv::functions::exp" [Var](const Var &x) except + Var func_log "primitiv::functions::log" [Var](const Var &x) except + - Var func_ipow "primitiv::functions::ipow" [Var](const Var &x, int k) except + + Var func_pown "primitiv::functions::pown" [Var](const Var &x, int k) except + Var func_pow "primitiv::functions::pow" [Var](const Var &x, float k) except + Var func_pow "primitiv::functions::pow" [Var](float x, const Var &k) except + Var func_pow "primitiv::functions::pow" [Var](const Var &x, const Var &k) except + @@ -53,6 +53,8 @@ cdef extern from "primitiv/functions.h": Var func_softmax_cross_entropy "primitiv::functions::softmax_cross_entropy" [Var](const Var &x, const Var &t, unsigned dim) except + Var func_softmax_cross_entropy "primitiv::functions::softmax_cross_entropy" [Var](const Var &x, const vector[unsigned] &ids, unsigned dim) except + Var func_stop_gradient "primitiv::functions::stop_gradient" [Var](const Var &x) except + + Var func_conv2d "primitiv::functions::conv2d" [Var](const Var &x, const Var &w, unsigned padding0, unsigned padding1, unsigned stride0, unsigned stride1, unsigned dilation0, unsigned dilation1) except + + Var func_max_pool2d "primitiv::functions::max_pool2d" [Var](const Var &x, unsigned window0, unsigned window1, unsigned padding0, unsigned padding1, unsigned stride0, unsigned stride1) except + CppTensor func_constant_tensor "primitiv::functions::constant_tensor" (const CppShape &shape, float k, CppDevice *dev) except + CppNode func_constant_node "primitiv::functions::constant_node" (const CppShape &shape, float k, CppDevice *dev, CppGraph *g) except + @@ -80,13 +82,13 @@ cdef extern from "primitiv/functions.h": Var func_divide "primitiv::functions::divide" [Var](const Var &a, const Var &b) except + -cdef extern from "primitiv/functions.h": +cdef extern from "primitiv/core/functions.h": Var func_batch_sum "primitiv::functions::batch::sum" [Var](const Var &x) except + Var func_batch_mean "primitiv::functions::batch::mean" [Var](const Var &x) except + Var func_batch_normalize "primitiv::functions::batch::normalize" [Var](const Var &x) except + -cdef extern from "primitiv/functions.h": +cdef extern from "primitiv/core/functions.h": CppNode func_random_bernoulli_node "primitiv::functions::random::bernoulli_node" (const CppShape &shape, float p, CppDevice *dev, CppGraph *g) except + CppTensor func_random_bernoulli_tensor "primitiv::functions::random::bernoulli_tensor" (const CppShape &shape, float p, CppDevice *dev) except + diff --git a/primitiv/_function.pyx b/primitiv/_function.pyx index ee7a6c6..e0fe574 100644 --- a/primitiv/_function.pyx +++ b/primitiv/_function.pyx @@ -16,15 +16,19 @@ import numpy as np class functions: @staticmethod - def raw_input(shape, vector[float] data, Device dev = None, Graph g = None): + def raw_input(shape, vector[float] data, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_input_node(normShape(shape).wrapped, data, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) # NOTE(vbkaisetsu) # This function takes an np.ndarray or a list of np.ndarray # instead of a vector. @staticmethod - def input(data, Device dev = None, Graph g = None): + def input(data, Device device = None, Graph graph = None): # NOTE(vbkaisetsu, odashi): # In this function, we don't check whether each ndarray is empty # (i.e., it doesn't have any elements) or not. @@ -42,16 +46,20 @@ class functions: shape = Shape(data[0].shape, len(data)) else: raise TypeError("`data` has incorrect type.") - return functions.raw_input(shape, ndarrays_to_vector(data), dev, g) + return functions.raw_input(shape, ndarrays_to_vector(data), device, graph) @staticmethod - def parameter(Parameter param, Graph g = None): - return wrapNode(func_parameter_node(param.wrapped[0], get_cpp_graph(g))) + def parameter(Parameter param, Graph graph = None): + if graph is None: + graph = Graph.get_default() + return wrapNode(func_parameter_node(param.wrapped[0], get_cpp_graph(graph))) @staticmethod - def copy(Node x, Device dev = None): - return wrapNode(func_copy(x.wrapped, get_cpp_device(dev))) + def copy(Node x, Device device = None): + if device is None: + device = Device.get_default() + return wrapNode(func_copy(x.wrapped, get_cpp_device(device))) @staticmethod def pick(Node x, vector[unsigned] ids, unsigned dim): @@ -70,8 +78,8 @@ class functions: return wrapNode(func_concat(vec, dim)) @staticmethod - def reshape(Node x, Shape new_shape): - return wrapNode(func_reshape(x.wrapped, new_shape.wrapped)) + def reshape(Node x, new_shape): + return wrapNode(func_reshape(x.wrapped, normShape(new_shape).wrapped)) @staticmethod def flatten(Node x): @@ -100,7 +108,7 @@ class functions: @staticmethod def pow(x, k): if isinstance(x, Node) and isinstance(k, int) and -0x80000000 <= k <= 0x7fffffff: - return wrapNode(func_ipow(( x).wrapped, k)) + return wrapNode(func_pown(( x).wrapped, k)) elif isinstance(x, Node) and isinstance(k, (int, float)): return wrapNode(func_pow(( x).wrapped, k)) elif isinstance(x, (int, float)) and isinstance(k, Node): @@ -206,23 +214,59 @@ class functions: return wrapNode(func_stop_gradient(x.wrapped)) @staticmethod - def constant(shape, float k, Device dev = None, Graph g = None): + def conv2d(Node x, Node w, + unsigned padding0, unsigned padding1, + unsigned stride0, unsigned stride1, + unsigned dilation0, unsigned dilation1): + return wrapNode(func_conv2d(x.wrapped, w.wrapped, + padding0, padding1, + stride0, stride1, + dilation0, dilation1)) + + @staticmethod + def max_pool2d(Node x, + unsigned window0, unsigned window1, + unsigned padding0, unsigned padding1, + unsigned stride0, unsigned stride1): + return wrapNode(func_max_pool2d(x.wrapped, + window0, window1, + padding0, padding1, + stride0, stride1)) + + @staticmethod + def constant(shape, float k, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_constant_node(normShape(shape).wrapped, k, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def zeros(shape, Device dev = None, Graph g = None): + def zeros(shape, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_zeros_node(normShape(shape).wrapped, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def ones(shape, Device dev = None, Graph g = None): + def ones(shape, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_ones_node(normShape(shape).wrapped, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def identity(unsigned size, Device dev = None, Graph g = None): - return wrapNode(func_identity_node(size, get_cpp_device(dev), get_cpp_graph(g))) + def identity(unsigned size, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() + return wrapNode(func_identity_node(size, get_cpp_device(device), get_cpp_graph(graph))) class batch: @staticmethod @@ -239,29 +283,49 @@ class functions: class random: @staticmethod - def bernoulli(shape, float p, Device dev = None, Graph g = None): + def bernoulli(shape, float p, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_random_bernoulli_node(normShape(shape).wrapped, p, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def uniform(shape, float lower, float upper, Device dev = None, Graph g = None): + def uniform(shape, float lower, float upper, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_random_uniform_node(normShape(shape).wrapped, lower, upper, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def normal(shape, float mean, float sd, Device dev = None, Graph g = None): + def normal(shape, float mean, float sd, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_random_normal_node(normShape(shape).wrapped, mean, sd, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def log_normal(shape, float mean, float sd, Device dev = None, Graph g = None): + def log_normal(shape, float mean, float sd, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_random_log_normal_node(normShape(shape).wrapped, mean, sd, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod - def gumbel(shape, float mu, float beta, Device dev = None, Graph g = None): + def gumbel(shape, float mu, float beta, Device device = None, Graph graph = None): + if device is None: + device = Device.get_default() + if graph is None: + graph = Graph.get_default() return wrapNode(func_random_gumbel_node(normShape(shape).wrapped, mu, beta, - get_cpp_device(dev), get_cpp_graph(g))) + get_cpp_device(device), get_cpp_graph(graph))) @staticmethod def dropout(Node x, float rate, bool enabled): @@ -271,15 +335,17 @@ class functions: class tensor_functions: @staticmethod - def raw_input(shape, vector[float] data, Device dev = None): + def raw_input(shape, vector[float] data, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_input_tensor(normShape(shape).wrapped, data, - get_cpp_device(dev)))) + get_cpp_device(device)))) # NOTE(vbkaisetsu) # This function takes an np.ndarray or a list of np.ndarray # instead of a vector. @staticmethod - def input(data, Device dev = None): + def input(data, Device device = None): # NOTE(vbkaisetsu, odashi): # In this function, we don't check whether each ndarray is empty # (i.e., it doesn't have any elements) or not. @@ -297,15 +363,17 @@ class tensor_functions: shape = Shape(data[0].shape, len(data)) else: raise TypeError("`data` has incorrect type.") - return tensor_functions.raw_input(shape, ndarrays_to_vector(data), dev) + return tensor_functions.raw_input(shape, ndarrays_to_vector(data), device) @staticmethod def parameter(Parameter param): return Tensor.get_wrapper_with_new(new CppTensor(func_parameter_tensor(param.wrapped[0]))) @staticmethod - def copy(Tensor x, Device dev = None): - return Tensor.get_wrapper_with_new(new CppTensor(func_copy(x.wrapped[0], get_cpp_device(dev)))) + def copy(Tensor x, Device device = None): + if device is None: + device = Device.get_default() + return Tensor.get_wrapper_with_new(new CppTensor(func_copy(x.wrapped[0], get_cpp_device(device)))) @staticmethod def pick(Tensor x, vector[unsigned] ids, unsigned dim): @@ -324,8 +392,8 @@ class tensor_functions: return Tensor.get_wrapper_with_new(new CppTensor(func_concat(vec, dim))) @staticmethod - def reshape(Tensor x, Shape new_shape): - return Tensor.get_wrapper_with_new(new CppTensor(func_reshape(x.wrapped[0], new_shape.wrapped))) + def reshape(Tensor x, new_shape): + return Tensor.get_wrapper_with_new(new CppTensor(func_reshape(x.wrapped[0], normShape(new_shape).wrapped))) @staticmethod def flatten(Tensor x): @@ -354,7 +422,7 @@ class tensor_functions: @staticmethod def pow(x, k): if isinstance(x, Tensor) and isinstance(k, int) and -0x80000000 <= k <= 0x7fffffff: - return Tensor.get_wrapper_with_new(new CppTensor(func_ipow(( x).wrapped[0], k))) + return Tensor.get_wrapper_with_new(new CppTensor(func_pown(( x).wrapped[0], k))) elif isinstance(x, Tensor) and isinstance(k, (int, float)): return Tensor.get_wrapper_with_new(new CppTensor(func_pow(( x).wrapped[0], k))) elif isinstance(x, (int, float)) and isinstance(k, Tensor): @@ -460,23 +528,51 @@ class tensor_functions: return Tensor.get_wrapper_with_new(new CppTensor(func_stop_gradient(x.wrapped[0]))) @staticmethod - def constant(shape, float k, Device dev = None): + def conv2d(Tensor x, Tensor w, + unsigned padding0, unsigned padding1, + unsigned stride0, unsigned stride1, + unsigned dilation0, unsigned dilation1): + return Tensor.get_wrapper_with_new(new CppTensor(func_conv2d(x.wrapped[0], w.wrapped[0], + padding0, padding1, + stride0, stride1, + dilation0, dilation1))) + + @staticmethod + def max_pool2d(Tensor x, + unsigned window0, unsigned window1, + unsigned padding0, unsigned padding1, + unsigned stride0, unsigned stride1): + return Tensor.get_wrapper_with_new(new CppTensor(func_max_pool2d(x.wrapped[0], + window0, window1, + padding0, padding1, + stride0, stride1))) + + @staticmethod + def constant(shape, float k, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_constant_tensor(normShape(shape).wrapped, k, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod - def zeros(shape, Device dev = None): + def zeros(shape, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_zeros_tensor(normShape(shape).wrapped, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod - def ones(shape, Device dev = None): + def ones(shape, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_ones_tensor(normShape(shape).wrapped, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod - def identity(unsigned size, Device dev = None): - return Tensor.get_wrapper_with_new(new CppTensor(func_identity_tensor(size, get_cpp_device(dev)))) + def identity(unsigned size, Device device = None): + if device is None: + device = Device.get_default() + return Tensor.get_wrapper_with_new(new CppTensor(func_identity_tensor(size, get_cpp_device(device)))) class batch: @staticmethod @@ -493,29 +589,39 @@ class tensor_functions: class random: @staticmethod - def bernoulli(shape, float p, Device dev = None): + def bernoulli(shape, float p, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_random_bernoulli_tensor(normShape(shape).wrapped, p, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod - def uniform(shape, float lower, float upper, Device dev = None): + def uniform(shape, float lower, float upper, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_random_uniform_tensor(normShape(shape).wrapped, lower, upper, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod - def normal(shape, float mean, float sd, Device dev = None): + def normal(shape, float mean, float sd, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_random_normal_tensor(normShape(shape).wrapped, mean, sd, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod - def log_normal(shape, float mean, float sd, Device dev = None): + def log_normal(shape, float mean, float sd, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_random_log_normal_tensor(normShape(shape).wrapped, mean, sd, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod - def gumbel(shape, float mu, float beta, Device dev = None): + def gumbel(shape, float mu, float beta, Device device = None): + if device is None: + device = Device.get_default() return Tensor.get_wrapper_with_new(new CppTensor(func_random_gumbel_tensor(normShape(shape).wrapped, mu, beta, - get_cpp_device(dev)))) + get_cpp_device(device)))) @staticmethod def dropout(Tensor x, float rate, bool enabled): diff --git a/primitiv/_graph.pxd b/primitiv/_graph.pxd index e5bb875..3c60c83 100644 --- a/primitiv/_graph.pxd +++ b/primitiv/_graph.pxd @@ -6,7 +6,7 @@ from primitiv._shape cimport CppShape from primitiv._tensor cimport CppTensor -cdef extern from "primitiv/graph.h" nogil: +cdef extern from "primitiv/core/graph.h" nogil: cdef cppclass CppNode "primitiv::Node": CppNode(CppNode &&src) except + CppNode() except + @@ -23,11 +23,9 @@ cdef extern from "primitiv/graph.h" nogil: void backward() except + -cdef extern from "primitiv/graph.h" nogil: +cdef extern from "primitiv/core/graph.h" nogil: cdef cppclass CppGraph "primitiv::Graph": CppGraph() except + - @staticmethod - void set_default(CppGraph &g) except + void clear() except + const CppTensor &forward(const CppNode &node) except + void backward(const CppNode &node) except + diff --git a/primitiv/_graph.pyx b/primitiv/_graph.pyx index b9bab02..32e5fb6 100644 --- a/primitiv/_graph.pyx +++ b/primitiv/_graph.pyx @@ -6,7 +6,7 @@ from primitiv._shape cimport wrapShape from primitiv._tensor cimport Tensor from primitiv._function cimport ( func_positive, func_negative, func_add, func_subtract, func_multiply, - func_divide, func_pow, func_ipow, func_matmul, + func_divide, func_pow, func_pown, func_matmul, ) from primitiv.config cimport pystr_to_cppstr, cppstr_to_pystr @@ -23,6 +23,8 @@ import numpy as np # It means that users can not compare instances by using "is" operator. cdef object py_primitiv_graph_weak_dict = WeakValueDictionary() +cdef object py_primitiv_default_graph = None + cdef class Node: """Pointer of a node in the computation graph. @@ -238,7 +240,7 @@ cdef class Node: if mod is not None: return NotImplemented if isinstance(right, int) and -0x80000000 <= right <= 0x7fffffff: - return wrapNode(func_ipow(( left).wrapped, right)) + return wrapNode(func_pown(( left).wrapped, right)) elif isinstance(right, (int, float)): return wrapNode(func_pow(( left).wrapped, right)) elif isinstance(left, (int, float)): @@ -275,14 +277,27 @@ cdef class Graph: self.wrapped = NULL @staticmethod - def set_default(Graph g): + def set_default(Graph graph): """Specifies a new default graph. - :param g: Reference of the new default graph. - :type g: primitiv.Graph + :param graph: Reference of the new default graph. + :type graph: primitiv.Graph + + """ + global py_primitiv_default_graph + py_primitiv_default_graph = graph + + @staticmethod + def get_default(): + """Retrieves the current default graph. + + :return: The current default graph + :rtype: primitiv.Graph """ - CppGraph.set_default(g.wrapped[0]) + if py_primitiv_default_graph is None: + raise RuntimeError("Default object is null.") + return py_primitiv_default_graph def clear(self): """Clear all functions in the graph. diff --git a/primitiv/_initializer.pxd b/primitiv/_initializer.pxd index 86c233b..a352794 100644 --- a/primitiv/_initializer.pxd +++ b/primitiv/_initializer.pxd @@ -1,7 +1,7 @@ from primitiv._tensor cimport CppTensor -cdef extern from "primitiv/initializer.h": +cdef extern from "primitiv/core/initializer.h": cdef cppclass CppInitializer "primitiv::Initializer": CppInitializer() except + void apply(CppTensor &x) except + diff --git a/primitiv/_model.pxd b/primitiv/_model.pxd index a7796a2..39c4aac 100644 --- a/primitiv/_model.pxd +++ b/primitiv/_model.pxd @@ -7,7 +7,7 @@ from primitiv._device cimport CppDevice from primitiv._parameter cimport CppParameter -cdef extern from "primitiv/model.h": +cdef extern from "primitiv/core/model.h": cdef cppclass CppModel "primitiv::Model": CppModel() except + void load(string &path, bool with_stats, CppDevice *device) except + diff --git a/primitiv/_model.pyx b/primitiv/_model.pyx index 3372191..b7ffa9c 100644 --- a/primitiv/_model.pyx +++ b/primitiv/_model.pyx @@ -40,6 +40,8 @@ cdef class Model: :type device: bool or None """ + if device is None: + device = Device.get_default() self.wrapped.load(pystr_to_cppstr(path), with_stats, get_cpp_device(device)) @@ -65,6 +67,9 @@ cdef class Model: ``name`` should not be overlapped with all registered parameters and submodels. + This function does not modify attribute information of this object. + To set ``arg`` as an attribule, use ``__setattr__`` instead. + """ if isinstance(arg, Parameter): self.wrapped.add(pystr_to_cppstr(name), ( arg).wrapped[0]) @@ -74,42 +79,26 @@ cdef class Model: raise TypeError("Argument 'arg' has incorrect type (Parameter or Model)") self.added.append(arg) - def scan_attributes(self): - """Registers all parameter and model members in this model. - - This method searches all Parameter and Model objects defined as the - attributes in the model object, and calls add() for each parameter/model - using corresponding attribute key to the name argument. - - Example: - - >>> class ParentModel(Model): - ... def __init__(self): - ... self.param1 = Parameter() - ... self.param2 = Parameter() - ... self.submodel1 = SubModel1() # Sub class of Model - ... self.submodel2 = SubModel2() # Sub class of Model - ... self.scan_attributes() - - is equivalent to: + def __setattr__(self, key, value): + """Set attribute - >>> class ParentModel(Model): - ... def __init__(self): - ... self.param1 = Parameter() - ... self.param2 = Parameter() - ... self.submodel1 = SubModel1() - ... self.submodel2 = SubModel2() - ... self.add("param1", self.param1) - ... self.add("param2", self.param2) - ... self.add("submodel1", self.submodel1) - ... self.add("submodel2", self.submodel2) + If Parameter or Model is set, add(key, value) is additionally + called to register a new parameter. Otherwise, a value is + normally set to this model. """ - for k, v in self.__dict__.items(): - if isinstance(v, Parameter) and v not in self.added: - self.add(k, v) - if isinstance(v, Model) and v not in self.added: - self.add(k, v) + if isinstance(value, Parameter) and value not in self.added: + self.add(key, value) + if isinstance(value, Model) and value not in self.added: + self.add(key, value) + self.__dict__[key] = value + + def __delattr__(self, key): + # NOTE(vbkaisetsu): __delattr__ is not called when the parent object is deleted. + item = self.__dict__[key] + if isinstance(item, Parameter) or isinstance(item, Model): + raise TypeError("Parameter and Model are not deletable.") + del self.__dict__[key] def __getitem__(self, key): """Retrieves a parameter or a model in this model. diff --git a/primitiv/_optimizer.pxd b/primitiv/_optimizer.pxd index 23d9e37..ac141fb 100644 --- a/primitiv/_optimizer.pxd +++ b/primitiv/_optimizer.pxd @@ -10,7 +10,7 @@ from primitiv._parameter cimport CppParameter, Parameter from primitiv._shape cimport CppShape -cdef extern from "primitiv/optimizer.h": +cdef extern from "primitiv/core/optimizer.h": cdef cppclass CppOptimizer "primitiv::Optimizer": CppOptimizer(CppOptimizer &&) except + CppOptimizer() except + diff --git a/primitiv/_parameter.pxd b/primitiv/_parameter.pxd index 02f155c..4d240d2 100644 --- a/primitiv/_parameter.pxd +++ b/primitiv/_parameter.pxd @@ -9,7 +9,7 @@ from primitiv._device cimport CppDevice from primitiv._initializer cimport CppInitializer, Initializer -cdef extern from "primitiv/parameter.h": +cdef extern from "primitiv/core/parameter.h": cdef cppclass CppParameter "primitiv::Parameter": CppParameter() except + CppParameter(const CppShape &shape, const vector[float] &value, CppDevice *device) except + diff --git a/primitiv/_parameter.pyx b/primitiv/_parameter.pyx index 30dc889..c0e5737 100644 --- a/primitiv/_parameter.pyx +++ b/primitiv/_parameter.pyx @@ -81,6 +81,8 @@ cdef class Parameter: :type device: primitiv.Device or None """ + if device is None: + device = Device.get_default() self.wrapped.init(normShape(shape).wrapped, initializer.wrapped[0], get_cpp_device(device)) return @@ -97,6 +99,8 @@ cdef class Parameter: :type device: primitiv.Device or None """ + if device is None: + device = Device.get_default() self.wrapped.load(pystr_to_cppstr(path), with_stats, get_cpp_device(device)) return diff --git a/primitiv/_shape.pxd b/primitiv/_shape.pxd index a0409fa..e5df1d7 100644 --- a/primitiv/_shape.pxd +++ b/primitiv/_shape.pxd @@ -3,7 +3,7 @@ from libcpp.string cimport string from libcpp cimport bool -cdef extern from "primitiv/shape.h": +cdef extern from "primitiv/core/shape.h": cdef cppclass CppShape "primitiv::Shape": CppShape() except + CppShape(vector[unsigned] &dims, unsigned batch) except + @@ -20,7 +20,7 @@ cdef extern from "primitiv/shape.h": bool has_batch() except + bool has_compatible_batch(const CppShape &rhs) except + bool is_scalar() except + - bool is_row_vector() except + + bool is_column_vector() except + bool is_matrix() except + bool has_same_dims(const CppShape &rhs) except + bool has_same_loo_dims(const CppShape &rhs, unsigned dim) except + @@ -32,6 +32,8 @@ cdef extern from "primitiv/shape.h": cdef class Shape: cdef CppShape wrapped + @staticmethod + cdef Shape get_wrapper(CppShape wrapped) cdef inline Shape wrapShape(CppShape wrapped) except +: diff --git a/primitiv/_shape.pyx b/primitiv/_shape.pyx index 23c1a37..047307a 100644 --- a/primitiv/_shape.pyx +++ b/primitiv/_shape.pyx @@ -131,14 +131,14 @@ cdef class Shape: """ return self.wrapped.is_scalar() - def is_row_vector(self): - """Checks whether the shape is a row vector or not. + def is_column_vector(self): + """Checks whether the shape is a column vector or not. - :return: ``True`` if the shape is a row vector, ``False`` otherwise. + :return: ``True`` if the shape is a column vector, ``False`` otherwise. :rtype: bool """ - return self.wrapped.is_row_vector() + return self.wrapped.is_column_vector() def is_matrix(self): """Checks whether the shape is a vector or a matrix, or not. @@ -186,8 +186,7 @@ cdef class Shape: :rtype: primitiv.Shape """ - self.wrapped.resize_dim(dim, m) - return self + return Shape.get_wrapper(self.wrapped.resize_dim(dim, m)) def resize_batch(self, unsigned batch): """Creates a new shape which have specified batch size. @@ -198,8 +197,7 @@ cdef class Shape: :rtype: primitiv.Shape """ - self.wrapped.resize_batch(batch) - return self + return Shape.get_wrapper(self.wrapped.resize_batch(batch)) def update_dim(self, unsigned dim, unsigned m): """Directly updates a specified dimension. @@ -226,3 +224,9 @@ cdef class Shape: def __deepcopy__(self, memo): raise NotImplementedError(type(self).__name__ + " does not support `__deepcopy__` for now.") + + @staticmethod + cdef Shape get_wrapper(CppShape wrapped): + cdef Shape shape = Shape.__new__(Shape) + shape.wrapped = wrapped + return shape diff --git a/primitiv/_tensor.pxd b/primitiv/_tensor.pxd index e9040cf..fdd91e3 100644 --- a/primitiv/_tensor.pxd +++ b/primitiv/_tensor.pxd @@ -5,7 +5,7 @@ from primitiv._device cimport CppDevice from primitiv._shape cimport CppShape -cdef extern from "primitiv/tensor.h" nogil: +cdef extern from "primitiv/core/tensor.h" nogil: cdef cppclass CppTensor "primitiv::Tensor": CppTensor(CppTensor &&src) except + CppTensor() except + diff --git a/primitiv/_tensor.pyx b/primitiv/_tensor.pyx index 9711875..ad40dd7 100644 --- a/primitiv/_tensor.pyx +++ b/primitiv/_tensor.pyx @@ -5,7 +5,7 @@ from primitiv._device cimport Device from primitiv._shape cimport Shape, wrapShape, normShape from primitiv._function cimport ( func_positive, func_negative, func_add, func_subtract, func_multiply, - func_divide, func_pow, func_ipow, func_matmul, + func_divide, func_pow, func_pown, func_matmul, ) from weakref import WeakValueDictionary @@ -259,7 +259,7 @@ cdef class Tensor: if mod is not None: return NotImplemented if isinstance(right, int) and -0x80000000 <= right <= 0x7fffffff: - return Tensor.get_wrapper_with_new(new CppTensor(func_ipow(( left).wrapped[0], right))) + return Tensor.get_wrapper_with_new(new CppTensor(func_pown(( left).wrapped[0], right))) elif isinstance(right, (int, float)): return Tensor.get_wrapper_with_new(new CppTensor(func_pow(( left).wrapped[0], right))) elif isinstance(left, (int, float)): diff --git a/primitiv/devices/__init__.py b/primitiv/devices/__init__.py index 72722dc..0de62b9 100644 --- a/primitiv/devices/__init__.py +++ b/primitiv/devices/__init__.py @@ -8,10 +8,16 @@ except ImportError: pass +try: + from primitiv.devices._eigen_device import Eigen + __all__.append("Eigen") +#except ModuleNotFoundError: +except ImportError: + pass + try: from primitiv.devices._opencl_device import OpenCL __all__.append("OpenCL") #except ModuleNotFoundError: except ImportError: pass - diff --git a/primitiv/devices/_cuda_device.pxd b/primitiv/devices/_cuda_device.pxd index 486e1ee..19c147b 100644 --- a/primitiv/devices/_cuda_device.pxd +++ b/primitiv/devices/_cuda_device.pxd @@ -1,12 +1,14 @@ from primitiv._device cimport CppDevice, Device -cdef extern from "primitiv/cuda_device.h": +cdef extern from "primitiv/devices/cuda/device.h": cdef cppclass CppCUDA "primitiv::devices::CUDA" (CppDevice): CppCUDA(unsigned device_id) except + CppCUDA(unsigned device_id, unsigned rng_seed) except + @staticmethod unsigned num_devices() except + + @staticmethod + unsigned check_support(unsigned device_id) except + cdef class CUDA(Device): diff --git a/primitiv/devices/_cuda_device.pyx b/primitiv/devices/_cuda_device.pyx index c4d970e..cb5669f 100644 --- a/primitiv/devices/_cuda_device.pyx +++ b/primitiv/devices/_cuda_device.pyx @@ -35,3 +35,17 @@ cdef class CUDA(Device): """ return CppCUDA.num_devices() + + @staticmethod + def check_support(unsigned device_id): + """Checks whether the device corresponding to the specified ID is + supported. + + :param device_id: Device ID to check. + :type: int + :return: True if this class supports the specified device, + False otherwise. + :rtype: bool + + """ + return CppCUDA.check_support(device_id) diff --git a/primitiv/devices/_eigen_device.pxd b/primitiv/devices/_eigen_device.pxd new file mode 100644 index 0000000..84a2c4d --- /dev/null +++ b/primitiv/devices/_eigen_device.pxd @@ -0,0 +1,11 @@ +from primitiv._device cimport CppDevice, Device + + +cdef extern from "primitiv/devices/eigen/device.h": + cdef cppclass CppEigen "primitiv::devices::Eigen" (CppDevice): + CppEigen() except + + CppEigen(unsigned rng_seed) except + + + +cdef class Eigen(Device): + pass diff --git a/primitiv/devices/_eigen_device.pyx b/primitiv/devices/_eigen_device.pyx new file mode 100644 index 0000000..3c71cd2 --- /dev/null +++ b/primitiv/devices/_eigen_device.pyx @@ -0,0 +1,28 @@ +from primitiv._device cimport Device + + +cdef class Eigen(Device): + """Creates a Eigen object. + + """ + + def __init__(self, rng_seed = None): + """Creates a Eigen object. + + :param rng_seed: The seed value of internal random number generator. + :type rng_seed: int or None + + """ + if self.wrapped is not NULL: + raise TypeError("__init__() has already been called.") + if rng_seed is None: + self.wrapped = new CppEigen() + else: + self.wrapped = new CppEigen( rng_seed) + + Device.register_wrapper(self.wrapped, self) + + def __dealloc__(self): + if self.wrapped is not NULL: + del self.wrapped + self.wrapped = NULL diff --git a/primitiv/devices/_naive_device.pxd b/primitiv/devices/_naive_device.pxd index 9184f68..a247e3d 100644 --- a/primitiv/devices/_naive_device.pxd +++ b/primitiv/devices/_naive_device.pxd @@ -1,7 +1,7 @@ from primitiv._device cimport CppDevice, Device -cdef extern from "primitiv/naive_device.h": +cdef extern from "primitiv/devices/naive/device.h": cdef cppclass CppNaive "primitiv::devices::Naive" (CppDevice): CppNaive() except + CppNaive(unsigned rng_seed) except + diff --git a/primitiv/devices/_opencl_device.pxd b/primitiv/devices/_opencl_device.pxd index b75b1ff..f7788cf 100644 --- a/primitiv/devices/_opencl_device.pxd +++ b/primitiv/devices/_opencl_device.pxd @@ -1,7 +1,7 @@ from primitiv._device cimport CppDevice, Device -cdef extern from "primitiv/opencl_device.h": +cdef extern from "primitiv/devices/opencl/device.h": cdef cppclass CppOpenCL "primitiv::devices::OpenCL" (CppDevice): CppOpenCL(unsigned platform_id, unsigned device_id) except + CppOpenCL(unsigned platform_id, unsigned device_id, unsigned rng_seed) except + @@ -9,6 +9,8 @@ cdef extern from "primitiv/opencl_device.h": unsigned num_platforms() except + @staticmethod unsigned num_devices(unsigned platform_id) except + + @staticmethod + unsigned check_support(unsigned platform_id, unsigned device_id) except + cdef class OpenCL(Device): diff --git a/primitiv/devices/_opencl_device.pyx b/primitiv/devices/_opencl_device.pyx index 2c3c72f..95d723e 100644 --- a/primitiv/devices/_opencl_device.pyx +++ b/primitiv/devices/_opencl_device.pyx @@ -49,3 +49,19 @@ cdef class OpenCL(Device): """ return CppOpenCL.num_devices(platform_id) + + @staticmethod + def check_support(unsigned platform_id, unsigned device_id): + """Checks whether the device corresponding to the specified ID is + supported. + + :param platform_id: Platform ID to check. + :type: int + :param device_id: Device ID to check. + :type: int + :return: True if this class supports the specified device, + False otherwise. + :rtype: bool + + """ + return CppOpenCL.check_support(platform_id, device_id) diff --git a/primitiv/initializers/__init__.py b/primitiv/initializers/__init__.py index 715640b..7c55bc8 100644 --- a/primitiv/initializers/__init__.py +++ b/primitiv/initializers/__init__.py @@ -4,6 +4,9 @@ from primitiv.initializers._initializer_impl import Identity from primitiv.initializers._initializer_impl import XavierUniform from primitiv.initializers._initializer_impl import XavierNormal +from primitiv.initializers._initializer_impl import XavierUniformConv2D +from primitiv.initializers._initializer_impl import XavierNormalConv2D + __all__ = [ "Constant", @@ -12,4 +15,6 @@ "Identity", "XavierUniform", "XavierNormal", + "XavierUniformConv2D", + "XavierNormalConv2D", ] diff --git a/primitiv/initializers/_initializer_impl.pxd b/primitiv/initializers/_initializer_impl.pxd index 9648168..433b1a5 100644 --- a/primitiv/initializers/_initializer_impl.pxd +++ b/primitiv/initializers/_initializer_impl.pxd @@ -1,7 +1,7 @@ from primitiv._initializer cimport CppInitializer, Initializer -cdef extern from "primitiv/initializer_impl.h": +cdef extern from "primitiv/core/initializer_impl.h": cdef cppclass CppConstant "primitiv::initializers::Constant" (CppInitializer): CppConstant(float k) @@ -20,6 +20,12 @@ cdef extern from "primitiv/initializer_impl.h": cdef cppclass CppXavierNormal "primitiv::initializers::XavierNormal" (CppInitializer): CppXavierNormal(float scale) + cdef cppclass CppXavierUniformConv2D "primitiv::initializers::XavierUniformConv2D" (CppInitializer): + CppXavierUniformConv2D(float scale) + + cdef cppclass CppXavierNormalConv2D "primitiv::initializers::XavierNormalConv2D" (CppInitializer): + CppXavierNormalConv2D(float scale) + cdef class Constant(Initializer): pass @@ -38,3 +44,9 @@ cdef class XavierUniform(Initializer): cdef class XavierNormal(Initializer): pass + +cdef class XavierUniformConv2D(Initializer): + pass + +cdef class XavierNormalConv2D(Initializer): + pass diff --git a/primitiv/initializers/_initializer_impl.pyx b/primitiv/initializers/_initializer_impl.pyx index 75ce94f..c8106c1 100644 --- a/primitiv/initializers/_initializer_impl.pyx +++ b/primitiv/initializers/_initializer_impl.pyx @@ -107,7 +107,7 @@ cdef class XavierUniform(Initializer): def __init__(self, scale = 1.0): """Crates a new initializer object. - :param scale: Scale of the distribusion. + :param scale: Additional scaling factor of the uniform distribution. :type scale: float """ @@ -147,3 +147,53 @@ cdef class XavierNormal(Initializer): temp = self.wrapped_newed del temp self.wrapped_newed = NULL + + +cdef class XavierUniformConv2D(Initializer): + """The Xavier initialization with the uniform distribution for conv2d filters. + + """ + + def __init__(self, scale = 1.0): + """Creates a new `XavierUniformConv2D` initializer. + + :param scale: Additional scaling factor of the uniform distribution. + :type scale: float + + """ + if self.wrapped_newed is not NULL: + raise TypeError("__init__() has already been called.") + self.wrapped_newed = new CppXavierUniformConv2D(scale) + self.wrapped = self.wrapped_newed + + def __dealloc__(self): + cdef CppXavierUniformConv2D *temp + if self.wrapped_newed is not NULL: + temp = self.wrapped_newed + del temp + self.wrapped_newed = NULL + + +cdef class XavierNormalConv2D(Initializer): + """The Xavier initialization with the normal distribution for conv2d filters. + + """ + + def __init__(self, scale = 1.0): + """Creates a new `XavierNormalConv2D` initializer. + + :param scale: Additional scaling factor of the normal distribution. + :type scale: float + + """ + if self.wrapped_newed is not NULL: + raise TypeError("__init__() has already been called.") + self.wrapped_newed = new CppXavierNormalConv2D(scale) + self.wrapped = self.wrapped_newed + + def __dealloc__(self): + cdef CppXavierNormalConv2D *temp + if self.wrapped_newed is not NULL: + temp = self.wrapped_newed + del temp + self.wrapped_newed = NULL diff --git a/primitiv/optimizers/_optimizer_impl.pxd b/primitiv/optimizers/_optimizer_impl.pxd index 5a349a4..2654281 100644 --- a/primitiv/optimizers/_optimizer_impl.pxd +++ b/primitiv/optimizers/_optimizer_impl.pxd @@ -5,7 +5,7 @@ from primitiv._device cimport CppDevice from primitiv._optimizer cimport CppOptimizer, Optimizer -cdef extern from "primitiv/optimizer_impl.h": +cdef extern from "primitiv/core/optimizer_impl.h": cdef cppclass CppSGD "primitiv::optimizers::SGD" (CppOptimizer): CppSGD(float eta) float eta() diff --git a/primitiv/py_optimizer.h b/primitiv/py_optimizer.h index 3b90636..2fbec33 100644 --- a/primitiv/py_optimizer.h +++ b/primitiv/py_optimizer.h @@ -1,7 +1,7 @@ #ifndef PRIMITIV_PYTHON_PY_OPTIMIZER_H_ #define PRIMITIV_PYTHON_PY_OPTIMIZER_H_ -#include +#include #include __PYX_EXTERN_C int primitiv_python_optimizer_get_configs( @@ -34,7 +34,7 @@ class PyOptimizer : public primitiv::Optimizer { if (ret == -1) { // NOTE(vbkaisetsu): This is just a trigger of throwing an error. // This message is not passed to Python. - THROW_ERROR("error: get_configs"); + PRIMITIV_THROW_ERROR("error: get_configs"); } Optimizer::get_configs(uint_configs, float_configs); } @@ -46,7 +46,7 @@ class PyOptimizer : public primitiv::Optimizer { if (ret == -1) { // NOTE(vbkaisetsu): This is just a trigger of throwing an error. // This message is not passed to Python. - THROW_ERROR("error: set_configs"); + PRIMITIV_THROW_ERROR("error: set_configs"); } } @@ -55,7 +55,7 @@ class PyOptimizer : public primitiv::Optimizer { if (ret == -1) { // NOTE(vbkaisetsu): This is just a trigger of throwing an error. // This message is not passed to Python. - THROW_ERROR("error: configure_parameter"); + PRIMITIV_THROW_ERROR("error: configure_parameter"); } } @@ -64,7 +64,7 @@ class PyOptimizer : public primitiv::Optimizer { if (ret == -1) { // NOTE(vbkaisetsu): This is just a trigger of throwing an error. // This message is not passed to Python. - THROW_ERROR("error: update_parameter"); + PRIMITIV_THROW_ERROR("error: update_parameter"); } } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e9940fa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +numpy>=1.16.1 +cython>=0.29.5 +cmake>=0.9.0 +scikit-build>=0.6.1 diff --git a/setup.py b/setup.py index d040456..b49abb9 100755 --- a/setup.py +++ b/setup.py @@ -3,16 +3,20 @@ import os import sys +from distutils.dir_util import copy_tree from setuptools.extension import Extension import numpy as np from Cython.Build import build_ext -VERSION = "0.3.0" +VERSION = "0.4.0" SUBMODULE_DIR = "primitiv-core" +EIGEN_DIR = "eigen-headers" + SUBMODULE_CMAKELIST = os.path.join(SUBMODULE_DIR, "CMakeLists.txt") +EIGEN_HEADER_DIR = os.path.join(EIGEN_DIR, "Eigen") build_number = os.getenv("PRIMITIV_PYTHON_BUILD_NUMBER") if build_number is not None: @@ -20,6 +24,14 @@ else: version_full = VERSION +bundle_eigen_headers = False +if "--bundle-eigen-headers" in sys.argv: + i = sys.argv.index("--bundle-eigen-headers") + sys.argv.pop(i) + eigen_path = sys.argv.pop(i) + copy_tree(eigen_path, EIGEN_DIR) + bundle_eigen_headers = True + dirname = os.path.dirname(os.path.abspath(__file__)) if "--no-build-core-library" in sys.argv: @@ -28,7 +40,15 @@ else: build_core = os.path.exists(os.path.join(dirname, SUBMODULE_CMAKELIST)) +eigen_bundled_exists = os.path.exists(os.path.join(dirname, EIGEN_HEADER_DIR)) +if "--disable-eigen" in sys.argv: + enable_eigen = False + sys.argv.remove("--disable-eigen") +else: + enable_eigen = eigen_bundled_exists + if build_core: + import skbuild from skbuild import setup else: from setuptools import setup @@ -51,6 +71,10 @@ enable_cuda = True sys.argv.remove("--enable-cuda") +if "--enable-eigen" in sys.argv: + enable_eigen = True + sys.argv.remove("--enable-eigen") + enable_opencl = False if "--enable-opencl" in sys.argv: enable_opencl = True @@ -65,10 +89,10 @@ def ext_common_args(*args, libraries=[], **kwargs): *args, **kwargs, language="c++", libraries=libs, - library_dirs=["_skbuild/cmake-install/lib"], + library_dirs=[os.path.join(skbuild.constants.CMAKE_INSTALL_DIR, "lib")], include_dirs=[ np.get_include(), - "_skbuild/cmake-install/include", + os.path.join(skbuild.constants.CMAKE_INSTALL_DIR, "include"), os.path.join(dirname, "primitiv"), ], extra_compile_args=["-std=c++11"], @@ -130,12 +154,20 @@ def ext_common_args(*args, libraries=[], **kwargs): ) ) +if enable_eigen: + ext_modules.append( + ext_common_args( + "primitiv.devices._eigen_device", + sources=["primitiv/devices/_eigen_device.pyx"], + ) + ) + if enable_opencl: ext_modules.append( ext_common_args( "primitiv.devices._opencl_device", libraries=[ - "clBLAS", + "clblast", "OpenCL", ], sources=["primitiv/devices/_opencl_device.pyx"], @@ -146,10 +178,29 @@ def ext_common_args(*args, libraries=[], **kwargs): if build_core: setup_kwargs["cmake_source_dir"] = SUBMODULE_DIR setup_kwargs["cmake_install_dir"] = "./" - setup_kwargs["setup_requires"] = ["scikit-build"] + setup_kwargs["setup_requires"] = [ + "cmake>=0.9.0", + "cython>=0.29.5", + "scikit-build>=0.6.1", + ] setup_kwargs["cmake_args"] = ["-DPRIMITIV_BUILD_STATIC_LIBRARY=ON"] + if sys.platform == "darwin": + # NOTE(vbkaisetsu): + # scikit-build adds -DCMAKE_OSX_DEPLOYMENT_TARGET with the default target if it does not + # set manually. However scikit-build does not check cmake_args argument of setup() + # for the target. + try: + cmake_args_pos = sys.argv.index("--") + except ValueError: + cmake_args_pos = len(sys.argv) + sys.argv.append("--") + sys.argv.insert(cmake_args_pos + 1, "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.12") if enable_cuda: setup_kwargs["cmake_args"].append("-DPRIMITIV_USE_CUDA=ON") + if enable_eigen: + setup_kwargs["cmake_args"].append("-DPRIMITIV_USE_EIGEN=ON") + if eigen_bundled_exists: + setup_kwargs["cmake_args"].append("-DEIGEN3_INCLUDE_DIR=%s" % os.path.join(dirname, EIGEN_DIR)) if enable_opencl: setup_kwargs["cmake_args"].append("-DPRIMITIV_USE_OPENCL=ON") @@ -158,6 +209,9 @@ def ext_common_args(*args, libraries=[], **kwargs): print("recursive-include primitiv *.pyx *.pxd", file=fp) if bundle_core_library: print("recursive-include %s *" % SUBMODULE_DIR, file=fp) + if bundle_eigen_headers: + print("include %s/COPYING.* %s/README.md" % (EIGEN_DIR, EIGEN_DIR), file=fp) + print("recursive-include %s *" % EIGEN_HEADER_DIR, file=fp) setup( name="primitiv", @@ -187,8 +241,7 @@ def ext_common_args(*args, libraries=[], **kwargs): "primitiv.optimizers", ], install_requires=[ - "cython", - "numpy", + "numpy>=1.16.1", ], **setup_kwargs, ) diff --git a/tests/graph_test.py b/tests/graph_test.py new file mode 100644 index 0000000..ca0de13 --- /dev/null +++ b/tests/graph_test.py @@ -0,0 +1,420 @@ +import unittest + +from primitiv import devices as D +from primitiv import functions as F +from primitiv import initializers as I +from primitiv import tensor_functions as tF +from primitiv import Device +from primitiv import Graph +from primitiv import Node +from primitiv import Parameter +from primitiv import Shape + +import numpy as np + + +class GraphTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.dev = D.Naive() + cls.dev2 = D.Naive() + + @classmethod + def tearDownClass(cls): + pass + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_GraphTest_CheckInvalidNode(self): + node = Node() + self.assertFalse(node.valid()) + with self.assertRaises(RuntimeError): + node.graph() + with self.assertRaises(RuntimeError): + node.operator_id() + with self.assertRaises(RuntimeError): + node.value_id() + with self.assertRaises(RuntimeError): + node.shape() + with self.assertRaises(RuntimeError): + node.device() + with self.assertRaises(RuntimeError): + node.to_float() + with self.assertRaises(RuntimeError): + node.to_list() + with self.assertRaises(RuntimeError): + node.to_ndarrays() + with self.assertRaises(RuntimeError): + node.backward() + + def test_GraphTest_CheckMultipleDevices(self): + Device.set_default(GraphTest.dev) + + g = Graph() + Graph.set_default(g) + + data1 = [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4] + data2 = [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2] + data3 = [1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6] + grad = [12, 1] + x1 = F.raw_input(Shape([2, 2], 3), data1) + x2 = F.raw_input(Shape([2, 2], 3), data2, GraphTest.dev2) + x3 = F.copy(x1, GraphTest.dev2) + x2 + self.assertEqual(Shape([2, 2], 3), x3.shape()) + self.assertIs(GraphTest.dev, x1.device()) + self.assertIs(GraphTest.dev2, x2.device()) + self.assertIs(GraphTest.dev2, x3.device()) + g.forward(x3) + self.assertEqual(data1, g.forward(x1).to_list()) + self.assertEqual(data1, x1.to_list()) + self.assertEqual(data2, g.forward(x2).to_list()) + self.assertEqual(data2, x2.to_list()) + self.assertEqual(data3, g.forward(x3).to_list()) + self.assertEqual(data3, x3.to_list()) + + def test_GraphTest_CheckInvalidMultipleDevices(self): + Device.set_default(GraphTest.dev) + + g = Graph() + Graph.set_default(g) + + dummy = [0] * 12 + x1 = F.raw_input(Shape([2, 2], 3), dummy) + x2 = F.raw_input(Shape([2, 2], 3), dummy, GraphTest.dev2) + x3 = x1 + x2 + with self.assertRaises(RuntimeError): + g.forward(x3) + + def test_GraphTest_CheckClear(self): + Device.set_default(GraphTest.dev) + + g = Graph() + Graph.set_default(g) + + self.assertEqual(0, g.num_operators()) + + F.raw_input([], [1]) + F.raw_input([], [1]) + self.assertEqual(2, g.num_operators()) + + g.clear() + self.assertEqual(0, g.num_operators()) + + F.raw_input([], [1]) + F.raw_input([], [1]) + F.raw_input([], [1]) + self.assertEqual(3, g.num_operators()) + + g.clear() + self.assertEqual(0, g.num_operators()) + + g.clear() + self.assertEqual(0, g.num_operators()) + + def test_GraphTest_CheckForwardBackward(self): + Device.set_default(GraphTest.dev) + + g = Graph() + Graph.set_default(g) + + data1 = [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4] + data3 = [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2] + + nodes = [] + nodes.append(F.raw_input(Shape([2, 2], 3), data1)) + nodes.append(F.ones([2, 2])) + nodes.append(F.raw_input(Shape([2, 2], 3), data3)) + nodes.append(nodes[0] + nodes[1]) + nodes.append(nodes[1] - nodes[2]) + nodes.append(nodes[3] * nodes[4]) + nodes.append(nodes[5] + 1) + nodes.append(F.sum(nodes[6], 0)) + nodes.append(F.sum(nodes[7], 1)) + nodes.append(F.batch.sum(nodes[8])) + + self.assertEqual(10, len(nodes)) + self.assertEqual(10, g.num_operators()) + + print(g.dump("dot")) + + expected_shapes = [ + Shape([2, 2], 3), Shape([2, 2]), Shape([2, 2], 3), + Shape([2, 2], 3), Shape([2, 2], 3), Shape([2, 2], 3), + Shape([2, 2], 3), + Shape([1, 2], 3), Shape([], 3), Shape([]), + ] + for i, node in enumerate(nodes): + self.assertEqual(expected_shapes[i], node.shape()) + self.assertIs(GraphTest.dev, node.device()) + + g.forward(nodes[-1]) + + expected_values = [ + [1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4], + [1, 1, 1, 1], + [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2], + [2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 4, 5], + [1, 1, 1, 1, 0, 0, 0, 0, -1, -1, -1, -1], + [2, 3, 4, 5, 0, 0, 0, 0, -2, -3, -4, -5], + [3, 4, 5, 6, 1, 1, 1, 1, -1, -2, -3, -4], + [7, 11, 2, 2, -3, -7], + [18, 4, -10], + [12], + ] + for i, node in enumerate(nodes): + val = g.forward(node) + self.assertTrue(val.valid()) + self.assertEqual(expected_values[i], val.to_list()) + self.assertEqual(expected_values[i], node.to_list()) + + def test_GraphTest_CheckXor(self): + Device.set_default(GraphTest.dev) + + w1 = Parameter([2, 2], I.Constant(0)) + w1.value += tF.raw_input([2, 2], [1, -1, 1, -1]) + b1 = Parameter([2], I.Constant(0)) + b1.value += tF.raw_input([2], [-1, -1]) + w2 = Parameter([1, 2], I.Constant(0)) + w2.value += tF.raw_input([1, 2], [1, 1]) + b2 = Parameter([], I.Constant(0)) + b2.value += tF.raw_input([], [1]) + + inputs = [1, 1, 1, -1, -1, 1, -1, -1] + outputs = [1, -1, -1, 1] + + g = Graph() + Graph.set_default(g) + + nodes = [] + + # sources + nodes.append(F.raw_input(Shape([2], 4), inputs)) + nodes.append(F.parameter(w1)) + nodes.append(F.parameter(b1)) + nodes.append(F.parameter(w2)) + nodes.append(F.parameter(b2)) + # calculation + nodes.append(F.matmul(nodes[1], nodes[0])); + nodes.append(nodes[5] + nodes[2]); + nodes.append(F.tanh(nodes[6])); + nodes.append(F.matmul(nodes[3], nodes[7])); + nodes.append(nodes[8] + nodes[4]); + # losses + nodes.append(F.raw_input(Shape([], 4), outputs)); + nodes.append(nodes[9] - nodes[10]); + nodes.append(nodes[11] * nodes[11]); + nodes.append(F.batch.sum(nodes[12])); + + self.assertEqual(len(nodes), g.num_operators()) + print(g.dump("dot")) + + g.forward(nodes[-1]) + + # Check all node values. + h1 = .76159416 # tanh(1) + h2 = .99505475 # tanh(3) + h3 = -.23346060 # tanh(1) - tanh(3) + h4 = -1.5231883 # -2 * tanh(1) + h5 = .76653940 # 1 + tanh(1) - tanh(3) + h6 = -.52318831 # 1 - 2 * tanh(1) + h7 = .47681169 # 2 - 2 * tanh(1) + expected_values = [ + [1, 1, 1, -1, -1, 1, -1, -1], + [1, -1, 1, -1], + [-1, -1], + [1, 1], + [1], + [2, -2, 0, 0, 0, 0, -2, 2], + [1, -3, -1, -1, -1, -1, -3, 1], + [h1, -h2, -h1, -h1, -h1, -h1, -h2, h1], + [h3, h4, h4, h3], + [h5, h6, h6, h5], + [1, -1, -1, 1], + [h3, h7, h7, h3], + [h3 * h3, h7 * h7, h7 * h7, h3 * h3], + [2 * (h3 * h3 + h7 * h7)], + ] + for i, node in enumerate(nodes): + val = g.forward(nodes[i]) + self.assertTrue(val.valid()) + self.assertTrue(np.isclose(expected_values[i], val.to_list()).all()) + self.assertTrue(np.isclose(expected_values[i], node.to_list()).all()) + + def test_GraphTest_CheckLSTM(self): + Device.set_default(GraphTest.dev) + + pWix = Parameter([2, 2], I.Constant(0)) + pWix.value += tF.raw_input([2, 2], [.3, .1, .5, .3]) + pWfx = Parameter([2, 2], I.Constant(0)) + pWfx.value += tF.raw_input([2, 2], [.4, .1, .5, .8]) + pWox = Parameter([2, 2], I.Constant(0)) + pWox.value += tF.raw_input([2, 2], [.5, .9, .9, .7]) + pWjx = Parameter([2, 2], I.Constant(0)) + pWjx.value += tF.raw_input([2, 2], [.2, .6, .9, .3]) + pWih = Parameter([2, 2], I.Constant(0)) + pWih.value += tF.raw_input([2, 2], [.2, .3, .3, .3]) + pWfh = Parameter([2, 2], I.Constant(0)) + pWfh.value += tF.raw_input([2, 2], [.8, .4, .8, .3]) + pWoh = Parameter([2, 2], I.Constant(0)) + pWoh.value += tF.raw_input([2, 2], [.6, .2, .2, .7]) + pWjh = Parameter([2, 2], I.Constant(0)) + pWjh.value += tF.raw_input([2, 2], [.6, .4, .9, .5]) + pbi = Parameter([2], I.Constant(0)) + pbf = Parameter([2], I.Constant(0)) + pbo = Parameter([2], I.Constant(0)) + pbj = Parameter([2], I.Constant(0)) + + g = Graph() + Graph.set_default(g) + + x = F.raw_input(Shape([2], 2), [2, -2, 0.5, -0.5]) + h = F.raw_input(Shape([2], 2), [-1, 1, -0.5, 0.5]) + c = F.zeros([2]) + Wfx = F.parameter(pWfx) + Wix = F.parameter(pWix) + Wox = F.parameter(pWox) + Wjx = F.parameter(pWjx) + Wih = F.parameter(pWih) + Wfh = F.parameter(pWfh) + Woh = F.parameter(pWoh) + Wjh = F.parameter(pWjh) + bi = F.parameter(pbi) + bf = F.parameter(pbf) + bo = F.parameter(pbo) + bj = F.parameter(pbj) + + i = F.sigmoid(F.matmul(Wix, x) + F.matmul(Wih, h) + bi) + f = F.sigmoid(F.matmul(Wfx, x) + F.matmul(Wfh, h) + bf) + o = F.sigmoid(F.matmul(Wox, x) + F.matmul(Woh, h) + bo) + j = F.tanh(F.matmul(Wjx, x) + F.matmul(Wjh, h) + bj) + cc = f * c + i * j + hh = o * F.tanh(cc) + + t = F.zeros([2]) + diff = hh - t + loss = diff * diff + sum_loss = F.batch.sum(F.sum(loss, 0)) + + self.assertEqual(45, g.num_operators()); + + loss_tensor = g.forward(loss) + sum_loss_tensor = g.forward(sum_loss) + sum_loss.backward() + + expected_losses = [ + 5.7667205e-03, 2.8605087e-02, 1.4819370e-03, 3.0073307e-03 + ] + expected_sum_loss = sum(expected_losses) + + self.assertTrue(np.isclose(expected_losses, loss_tensor.to_list()).all()) + self.assertTrue(np.isclose(expected_losses, loss.to_list()).all()) + self.assertAlmostEqual(expected_sum_loss, sum_loss_tensor.to_float()) + self.assertAlmostEqual(expected_sum_loss, sum_loss.to_float()) + + def print_node_val(name, value): + print("%s: value=%s" % (name, value.to_ndarrays())) + + print("VALUES:") + print_node_val("x", x) + print_node_val("h", h) + print_node_val("c", c) + print_node_val("Wix", Wix) + print_node_val("Wfx", Wfx) + print_node_val("Wox", Wox) + print_node_val("Wjx", Wjx) + print_node_val("Wih", Wih) + print_node_val("Wfh", Wfh) + print_node_val("Woh", Woh) + print_node_val("Wjh", Wjh) + print_node_val("bi", bi) + print_node_val("bf", bf) + print_node_val("bo", bo) + print_node_val("bj", bj) + print_node_val("i", i) + print_node_val("f", f) + print_node_val("o", o) + print_node_val("j", j) + print_node_val("cc", cc) + print_node_val("hh", hh) + print_node_val("t", t) + print_node_val("diff", diff) + print_node_val("loss", loss) + + def test_GraphTest_CheckConcatLSTM(self): + Device.set_default(GraphTest.dev) + + pWx = Parameter([8, 2], I.Constant(0)) + pWx.value += tF.raw_input([8, 2], [ + .3, .1, .4, .1, .5, .9, .2, .6, + .5, .3, .5, .8, .9, .7, .9, .3, + ]) + pWh = Parameter([8, 2], I.Constant(0)) + pWh.value += tF.raw_input([8, 2], [ + .2, .3, .8, .4, .6, .2, .6, .4, + .3, .3, .8, .3, .2, .7, .9, .5, + ]) + pb = Parameter([8], I.Constant(0)) + + g = Graph() + Graph.set_default(g) + + x = F.raw_input(Shape([2], 2), [2, -2, 0.5, -0.5]) + h = F.raw_input(Shape([2], 2), [-1, 1, -0.5, 0.5]) + c = F.zeros([2]) + Wx = F.parameter(pWx) + Wh = F.parameter(pWh) + b = F.parameter(pb) + + u = F.matmul(Wx, x) + F.matmul(Wh, h) + b + i = F.sigmoid(F.slice(u, 0, 0, 2)) + f = F.sigmoid(F.slice(u, 0, 2, 4)) + o = F.sigmoid(F.slice(u, 0, 4, 6)) + j = F.tanh(F.slice(u, 0, 6, 8)) + cc = f * c + i * j + hh = o * F.tanh(cc) + + t = F.zeros([2]) + diff = hh - t + loss = diff * diff + sum_loss = F.batch.sum(F.sum(loss, 0)) + + self.assertEqual(28, g.num_operators()) + + loss_tensor = g.forward(loss) + sum_loss_tensor = g.forward(sum_loss) + sum_loss.backward() + + expected_losses = [ + 5.7667205e-03, 2.8605087e-02, 1.4819370e-03, 3.0073307e-03 + ] + expected_sum_loss = sum(expected_losses) + + self.assertTrue(np.isclose(expected_losses, loss_tensor.to_list()).all()) + self.assertTrue(np.isclose(expected_losses, loss.to_list()).all()); + self.assertAlmostEqual(expected_sum_loss, sum_loss_tensor.to_float()); + self.assertAlmostEqual(expected_sum_loss, sum_loss.to_float()); + + def print_node_val(name, value): + print("%s: value=%s" % (name, value.to_ndarrays())) + + print("VALUES:") + print_node_val("x", x) + print_node_val("h", h) + print_node_val("c", c) + print_node_val("Wx", Wx) + print_node_val("Wh", Wh) + print_node_val("b", b) + print_node_val("i", i) + print_node_val("f", f) + print_node_val("o", o) + print_node_val("j", j) + print_node_val("cc", cc) + print_node_val("hh", hh) + print_node_val("t", t) + print_node_val("diff", diff) + print_node_val("loss", loss) diff --git a/tests/instance_match.py b/tests/instance_match.py index 2e57c9a..8e75388 100644 --- a/tests/instance_match.py +++ b/tests/instance_match.py @@ -43,7 +43,7 @@ def test_device_instance(self): my_device = Naive() self.assertIsNot(my_device, self.device) - node = F.raw_input([], [0], dev=my_device) + node = F.raw_input([], [0], device=my_device) dev = node.device() self.assertIs(dev, my_device) diff --git a/tests/model.py b/tests/model.py index 9139780..9206f63 100644 --- a/tests/model.py +++ b/tests/model.py @@ -38,40 +38,38 @@ def tearDown(self): def test_model_load_save(self): submodel = TestModel() - submodel.sp1 = Parameter([2, 4], I.Constant(0)) - submodel.sp1.value = tF.input(np.array([[0,1,2,3],[4,5,6,7]])) - submodel.sp2 = Parameter([2, 4], I.Constant(0)) - submodel.sp2.value = tF.input(np.array([[9,8,7,6],[5,4,3,2]])) - submodel.add("sp1", submodel.sp1) - submodel.add("sp2", submodel.sp2) + sp1 = Parameter([2, 4], I.Constant(0)) + sp1.value = tF.input(np.array([[0,1,2,3],[4,5,6,7]])) + sp2 = Parameter([2, 4], I.Constant(0)) + sp2.value = tF.input(np.array([[9,8,7,6],[5,4,3,2]])) + submodel.add("sp1", sp1) + submodel.add("sp2", sp2) parentmodel = TestModel() - parentmodel.p1 = Parameter([4, 2], I.Constant(0)) - parentmodel.p1.value = tF.input(np.array([[0,1],[2,3],[4,5],[6,7]])) - parentmodel.p2 = Parameter([4, 2], I.Constant(0)) - parentmodel.p2.value = tF.input(np.array([[9,8],[7,6],[5,4],[3,2]])) - parentmodel.sub = submodel - parentmodel.add("p1", parentmodel.p1) - parentmodel.add("p2", parentmodel.p2) - parentmodel.add("sub", parentmodel.sub) + p1 = Parameter([4, 2], I.Constant(0)) + p1.value = tF.input(np.array([[0,1],[2,3],[4,5],[6,7]])) + p2 = Parameter([4, 2], I.Constant(0)) + p2.value = tF.input(np.array([[9,8],[7,6],[5,4],[3,2]])) + parentmodel.add("p1", p1) + parentmodel.add("p2", p2) + parentmodel.add("sub", submodel) submodel_load = TestModel() - submodel_load.sp1 = Parameter() - submodel_load.sp2 = Parameter() - submodel_load.add("sp1", submodel_load.sp1) - submodel_load.add("sp2", submodel_load.sp2) + sp1 = Parameter() + sp2 = Parameter() + submodel_load.add("sp1", sp1) + submodel_load.add("sp2", sp2) parentmodel_load = TestModel() - parentmodel_load.p1 = Parameter() - parentmodel_load.p2 = Parameter() - parentmodel_load.sub = submodel_load - parentmodel_load.add("p1", parentmodel_load.p1) - parentmodel_load.add("p2", parentmodel_load.p2) - parentmodel_load.add("sub", parentmodel_load.sub) + p1 = Parameter() + p2 = Parameter() + parentmodel_load.add("p1", p1) + parentmodel_load.add("p2", p2) + parentmodel_load.add("sub", submodel_load) with tempfile.NamedTemporaryFile() as fp: parentmodel.save(fp.name) parentmodel_load.load(fp.name) - self.assertTrue((parentmodel_load.p1.value.to_ndarrays()[0] == np.array([[0,1],[2,3],[4,5],[6,7]])).all()) - self.assertTrue((parentmodel_load.p2.value.to_ndarrays()[0] == np.array([[9,8],[7,6],[5,4],[3,2]])).all()) - self.assertTrue((parentmodel_load.sub.sp1.value.to_ndarrays()[0] == np.array([[0,1,2,3],[4,5,6,7]])).all()) - self.assertTrue((parentmodel_load.sub.sp2.value.to_ndarrays()[0] == np.array([[9,8,7,6],[5,4,3,2]])).all()) + self.assertTrue((parentmodel_load["p1"].value.to_ndarrays()[0] == np.array([[0,1],[2,3],[4,5],[6,7]])).all()) + self.assertTrue((parentmodel_load["p2"].value.to_ndarrays()[0] == np.array([[9,8],[7,6],[5,4],[3,2]])).all()) + self.assertTrue((parentmodel_load["sub", "sp1"].value.to_ndarrays()[0] == np.array([[0,1,2,3],[4,5,6,7]])).all()) + self.assertTrue((parentmodel_load["sub", "sp2"].value.to_ndarrays()[0] == np.array([[9,8,7,6],[5,4,3,2]])).all()) def test_model_parameter(self): model = Model() @@ -127,46 +125,53 @@ def test_model_invalid_operation(self): model1[(0, 1)] with self.assertRaises(TypeError): model1[[0, 1]] - + model3 = TestModel() + model3.p = Parameter() + model3.m = TestModel() + model3.a = "test" + del model3.a + self.assertNotIn("a", model3.__dict__) + with self.assertRaises(TypeError): + del model3.p + self.assertIn("p", model3.__dict__) + with self.assertRaises(TypeError): + del model3.m + self.assertIn("m", model3.__dict__) def test_model_get_all_parameters(self): submodel = TestModel() - submodel.sp1 = Parameter() - submodel.sp2 = Parameter() - submodel.add("sp1", submodel.sp1) - submodel.add("sp2", submodel.sp2) + sp1 = Parameter() + sp2 = Parameter() + submodel.add("sp1", sp1) + submodel.add("sp2", sp2) parentmodel = TestModel() - parentmodel.p1 = Parameter() - parentmodel.p2 = Parameter() - parentmodel.sub = submodel - parentmodel.add("p1", parentmodel.p1) - parentmodel.add("p2", parentmodel.p2) - parentmodel.add("sub", parentmodel.sub) + p1 = Parameter() + p2 = Parameter() + sub = submodel + parentmodel.add("p1", p1) + parentmodel.add("p2", p2) + parentmodel.add("sub", sub) params = parentmodel.get_all_parameters() - self.assertIs(params[("p1",)], parentmodel.p1) - self.assertIs(params[("p2",)], parentmodel.p2) - self.assertIs(params[("sub", "sp1")], parentmodel.sub.sp1) - self.assertIs(params[("sub", "sp2")], parentmodel.sub.sp2) + self.assertIs(params[("p1",)], p1) + self.assertIs(params[("p2",)], p2) + self.assertIs(params[("sub", "sp1")], sp1) + self.assertIs(params[("sub", "sp2")], sp2) - def test_model_scan_attributes(self): + def test_model_setattr(self): model = TestModel() model.p1 = Parameter() - model.add("p1_manual", model.p1) model.p2 = Parameter() model.p3 = Parameter() model.m1 = TestModel() - model.add("m1_manual", model.m1) model.m2 = TestModel() model.m3 = TestModel() - model.scan_attributes() - self.assertIs(model["p1_manual"], model.p1) + self.assertIs(model["p1"], model.p1) self.assertIs(model["p2"], model.p2) self.assertIs(model["p3"], model.p3) model.p4 = Parameter() - self.assertIs(model["m1_manual"], model.m1) + self.assertIs(model["m1"], model.m1) self.assertIs(model["m2"], model.m2) self.assertIs(model["m3"], model.m3) model.m4 = TestModel() - model.scan_attributes() self.assertIs(model["p4"], model.p4) self.assertIs(model["m4"], model.m4) diff --git a/tests/model_test.py b/tests/model_test.py new file mode 100644 index 0000000..ee8f0df --- /dev/null +++ b/tests/model_test.py @@ -0,0 +1,520 @@ +import tempfile +import unittest + +from primitiv import Device +from primitiv import Model +from primitiv import Parameter +from primitiv import Shape +from primitiv.devices import Naive +from primitiv import initializers as I +from primitiv import tensor_functions as tF + + +class ModelTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.device = Naive() + + @classmethod + def tearDownClass(cls): + pass + + def setUp(self): + Device.set_default(ModelTest.device) + + def tearDown(self): + pass + + def test_ModelTest_CheckAddParameter(self): + m = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + + m.add("p1", p1) + m.add("p1", p1) + with self.assertRaises(RuntimeError): + m.add("x", p1) + + with self.assertRaises(RuntimeError): + m.add("p1", p2) + m.add("p2", p2) + m.add("p2", p2) + with self.assertRaises(RuntimeError): + m.add("x", p2) + + with self.assertRaises(RuntimeError): + m.add("p1", p3) + with self.assertRaises(RuntimeError): + m.add("p2", p3) + m.add("p3", p3) + m.add("p3", p3) + with self.assertRaises(RuntimeError): + m.add("x", p3) + + def test_ModelTest_CheckAddSubmodel(self): + m = Model() + sm1 = Model() + sm2 = Model() + p1 = Parameter() + p2 = Parameter() + + m.add("p1", p1) + m.add("sm1", sm1) + m.add("sm1", sm1) + with self.assertRaises(RuntimeError): + m.add("x", sm1) + + with self.assertRaises(RuntimeError): + m.add("p1", p2) + with self.assertRaises(RuntimeError): + m.add("sm1", p2) + with self.assertRaises(RuntimeError): + m.add("p1", sm2) + with self.assertRaises(RuntimeError): + m.add("sm1", sm2) + + m.add("p2", p2) + m.add("sm2", sm2) + m.add("sm2", sm2) + with self.assertRaises(RuntimeError): + m.add("x", sm2) + + def test_ModelTest_CheckAddSubmodelCycle(self): + m1 = Model() + m2 = Model() + m3 = Model() + m4 = Model() + + with self.assertRaises(RuntimeError): + m1.add("self", m1) + + m1.add("m2", m2) + with self.assertRaises(RuntimeError): + m2.add("m1", m1) + + m2.add("m3", m3) + with self.assertRaises(RuntimeError): + m3.add("m1", m1) + with self.assertRaises(RuntimeError): + m3.add("m2", m2) + + m2.add("m4", m4) + with self.assertRaises(RuntimeError): + m4.add("m1", m1) + with self.assertRaises(RuntimeError): + m4.add("m2", m2) + + m4.add("m3", m3) + + def test_ModelTest_CheckGetParameteer(self): + m = Model() + sm = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + m.add("p1", p1) + m.add("p2", p2) + sm.add("p3", p3) + m.add("sm", sm) + + self.assertIs(p1, m["p1"]) + self.assertIs(p2, m["p2"]) + with self.assertRaises(TypeError): + m["p3"] + self.assertIs(sm, m["sm"]) + with self.assertRaises(TypeError): + m["x"] + + def test_ModelTest_CheckGetParameterRecursiveByTuple(self): + m = Model() + sm = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + m.add("p1", p1) + sm.add("p2", p2) + sm.add("p3", p3) + m.add("sm", sm) + + self.assertIs(p1, m["p1"]) + self.assertIs(p2, m["sm", "p2"]) + self.assertIs(p3, m["sm", "p3"]) + self.assertIs(p2, sm["p2"]) + self.assertIs(p3, sm["p3"]) + with self.assertRaises(TypeError): + m["p2"] + with self.assertRaises(TypeError): + m["p3"] + m["sm"] + with self.assertRaises(TypeError): + m["sm", "p1"] + with self.assertRaises(TypeError): + sm["p1"] + with self.assertRaises(TypeError): + m["x"] + + def test_ModelTest_CheckGetSubmodel(self): + m = Model() + sm1 = Model() + sm2 = Model() + ssm = Model() + p = Parameter() + m.add("p", p) + m.add("sm1", sm1) + m.add("sm2", sm2) + sm1.add("ssm", ssm) + + self.assertIs(sm1, m["sm1"]); + self.assertIs(sm2, m["sm2"]); + with self.assertRaises(TypeError): + m["ssm"] + m["p"] + + def test_ModelTest_CheckGetSubmodelRecursiveByTuple(self): + m = Model() + sm1 = Model() + sm2 = Model() + ssm = Model() + p = Parameter() + m.add("p", p) + m.add("sm1", sm1) + m.add("sm2", sm2) + sm1.add("ssm", ssm) + + self.assertIs(sm1, m["sm1"]); + self.assertIs(sm2, m["sm2"]); + self.assertIs(ssm, m["sm1", "ssm"]); + self.assertIs(ssm, sm1["ssm"]); + m["p"] + with self.assertRaises(TypeError): + m["ssm"] + with self.assertRaises(TypeError): + m["sm2", "ssm"] + with self.assertRaises(TypeError): + m["x"] + + def test_ModelTest_CheckGetAllParameters(self): + m = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + m.add("p1", p1) + m.add("p2", p2) + m.add("p3", p3) + params = m.get_all_parameters() + self.assertEqual(3, len(params)) + self.assertIsInstance(params, dict) + self.assertIs(p1, params[("p1",)]) + self.assertIs(p2, params[("p2",)]) + self.assertIs(p3, params[("p3",)]) + + def test_ModelTest_CheckGetAllParametersWithSubmodels(self): + m1 = Model() + m2 = Model() + m3 = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + m1.add("p", p1) + m2.add("p", p2) + m3.add("p", p3) + m1.add("sm", m2) + m2.add("sm", m3) + + params1 = m1.get_all_parameters() + self.assertEqual(3, len(params1)) + self.assertIsInstance(params1, dict) + self.assertIs(p1, params1[("p",)]) + self.assertIs(p2, params1[("sm", "p",)]) + self.assertIs(p3, params1[("sm", "sm", "p",)]) + + params2 = m2.get_all_parameters() + self.assertEqual(2, len(params2)) + self.assertIsInstance(params2, dict) + self.assertIs(p2, params2[("p",)]) + self.assertIs(p3, params2[("sm", "p",)]) + + params3 = m3.get_all_parameters() + self.assertEqual(1, len(params3)) + self.assertIsInstance(params3, dict) + self.assertIs(p3, params3[("p",)]) + + def test_ModelTest_CheckGetTrainableParameters(self): + m = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + m.add("p1", p1) + m.add("p2", p2) + m.add("p3", p3) + params = m.get_trainable_parameters() + self.assertEqual(3, len(params)) + self.assertIsInstance(params, dict) + self.assertIs(p1, params[("p1",)]); + self.assertIs(p2, params[("p2",)]); + self.assertIs(p3, params[("p3",)]); + + def test_ModelTest_CheckGetTrainableParametersWithSubmodels(self): + m1 = Model() + m2 = Model() + m3 = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + m1.add("p", p1) + m2.add("p", p2) + m3.add("p", p3) + m1.add("sm", m2) + m2.add("sm", m3) + + params1 = m1.get_trainable_parameters() + self.assertEqual(3, len(params1)) + self.assertIsInstance(params1, dict) + self.assertIs(p1, params1[("p",)]) + self.assertIs(p2, params1[("sm", "p",)]) + self.assertIs(p3, params1[("sm", "sm", "p",)]) + + params2 = m2.get_trainable_parameters() + self.assertEqual(2, len(params2)) + self.assertIsInstance(params2, dict) + self.assertIs(p2, params2[("p",)]) + self.assertIs(p3, params2[("sm", "p",)]) + + params3 = m3.get_trainable_parameters() + self.assertEqual(1, len(params3)) + self.assertIsInstance(params3, dict) + self.assertIs(p3, params3[("p",)]) + + + def test_ModelTest_CheckSaveLoad_Same(self): + shape = Shape([2, 2]) + values1 = [1, 2, 3, 4] + values2 = [5, 6, 7, 8] + tmp = tempfile.NamedTemporaryFile() + + m1 = Model() + m2 = Model() + p1 = Parameter(shape, I.Constant(0)) + p1.value += tF.raw_input(shape, values1) + p2 = Parameter(shape, I.Constant(0)) + p2.value += tF.raw_input(shape, values2) + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.save(tmp.name) + + m1 = Model() + m2 = Model() + p1 = Parameter() + p2 = Parameter() + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.load(tmp.name) + + self.assertTrue(p1.valid()) + self.assertTrue(p2.valid()) + self.assertEqual(shape, p1.shape()) + self.assertEqual(shape, p2.shape()) + self.assertEqual(values1, p1.value.to_list()) + self.assertEqual(values2, p2.value.to_list()) + + def test_ModelTest_CheckSaveLoad_Insufficient(self): + shape = Shape([2, 2]) + values1 = [1, 2, 3, 4] + values2 = [5, 6, 7, 8] + tmp = tempfile.NamedTemporaryFile() + + m1 = Model() + m2 = Model() + p1 = Parameter(shape, I.Constant(0)) + p1.value += tF.raw_input(shape, values1) + p2 = Parameter(shape, I.Constant(0)) + p2.value += tF.raw_input(shape, values2) + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.save(tmp.name) + + m1 = Model() + m2 = Model() + p1 = Parameter() + m1.add("p", p1) + m1.add("sm", m2) + + with self.assertRaises(RuntimeError): + m1.load(tmp.name) + + def test_ModelTest_CheckSaveLoad_Excessive(self): + shape = Shape([2, 2]) + values1 = [1, 2, 3, 4] + values2 = [5, 6, 7, 8] + tmp = tempfile.NamedTemporaryFile() + + m1 = Model() + m2 = Model() + p1 = Parameter(shape, I.Constant(0)) + p1.value += tF.raw_input(shape, values1) + p2 = Parameter(shape, I.Constant(0)) + p2.value += tF.raw_input(shape, values2) + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.save(tmp.name) + + m1 = Model() + m2 = Model() + p1 = Parameter() + p2 = Parameter() + p3 = Parameter() + m1.add("p", p1) + m2.add("p", p2) + m2.add("pp", p3) + m1.add("sm", m2) + + m1.load(tmp.name) + + self.assertTrue(p1.valid()) + self.assertTrue(p2.valid()) + self.assertFalse(p3.valid()) + self.assertEqual(shape, p1.shape()) + self.assertEqual(shape, p2.shape()) + self.assertEqual(values1, p1.value.to_list()) + self.assertEqual(values2, p2.value.to_list()) + + def test_ModelTest_CheckSaveLoadWithStats(self): + shape = Shape([2, 2]) + values1 = [1, 2, 3, 4] + values2 = [5, 6, 7, 8] + stats1 = [10, 20, 30, 40] + stats2 = [50, 60, 70, 80] + tmp = tempfile.NamedTemporaryFile() + + m1 = Model() + m2 = Model() + p1 = Parameter(shape, I.Constant(0)) + p1.value += tF.raw_input(shape, values1) + p2 = Parameter(shape, I.Constant(0)) + p2.value += tF.raw_input(shape, values2) + p1.add_stats("a", shape) + p2.add_stats("b", shape) + p1.stats["a"].reset_by_vector(stats1); + p2.stats["b"].reset_by_vector(stats2); + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.save(tmp.name) + + m1 = Model() + m2 = Model() + p1 = Parameter() + p2 = Parameter() + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.load(tmp.name) + + self.assertTrue(p1.valid()) + self.assertTrue(p2.valid()) + self.assertEqual(shape, p1.shape()) + self.assertEqual(shape, p2.shape()) + self.assertEqual(values1, p1.value.to_list()) + self.assertEqual(values2, p2.value.to_list()) + self.assertTrue("a" in p1.stats) + self.assertTrue("b" in p2.stats) + self.assertEqual(stats1, p1.stats["a"].to_list()) + self.assertEqual(stats2, p2.stats["b"].to_list()) + + def test_ModelTest_CheckSaveWithoutStats(self): + shape = Shape([2, 2]) + values1 = [1, 2, 3, 4] + values2 = [5, 6, 7, 8] + stats1 = [10, 20, 30, 40] + stats2 = [50, 60, 70, 80] + tmp = tempfile.NamedTemporaryFile() + + m1 = Model() + m2 = Model() + p1 = Parameter(shape, I.Constant(0)) + p1.value += tF.raw_input(shape, values1) + p2 = Parameter(shape, I.Constant(0)) + p2.value += tF.raw_input(shape, values2) + p1.add_stats("a", shape) + p2.add_stats("b", shape) + p1.stats["a"].reset_by_vector(stats1) + p2.stats["b"].reset_by_vector(stats2) + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.save(tmp.name, False) + + m1 = Model() + m2 = Model() + p1 = Parameter() + p2 = Parameter() + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.load(tmp.name) + + self.assertTrue(p1.valid()) + self.assertTrue(p2.valid()) + self.assertEqual(shape, p1.shape()) + self.assertEqual(shape, p2.shape()) + self.assertEqual(values1, p1.value.to_list()) + self.assertEqual(values2, p2.value.to_list()) + self.assertFalse("a" in p1.stats) + self.assertFalse("b" in p2.stats) + + def test_ModelTest_CheckLoadWithoutStats(self): + shape = Shape([2, 2]) + values1 = [1, 2, 3, 4] + values2 = [5, 6, 7, 8] + stats1 = [10, 20, 30, 40] + stats2 = [50, 60, 70, 80] + tmp = tempfile.NamedTemporaryFile() + + m1 = Model() + m2 = Model() + p1 = Parameter(shape, I.Constant(0)) + p1.value += tF.raw_input(shape, values1) + p2 = Parameter(shape, I.Constant(0)) + p2.value += tF.raw_input(shape, values2) + p1.add_stats("a", shape) + p2.add_stats("b", shape) + p1.stats["a"].reset_by_vector(stats1) + p2.stats["b"].reset_by_vector(stats2) + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.save(tmp.name) + + m1 = Model() + m2 = Model() + p1 = Parameter() + p2 = Parameter() + m1.add("p", p1) + m2.add("p", p2) + m1.add("sm", m2) + + m1.load(tmp.name, False) + + self.assertTrue(p1.valid()) + self.assertTrue(p2.valid()) + self.assertEqual(shape, p1.shape()) + self.assertEqual(shape, p2.shape()) + self.assertEqual(values1, p1.value.to_list()) + self.assertEqual(values2, p2.value.to_list()) + self.assertFalse("a" in p1.stats) + self.assertFalse("b" in p2.stats) diff --git a/tests/node_functions.py b/tests/node_functions.py index 8abbb31..a95191e 100644 --- a/tests/node_functions.py +++ b/tests/node_functions.py @@ -88,6 +88,5 @@ def test_node_pow(self): x = F.input(input_arr) self.assertTrue(((x ** 0x7fffffff).to_ndarrays()[0] == np.array([1, -1])).all()) self.assertTrue(((x ** -0x80000000).to_ndarrays()[0] == np.array([1, 1])).all()) - self.assertTrue(np.isnan((x ** 0x80000000).to_ndarrays()[0]).any()) - self.assertTrue(np.isnan((x ** -0x80000001).to_ndarrays()[0]).any()) + self.assertRaises(TypeError, lambda: pow(x, y, 2)) diff --git a/tests/optimizer.py b/tests/optimizer.py index b07f410..318e013 100644 --- a/tests/optimizer.py +++ b/tests/optimizer.py @@ -13,7 +13,6 @@ class TestModel(Model): def __init__(self): self.param = Parameter([5], I.Constant(0)) self.param.gradient = tF.raw_input([5], [1, 2, 3, 4, 5]) - self.scan_attributes() class Optimizer(unittest.TestCase): diff --git a/tests/tensor_forward_test.py b/tests/tensor_forward_test.py new file mode 100644 index 0000000..14b1992 --- /dev/null +++ b/tests/tensor_forward_test.py @@ -0,0 +1,2496 @@ +import math +import random +import sys +import unittest + +from primitiv import Device +from primitiv import Shape +from primitiv import Parameter +from primitiv import Tensor +from primitiv import initializers as I +from primitiv import tensor_functions as tF + +import numpy as np +from . import test_utils + + +class TensorForwardTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.devices = test_utils.available_devices() + + @classmethod + def tearDownClass(cls): + pass + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_TensorForwardTest_CheckInputByVector(self): + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + for dev in TensorForwardTest.devices: + y = tF.raw_input(Shape([2, 2], 3), data, dev) + self.assertEqual(Shape([2, 2], 3), y.shape()) + self.assertIs(dev, y.device()) + self.assertEqual(data, y.to_list()) + + def test_TensorForwardTest_CheckInputByParameter(self): + data = [1, 2, 3, 4] + for dev in TensorForwardTest.devices: + param = Parameter([2, 2], I.Constant(0), dev) + param.value += tF.raw_input([2, 2], data, dev) + y = tF.parameter(param) + self.assertEqual(Shape([2, 2]), y.shape()) + self.assertIs(dev, y.device()) + self.assertEqual(data, y.to_list()) + + def test_TensorForwardTest_CheckInputByNdArray(self): + y_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + x_data = [ + np.array([[1, 3], [2, 4]]), + np.array([[5, 7], [6, 8]]), + np.array([[9, 11], [10, 12]]), + ] + for dev in TensorForwardTest.devices: + y = tF.input(x_data, dev) + self.assertEqual(Shape([2, 2], 3), y.shape()) + self.assertIs(dev, y.device()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckCopy(self): + i = 0 + for dev in TensorForwardTest.devices: + for dev2 in TensorForwardTest.devices: + data = list(range(i, i + 12)) + print(data) + i += 12 + x = tF.raw_input(Shape([2, 2], 3), data, dev) + y = tF.copy(x, dev2) + self.assertEqual(Shape([2, 2], 3), y.shape()) + self.assertIs(dev, x.device()) + self.assertIs(dev2, y.device()) + self.assertEqual(x.to_list(), y.to_list()) + y *= 2 + self.assertNotEqual(x.to_list(), y.to_list()) + + def test_TensorForwardTest_CheckInvalidCopy(self): + for dev in TensorForwardTest.devices: + with self.assertRaises(RuntimeError): + tF.copy(Tensor(), dev) + + def test_TensorForwardTest_CheckIdentity(self): + test_cases = [ + (1, Shape(), [1]), + (2, Shape([2, 2]), [1, 0, 0, 1]), + (3, Shape([3, 3]), [1, 0, 0, 0, 1, 0, 0, 0, 1]), + (4, Shape([4, 4]), [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]), + ] + for dev in TensorForwardTest.devices: + Device.set_default(dev) + for tc in test_cases: + y = tF.identity(tc[0]) + self.assertEqual(tc[1], y.shape()) + self.assertEqual(tc[2], y.to_list()) + + def test_TensorForwardTest_CheckInvalidIdentity(self): + for dev in TensorForwardTest.devices: + Device.set_default(dev) + with self.assertRaises(RuntimeError): + tF.identity(0) + + def test_TensorForwardTest_CheckPickNN(self): + test_cases = [ + (Shape([2, 2, 2], 3), 0, [0, 0, 0], + Shape([1, 2, 2], 3), + [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]), + (Shape([2, 2, 2], 3), 0, [1, 0, 1], + Shape([1, 2, 2], 3), + [1, 3, 5, 7, 8, 10, 12, 14, 17, 19, 21, 23]), + (Shape([2, 2, 2], 3), 0, [0], + Shape([1, 2, 2], 3), + [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]), + (Shape([2, 2, 2]), 0, [0, 1, 0], + Shape([1, 2, 2], 3), + [0, 2, 4, 6, 1, 3, 5, 7, 0, 2, 4, 6]), + (Shape([2, 2, 2], 3), 1, [0, 0, 0], + Shape([2, 1, 2], 3), + [0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21]), + (Shape([2, 2, 2], 3), 2, [0, 0, 0], + Shape([2, 2, 1], 3), + [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19]), + ] + for dev in TensorForwardTest.devices: + for tc in test_cases: + print("x_shape =", tc[0], + ", dim =", tc[1], ", ids = [", file=sys.stderr) + print(tc[2], file=sys.stderr) + print("]", file=sys.stderr) + x_data = list(range(tc[0].size())) + x = tF.raw_input(tc[0], x_data, dev) + y = tF.pick(x, tc[2], tc[1]) + self.assertEqual(tc[3], y.shape()) + self.assertEqual(tc[4], y.to_list()) + + def test_TensorForwardTest_CheckInvalidPick(self): + test_cases = [ + (0, []), + (0, [2]), + (0, [0, 1]), + (0, [0, 1, 2]), + (1, [2]), + (2, [2]), + (3, [1]), + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2, 2], 3), [0] * 24, dev) + for tc in test_cases: + with self.assertRaises(RuntimeError): + tF.pick(x, tc[1], tc[0]) + + def test_TensorForwardTest_CheckSlice(self): + x_data = list(range(3 * 3 * 2 * 4)) + test_cases = [ + (0, 0, 1, Shape([1, 3, 2], 4), + [0, 3, 6, 9, 12, 15, + 18, 21, 24, 27, 30, 33, + 36, 39, 42, 45, 48, 51, + 54, 57, 60, 63, 66, 69]), + (1, 0, 1, Shape([3, 1, 2], 4), + [0, 1, 2, 9, 10, 11, + 18, 19, 20, 27, 28, 29, + 36, 37, 38, 45, 46, 47, + 54, 55, 56, 63, 64, 65]), + (2, 0, 1, Shape([3, 3, 1], 4), + [0, 1, 2, 3, 4, 5, 6, 7, 8, + 18, 19, 20, 21, 22, 23, 24, 25, 26, + 36, 37, 38, 39, 40, 41, 42, 43, 44, + 54, 55, 56, 57, 58, 59, 60, 61, 62]), + (0, 1, 2, Shape([1, 3, 2], 4), + [1, 4, 7, 10, 13, 16, + 19, 22, 25, 28, 31, 34, + 37, 40, 43, 46, 49, 52, + 55, 58, 61, 64, 67, 70]), + (1, 1, 2, Shape([3, 1, 2], 4), + [3, 4, 5, 12, 13, 14, + 21, 22, 23, 30, 31, 32, + 39, 40, 41, 48, 49, 50, + 57, 58, 59, 66, 67, 68]), + (2, 1, 2, Shape([3, 3, 1], 4), + [9, 10, 11, 12, 13, 14, 15, 16, 17, + 27, 28, 29, 30, 31, 32, 33, 34, 35, + 45, 46, 47, 48, 49, 50, 51, 52, 53, + 63, 64, 65, 66, 67, 68, 69, 70, 71]), + (0, 2, 3, Shape([1, 3, 2], 4), + [2, 5, 8, 11, 14, 17, + 20, 23, 26, 29, 32, 35, + 38, 41, 44, 47, 50, 53, + 56, 59, 62, 65, 68, 71]), + (1, 2, 3, Shape([3, 1, 2], 4), + [6, 7, 8, 15, 16, 17, + 24, 25, 26, 33, 34, 35, + 42, 43, 44, 51, 52, 53, + 60, 61, 62, 69, 70, 71]), + (3, 0, 1, Shape([3, 3, 2], 4), x_data), + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([3, 3, 2], 4), x_data, dev) + for tc in test_cases: + print("dim =", tc[0], ", lower =", tc[1], + ", upper =", tc[2], file=sys.stderr) + y = tF.slice(x, tc[0], tc[1], tc[2]) + self.assertEqual(tc[3], y.shape()) + self.assertEqual(tc[4], y.to_list()) + + def test_TensorForwardTest_CheckInvalidSlice(self): + test_cases = [ + (0, 0, 0), (0, 1, 0), (0, 0, 4), (0, 3, 4), + (1, 0, 0), (1, 1, 0), (1, 0, 4), (1, 3, 4), + (2, 0, 0), (2, 1, 0), (2, 0, 2), (2, 1, 2), + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input([3, 3], [3] * 9) + for tc in test_cases: + with self.assertRaises(RuntimeError): + tF.slice(x, tc[0], tc[1], tc[2]) + + def test_TensorForwardTest_CheckConcatN_3x3(self): + y_data = [ + 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, + ] + for dev in TensorForwardTest.devices: + a = tF.raw_input([1, 3], [1, 1, 1], dev) + b = tF.raw_input([2, 3], [2, 3, 2, 3, 2, 3], dev) + c = tF.raw_input([3, 3], [4, 5, 6, 4, 5, 6, 4, 5, 6], dev) + y = tF.concat([a, b, c], 0) + self.assertEqual(Shape([6, 3]), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckConcat5x4(self): + shapes = [ + Shape([20]), + Shape([5, 4]), + Shape([5, 1, 4]), + ] + y_data = [ + 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, + ] + for dev in TensorForwardTest.devices: + a = tF.raw_input([5], [1, 1, 1, 1, 1], dev) + b = tF.raw_input([5], [2, 2, 2, 2, 2], dev) + c = tF.raw_input([5], [3, 3, 3, 3, 3], dev) + d = tF.raw_input([5], [4, 4, 4, 4, 4], dev) + for i in range(3): + y = tF.concat([a, b, c, d], i) + self.assertEqual(shapes[i], y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckConcat2_2_2x2(self): + a_data = [ + 1, 2, 3, 4, 5, 6, 7, 8, + 11, 22, 33, 44, 55, 66, 77, 88, + ] + b_data = [ + -1, -2, -3, -4, -5, -6, -7, -8, + -11, -22, -33, -44, -55, -66, -77, -88, + ] + shapes = [ + Shape([4, 2, 2], 2), + Shape([2, 4, 2], 2), + Shape([2, 2, 4], 2), + Shape([2, 2, 2, 2], 2), + Shape([2, 2, 2, 1, 2], 2), + ] + y_data = [ + [1, 2, -1, -2, 3, 4, -3, -4, + 5, 6, -5, -6, 7, 8, -7, -8, + 11, 22, -11, -22, 33, 44, -33, -44, + 55, 66, -55, -66, 77, 88, -77, -88], + [1, 2, 3, 4, -1, -2, -3, -4, + 5, 6, 7, 8, -5, -6, -7, -8, + 11, 22, 33, 44, -11, -22, -33, -44, + 55, 66, 77, 88, -55, -66, -77, -88], + [1, 2, 3, 4, 5, 6, 7, 8, + -1, -2, -3, -4, -5, -6, -7, -8, + 11, 22, 33, 44, 55, 66, 77, 88, + -11, -22, -33, -44, -55, -66, -77, -88], + ] + for dev in TensorForwardTest.devices: + a = tF.raw_input(Shape([2, 2, 2], 2), a_data, dev) + b = tF.raw_input(Shape([2, 2, 2], 2), b_data, dev) + for i in range(5): + y = tF.concat([a, b], i) + self.assertEqual(shapes[i], y.shape()) + self.assertEqual(y_data[i if i < 2 else 2], y.to_list()) + + def test_TensorForwardTest_CheckConcatBatchBroadcast(self): + for dev in TensorForwardTest.devices: + y_data = [ + 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, + 11, 11, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, + ] + a = tF.raw_input(Shape([2, 1], 2), [1, 1, 11, 11], dev) + b = tF.raw_input([2, 2], [2, 2, 2, 2], dev) + c = tF.raw_input([2, 3], [3, 3, 3, 3, 3, 3], dev) + y = tF.concat([a, b, c], 1) + self.assertEqual(Shape([2, 6], 2), y.shape()) + self.assertEqual(y_data, y.to_list()) + + y_data = [ + 1, 1, 1, 2, 2, 3, 1, 1, 1, 2, 2, 3, + 1, 1, 1, 22, 22, 33, 1, 1, 1, 22, 22, 33, + ] + a = tF.raw_input([3, 2], [1, 1, 1, 1, 1, 1], dev) + b = tF.raw_input(Shape([2, 2], 2), + [2, 2, 2, 2, 22, 22, 22, 22], dev) + c = tF.raw_input(Shape([1, 2], 2), [3, 3, 33, 33], dev) + y = tF.concat([a, b, c], 0) + self.assertEqual(Shape([6, 2], 2), y.shape()) + self.assertEqual(y_data, y.to_list()) + + y_data = [1, 2, 3, 1, 2, 33, 1, 2, 333] + a = tF.raw_input([], [1], dev) + b = tF.raw_input([], [2], dev) + c = tF.raw_input(Shape([], 3), [3, 33, 333], dev) + y = tF.concat([a, b, c], 0) + self.assertEqual(Shape([3], 3), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckInvalidConcat(self): + for dev in TensorForwardTest.devices: + a = tF.zeros(Shape([1, 42], 2), dev) + b = tF.zeros(Shape([2, 42], 2), dev) + c = tF.zeros(Shape([1, 42], 3), dev) + d = tF.zeros([2, 42], dev) + + tF.concat([a, b], 0) + with self.assertRaises(RuntimeError): + tF.concat([a, b], 1) + with self.assertRaises(RuntimeError): + tF.concat([a, b], 2) + with self.assertRaises(RuntimeError): + tF.concat([a, c], 0) + with self.assertRaises(RuntimeError): + tF.concat([a, c], 1) + with self.assertRaises(RuntimeError): + tF.concat([a, c], 2) + with self.assertRaises(RuntimeError): + tF.concat([b, c], 0) + with self.assertRaises(RuntimeError): + tF.concat([b, c], 1) + with self.assertRaises(RuntimeError): + tF.concat([b, c], 2) + tF.concat([a, d], 0) + with self.assertRaises(RuntimeError): + tF.concat([a, d], 1) + with self.assertRaises(RuntimeError): + tF.concat([a, d], 2) + + def test_TensorForwardTest_CheckReshape(self): + shapes = [ + Shape([6]), Shape([1, 6]), Shape([1, 1, 6]), Shape([1, 1, 1, 6]), + Shape([2, 3]), Shape([2, 1, 3]), Shape([1, 2, 3]), + Shape([2, 1, 1, 3]), Shape([1, 2, 1, 3]), Shape([1, 1, 2, 3]), + Shape([3, 2]), Shape([3, 1, 2]), Shape([1, 3, 2]), + Shape([3, 1, 1, 2]), Shape([1, 3, 1, 2]), Shape([1, 1, 3, 2]), + ] + for dev in TensorForwardTest.devices: + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + a = tF.raw_input(Shape([6], 2), data, dev) + for shape in shapes: + y1 = tF.reshape(a, shape) + self.assertEqual(shape.resize_batch(2), y1.shape()) + self.assertEqual(data, y1.to_list()) + y2 = tF.reshape(a, shape.resize_batch(2)) + self.assertEqual(shape.resize_batch(2), y2.shape()) + self.assertEqual(data, y2.to_list()) + + def test_TensorForwardTest_CheckInvalidReshape(self): + for dev in TensorForwardTest.devices: + a = tF.zeros(Shape([6], 2), dev) + with self.assertRaises(RuntimeError): + tF.reshape(a, [7]) + with self.assertRaises(RuntimeError): + tF.reshape(a, Shape([6], 3)) + with self.assertRaises(RuntimeError): + tF.reshape(a, Shape([7], 3)) + + def test_TensorForwardTest_CheckFlatten(self): + shapes = [ + Shape([6]), Shape([1, 6]), Shape([1, 1, 6]), Shape([1, 1, 1, 6]), + Shape([2, 3]), Shape([2, 1, 3]), Shape([1, 2, 3]), + Shape([2, 1, 1, 3]), Shape([1, 2, 1, 3]), Shape([1, 1, 2, 3]), + Shape([3, 2]), Shape([3, 1, 2]), Shape([1, 3, 2]), + Shape([3, 1, 1, 2]), Shape([1, 3, 1, 2]), Shape([1, 1, 3, 2]), + ] + for dev in TensorForwardTest.devices: + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + for shape in shapes: + a = tF.raw_input(shape.resize_batch(2), data, dev) + y = tF.flatten(a) + self.assertEqual(Shape([6], 2), y.shape()) + self.assertEqual(data, y.to_list()) + + def test_TensorForwardTest_CheckDuplicate(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + y = +x + self.assertEqual(Shape([2, 2], 2), y.shape()) + self.assertTrue(np.isclose(x_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckNegate(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + y_data = [-1000, -100, -10, -1, -0.1, -0.01, -0.001, -0.0001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + y = -x + self.assertEqual(Shape([2, 2], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckAddConst(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k = 1 + y_data = [1001, 101, 11, 2, 1.1, 1.01, 1.001, 1.0001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + y1 = k + x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = x + k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckAddScalar(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [10, 1] + y_data = [1010, 110, 20, 11, 1.1, 1.01, 1.001, 1.0001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k + x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = x + k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckAddScalarBatchBroadcast(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [1] + y_data = [1001, 101, 11, 2, 1.1, 1.01, 1.001, 1.0001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input([], k_data, dev) + y1 = k + x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = x + k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + x_data = [1000, 100, 10, 1] + k_data = [10, 1] + y_data = [1010, 110, 20, 11, 1001, 101, 11, 2] + for dev in TensorForwardTest.devices: + x = tF.raw_input([2, 2], x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k + x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertEqual(y_data, y1.to_list()) + y2 = x + k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertEqual(y_data, y2.to_list()) + + def test_TensorForwardTest_CheckAdd(self): + a_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + b_data = [ 0, 100, 20, 3, 0.4, 0.05, 0.006, 0.0007] + y_data = [1000, 200, 30, 4, 0.5, 0.06, 0.007, 0.0008] + for dev in TensorForwardTest.devices: + a = tF.raw_input(Shape([2, 2], 2), a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a + b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = b + a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckAddBatchBroadcast(self): + a_data = [0, 1, 2, 3] + b_data = [0, 0, 0, 0, 4, 4, 4, 4] + y_data = [0, 1, 2, 3, 4, 5, 6, 7] + for dev in TensorForwardTest.devices: + a = tF.raw_input([2, 2], a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a + b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertEqual(y_data, y1.to_list()) + y2 = b + a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertEqual(y_data, y2.to_list()) + + def test_TensorForwardTest_CheckSubtractConst(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k = 1 + y1_data = [-999, -99, -9, 0, 0.9, 0.99, 0.999, 0.9999] + y2_data = [999, 99, 9, 0, -0.9, -0.99, -0.999, -0.9999] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + y1 = k - x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = x - k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckSubtractScalar(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [10, 1] + y1_data = [-990, -90, 0, 9, 0.9, 0.99, 0.999, 0.9999] + y2_data = [990, 90, 0, -9, -0.9, -0.99, -0.999, -0.9999] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k - x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = x - k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckSubtractScalarBatchBroadcast(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [1] + y1_data = [-999, -99, -9, 0, 0.9, 0.99, 0.999, 0.9999] + y2_data = [999, 99, 9, 0, -0.9, -0.99, -0.999, -0.9999] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input([], k_data, dev) + y1 = k - x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = x - k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + x_data = [1000, 100, 10, 1] + k_data = [10, 1] + y1_data = [-990, -90, 0, 9, -999, -99, -9, 0] + y2_data = [990, 90, 0, -9, 999, 99, 9, 0] + for dev in TensorForwardTest.devices: + x = tF.raw_input([2, 2], x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k - x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertEqual(y1_data, y1.to_list()) + y2 = x - k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertEqual(y2_data, y2.to_list()) + + def test_TensorForwardTest_CheckSubtract(self): + a_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + b_data = [ 0, 100, 20, 3, 0.4, 0.05, 0.006, 0.0007] + y1_data = [1000, 0, -10, -2, -0.3, -0.04, -0.005, -0.0006] + y2_data = [-1000, 0, 10, 2, 0.3, 0.04, 0.005, 0.0006] + for dev in TensorForwardTest.devices: + a = tF.raw_input(Shape([2, 2], 2), a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a - b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = b - a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckSubtractBatchBroadcast(self): + a_data = [0, 1, 2, 3] + b_data = [0, 0, 0, 0, 4, 4, 4, 4] + y1_data = [0, 1, 2, 3, -4, -3, -2, -1] + y2_data = [0, -1, -2, -3, 4, 3, 2, 1] + for dev in TensorForwardTest.devices: + a = tF.raw_input([2, 2], a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a - b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(y1_data, y1.to_list()) + y2 = b - a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(y2_data, y2.to_list()) + + def test_TensorForwardTest_CheckMultiplyConst(self): + x_data = [1000, -100, 10, -1, 0.1, -0.01, 0.001, -0.0001] + k = 10 + y_data = [10000, -1000, 100, -10, 1, -0.1, 0.01, -0.001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + y1 = k * x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = x * k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckMultiplyScalar(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [0.1, 10] + y_data = [100, 10, 1, 0.1, 1, 0.1, 0.01, 0.001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k * x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = x * k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckMultiplyScalarBatchBroadcast(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [10] + y_data = [10000, 1000, 100, 10, 1, 0.1, 0.01, 0.001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input([], k_data, dev) + y1 = k * x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = x * k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + x_data = [1000, 100, 10, 1] + k_data = [0.1, 10] + y_data = [100, 10, 1, 0.1, 10000, 1000, 100, 10] + for dev in TensorForwardTest.devices: + x = tF.raw_input([2, 2], x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k * x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = x * k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckMultiply(self): + a_data = [1000, -100, 10, -1, 0.1, -0.01, 0.001, -0.0001] + b_data = [0, 1, 2, 3, -4, -5, -6, -7] + y_data = [0, -100, 20, -3, -0.4, 0.05, -0.006, 0.0007] + for dev in TensorForwardTest.devices: + a = tF.raw_input(Shape([2, 2], 2), a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a * b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + y2 = b * a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckMultiplyBatchBroadcast(self): + a_data = [0, 1, 2, 3] + b_data = [1, 1, 1, 1, 0, 1, 2, 3] + y_data = [0, 1, 2, 3, 0, 1, 4, 9] + for dev in TensorForwardTest.devices: + a = tF.raw_input([2, 2], a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a * b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertEqual(y_data, y1.to_list()) + y2 = b * a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertEqual(y_data, y2.to_list()) + + def test_TensorForwardTest_CheckDivideConst(self): + x_data = [1000, -100, 10, -1, 0.1, -0.01, 0.001, -0.0001] + k = 10 + y1_data = [0.01, -0.1, 1, -10, 100, -1000, 10000, -100000] + y2_data = [ + 100, -10, 1, -0.1, 0.01, -0.001, 0.0001, -0.00001, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + y1 = k / x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = x / k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckDivideScalar(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [10, 0.1] + y1_data = [0.01, 0.1, 1, 10, 1, 10, 100, 1000] + y2_data = [100, 10, 1, 0.1, 1, 0.1, 0.01, 0.001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k / x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = x / k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckDivideScalarBatchBroadcast(self): + x_data = [1000, 100, 10, 1, 0.1, 0.01, 0.001, 0.0001] + k_data = [10] + y1_data = [0.01, 0.1, 1, 10, 100, 1000, 10000, 100000] + y2_data = [100, 10, 1, 0.1, 0.01, 0.001, 0.0001, 0.00001] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 2), x_data, dev) + k = tF.raw_input([], k_data, dev) + y1 = k / x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = x / k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + x_data = [1000, 100, 10, 1] + k_data = [10, 0.1] + y1_data = [0.01, 0.1, 1, 10, 0.0001, 0.001, 0.01, 0.1] + y2_data = [100, 10, 1, 0.1, 10000, 1000, 100, 10] + for dev in TensorForwardTest.devices: + x = tF.raw_input([2, 2], x_data, dev) + k = tF.raw_input(Shape([], 2), k_data, dev) + y1 = k / x + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = x / k + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckDivide(self): + a_data = [1000, -100, 10, -1, 0.1, -0.01, 0.001, -0.0001] + b_data = [1, 2, 3, 4, -5, -6, -7, -8] + y1_data = [ + 1000, -50, 10.0/3, -0.25, -0.02, 0.01/6, -0.001/7, 1.25e-5, + ] + y2_data = [0.001, -0.02, 0.3, -4, -50, 600, -7000, 80000] + for dev in TensorForwardTest.devices: + a = tF.raw_input(Shape([2, 2], 2), a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a / b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = b / a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckDivideBatchBroadcast(self): + a_data = [1, 2, 3, 4] + b_data = [1, 1, 1, 1, 1, 2, 3, 4] + y1_data = [1, 2, 3, 4, 1, 1, 1, 1] + y2_data = [1, 0.5, 1.0/3, 0.25, 1, 1, 1, 1] + for dev in TensorForwardTest.devices: + a = tF.raw_input([2, 2], a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y1 = a / b + self.assertEqual(Shape([2, 2], 2), y1.shape()) + self.assertTrue(np.isclose(y1_data, y1.to_list()).all()) + y2 = b / a + self.assertEqual(Shape([2, 2], 2), y2.shape()) + self.assertTrue(np.isclose(y2_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckInvalidArithmeticOps(self): + sa = [ + Shape([2, 2], 2), Shape([2, 2], 2), Shape([2, 2], 2), + ] + sb = [ + Shape([2, 2], 3), Shape([3, 3], 2), Shape([3, 3], 3), + ] + for dev in TensorForwardTest.devices: + for ssa, ssb in zip(sa, sb): + a = tF.zeros(ssa, dev) + b = tF.zeros(ssb, dev) + with self.assertRaises(RuntimeError): + a + b + with self.assertRaises(RuntimeError): + a - b + with self.assertRaises(RuntimeError): + a * b + with self.assertRaises(RuntimeError): + a / b + + def test_TensorForwardTest_CheckTranspose11(self): + for dev in TensorForwardTest.devices: + x_data = [42] + y_data = [42] + x = tF.raw_input([], x_data, dev) + y = tF.transpose(x) + self.assertEqual(Shape(), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckTransposeN1(self): + x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + y_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + for dev in TensorForwardTest.devices: + x = tF.raw_input([12], x_data, dev) + y = tF.transpose(x) + self.assertEqual(Shape([1, 12]), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckTranspose1N(self): + x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + y_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([1, 3], 4), x_data, dev) + y = tF.transpose(x) + self.assertEqual(Shape([3], 4), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckTransposeNN(self): + x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + y_data = [1, 3, 2, 4, 5, 7, 6, 8, 9, 11, 10, 12] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 3), x_data, dev) + y = tF.transpose(x) + self.assertEqual(Shape([2, 2], 3), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckTransposeMN(self): + x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + y_data = [1, 3, 5, 2, 4, 6, 7, 9, 11, 8, 10, 12] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.transpose(x) + self.assertEqual(Shape([3, 2], 2), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckInvalidTranspose(self): + for dev in TensorForwardTest.devices: + x = tF.zeros([2, 3, 4], dev) + with self.assertRaises(RuntimeError): + tF.transpose(x) + + def test_TensorForwardTest_CheckMatMulAA(self): + x_data = [1, 2, 3, 4, 1, 0, 0, 1, 0, 2, 3, 0] + y_data = [7, 10, 15, 22, 1, 0, 0, 1, 6, 0, 0, 6] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2], 3), x_data, dev) + y1 = tF.matmul(x, x) + y2 = x @ x + self.assertEqual(Shape([2, 2], 3), y1.shape()) + self.assertEqual(y_data, y1.to_list()) + self.assertEqual(Shape([2, 2], 3), y2.shape()) + self.assertEqual(y_data, y2.to_list()) + + def test_TensorForwardTest_CheckMatMulAB(self): + a_data = [ + 1, 1000, 1, + 10, 100, 100, + 100, 10, 10000, + 1000, 1, 1000000, + ] + b_data = [ + 0, 2, 4, 6, + 1, 3, 5, 7, + 8, 6, 4, 2, + 9, 7, 5, 3, + 2, 3, 5, 7, + 9, 4, 1, 0, + ] + y_data = [ + 6420, 246, 6040200, + 7531, 1357, 7050301, + 2468, 8642, 2040608, + 3579, 9753, 3050709, + 7532, 2357, 7050302, + 149, 9410, 10409, + ] + for dev in TensorForwardTest.devices: + a = tF.raw_input([3, 4], a_data, dev) + b = tF.raw_input([4, 6], b_data, dev) + y1 = tF.matmul(a, b) + y2 = a @ b + self.assertEqual(Shape([3, 6]), y1.shape()) + self.assertEqual(y_data, y1.to_list()) + self.assertEqual(Shape([3, 6]), y2.shape()) + self.assertEqual(y_data, y2.to_list()) + + def test_TensorForwardTest_CheckMatMulBatchBroadcast1N(self): + a_data = [10, 1000, 1, 100] + b_data = [1, 2, 3, 4, 5, 6, 7, 8] + y_data = [12, 1200, 34, 3400, 56, 5600, 78, 7800] + for dev in TensorForwardTest.devices: + a = tF.raw_input([2, 2], a_data, dev) + b = tF.raw_input(Shape([2, 2], 2), b_data, dev) + y = tF.matmul(a, b) + self.assertEqual(Shape([2, 2], 2), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckMatMulBatchBroadcastN1(self): + a_data = [1, 2, 3, 4, 5, 6, 7, 8] + b_data = [10, 1, 1000, 100] + y_data = [13, 24, 1300, 2400, 57, 68, 5700, 6800] + for dev in TensorForwardTest.devices: + a = tF.raw_input(Shape([2, 2], 2), a_data, dev) + b = tF.raw_input([2, 2], b_data, dev) + y = tF.matmul(a, b) + self.assertEqual(Shape([2, 2], 2), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckMatMulLarge(self): + N = 123 + a_data = [0] * (N * N) + b_data = [0] * (N * N) + y1_data = [0] * (N * N) + y2_data = [0] * (N * N) + k = 0 + for i in range(N): + k += i * i + for i in range(N): + for j in range(N): + a_data[i + j * N] = i + b_data[i + j * N] = j + y1_data[i + j * N] = N * i * j + y2_data[i + j * N] = k + for dev in TensorForwardTest.devices: + a = tF.raw_input(Shape([N, N]), a_data, dev) + b = tF.raw_input([N, N], b_data, dev) + y1 = tF.matmul(a, b) + y2 = tF.matmul(b, a) + self.assertEqual(Shape([N, N]), y1.shape()) + self.assertEqual(Shape([N, N]), y2.shape()) + self.assertEqual(y1_data, y1.to_list()) + self.assertEqual(y2_data, y2.to_list()) + + def test_TensorForwardTest_CheckInvalidMatMul(self): + for dev in TensorForwardTest.devices: + a = tF.zeros([2, 3], dev) + b = tF.zeros([], dev) + with self.assertRaises(RuntimeError): + tF.matmul(a, b) + a = tF.zeros([], dev) + b = tF.zeros([2, 3], dev) + with self.assertRaises(RuntimeError): + tF.matmul(a, b) + a = tF.zeros([2, 3, 4], dev) + b = tF.zeros([4], dev) + with self.assertRaises(RuntimeError): + tF.matmul(a, b) + a = tF.zeros([1, 2], dev) + b = tF.zeros([2, 3, 4], dev) + with self.assertRaises(RuntimeError): + tF.matmul(a, b) + a = tF.zeros([2, 3], dev) + b = tF.zeros([2, 3], dev) + with self.assertRaises(RuntimeError): + tF.matmul(a, b) + + def test_TensorForwardTest_CheckSqrt(self): + x_data = [ + 0, 1, 2, 3, 4, 5, + 0, 1, 4, 9, 16, 25, + ] + y_data = [ + 0, 1, 1.41421356, 1.73205041, 2, 2.23606798, + 0, 1, 2, 3, 4, 5, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.sqrt(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckExp(self): + x_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.5, -1, -2, -4, -8, + ] + y_data = [ + 1, 1.6487213, 2.7182818, 7.3890561, 54.598150, 2980.9580, + 1, .60653066, .36787944, .13533528, .018315639, .00033546263, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.exp(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckLog(self): + x_data = [ + 0.01, .5, 1, 2, 4, 8, + 0.01, .5, 1, 2, 4, 8, + ] + y_data = [ + -4.60517019, -0.69314718, 0, 0.69314718, 1.38629436, 2.07944154, + -4.60517019, -0.69314718, 0, 0.69314718, 1.38629436, 2.07944154, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.log(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckPow(self): + x_data = [ + 0.01, .5, 1, 2, 4, 8, + 0.01, .5, 1, 2, 4, 8, + ] + y_data = [ + 0.00001, 0.17677670, 1, 5.65685425, 32, 181.01933598, + 0.00001, 0.17677670, 1, 5.65685425, 32, 181.01933598, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y1 = tF.pow(x, 2.5) + y2 = x ** 2.5 + self.assertEqual(Shape([2, 3], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + self.assertEqual(Shape([2, 3], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckIPowPositive(self): + x_data = [ + 0.01, .5, 1, 2, 4, 8, + -0.01, -.5, -1, -2, -4, -8, + ] + y_data = [ + 0.000001, 0.125, 1, 8, 64, 512, + -0.000001, -0.125, -1, -8, -64, -512, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y1 = tF.pow(x, 3) + y2 = x ** 3 + self.assertEqual(Shape([2, 3], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + self.assertEqual(Shape([2, 3], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckIPowNegative(self): + x_data = [ + 0.01, .5, 1, 2, 4, 8, + -0.01, -.5, -1, -2, -4, -8, + ] + y_data = [ + 1000000, 8, 1, 0.125, 0.015625, 0.001953125, + -1000000, -8, -1, -0.125, -0.015625, -0.001953125, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y1 = tF.pow(x, -3) + y2 = x ** -3 + self.assertEqual(Shape([2, 3], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + self.assertEqual(Shape([2, 3], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckIPowUpperBound(self): + x_data = [ + 1, -1, 1, -1, 1, -1, + 1, -1, 1, -1, 1, -1, + ] + y_data = [ + 1, -1, 1, -1, 1, -1, + 1, -1, 1, -1, 1, -1, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y1 = tF.pow(x, 0x7fffffff) + y2 = x ** 0x7fffffff + self.assertEqual(Shape([2, 3], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + self.assertEqual(Shape([2, 3], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckIPowLowerBound(self): + x_data = [ + 1, -1, 1, -1, 1, -1, + 1, -1, 1, -1, 1, -1, + ] + y_data = [ + 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y1 = tF.pow(x, -2147483648) # 0x80000000 + y2 = x ** -2147483648 + self.assertEqual(Shape([2, 3], 2), y1.shape()) + self.assertTrue(np.isclose(y_data, y1.to_list()).all()) + self.assertEqual(Shape([2, 3], 2), y2.shape()) + self.assertTrue(np.isclose(y_data, y2.to_list()).all()) + + def test_TensorForwardTest_CheckIPowPositiveConvergence(self): + x_data = [ + 0.9999999, -0.9999999, 0.9999999, -0.9999999, 0.9999999, -0.9999999, + 0.9999999, -0.9999999, 0.9999999, -0.9999999, 0.9999999, -0.9999999, + ] + y_data = [ + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y1 = tF.pow(x, 0x7fffffff) + y2 = x ** 0x7fffffff + self.assertEqual(Shape([2, 3], 2), y1.shape()) + self.assertEqual(y_data, y1.to_list()) + self.assertEqual(Shape([2, 3], 2), y2.shape()) + self.assertEqual(y_data, y2.to_list()) + + def test_TensorForwardTest_CheckIPowNegativeConvergence(self): + x_data = [ + 1.000001, -1.000001, 1.000001, -1.000001, 1.000001, -1.000001, + 1.000001, -1.000001, 1.000001, -1.000001, 1.000001, -1.000001, + ] + y_data = [ + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y1 = tF.pow(x, -2147483648) # 0x80000000 + y2 = x ** -2147483648 + self.assertEqual(Shape([2, 3], 2), y1.shape()) + self.assertEqual(y_data, y1.to_list()) + self.assertEqual(Shape([2, 3], 2), y2.shape()) + self.assertEqual(y_data, y2.to_list()) + + def test_TensorForwardTest_CheckTanh(self): + x_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.5, -1, -2, -4, -8, + ] + y_data = [ + 0, .46211716, .76159416, .96402758, .99932930, .99999977, + 0, -.46211716, -.76159416, -.96402758, -.99932930, -.99999977, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.tanh(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckSigmoid(self): + x_data = [ + 0, .5, 1, 2, 3, 4, + 0, -.5, -1, -2, -3, -4, + ] + y_data = [ + .5, .62245933, .73105858, .88079708, .95257413, .98201379, + .5, .37754067, .26894142, .11920292, .047425873, .017986210, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.sigmoid(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckSoftplus(self): + x_data = [ + 0, .5, 1, 2, 3, 4, + 0, -.5, -1, -2, -3, -4, + ] + y_data = [ + .69314718, .97407698, 1.3132617, 2.1269280, 3.0485874, 4.0181499, + .69314718, .47407698, .31326169, .12692801, .048587352, .018149928, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.softplus(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list(), 0, 1e-6).all()) + + def test_TensorForwardTest_CheckSin(self): + x_data = [ + 0, .5, 1, 2, 3, 4, + 0, -.5, -1, -2, -3, -4, + ] + y_data = [ + 0, .47942554, .84147098, .90929743, .14112001, -.75680250, + 0, -.47942554, -.84147098, -.90929743, -.14112001, .75680250, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.sin(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckCos(self): + x_data = [ + 0, .5, 1, 2, 3, 4, + 0, -.5, -1, -2, -3, -4, + ] + y_data = [ + 1, .87758256, .54030231, -.41614684, -.98999250, -.65364362, + 1, .87758256, .54030231, -.41614684, -.98999250, -.65364362, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.cos(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckTan(self): + x_data = [ + 0, .5, 1, 2, 3, 4, + 0, -.5, -1, -2, -3, -4, + ] + y_data = [ + 0, .54630249, 1.5574077, -2.1850399, -.14254654, 1.1578213, + 0, -.54630249, -1.5574077, 2.1850399, .14254654, -1.1578213, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.tan(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckReLU(self): + x_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.5, -1, -2, -4, -8, + ] + y_data = [ + 0, .5, 1, 2, 4, 8, + 0, 0, 0, 0, 0, 0, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.relu(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckLReLU(self): + x_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.5, -1, -2, -4, -8, + ] + y_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.005, -.01, -.02, -.04, -.08, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.lrelu(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckPReLU(self): + ks = [.01, .1, 1., 10., 100., -.01, -.1, -1., -10., -100.] + for dev in TensorForwardTest.devices: + for k in ks: + x_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.5, -1, -2, -4, -8, + ] + y_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.5 * k, -k, -2 * k, -4 * k, -8 * k, + ] + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.prelu(x, k) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckELU(self): + ks = [.01, .1, 1., 10., 100., -.01, -.1, -1., -10., -100.] + for dev in TensorForwardTest.devices: + for k in ks: + x_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.5, -1, -2, -4, -8, + ] + y_data = [ + 0, .5, 1, 2, 4, 8, + 0, -.39346934 * k, -.63212056 * k, + -.86466472 * k, -.98168436 * k, -.99966454 * k, + ] + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.elu(x, k) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def test_TensorForwardTest_CheckSum(self): + x_data = [ + 1, 2, 3, 4, 5, 6, 7, 8, -1, -2, -3, -4, -5, -6, -7, -8, + ] + shape = [ + Shape([1, 2, 2], 2), + Shape([2, 1, 2], 2), + Shape([2, 2], 2), + Shape([2, 2, 2], 2), + ] + y_data = [ + [3, 7, 11, 15, -3, -7, -11, -15], + [4, 6, 12, 14, -4, -6, -12, -14], + [6, 8, 10, 12, -6, -8, -10, -12], + [1, 2, 3, 4, 5, 6, 7, 8, -1, -2, -3, -4, -5, -6, -7, -8], + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2, 2], 2), x_data, dev) + for i in range(4): + y = tF.sum(x, i) + self.assertEqual(shape[i], y.shape()) + self.assertEqual(y_data[i], y.to_list()) + + def test_TensorForwardTest_CheckSum2(self): + ns = [ + 1, 2, 3, 15, 16, 17, 255, 256, 257, 1023, 1024, 1025, 65535, 65536, 65537, + ] + for dev in TensorForwardTest.devices: + for n in ns: + x = tF.constant([n], 1) + y = tF.sum(x, 0) + self.assertEqual(Shape(), y.shape()) + self.assertEqual(n, y.to_float()) + + def test_TensorForwardTest_CheckLogSumExp(self): + x_data = [ + 1, 2, 3, 4, 5, 6, 7, 8, -1, -2, -3, -4, -5, -6, -7, -8, + ] + shape = [ + Shape([1, 2, 2], 2), + Shape([2, 1, 2], 2), + Shape([2, 2], 2), + Shape([2, 2, 2], 2), + ] + y_data = [ + [2.31326169, 4.31326169, 6.31326169, 8.31326169, + -0.68673831, -2.68673831, -4.68673831, -6.68673831], + [3.12692801, 4.12692801, 7.12692801, 8.12692801, + -0.87307199, -1.87307199, -4.87307199, -5.87307199], + [5.01814993, 6.01814993, 7.01814993, 8.01814993, + -0.98185007, -1.98185007, -2.98185007, -3.98185007], + [1, 2, 3, 4, 5, 6, 7, 8, -1, -2, -3, -4, -5, -6, -7, -8], + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2, 2], 2), x_data, dev) + for i in range(4): + y = tF.logsumexp(x, i) + self.assertEqual(shape[i], y.shape()) + self.assertTrue(np.isclose(y_data[i], y.to_list()).all()) + + def test_TensorForwardTest_CheckLogSumExp2(self): + ns = [ + 1, 2, 3, 15, 16, 17, 255, 256, 257, 1023, + 1024, 1025, 65535, 65536, 65537, + ] + for dev in TensorForwardTest.devices: + for n in ns: + for k in [-5, -1, 0, 1, 5]: + x = tF.constant([n], k, dev) + y = tF.logsumexp(x, 0) + self.assertEqual(Shape(), y.shape()) + self.assertTrue(np.isclose( + [k + math.log(n)], y.to_list(), 0, 1e-3)) + + def test_TensorForwardTest_CheckLogSoftmax(self): + x_data = [ + 1, 2, 3, 4, 5, 6, 7, 8, -1, -2, -3, -4, -5, -6, -7, -8, + ] + y_data = [ + [-1.31326169, -0.31326169, -1.31326169, -0.31326169, + -1.31326169, -0.31326169, -1.31326169, -0.31326169, + -0.31326169, -1.31326169, -0.31326169, -1.31326169, + -0.31326169, -1.31326169, -0.31326169, -1.31326169], + [-2.12692801, -2.12692801, -0.12692801, -0.12692801, + -2.12692801, -2.12692801, -0.12692801, -0.12692801, + -0.12692801, -0.12692801, -2.12692801, -2.12692801, + -0.12692801, -0.12692801, -2.12692801, -2.12692801], + [-4.01814993, -4.01814993, -4.01814993, -4.01814993, + -0.01814993, -0.01814993, -0.01814993, -0.01814993, + -0.01814993, -0.01814993, -0.01814993, -0.01814993, + -4.01814993, -4.01814993, -4.01814993, -4.01814993], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2, 2], 2), x_data, dev) + for i in range(4): + y = tF.log_softmax(x, i) + self.assertEqual(Shape([2, 2, 2], 2), y.shape()) + self.assertTrue(np.isclose(y_data[i], y.to_list(), 0, 1e-6).all()) + + def test_TensorForwardTest_CheckLogSoftmax2(self): + ns = [ + 1, 2, 3, 15, 16, 17, 255, 256, 257, 1023, + 1024, 1025, 65535, 65536, 65537, + ] + for dev in TensorForwardTest.devices: + for n in ns: + for k in [-5, -1, 0, 1, 5]: + x = tF.constant([n], k, dev) + y = tF.log_softmax(x, 0) + self.assertEqual(Shape([n]), y.shape()) + self.assertTrue( + np.isclose([-math.log(n)] * n, y.to_list(), 0, 1e-3).all()) + + def test_TensorForwardTest_CheckSoftmax(self): + x_data = [ + 1, 2, 3, 4, 5, 6, 7, 8, -1, -2, -3, -4, -5, -6, -7, -8, + ] + y_data = [ + [0.26894142, 0.73105858, 0.26894142, 0.73105858, + 0.26894142, 0.73105858, 0.26894142, 0.73105858, + 0.73105858, 0.26894142, 0.73105858, 0.26894142, + 0.73105858, 0.26894142, 0.73105858, 0.26894142], + [0.11920292, 0.11920292, 0.88079708, 0.88079708, + 0.11920292, 0.11920292, 0.88079708, 0.88079708, + 0.88079708, 0.88079708, 0.11920292, 0.11920292, + 0.88079708, 0.88079708, 0.11920292, 0.11920292], + [0.01798621, 0.01798621, 0.01798621, 0.01798621, + 0.98201379, 0.98201379, 0.98201379, 0.98201379, + 0.98201379, 0.98201379, 0.98201379, 0.98201379, + 0.01798621, 0.01798621, 0.01798621, 0.01798621], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2, 2], 2), x_data, dev) + for i in range(4): + y = tF.softmax(x, i) + self.assertEqual(Shape([2, 2, 2], 2), y.shape()) + self.assertTrue(np.isclose(y_data[i], y.to_list(), 0, 1e-6).all()) + + def test_TensorForwardTest_CheckSoftmax2(self): + ns = [ + 1, 2, 3, 15, 16, 17, 255, 256, 257, 1023, + 1024, 1025, 65535, 65536, 65537, + ] + for dev in TensorForwardTest.devices: + for n in ns: + for k in [-5, -1, 0, 1, 5]: + x = tF.constant([n], k, dev) + y = tF.softmax(x, 0) + self.assertEqual(Shape([n]), y.shape()) + self.assertTrue( + np.isclose([1./n] * n, y.to_list(), 0, 1e-6).all()) + + def test_TensorForwardTest_CheckBroadcast(self): + test_cases = [ + (0, 1, Shape([]), [1]), + (0, 20, Shape([20]), [1] * 20), + (1, 50, Shape([1, 50]), [1] * 50), + (2, 100, Shape([1, 1, 100]), [1] * 100), + ] + for dev in TensorForwardTest.devices: + for tc in test_cases: + x = tF.constant([], 1, dev) + y = tF.broadcast(x, tc[0], tc[1]) + self.assertEqual(tc[2], y.shape()) + self.assertEqual(tc[3], y.to_list()) + + def test_TensorForwardTest_CheckBroadcast2(self): + test_cases = [ + (1, 1, Shape([2], 3), [1, 2, 3, 4, 5, 6]), + (2, 1, Shape([2], 3), [1, 2, 3, 4, 5, 6]), + (1, 2, Shape([2, 2], 3), [1, 2, 1, 2, 3, 4, 3, 4, 5, 6, 5, 6]), + (2, 2, Shape([2, 1, 2], 3), [1, 2, 1, 2, 3, 4, 3, 4, 5, 6, 5, 6]), + ] + for dev in TensorForwardTest.devices: + for tc in test_cases: + x = tF.raw_input(Shape([2], 3), [1, 2, 3, 4, 5, 6], dev) + y = tF.broadcast(x, tc[0], tc[1]) + self.assertEqual(tc[2], y.shape()) + self.assertEqual(tc[3], y.to_list()) + + def test_TensorForwardTest_CheckBroadcast3(self): + test_cases = [ + (0, 1, Shape([1, 2, 1, 2], 2), + [1, 2, 3, 4, 5, 6, 7, 8]), + (2, 1, Shape([1, 2, 1, 2], 2), + [1, 2, 3, 4, 5, 6, 7, 8]), + (4, 1, Shape([1, 2, 1, 2], 2), + [1, 2, 3, 4, 5, 6, 7, 8]), + (0, 2, Shape([2, 2, 1, 2], 2), + [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8]), + (2, 2, Shape([1, 2, 2 ,2], 2), + [1, 2, 1, 2, 3, 4, 3, 4, 5, 6, 5, 6, 7, 8, 7, 8]), + (4, 2, Shape([1, 2, 1, 2, 2], 2), + [1, 2, 3, 4, 1, 2, 3, 4, 5, 6, 7, 8, 5, 6, 7, 8]), + ] + for dev in TensorForwardTest.devices: + for tc in test_cases: + x = tF.raw_input( + Shape([1, 2, 1, 2], 2), [1, 2, 3, 4, 5, 6, 7, 8], dev) + y = tF.broadcast(x, tc[0], tc[1]) + self.assertEqual(tc[2], y.shape()) + self.assertEqual(tc[3], y.to_list()) + + def test_TensorForwardTest_CheckInvalidBroadcast(self): + for dev in TensorForwardTest.devices: + x = tF.zeros([1, 2], dev) + with self.assertRaises(RuntimeError): + tF.broadcast(x, 0, 0) + with self.assertRaises(RuntimeError): + tF.broadcast(x, 1, 0) + with self.assertRaises(RuntimeError): + tF.broadcast(x, 1, 1) + with self.assertRaises(RuntimeError): + tF.broadcast(x, 1, 3) + with self.assertRaises(RuntimeError): + tF.broadcast(x, 2, 0) + + def test_TensorForwardTest_CheckBatchSum(self): + x_data = [ + 1, 2, 3, 4, 5, 6, 7, 8, + -2, -4, -6, -8, -10, -12, -14, -16, + ] + y_data = [ + -1, -2, -3, -4, -5, -6, -7, -8, + ] + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 2, 2], 2), x_data, dev) + y = tF.batch.sum(x) + self.assertEqual(Shape([2, 2, 2]), y.shape()) + self.assertEqual(y_data, y.to_list()) + + def test_TensorForwardTest_CheckSoftmaxCrossEntropy(self): + x_data = [ + [-1, 0, 1, 1, 0, 0, 0, 0, 1], + [-1, 1, 0, 0, 0, 0, 1, 0, 1], + ] + t_data = [ + [1./3, 1./3, 1./3, .5, .25, .25, 0, 0, 1], + [1./3, .5, 0, 1./3, .25, 0, 1./3, .25, 1], + ] + y_data = [ + [1.40760596, 1.05144471, 0.55144471], + [1.40760596, 1.05144471, 0.55144471], + ] + shape = [Shape([1, 3]), Shape([3])] + for dev in TensorForwardTest.devices: + for dim in [0, 1]: + x = tF.raw_input([3, 3], x_data[dim], dev) + t = tF.raw_input([3, 3], t_data[dim], dev) + y = tF.softmax_cross_entropy(x, t, dim) + self.assertEqual(shape[dim], y.shape()) + self.assertTrue(np.isclose(y_data[dim], y.to_list()).all()) + + def test_TensorForwardTest_CheckSoftmaxCrossEntropyBatchBroadcast(self): + test_cases = [ + ([-1, 0, 1], + [1, 0, 0, 0, 1, 0, 0, 0, 1], + [2.40760596, 1.40760596, 0.40760596], + Shape([3]), Shape([3], 3), Shape([], 3)), + ([-1, 0, 1, 1, -1, 0, 0, 1, -1], + [1, 0, 0], + [2.40760596, 0.40760596, 1.40760596], + Shape([3], 3), Shape([3]), Shape([], 3)), + ] + for dev in TensorForwardTest.devices: + for tc in test_cases: + x = tF.raw_input(tc[3], tc[0], dev) + t = tF.raw_input(tc[4], tc[1], dev) + y = tF.softmax_cross_entropy(x, t, 0) + self.assertEqual(tc[5], y.shape()) + self.assertTrue(np.isclose(tc[2], y.to_list()).all()) + + def test_TensorForwardTest_CheckInvalidSoftmaxCrossEntropy(self): + for dev in TensorForwardTest.devices: + x = tF.constant([2, 2], .5, dev) + t = tF.constant([2, 3], .5, dev) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 0) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 1) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 2) + x = tF.constant(Shape([2, 2], 2), .5, dev) + t = tF.constant(Shape([2, 3], 3), .5, dev) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 0) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 1) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 2) + + def test_TensorForwardTest_CheckSparseSoftmaxCrossEntropy(self): + test_cases = [ + ([-1, 0, 1, 1, -1, 0, 0, 1, -1], + 0, [0], Shape([3, 3]), Shape([1, 3]), + [2.40760596, 0.40760596, 1.40760596]), + ([-1, 0, 1, 1, -1, 0, 0, 1, -1], + 1, [1], Shape([3, 3]), Shape([3]), + [0.40760596, 2.40760596, 1.40760596]), + ([-1, 0, 1, 1, -1, 0, 0, 1, -1], + 2, [0], Shape([3, 3]), Shape([3, 3]), + [0, 0, 0, 0, 0, 0, 0, 0, 0]), + ([-1, 0, 1, 1, -1, 0, 0, 1, -1, -2, 0, 2, 2, -2, 0, 0, 2, -2], + 0, [0, 1], Shape([3, 3], 2), Shape([1, 3], 2), + [2.40760596, 0.40760596, 1.40760596, + 2.14293163, 4.14293163, 0.14293163]), + ([-1, 0, 1, 1, -1, 0, 0, 1, -1, -2, 0, 2, 2, -2, 0, 0, 2, -2], + 0, [0], Shape([3, 3], 2), Shape([1, 3], 2), + [2.40760596, 0.40760596, 1.40760596, + 4.14293163, 0.14293163, 2.14293163]), + ([-1, 0, 1, 1, -1, 0, 0, 1, -1], + 0, [0, 1], Shape([3, 3]), Shape([1, 3], 2), + [2.40760596, 0.40760596, 1.40760596, + 1.40760596, 2.40760596, 0.40760596]), + ] + for dev in TensorForwardTest.devices: + for tc in test_cases: + x = tF.raw_input(tc[3], tc[0], dev) + y = tF.softmax_cross_entropy(x, tc[2], tc[1]) + self.assertEqual(tc[4], y.shape()) + self.assertTrue(np.isclose(tc[5], y.to_list(), 0, 1e-6).all()) + + def test_TensorForwardTest_CheckInvalidSparseSoftmaxCrossEntropy(self): + for dev in TensorForwardTest.devices: + x = tF.constant([2, 2], .5, dev) + t = [2] + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 0) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 1) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 2) + x = tF.constant(Shape([2, 2], 2), .5, dev) + t = [0, 0, 0] + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 0) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 1) + with self.assertRaises(RuntimeError): + tF.softmax_cross_entropy(x, t, 2) + + def test_TensorForwardTest_CheckStopGradient(self): + x_data = [ + 0, .5, 1, 2, 3, 4, + 0, -.5, -1, -2, -3, -4, + ] + y_data = x_data + for dev in TensorForwardTest.devices: + x = tF.raw_input(Shape([2, 3], 2), x_data, dev) + y = tF.stop_gradient(x) + self.assertEqual(Shape([2, 3], 2), y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all()) + + def run_test_conv2d(self, x_shape, x_data, w_shape, w_data, y_shape, y_data, pad0, pad1, str0, str1, dil0, dil1): + for dev in TensorForwardTest.devices: + try: + x = tF.raw_input(x_shape, x_data, dev) + w = tF.raw_input(w_shape, w_data, dev) + y = tF.conv2d(x, w, pad0, pad1, str0, str1, dil0, dil1) + self.assertEqual(y_shape, y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all) + except RuntimeError as e: + # TODO(vbkaisetsu): + # We have to implement a better method to detect + # NotImplementedError in Python + if "Not implemented" not in str(e): + raise + + def test_TensorForwardTest_CheckConv2D_1x1x1_1x1x1x1(self): + x_data = [123] + w_data = [42] + y_data = [123 * 42] + x_shape = Shape([]) + w_shape = Shape([]) + y_shape = Shape([]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x1x1_1x1x1x1(self): + x_data = list(range(1, 5 + 1)) + w_data = [42] + y_data = [42, 84, 126, 168, 210] + x_shape = Shape([5]) + w_shape = Shape([]) + y_shape = Shape([5]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x1x1_2x1x1x1(self): + x_data = list(range(1, 5 + 1)) + w_data = list(range(1, 2 + 1)) + y_data = [4, 7, 10, 13] + x_shape = Shape([5]) + w_shape = Shape([2]) + y_shape = Shape([4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x1x1_5x1x1x1(self): + x_data = list(range(1, 5 + 1)) + w_data = list(range(1, 5 + 1)) + y_data = [35] + x_shape = Shape([5]) + w_shape = Shape([5]) + y_shape = Shape([]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_1x5x1_1x1x1x1(self): + x_data = list(range(1, 5 + 1)) + w_data = [42] + y_data = [42, 84, 126, 168, 210] + x_shape = Shape([1, 5]) + w_shape = Shape([]) + y_shape = Shape([1, 5]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_1x5x1_1x2x1x1(self): + x_data = list(range(1, 5 + 1)) + w_data = list(range(1, 2 + 1)) + y_data = [4, 7, 10, 13] + x_shape = Shape([1, 5]) + w_shape = Shape([1, 2]) + y_shape = Shape([1, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_1x5x1_1x5x1x1(self): + x_data = list(range(1, 5 + 1)) + w_data = list(range(1, 5 + 1)) + y_data = [35] + x_shape = Shape([1, 5]) + w_shape = Shape([1, 5]) + y_shape = Shape([]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_1x1x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = [42] + y_data = [ + 42, 84, 126, 168, 210, + 252, 294, 336, 378, 420, + 462, 504, 546, 588, 630, + 672, 714, 756, 798, 840, + 882, 924, 966, 1008, 1050, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([]) + y_shape = Shape([5, 5]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x1x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 + 1)) + y_data = [ + 4, 7, 10, 13, + 19, 22, 25, 28, + 34, 37, 40, 43, + 49, 52, 55, 58, + 64, 67, 70, 73, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2]) + y_shape = Shape([4, 5]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_5x1x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 5 + 1)) + y_data = [ + 35, + 110, + 185, + 260, + 335, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([5]) + y_shape = Shape([1, 5]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_1x2x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 + 1)) + y_data = [ + 8, 11, 14, 17, 20, + 23, 26, 29, 32, 35, + 38, 41, 44, 47, 50, + 53, 56, 59, 62, 65, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([1, 2]) + y_shape = Shape([5, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 29, 39, 49, 59, + 79, 89, 99, 109, + 129, 139, 149, 159, + 179, 189, 199, 209, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([4, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_5x2x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 5 * 2 + 1)) + y_data = [ + 220, + 495, + 770, + 1045, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([5, 2]) + y_shape = Shape([1, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_1x5x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 1 * 5 + 1)) + y_data = [ + 115, 130, 145, 160, 175, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([1, 5]) + y_shape = Shape([5]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x5x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 5 + 1)) + y_data = [ + 430, 485, 540, 595, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 5]) + y_shape = Shape([4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_5x5x1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 5 * 5 + 1)) + y_data = [2925] + x_shape = Shape([5, 5]) + w_shape = Shape([5, 5]) + y_shape = Shape([]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x3_2x2x3x1(self): + x_data = list(range(1, 5 * 5 * 3 + 1)) + w_data = list(range(1, 2 * 2 * 3 + 1)) + y_data = [ + 3029, 3107, 3185, 3263, + 3419, 3497, 3575, 3653, + 3809, 3887, 3965, 4043, + 4199, 4277, 4355, 4433, + ] + x_shape = Shape([5, 5, 3]) + w_shape = Shape([2, 2, 3]) + y_shape = Shape([4, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x3(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 * 3 + 1)) + y_data = [ + # channel 1 + 29, 39, 49, 59, + 79, 89, 99, 109, + 129, 139, 149, 159, + 179, 189, 199, 209, + # channel 2 + 93, 119, 145, 171, + 223, 249, 275, 301, + 353, 379, 405, 431, + 483, 509, 535, 561, + # channel 3 + 157, 199, 241, 283, + 367, 409, 451, 493, + 577, 619, 661, 703, + 787, 829, 871, 913, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2, 1, 3]) + y_shape = Shape([4, 4, 3]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x3_2x2x3x3(self): + x_data = list(range(1, 5 * 5 * 3 + 1)) + w_data = list(range(1, 2 * 2 * 3 * 3 + 1)) + y_data = [ + # channel 1 + 3029, 3107, 3185, 3263, + 3419, 3497, 3575, 3653, + 3809, 3887, 3965, 4043, + 4199, 4277, 4355, 4433, + # channel 2 + 7205, 7427, 7649, 7871, + 8315, 8537, 8759, 8981, + 9425, 9647, 9869, 10091, + 10535, 10757, 10979, 11201, + # channel 3 + 11381, 11747, 12113, 12479, + 13211, 13577, 13943, 14309, + 15041, 15407, 15773, 16139, + 16871, 17237, 17603, 17969, + ] + x_shape = Shape([5, 5, 3]) + w_shape = Shape([2, 2, 3, 3]) + y_shape = Shape([4, 4, 3]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Padding10(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 9, 29, 39, 49, 59, 40, + 29, 79, 89, 99, 109, 70, + 49, 129, 139, 149, 159, 100, + 69, 179, 189, 199, 209, 130, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([6, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 1, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Padding01(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 4, 7, 10, 13, + 29, 39, 49, 59, + 79, 89, 99, 109, + 129, 139, 149, 159, + 179, 189, 199, 209, + 150, 157, 164, 171, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([4, 6]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 1, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Padding11(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 1, 4, 7, 10, 13, 10, + 9, 29, 39, 49, 59, 40, + 29, 79, 89, 99, 109, 70, + 49, 129, 139, 149, 159, 100, + 69, 179, 189, 199, 209, 130, + 63, 150, 157, 164, 171, 100, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([6, 6]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 1, 1, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Stride21(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 29, 49, + 79, 99, + 129, 149, + 179, 199, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([2, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 2, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Stride12(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 29, 39, 49, 59, + 129, 139, 149, 159, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([4, 2]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 2, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Stride22(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 29, 49, + 129, 149, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([2, 2]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 2, 2, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Dilation21(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 33, 43, 53, + 83, 93, 103, + 133, 143, 153, + 183, 193, 203, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([3, 4]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 2, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Dilation12(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 44, 54, 64, 74, + 94, 104, 114, 124, + 144, 154, 164, 174, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([4, 3]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 2) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_Dilation22(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + 48, 58, 68, + 98, 108, 118, + 148, 158, 168, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2]) + y_shape = Shape([3, 3]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 2, 2) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_N1(self): + x_data = list(range(1, 5 * 5 * 3 + 1)) + w_data = list(range(1, 2 * 2 + 1)) + y_data = [ + # minibatch 1 + 29, 39, 49, 59, + 79, 89, 99, 109, + 129, 139, 149, 159, + 179, 189, 199, 209, + # minibatch 2 + 279, 289, 299, 309, + 329, 339, 349, 359, + 379, 389, 399, 409, + 429, 439, 449, 459, + # minibatch 3 + 529, 539, 549, 559, + 579, 589, 599, 609, + 629, 639, 649, 659, + 679, 689, 699, 709, + ] + x_shape = Shape([5, 5], 3) + w_shape = Shape([2, 2]) + y_shape = Shape([4, 4], 3) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_1N(self): + x_data = list(range(1, 5 * 5 + 1)) + w_data = list(range(1, 2 * 2 * 3 + 1)) + y_data = [ + # minibatch 1 + 29, 39, 49, 59, + 79, 89, 99, 109, + 129, 139, 149, 159, + 179, 189, 199, 209, + # minibatch 2 + 93, 119, 145, 171, + 223, 249, 275, 301, + 353, 379, 405, 431, + 483, 509, 535, 561, + # minibatch 3 + 157, 199, 241, 283, + 367, 409, 451, 493, + 577, 619, 661, 703, + 787, 829, 871, 913, + ] + x_shape = Shape([5, 5]) + w_shape = Shape([2, 2], 3) + y_shape = Shape([4, 4], 3) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_5x5x1_2x2x1x1_NN(self): + x_data = list(range(1, 5 * 5 * 3 + 1)) + w_data = list(range(1, 2 * 2 * 3 + 1)) + y_data = [ + # minibatch 1 + 29, 39, 49, 59, + 79, 89, 99, 109, + 129, 139, 149, 159, + 179, 189, 199, 209, + # minibatch 2 + 743, 769, 795, 821, + 873, 899, 925, 951, + 1003, 1029, 1055, 1081, + 1133, 1159, 1185, 1211, + # minibatch 3 + 2257, 2299, 2341, 2383, + 2467, 2509, 2551, 2593, + 2677, 2719, 2761, 2803, + 2887, 2929, 2971, 3013, + ] + x_shape = Shape([5, 5], 3) + w_shape = Shape([2, 2], 3) + y_shape = Shape([4, 4], 3) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 0, 0, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckConv2D_VGG16FirstLayer(self): + x_data = [1] * (224 * 224 * 3) + w_data = [1] * (3 * 3 * 3 * 64) + y_data = [27] * (224 * 224 * 64) + for b in range(64): + y_data[0 + b * 224 * 224] += 3; + y_data[223 + b * 224 * 224] += 3; + y_data[223 * 224 + b * 224 * 224] += 3; + y_data[223 * 224 + 223 + b * 224 * 224] += 3; + for i in range(224): + y_data[i + b * 224 * 224] -= 3 * 3 + y_data[223 * 224 + i + b * 224 * 224] -= 3 * 3 + y_data[i * 224 + b * 224 * 224] -= 3 * 3 + y_data[i * 224 + 223 + b * 224 * 224] -= 3 * 3 + + x_shape = Shape([224, 224, 3]) + w_shape = Shape([3, 3, 3, 64]) + y_shape = Shape([224, 224, 64]) + self.run_test_conv2d(x_shape, x_data, w_shape, w_data, y_shape, y_data, 1, 1, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckInvalidConv2D(self): + test_cases = [ + # invalid #dimensions + (Shape([1, 1, 1, 2]), Shape([]), 0, 0, 1, 1, 1, 1, False), + (Shape([]), Shape([1, 1, 1, 1, 2]), 0, 0, 1, 1, 1, 1, False), + # zero-stride/dilation + (Shape([]), Shape([]), 0, 0, 1, 1, 1, 1, True), + (Shape([]), Shape([]), 0, 0, 0, 1, 1, 1, False), + (Shape([]), Shape([]), 0, 0, 1, 0, 1, 1, False), + (Shape([]), Shape([]), 0, 0, 1, 1, 0, 1, False), + (Shape([]), Shape([]), 0, 0, 1, 1, 1, 0, False), + # minibatches mismatching + (Shape([], 2), Shape([], 2), 0, 0, 1, 1, 1, 1, True), + (Shape([], 3), Shape([], 3), 0, 0, 1, 1, 1, 1, True), + (Shape([], 2), Shape([], 3), 0, 0, 1, 1, 1, 1, False), + # channels mismatching + (Shape([3, 3, 42]), Shape([3, 3, 42]), 0, 0, 1, 1, 1, 1, True), + (Shape([3, 3, 42]), Shape([3, 3, 43]), 0, 0, 1, 1, 1, 1, False), + # sizes mismatching + (Shape([3, 3]), Shape([3, 3]), 0, 0, 1, 1, 1, 1, True), + (Shape([3, 3]), Shape([4, 3]), 0, 0, 1, 1, 1, 1, False), + (Shape([3, 3]), Shape([3, 4]), 0, 0, 1, 1, 1, 1, False), + (Shape([3, 3]), Shape([4, 4]), 0, 0, 1, 1, 1, 1, False), + # sizes mismatching with padding + (Shape([3, 3]), Shape([5, 5]), 1, 1, 1, 1, 1, 1, True), + (Shape([3, 3]), Shape([6, 5]), 1, 1, 1, 1, 1, 1, False), + (Shape([3, 3]), Shape([5, 6]), 1, 1, 1, 1, 1, 1, False), + (Shape([3, 3]), Shape([6, 6]), 1, 1, 1, 1, 1, 1, False), + # sizes mismatching with stride + (Shape([3, 3]), Shape([3, 3]), 0, 0, 2, 2, 1, 1, True), + (Shape([3, 3]), Shape([4, 3]), 0, 0, 2, 2, 1, 1, False), + (Shape([3, 3]), Shape([3, 4]), 0, 0, 2, 2, 1, 1, False), + (Shape([3, 3]), Shape([4, 4]), 0, 0, 2, 2, 1, 1, False), + # sizes mismatching with dilation + (Shape([3, 3]), Shape([2, 2]), 0, 0, 1, 1, 2, 2, True), + (Shape([2, 3]), Shape([2, 2]), 0, 0, 1, 1, 2, 2, False), + (Shape([3, 2]), Shape([2, 2]), 0, 0, 1, 1, 2, 2, False), + (Shape([2, 2]), Shape([2, 2]), 0, 0, 1, 1, 2, 2, False), + (Shape([3, 3]), Shape([2, 2]), 0, 0, 1, 1, 3, 2, False), + (Shape([3, 3]), Shape([2, 2]), 0, 0, 1, 1, 2, 3, False), + (Shape([3, 3]), Shape([2, 2]), 0, 0, 1, 1, 3, 3, False), + ] + + for dev in TensorForwardTest.devices: + for tc in test_cases: + x = tF.constant(tc[0], 0, dev) + w = tF.constant(tc[1], 0, dev) + if tc[8]: + try: + tF.conv2d(x, w, tc[2], tc[3], tc[4], tc[5], tc[6], tc[7]) + except RuntimeError as e: + # TODO(vbkaisetsu): + # We have to implement a better method to detect + # NotImplementedError in Python + if "Not implemented" not in str(e): + raise + else: + with self.assertRaises(RuntimeError) as e: + tF.conv2d(x, w, tc[2], tc[3], tc[4], tc[5], tc[6], tc[7]) + + def run_test_max_pool2d(self, x_shape, x_data, y_shape, y_data, win0, win1, pad0, pad1, str0, str1): + for dev in TensorForwardTest.devices: + try: + print(dev) + x = tF.raw_input(x_shape, x_data, dev) + y = tF.max_pool2d(x, win0, win1, pad0, pad1, str0, str1) + self.assertEqual(y_shape, y.shape()) + self.assertTrue(np.isclose(y_data, y.to_list()).all) + except RuntimeError as e: + # TODO(vbkaisetsu): + # We have to implement a better method to detect + # NotImplementedError in Python + if "Not implemented" not in str(e): + raise + + def test_TensorForwardTest_CheckMaxPool2D_1x1x1_1x1(self): + x_data = [123] + y_data = [123] + x_shape = Shape([]) + y_shape = Shape([]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x1x1_1x1(self): + x_data = list(range(1, 5 + 1)) + y_data = [1, 2, 3, 4, 5] + x_shape = Shape([5]) + y_shape = Shape([5]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x1x1_2x1(self): + x_data = list(range(1, 5 + 1)) + y_data = [2, 3, 4, 5] + x_shape = Shape([5]) + y_shape = Shape([4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x1x1_5x1(self): + x_data = list(range(1, 5 + 1)) + y_data = [5] + x_shape = Shape([5]) + y_shape = Shape([]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 5, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_1x5x1_1x1(self): + x_data = list(range(1, 5 + 1)) + y_data = [1, 2, 3, 4, 5] + x_shape = Shape([1, 5]) + y_shape = Shape([1, 5]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_1x5x1_1x2(self): + x_data = list(range(1, 5 + 1)) + y_data = [2, 3, 4, 5] + x_shape = Shape([1, 5]) + y_shape = Shape([1, 4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 2, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_1x5x1_1x5(self): + x_data = list(range(1, 5 + 1)) + y_data = [5] + x_shape = Shape([1, 5]) + y_shape = Shape([]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 5, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_1x1(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([5, 5]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x1(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 2, 3, 4, 5, + 7, 8, 9, 10, + 12, 13, 14, 15, + 17, 18, 19, 20, + 22, 23, 24, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([4, 5]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_5x1(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 5, + 10, + 15, + 20, + 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([1, 5]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 5, 1, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_1x2(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([5, 4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 2, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 7, 8, 9, 10, + 12, 13, 14, 15, + 17, 18, 19, 20, + 22, 23, 24, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([4, 4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_5x2(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 10, + 15, + 20, + 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([1, 4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 5, 2, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_1x5(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 21, 22, 23, 24, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([5]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 1, 5, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x5(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 22, 23, 24, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 5, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_5x5(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [25] + x_shape = Shape([5, 5]) + y_shape = Shape([]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 5, 5, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x3_2x2(self): + x_data = list(range(1, 5 * 5 * 3 + 1)) + y_data = [ + # channel 1 + 7, 8, 9, 10, + 12, 13, 14, 15, + 17, 18, 19, 20, + 22, 23, 24, 25, + # channel 2 + 32, 33, 34, 35, + 37, 38, 39, 40, + 42, 43, 44, 45, + 47, 48, 49, 50, + # channel 3 + 57, 58, 59, 60, + 62, 63, 64, 65, + 67, 68, 69, 70, + 72, 73, 74, 75, + ] + x_shape = Shape([5, 5, 3]) + y_shape = Shape([4, 4, 3]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2_Padding10(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 6, 7, 8, 9, 10, 10, + 11, 12, 13, 14, 15, 15, + 16, 17, 18, 19, 20, 20, + 21, 22, 23, 24, 25, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([6, 4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 1, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2_Padding01(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 2, 3, 4, 5, + 7, 8, 9, 10, + 12, 13, 14, 15, + 17, 18, 19, 20, + 22, 23, 24, 25, + 22, 23, 24, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([4, 6]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 1, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2_Padding11(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 1, 2, 3, 4, 5, 5, + 6, 7, 8, 9, 10, 10, + 11, 12, 13, 14, 15, 15, + 16, 17, 18, 19, 20, 20, + 21, 22, 23, 24, 25, 25, + 21, 22, 23, 24, 25, 25, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([6, 6]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 1, 1, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2_Stride21(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 7, 9, + 12, 14, + 17, 19, + 22, 24, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([2, 4]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 0, 2, 1) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2_Stride12(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 7, 8, 9, 10, + 17, 18, 19, 20, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([4, 2]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 0, 1, 2) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2_Stride22(self): + x_data = list(range(1, 5 * 5 + 1)) + y_data = [ + 7, 9, + 17, 19, + ] + x_shape = Shape([5, 5]) + y_shape = Shape([2, 2]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 0, 2, 2) + + def test_TensorForwardTest_CheckMaxPool2D_5x5x1_2x2_N(self): + x_data = list(range(1, 5 * 5 * 3 + 1)) + y_data = [ + # minibatch 1 + 7, 8, 9, 10, + 12, 13, 14, 15, + 17, 18, 19, 20, + 22, 23, 24, 25, + # minibatch 2 + 32, 33, 34, 35, + 37, 38, 39, 40, + 42, 43, 44, 45, + 47, 48, 49, 50, + # minibatch 3 + 57, 58, 59, 60, + 62, 63, 64, 65, + 67, 68, 69, 70, + 72, 73, 74, 75, + ] + x_shape = Shape([5, 5], 3) + y_shape = Shape([4, 4], 3) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 0, 1, 1) + + def test_TensorForwardTest_CheckMaxPool2D_VGG16ThirdLayer(self): + # NOTE(odashi): 224*224*64 < 2^23 (float precision) + x_data = list(range(1, 224 * 224 * 64 + 1)) + y_data = [0] * (112 * 112 * 64) + for b in range(64): + b_ofs = b * 224 * 224 + for x in range(112): + x_ofs = b_ofs + (2 * x + 1) * 224 + for y in range(112): + y_data[y + b * 112 * 112 + x * 112] = x_ofs + 2 * y + 2 + x_shape = Shape([224, 224, 64]) + y_shape = Shape([112, 112, 64]) + self.run_test_max_pool2d(x_shape, x_data, y_shape, y_data, 2, 2, 0, 0, 2, 2) + + def test_TensorForwardTest_CheckInvalidPool2D(self): + test_cases = [ + # invalid #dimensions + (Shape([1, 1, 1, 2]), 1, 1, 0, 0, 1, 1, False), + # zero-window/stride + (Shape([]), 1, 1, 0, 0, 1, 1, True), + (Shape([]), 0, 1, 0, 0, 1, 1, False), + (Shape([]), 1, 0, 0, 0, 1, 1, False), + (Shape([]), 1, 1, 0, 0, 0, 1, False), + (Shape([]), 1, 1, 0, 0, 1, 0, False), + # sizes mismatching + (Shape([3, 3]), 3, 3, 0, 0, 1, 1, True), + (Shape([3, 3]), 4, 3, 0, 0, 1, 1, False), + (Shape([3, 3]), 3, 4, 0, 0, 1, 1, False), + (Shape([3, 3]), 4, 4, 0, 0, 1, 1, False), + # sizes mismatching with padding + (Shape([3, 3]), 5, 5, 1, 1, 1, 1, True), + (Shape([3, 3]), 6, 5, 1, 1, 1, 1, False), + (Shape([3, 3]), 5, 6, 1, 1, 1, 1, False), + (Shape([3, 3]), 6, 6, 1, 1, 1, 1, False), + # sizes mismatching with stride + (Shape([3, 3]), 3, 3, 0, 0, 2, 2, True), + (Shape([3, 3]), 4, 3, 0, 0, 2, 2, False), + (Shape([3, 3]), 3, 4, 0, 0, 2, 2, False), + (Shape([3, 3]), 4, 4, 0, 0, 2, 2, False), + ] + for dev in TensorForwardTest.devices: + for tc in test_cases: + x = tF.constant(tc[0], 0, dev) + if tc[7]: + try: + tF.max_pool2d(x, tc[1], tc[2], tc[3], tc[4], tc[5], tc[6]) + except RuntimeError as e: + # TODO(vbkaisetsu): + # We have to implement a better method to detect + # NotImplementedError in Python + if "Not implemented" not in str(e): + raise + else: + with self.assertRaises(RuntimeError) as e: + tF.max_pool2d(x, tc[1], tc[2], tc[3], tc[4], tc[5], tc[6]) diff --git a/tests/tensor_functions.py b/tests/tensor_functions.py index 5ca21b4..1a527ec 100644 --- a/tests/tensor_functions.py +++ b/tests/tensor_functions.py @@ -85,8 +85,7 @@ def test_tensor_pow(self): x = tF.input(input_arr) self.assertTrue(((x ** 0x7fffffff).to_ndarrays()[0] == np.array([1, -1])).all()) self.assertTrue(((x ** -0x80000000).to_ndarrays()[0] == np.array([1, 1])).all()) - self.assertTrue(np.isnan((x ** 0x80000000).to_ndarrays()[0]).any()) - self.assertTrue(np.isnan((x ** -0x80000001).to_ndarrays()[0]).any()) + self.assertRaises(TypeError, lambda: pow(x, y, 2)) def test_tensor_iadd(self): diff --git a/tests/tensor_test.py b/tests/tensor_test.py new file mode 100644 index 0000000..39ec405 --- /dev/null +++ b/tests/tensor_test.py @@ -0,0 +1,323 @@ +import random +import unittest + +from primitiv import Shape +from primitiv import Tensor +from primitiv import tensor_functions as tF + +import numpy as np +from . import test_utils + + +class TensorTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.devices = test_utils.available_devices() + + @classmethod + def tearDownClass(cls): + pass + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_TensorTest_CheckInvalid(self): + x = Tensor() + self.assertFalse(x.valid()) + with self.assertRaises(RuntimeError): + x.shape() + with self.assertRaises(RuntimeError): + x.device() + with self.assertRaises(RuntimeError): + x.to_float() + with self.assertRaises(RuntimeError): + x.to_list() + with self.assertRaises(RuntimeError): + x.to_ndarrays() + + def test_TensorTest_CheckNewScalarWithData(self): + for dev in TensorTest.devices: + x = tF.raw_input([], [1], dev) + x_ndarray = [ + np.array([1]), + ] + self.assertTrue(x.valid()) + self.assertIs(dev, x.device()) + self.assertEqual(Shape(), x.shape()) + self.assertEqual([1], x.to_list()) + self.assertEqual(1.0, x.to_float()) + self.assertEqual(x_ndarray, x.to_ndarrays()) + + def test_TensorTest_CheckNewMatrixWithData(self): + for dev in TensorTest.devices: + data = [1, 2, 3, 4, 5, 6] + data_ndarray = [ + np.array([[1, 3, 5], [2, 4, 6]]), + ] + x = tF.raw_input([2, 3], data, dev) + self.assertTrue(x.valid()) + self.assertIs(dev, x.device()) + self.assertEqual(Shape([2, 3]), x.shape()) + self.assertEqual(data, x.to_list()) + with self.assertRaises(RuntimeError): + x.to_float() + self.assertTrue(np.array_equal(data_ndarray, x.to_ndarrays())) + + def test_TensorTest_CheckNewMatrixMinibatchWithData(self): + for dev in TensorTest.devices: + data = [ + 3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, + 9, 7, 9, 3, 2, 3, 8, 4, 6, 2, 6, 4, + ] + data_ndarray = [ + np.array([[3, 4, 5], [1, 1, 9]]), + np.array([[2, 5, 5], [6, 3, 8]]), + np.array([[9, 9, 2], [7, 3, 3]]), + np.array([[8, 6, 6], [4, 2, 4]]), + ] + x = tF.raw_input(Shape([2, 3], 4), data, dev) + self.assertTrue(x.valid()) + self.assertIs(dev, x.device()) + self.assertEqual(Shape([2, 3], 4), x.shape()) + self.assertEqual(data, x.to_list()) + with self.assertRaises(RuntimeError): + x.to_float() + self.assertTrue(np.array_equal(data_ndarray, x.to_ndarrays())) + + def test_TensorTest_CheckCopyValidToNew(self): + for dev in TensorTest.devices: + print(dev) + tmp = tF.raw_input(Shape([2], 3), [1, 2, 3, 4, 5, 6], dev) + x = Tensor(tmp) + self.assertTrue(x.valid()) + self.assertTrue(tmp.valid()) + self.assertEqual(Shape([2], 3), x.shape()) + self.assertEqual(Shape([2], 3), tmp.shape()) + self.assertEqual([1, 2, 3, 4, 5, 6], x.to_list()) + self.assertEqual([1, 2, 3, 4, 5, 6], tmp.to_list()) + + def test_TensorTest_CheckCopyInvalidToNew(self): + for dev in TensorTest.devices: + tmp = Tensor() + x = Tensor(tmp) + self.assertFalse(x.valid()) + self.assertFalse(tmp.valid()) + + def test_TesnorTest_CheckResetValuesByConstant(self): + for dev in TensorTest.devices: + x = tF.raw_input(Shape([2, 2], 2), [42] * 8, dev) + self.assertEqual([42] * 8, x.to_list()) + + x = tF.raw_input(Shape([2, 2], 2), [0] * 8, dev) + x.reset(42) + self.assertEqual([42] * 8, x.to_list()) + + x = tF.raw_input(Shape([2, 2], 2), [123] * 8, dev) + copied = Tensor(x) + + x.reset(42) + self.assertEqual([42] * 8, x.to_list()) + self.assertEqual([123] * 8, copied.to_list()) + + def test_TensorTest_CheckResetValuesByVector(self): + for dev in TensorTest.devices: + data = [1, 2, 3, 4, 5, 6, 7, 8] + x = tF.raw_input(Shape([2, 2], 2), data, dev) + self.assertEqual(data, x.to_list()) + + data = [1, 2, 3, 4, 5, 6, 7, 8] + x = tF.raw_input(Shape([2, 2], 2), [0] * 8, dev) + x.reset_by_vector(data) + self.assertEqual(data, x.to_list()) + + data = [1, 2, 3, 4, 5, 6, 7, 8] + x = tF.raw_input(Shape([2, 2], 2), [123] * 8, dev) + copied = Tensor(x) + + x.reset_by_vector(data) + self.assertEqual(data, x.to_list()) + self.assertEqual([123] * 8, copied.to_list()) + + def test_TensorTest_InplaceMultiplyConst(self): + for dev in TensorTest.devices: + x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + y_data = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24] + x = tF.raw_input(Shape([2, 2], 3), x_data, dev) + x *= 2 + self.assertEqual(y_data, x.to_list()) + for dev in TensorTest.devices: + x_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + y_data = [.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5, 6] + x = tF.raw_input(Shape([2, 2], 3), x_data, dev) + x *= .5 + self.assertEqual(y_data, x.to_list()) + + def test_TensorTest_CheckInplaceAddNN(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + b_data = [0, -1, -2, -3, -3, -4, -5, -6, -6, -7, -8, -9] + y_data = [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3] + a = tF.raw_input(Shape([2, 2], 3), a_data, dev) + b = tF.raw_input(Shape([2, 2], 3), b_data, dev) + a += b + self.assertEqual(y_data, a.to_list()) + + def test_TensorTest_CheckInplaceAdd1N(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4] + b_data = [0, -1, -2, -3, -3, -4, -5, -6, -6, -7, -8, -9] + y_data = [-8, -10, -12, -14] + a = tF.raw_input([2, 2], a_data, dev) + b = tF.raw_input(Shape([2, 2], 3), b_data, dev) + a += b + self.assertEqual(y_data, a.to_list()) + + def test_TensorTest_CheckInplaceAddN1(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + b_data = [0, -1, -2, -3] + y_data = [1, 1, 1, 1, 5, 5, 5, 5, 9, 9, 9, 9] + a = tF.raw_input(Shape([2, 2], 3), a_data, dev) + b = tF.raw_input([2, 2], b_data, dev) + a += b + self.assertEqual(y_data, a.to_list()) + + def test_TensorTest_CheckInplaceAdd(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + b_data = [0, -1, -2, -3, -3, -4, -5, -6, -6, -7, -8, -9] + y_data = [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3] + a = tF.raw_input(Shape([2, 2], 3), a_data, dev) + b = tF.raw_input(Shape([2, 2], 3), b_data, dev) + + copied = Tensor(a) + ref_a = a + + a += b + self.assertEqual(y_data, a.to_list()) + + self.assertEqual(y_data, a.to_list()); + self.assertEqual(a_data, copied.to_list()); + self.assertIs(ref_a, a) + + def test_TensorTest_CheckInplaceSubtractNN(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + b_data = [0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9] + y_data = [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3] + a = tF.raw_input(Shape([2, 2], 3), a_data, dev) + b = tF.raw_input(Shape([2, 2], 3), b_data, dev) + a -= b + self.assertEqual(y_data, a.to_list()) + + def test_TensorTest_CheckInplaceSubtract1N(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4] + b_data = [0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9] + y_data = [-8, -10, -12, -14] + a = tF.raw_input([2, 2], a_data, dev) + b = tF.raw_input(Shape([2, 2], 3), b_data, dev) + a -= b + self.assertEqual(y_data, a.to_list()) + + def test_TensorTest_CheckInplaceSubtractN1(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + b_data = [0, 1, 2, 3] + y_data = [1, 1, 1, 1, 5, 5, 5, 5, 9, 9, 9, 9] + a = tF.raw_input(Shape([2, 2], 3), a_data, dev) + b = tF.raw_input([2, 2], b_data, dev) + a -= b + self.assertEqual(y_data, a.to_list()) + + def test_TensorTest_CheckInplaceSubtract(self): + for dev in TensorTest.devices: + a_data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + b_data = [0, 1, 2, 3, 3, 4, 5, 6, 6, 7, 8, 9] + y_data = [1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3] + a = tF.raw_input(Shape([2, 2], 3), a_data, dev) + b = tF.raw_input(Shape([2, 2], 3), b_data, dev) + + copied = Tensor(a) + ref_a = a + + a -= b + self.assertEqual(y_data, a.to_list()) + + self.assertEqual(y_data, a.to_list()); + self.assertEqual(a_data, copied.to_list()); + self.assertIs(ref_a, a) + + def test_TensorTest_CheckInvalidInplaceOps(self): + for dev in TensorTest.devices: + shapes = [ + Shape(), + Shape([], 3), + Shape([2, 2], 2), + ] + a = tF.raw_input(Shape([2, 2], 3), [0] * 12, dev) + + for shape in shapes: + b = tF.raw_input(shape, [0] * shape.size(), dev) + with self.assertRaises(RuntimeError): + a += b + with self.assertRaises(RuntimeError): + a -= b + + def test_TensorTest_CheckArgMaxDims(self): + data = [ + 0, 1, 2, 6, 7, 8, 3, 4, 5, -3, -4, -5, 0, -1, -2, -6, -7, -8, + ] + expected = [ + [2, 2, 2, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ] + for dev in TensorTest.devices: + a = tF.raw_input(Shape([3, 3], 2), data, dev) + for i, exp in enumerate(expected): + self.assertEqual(exp, a.argmax(i)) + + def test_TensorTest_CheckArgMaxLarge(self): + ns = [ + 1, 2, 3, 15, 16, 17, 255, 256, 257, 1023, 1024, + 1025, 65535, 65536, 65537, + ] + for dev in TensorTest.devices: + for n in ns: + data = list(range(n)) + random.shuffle(data) + pos = data.index(n - 1) + a = tF.raw_input([n], data, dev) + self.assertEqual(pos, a.argmax(0)[0]) + + def test_TensorTest_CheckArgMinDims(self): + data = [ + 3, 4, 5, 0, 1, 2, 6, 7, 8, 0, -1, -2, -6, -7, -8, -3, -4, -5, + ] + expected = [ + [0, 0, 0, 2, 2, 2], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ] + for dev in TensorTest.devices: + a = tF.raw_input(Shape([3, 3], 2), data, dev) + for i, exp in enumerate(expected): + self.assertEqual(exp, a.argmin(i)) + + def test_TensorTest_CheckArgMinLarge(self): + ns = [ + 1, 2, 3, 15, 16, 17, 255, 256, 257, 1023, 1024, + 1025, 65535, 65536, 65537, + ] + for dev in TensorTest.devices: + for n in ns: + data = list(range(n)) + random.shuffle(data) + pos = data.index(0) + a = tF.raw_input([n], data, dev) + self.assertEqual(pos, a.argmin(0)[0]) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..df689d4 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,74 @@ +import sys + + +def available_devices(): + devices = [] + devices.extend(available_naive_devices()) + devices.extend(available_cuda_devices()) + devices.extend(available_eigen_devices()) + devices.extend(available_opencl_devices()) + for dev in devices: + dev.dump_description() + return devices + + +def available_naive_devices(): + from primitiv.devices import Naive + return [ + Naive(), + Naive(), + ] + + +def available_cuda_devices(): + try: + from primitiv.devices import CUDA + except ImportError: + return [] + devices = [] + num_devs = CUDA.num_devices() + num_avail_devs = 0 + for dev_id in range(num_devs): + if CUDA.check_support(dev_id): + devices.append(CUDA(dev_id)) + num_avail_devs += 1 + if len(devices) == 1: + devices.append(CUDA(dev_id)) + if num_devs - num_avail_devs: + print("%d CUDA device(s) are not supported." + % (num_devs - num_avail_devs), file=sys.stderr) + return devices + + +def available_eigen_devices(): + try: + from primitiv.devices import Eigen + except ImportError: + return [] + return [ + Eigen(), + Eigen(), + ] + + +def available_opencl_devices(): + try: + from primitiv.devices import OpenCL + except ImportError: + return [] + devices = [] + num_pfs = OpenCL.num_platforms(); + num_avail_devs = 0 + num_devs = 0 + for pf_id in range(num_pfs): + num_devs += OpenCL.num_devices(pf_id) + for dev_id in range(num_devs): + if OpenCL.check_support(pf_id, dev_id): + devices.append(OpenCL(pf_id, dev_id)) + num_avail_devs += 1 + if len(devices) == 1: + devices.append(OpenCL(pf_id, dev_id)) + if num_devs != num_avail_devs: + print("%d OpenCL device(s) are not supported." + % (num_devs - num_avail_devs), file=sys.stderr) + return devices