Skip to content

[IPU] update to sdk2.6.0 #44673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 29, 2022
Merged

[IPU] update to sdk2.6.0 #44673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e9c4110
update to sdk2.6.0
gglin001 Jul 27, 2022
9025c79
Merge branch 'develop' into ipu_commit/sdk260
gglin001 Jul 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions paddle/fluid/platform/device/ipu/ipu_compiler.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
#include <popart/adaptive.hpp>
#include <popart/optimizer.hpp>
#include <popart/sgd.hpp>

#include "paddle/utils/blank.h"
#include <popart/voiddata.hpp>

#include "paddle/fluid/framework/ir/graph_helper.h"
#include "paddle/fluid/platform/device/ipu/ipu_names.h"
#include "paddle/fluid/platform/device/ipu/ipu_strategy.h"
#include "paddle/fluid/platform/device/ipu/ipu_utils.h"
#include "paddle/utils/blank.h"

namespace paddle {
namespace platform {
Expand Down
16 changes: 6 additions & 10 deletions paddle/fluid/platform/device/ipu/ipu_executor.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ limitations under the License. */

#include "paddle/fluid/platform/device/ipu/ipu_executor.h"

#include <popart/devicemanager.hpp>
#include <popdist/popdist_poplar.hpp>

#include "paddle/fluid/framework/operator.h"
#include "paddle/fluid/platform/device/ipu/ipu_compiler.h"
#include "paddle/fluid/platform/device/ipu/ipu_names.h"
Expand Down Expand Up @@ -204,9 +207,7 @@ void Executor::Run(const std::vector<const Tensor *> &inputs,
}
VLOG(10) << "Prepared inputs/anchors";

if (ipu_strategy_->is_training && compiler_resources_->with_lr_sched &&
!(ipu_strategy_->popart_options.createImplicitPipeliningFwdOnlyProgram &&
ipu_strategy_->runtime_options.enable_eval)) {
if (ipu_strategy_->is_training && compiler_resources_->with_lr_sched) {
popart::Optimizer *optimizer;
if (ipu_strategy_->runtime_options.enable_eval) {
VLOG(10) << "Switch optimizer to eval mode";
Expand All @@ -229,12 +230,7 @@ void Executor::Run(const std::vector<const Tensor *> &inputs,

popart::StepIO stepio(popart_inputs, popart_anchors);
VLOG(10) << "Running...";
if (ipu_strategy_->popart_options.createImplicitPipeliningFwdOnlyProgram &&
ipu_strategy_->runtime_options.enable_eval) {
session_->run("implicitPipeliningFwdOnly", stepio);
} else {
session_->run(stepio);
}
session_->run(stepio);
VLOG(10) << "Running...done";
}

Expand Down Expand Up @@ -286,7 +282,7 @@ void Executor::AcquireDevice() {
VLOG(10) << "Create distribution device...";
auto ipus_per_replica = ipu_strategy_->num_ipus /
ipu_strategy_->popart_options.replicatedGraphCount;
auto device_id = popdist_get_device(ipus_per_replica);
auto device_id = popdist::getDeviceId(ipus_per_replica);
device_ = popart::DeviceManager::createDeviceManager().acquireDeviceById(
device_id);
PADDLE_ENFORCE_NOT_NULL(
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/platform/device/ipu/ipu_executor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ limitations under the License. */
#include <popart/names.hpp>
#include <popart/patterns/patterns.hpp>
#include <popart/session.hpp>
#include <popart/stepio.hpp>
#include <popart/tensorinfo.hpp>
#include <popdist/popdist_poplar.hpp>

#include "paddle/fluid/platform/device/ipu/ipu_utils.h"

Expand Down
2 changes: 0 additions & 2 deletions paddle/fluid/platform/device/ipu/ipu_strategy.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,6 @@ IpuStrategy::IpuStrategy() {
ADD_POPART_BOOL_OPTION_ALIAS(
schedule_non_weight_update_gradient_consumers_early,
scheduleNonWeightUpdateGradientConsumersEarly);
ADD_POPART_BOOL_OPTION_ALIAS(create_implicit_pipelining_fwd_only_program,
createImplicitPipeliningFwdOnlyProgram);

ADD_POPART_DOUBLE_OPTION_ALIAS(outline_sequence_break_cost,
outlineSequenceBreakCost);
Expand Down
6 changes: 3 additions & 3 deletions paddle/fluid/platform/device/ipu/supported_ops_autogen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,12 @@ OP_DECL(popart_dynamicupdate_v2, aiGraphcoreOpset.dynamicupdate, ARG(INT_VEC,axe
OP_DECL(popart_dynamiczero_v2, aiGraphcoreOpset.dynamiczero, ARG(INT_VEC,axes) ARG(INT_VEC,sizes) ) // NOLINT
OP_DECL(popart_dynamicadd_v2, aiGraphcoreOpset.dynamicadd, ARG(INT_VEC,axes) ARG(INT_VEC,sizes) ) // NOLINT
OP_DECL(popart_sequenceslice_v2, aiGraphcoreOpset.sequenceslice, ARG(INT,zeroUnused) ) // NOLINT
OP_DECL(popart_replicatedallreduce_v2, aiGraphcoreOpset.replicatedallreduce, OPT_ARG(INT_VEC,commGroup) ) // NOLINT
OP_DECL(popart_l1loss_v2, aiGraphcoreOpset.l1loss, ARG(FLOAT,lambda) SIG_ARG(INT32,popart::ReductionType,reduction) ) // NOLINT
OP_DECL(popart_nllloss_v2, aiGraphcoreOpset.nllloss, SIG_ARG(INT32,popart::ReductionType,reduction) OPT_ARG(INT32,ignoreIndex) ARG(BOOL,inputIsLogProbability) ) // NOLINT
OP_DECL(popart_identityloss_v2, aiGraphcoreOpset.identityloss, SIG_ARG(INT32,popart::ReductionType,reduction) ) // NOLINT
OP_DECL(popart_tensorremap_v2, aiGraphcoreOpset.tensorremap, ARG(INT,remap_type) ) // NOLINT
OP_DECL(popart_ctcloss_v2, aiGraphcoreOpset.ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ) // NOLINT
OP_DECL(popart__ctcloss_v2, aiGraphcoreOpset._ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ) // NOLINT
OP_DECL(popart_ctcloss_v2, aiGraphcoreOpset.ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ARG(BOOL,zeroInfinity) ) // NOLINT
OP_DECL(popart__ctcloss_v2, aiGraphcoreOpset._ctcloss, SIG_ARG(INT32,popart::ReductionType,reduction) ARG(INT,blank) ARG(STRING,outDataType) ARG(BOOL,zeroInfinity) ) // NOLINT
OP_DECL(popart_ctcbeamsearchdecoder_v2, aiGraphcoreOpset.ctcbeamsearchdecoder, ARG(INT,blank) ARG(INT,beamWidth) ARG(INT,topPaths) ) // NOLINT
OP_DECL(popart_shapeddropout_v2, aiGraphcoreOpset.shapeddropout, ARG(INT_VEC,shape) ARG(FLOAT,ratio) ) // NOLINT
OP_DECL(popart_atan2_v2, aiGraphcoreOpset.atan2, NONE) // NOLINT
Expand All @@ -48,6 +47,7 @@ OP_DECL(popart_log1p_v2, aiGraphcoreOpset.log1p, NONE) // NOLINT
OP_DECL(popart_fmod_v2, aiGraphcoreOpset.fmod, NONE) // NOLINT
OP_DECL(popart_remainder_v2, aiGraphcoreOpset.remainder, NONE) // NOLINT
OP_DECL(popart_reverse_v2, aiGraphcoreOpset.reverse, ARG(INT_VEC,dimensions) ) // NOLINT
OP_DECL(popart_slice_v2, aiGraphcoreOpset.slice, ARG(INT_VEC,ends) ARG(INT_VEC,starts) ARG(INT_VEC,axes) ) // NOLINT
OP_DECL(popart_bitwisenot_v2, aiGraphcoreOpset.bitwisenot, NONE) // NOLINT
OP_DECL(popart_bitwiseand_v2, aiGraphcoreOpset.bitwiseand, NONE) // NOLINT
OP_DECL(popart_bitwiseor_v2, aiGraphcoreOpset.bitwiseor, NONE) // NOLINT
Expand Down
1 change: 1 addition & 0 deletions python/paddle/fluid/tests/unittests/ipu/custom_ops/leaky_relu_ipu.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

#include <popart/opmanager.hpp>
#include <popart/opserialiser.hpp>
#include <popart/popx/opx.hpp>
#include <popart/popx/opxmanager.hpp>
#include <popart/shapeinference.hpp>

Expand Down
38 changes: 15 additions & 23 deletions tools/dockerfile/Dockerfile.ipu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,37 +6,29 @@
# run a container
# docker run --ulimit memlock=-1:-1 --net=host --cap-add=IPC_LOCK --device=/dev/infiniband/ --ipc=host --rm -it paddlepaddle/paddle:latest-dev-ipu bash

FROM graphcore/poplar:2.5.1
FROM graphcore/poplar:2.6.0
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>

# ENV variables
ARG WITH_AVX
ENV HOME /root
ENV WITH_AVX=${WITH_AVX:-ON}

# Add bash enhancements
COPY paddle/scripts/docker/root/ /root/
# install pkgs
RUN apt-get update && apt-get install -y apt-utils
RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test
RUN apt-get update && apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip make libgcc-s1 sudo openssh-server \
coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev \
bison graphviz libjpeg-dev zlib1g zlib1g-dev automake locales swig net-tools libtool module-init-tools numactl libnuma-dev \
openssl libffi-dev pciutils libblas-dev gfortran libblas3 liblapack-dev liblapack3 default-jre screen tmux gdb lldb gcc g++
coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev \
bison graphviz libjpeg-dev zlib1g zlib1g-dev automake locales swig net-tools libtool module-init-tools numactl libnuma-dev \
openssl libffi-dev pciutils libblas-dev gfortran libblas3 liblapack-dev liblapack3 default-jre screen tmux gdb lldb gcc g++
RUN apt-get update && apt-get install -y rdma-core librdmacm1
RUN apt-get update && apt-get install libspdlog-dev

# Downgrade gcc&&g++
WORKDIR /usr/bin
COPY tools/dockerfile/build_scripts /build_scripts
RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts
RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++
RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc
RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++
ENV PATH=/usr/local/gcc-8.2/bin:$PATH

# install g++-8
RUN apt install g++-8 gcc-8 -y
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里使用 apt install g++-8, 对于 ubuntu18.04 来说, 目前是 8.4, 之前手动编译的版本是 8.2

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已经在新docker内编译ipu版本paddle并验证可以在物理卡上运行吗？
当前的源码改动是否影响在之前2.5.1的镜像的使用？

Copy link
Contributor Author

@gglin001 gglin001 Jul 29, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

已经在新docker内编译ipu版本paddle并验证可以在物理卡上运行吗？

yes

当前的源码改动是否影响在之前2.5.1的镜像的使用？

不支持 sdk2.5.1 以及以前的 sdk, 我们在 sdk 层面不打算向下兼容, 对于使用 ipu 的 paddle (python)代码应该是基本没有影响的

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

对于使用 gcc8.2 或者 gcc8.4 编译 paddle, 我的观察是没有差别的, 可以保证兼容性

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok，paddle官方目前编译器版本是8.2，8.4还没有验证，这点请知悉

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

嗯, 因为目前使用 ipu 版本的paddle 也只支持在特定的 docker 内使用, 我觉得影响是不大的, 这里之所以没有继续使用 gcc8.2, 是因为编译过程很慢/麻烦, 似乎也是没有必要的

RUN ln -sf /usr/bin/gcc-8 /usr/bin/gcc
RUN ln -sf /usr/bin/g++-8 /usr/bin/g++

# install cmake
WORKDIR /home
Expand All @@ -50,6 +42,7 @@ RUN cd /opt && wget -q https://repo.anaconda.com/miniconda/${CONDA_FILE} && chmo
RUN mkdir /opt/conda && ./${CONDA_FILE} -b -f -p "/opt/conda" && rm -rf ${CONDA_FILE}
ENV PATH=/opt/conda/bin:${PATH}
RUN conda init bash && conda install -n base jupyter jupyterlab
RUN conda install -n base spdlog==1.8.0 -y

# Install Go and glide
RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \
Expand Down Expand Up @@ -93,10 +86,9 @@ RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
make -j8 && make install && \
ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache

# clang-form 3.8.0
RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \
cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
# # clang-form 3.8.0
# RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
# tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \
# cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz

RUN apt-get clean -y
EXPOSE 22