cherry pick: 3.0.2 (#4082)

* Support the ARM platform of docker image (#3939)

* fix version in docker image and Revert 4068 (#4080)

* Revert "enhance docker build, using cache"

This reverts commit 9e2bec3.

* fix docker version

Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>

* Revert pr 3942 (#4075)

Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>

Co-authored-by: Yee <2520865+yixinglu@users.noreply.github.com>
Co-authored-by: Harris.Chu <1726587+HarrisChu@users.noreply.github.com>
Co-authored-by: Yichen Wang <18348405+Aiee@users.noreply.github.com>

Author: 4 people

.github/workflows/nightly.yml

@@ -82,8 +82,7 @@ jobs:
         with:
           context: .
           file: ./docker/Dockerfile.${{ matrix.service }}
-          # platforms: linux/amd64,linux/arm64
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           tags: |
             vesoft/nebula-${{ matrix.service }}:nightly
           push: true

.github/workflows/rc.yml

@@ -99,15 +99,15 @@ jobs:
         with:
           context: .
           file: ./docker/Dockerfile.${{ matrix.service }}
-          # platforms: linux/amd64,linux/arm64
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           tags: |
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
             ${{ steps.docker.outputs.tag }}
           push: true
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}
+            VERSON=${{ steps.tagname.outputs.tagnum }}
 
   test:
     name: test

docker/Dockerfile

@@ -1,6 +1,7 @@
 FROM vesoft/nebula-dev:centos7 as builder
 
 ARG BRANCH=master
+ARG VERSION=
 
 COPY . /home/nebula/BUILD
 

docker/Dockerfile.graphd

@@ -1,11 +1,14 @@
 FROM vesoft/nebula-dev:centos7 as builder
 
-COPY . /home/nebula/BUILD
-
 ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
 
 FROM centos:7
 

docker/Dockerfile.metad

@@ -1,11 +1,14 @@
 FROM vesoft/nebula-dev:centos7 as builder
 
 ARG BRANCH=master
+ARG VERSION=
 
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
 
 FROM centos:7
 

docker/Dockerfile.storaged

@@ -1,11 +1,14 @@
 FROM vesoft/nebula-dev:centos7 as builder
 
 ARG BRANCH=master
+ARG VERSION=
 
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
 
 FROM centos:7
 

docker/Dockerfile.tools

@@ -1,11 +1,14 @@
 FROM vesoft/nebula-dev:centos7 as builder
 
 ARG BRANCH=master
+ARG VERSION=
 
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \
-  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE
+  && [[ -z "${VERSION}" ]] \
+  && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+  || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
 
 FROM centos:7
 

src/clients/meta/MetaClient.cpp

@@ -2601,11 +2601,6 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
     }
   }
 
-  // TTL for clientAddrMap
-  // If multiple connections are created but do not authenticate, the clientAddrMap_ will keep
-  // growing. This is to clear the clientAddrMap_ regularly.
-  clearClientAddrMap();
-
   // info used in the agent, only set once
   // TOOD(spw): if we could add data path(disk) dynamicly in the future, it should be
   // reported every time it changes
@@ -3802,20 +3797,5 @@ Status MetaClient::verifyVersion() {
   return Status::OK();
 }
 
-void MetaClient::clearClientAddrMap() {
-  if (clientAddrMap_.size() == 0) {
-    return;
-  }
-
-  auto curTimestamp = time::WallClock::fastNowInSec();
-  for (auto it = clientAddrMap_.cbegin(); it != clientAddrMap_.cend();) {
-    // The clientAddr is expired
-    if (it->second < curTimestamp) {
-      it = clientAddrMap_.erase(it);
-    } else {
-      ++it;
-    }
-  }
-}
 }  // namespace meta
 }  // namespace nebula

src/clients/meta/MetaClient.h

@@ -156,7 +156,6 @@ using FTIndexMap = std::unordered_map<std::string, cpp2::FTIndex>;
 
 using SessionMap = std::unordered_map<SessionID, cpp2::Session>;
 
-using clientAddrMap = folly::ConcurrentHashMap<HostAddr, int64_t>;
 class MetaChangedListener {
  public:
   virtual ~MetaChangedListener() = default;
@@ -671,10 +670,6 @@ class MetaClient {
     return options_.localHost_.toString();
   }
 
-  clientAddrMap& getClientAddrMap() {
-    return clientAddrMap_;
-  }
-
  protected:
   // Return true if load succeeded.
   bool loadData();
@@ -761,9 +756,6 @@ class MetaClient {
 
   Status verifyVersion();
 
-  // Removes expired keys in the clientAddrMap_
-  void clearClientAddrMap();
-
  private:
   std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool_;
   std::shared_ptr<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>> clientsMan_;
@@ -844,18 +836,6 @@ class MetaClient {
   NameIndexMap tagNameIndexMap_;
   NameIndexMap edgeNameIndexMap_;
 
-  // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
-  // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
-  //
-  // The key here is the host of the client that sends the request, and the value indicates the
-  // expiration of the key because we don't want to keep the key forever.
-  //
-  // The assumption here is that there is ONLY ONE VERSION of the client in the host.
-  //
-  // This map will be updated when verifyVersion() is called. Only the clients since v2.6.0 will
-  // call verifyVersion(), thus we could determine whether the client version is lower than v2.6.0
-  clientAddrMap clientAddrMap_;
-
   // Global service client
   ServiceClientsList serviceClientList_;
   FTIndexMap fulltextIndexMap_;

src/graph/service/GraphService.cpp

@@ -24,9 +24,6 @@
 namespace nebula {
 namespace graph {
 
-// The default value is 28800 seconds
-const int64_t clientAddrTimeout = FLAGS_client_idle_timeout_secs;
-
 Status GraphService::init(std::shared_ptr<folly::IOThreadPoolExecutor> ioExecutor,
                           const HostAddr& hostAddr) {
   auto addrs = network::NetworkUtils::toHosts(FLAGS_meta_server_addrs);
@@ -72,10 +69,9 @@ folly::Future<AuthResponse> GraphService::future_authenticate(const std::string&
 
   auto ctx = std::make_unique<RequestContext<AuthResponse>>();
   auto future = ctx->future();
-  // Check username and password failed
-  // Check whether the client has called verifyClientVersion()
-  auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());
-  auto authResult = auth(username, password, clientAddr);
+
+  // check username and password failed
+  auto authResult = auth(username, password);
   if (!authResult.ok()) {
     ctx->resp().errorCode = ErrorCode::E_BAD_USERNAME_PASSWORD;
     ctx->resp().errorMsg.reset(new std::string(authResult.toString()));
@@ -207,24 +203,8 @@ folly::Future<std::string> GraphService::future_executeJsonWithParameter(
   });
 }
 
-Status GraphService::auth(const std::string& username,
-                          const std::string& password,
-                          const HostAddr& clientIp) {
+Status GraphService::auth(const std::string& username, const std::string& password) {
   auto metaClient = queryEngine_->metaClient();
-
-  // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
-  // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
-  //
-  // Only the clients since v2.6.0 will call verifyVersion(), thus we could determine whether the
-  // client version is lower than v2.6.0
-  auto clientAddrIt = metaClient->getClientAddrMap().find(clientIp);
-  if (clientAddrIt == metaClient->getClientAddrMap().end()) {
-    return Status::Error(
-        folly::sformat("The version of the client sending request from {} is lower than v2.6.0, "
-                       "please update the client.",
-                       clientIp.toString()));
-  }
-
   // Skip authentication if FLAGS_enable_authorize is false
   if (!FLAGS_enable_authorize) {
     return Status::OK();
@@ -266,13 +246,6 @@ folly::Future<cpp2::VerifyClientVersionResp> GraphService::future_verifyClientVe
     resp.error_code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED;
   }
 
-  // The client sent request has a version >= v2.6.0, mark the address as valid
-  auto* peer = getRequestContext()->getPeerAddress();
-  auto clientAddr = HostAddr(peer->getAddressStr(), peer->getPort());
-
-  auto ttlTimestamp = time::WallClock::fastNowInSec() + clientAddrTimeout;
-  auto clientAddrMap = &metaClient_->getClientAddrMap();
-  clientAddrMap->insert_or_assign(clientAddr, ttlTimestamp);
   return folly::makeFuture<cpp2::VerifyClientVersionResp>(std::move(resp));
 }
 }  // namespace graph

src/graph/service/GraphService.h

@@ -54,7 +54,7 @@ class GraphService final : public cpp2::GraphServiceSvIf {
   std::unique_ptr<meta::MetaClient> metaClient_;
 
  private:
-  Status auth(const std::string& username, const std::string& password, const HostAddr& clientIp);
+  Status auth(const std::string& username, const std::string& password);
 
   std::unique_ptr<GraphSessionManager> sessionManager_;
   std::unique_ptr<QueryEngine> queryEngine_;